ydbd_slice: add support for using raw config.yaml via --yaml-config and dynconfig.yaml via --yaml-dynconfig

- Add yaml_configurator instead of configurator
- Add warning for old style cluster.yaml
- Add dynconfig-generator for generating simple dynconfig.yaml
- Add --save-raw-cfg option for saving raw config from old cluster.yaml

Author: melazyk

ydb/tools/cfg/base.py

@@ -622,6 +622,10 @@ def domains(self):
             )
         return domains
 
+    @domains.setter
+    def domains(self, values):
+        self.__cluster_description["domains"] = values
+
     @property
     def domains_config(self):
         domains_config_dict = self.__cluster_description.get("domains_config", {})

ydb/tools/ydbd_slice/__init__.py

@@ -13,10 +13,10 @@
 from urllib3.exceptions import HTTPWarning
 
 from ydb.tools.cfg.walle import NopHostsInformationProvider
-from ydb.tools.ydbd_slice import nodes, handlers, cluster_description
+from ydb.tools.ydbd_slice import nodes, handlers, cluster_description, yaml_configurator
 from ydb.tools.ydbd_slice.kube import handlers as kube_handlers, docker
 
-warnings.filterwarnings("ignore", category=DeprecationWarning)
+# warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=HTTPWarning)
 
 
@@ -40,6 +40,9 @@
 \033[95msample-config\033[94m - get sample configuration for cluster:
     %(prog)s sample-config --cluster-type=block-4-2-8-nodes --output-file=cluster.yaml
 
+\033[95mdynconfig-generator\033[94m - generate simple dynamic configuration for cluster:
+    %(prog)s dynconfig-generator --yaml-config=config.yaml --output-file=dynconfig.yaml
+
 \033[95minstall\033[94m - full install process from scratch:
     %(prog)s install cluster.yaml --arcadia
 
@@ -254,9 +257,9 @@ def handler(signum, frame):
         raise Terminate(signum, frame)
 
 
-def safe_load_cluster_details(cluster_yaml, walle_provider):
+def safe_load_cluster_details(cluster_yaml, walle_provider, validator=None):
     try:
-        cluster_details = cluster_description.ClusterDetails(cluster_yaml, walle_provider)
+        cluster_details = cluster_description.ClusterDetails(cluster_yaml, walle_provider, validator=validator)
     except IOError as io_err:
         print('', file=sys.stderr)
         print("unable to open YAML params as a file, check args", file=sys.stderr)
@@ -310,8 +313,7 @@ def deduce_components_from_args(args, cluster_details):
     return result
 
 
-def deduce_nodes_from_args(args, walle_provider, ssh_user, ssh_key_path):
-    cluster_hosts = safe_load_cluster_details(args.cluster, walle_provider).hosts_names
+def deduce_nodes_from_args(args, cluster_hosts, ssh_user, ssh_key_path):
     result = cluster_hosts
 
     if args.nodes is not None:
@@ -547,6 +549,26 @@ def databases_config_path_args():
     return args
 
 
+def yaml_config_path_args():
+    args = argparse.ArgumentParser(add_help=False)
+    args.add_argument(
+        "--yaml-config",
+        metavar="YAML_CONFIG",
+        default="",
+        required=False,
+        help="Path to file with config.yaml configuration",
+    )
+    args.add_argument(
+        "--yaml-dynconfig",
+        metavar="YAML_DYNCONFIG",
+        default="",
+        required=False,
+        help="Path to file with dynconfig.yaml configuration",
+    )
+
+    return args
+
+
 def cluster_type_args():
     args = argparse.ArgumentParser(add_help=False)
     available_erasure_types = [
@@ -628,38 +650,65 @@ def dispatch_run(func, args, walle_provider, need_confirmation=False):
 
     logger.debug("run func '%s' with cmd args is '%s'", func.__name__, args)
 
-    cluster_details = safe_load_cluster_details(args.cluster, walle_provider)
-    components = deduce_components_from_args(args, cluster_details)
-
-    nodes = deduce_nodes_from_args(args, walle_provider, args.ssh_user, args.ssh_key_path)
-
     temp_dir = deduce_temp_dir_from_args(args)
+    kikimr_bin, kikimr_compressed_bin = deduce_kikimr_bin_from_args(args)
     clear_tmp = not args.dry_run and args.temp_dir is None
 
-    kikimr_bin, kikimr_compressed_bin = deduce_kikimr_bin_from_args(args)
+    if args.yaml_config and args.yaml_dynconfig:
+        configurator = yaml_configurator.YamlConfigurator(
+            args.cluster,
+            temp_dir,
+            kikimr_bin,
+            kikimr_compressed_bin,
+            args.yaml_config,
+            args.yaml_dynconfig
+        )
+        cluster_details = configurator.cluster_description
+    else:
+        cluster_details = safe_load_cluster_details(args.cluster, walle_provider)
+        configurator = cluster_description.Configurator(
+            cluster_details,
+            out_dir=temp_dir,
+            kikimr_bin=kikimr_bin,
+            kikimr_compressed_bin=kikimr_compressed_bin,
+            walle_provider=walle_provider
+        )
 
-    configurator = cluster_description.Configurator(
-        cluster_details,
-        out_dir=temp_dir,
-        kikimr_bin=kikimr_bin,
-        kikimr_compressed_bin=kikimr_compressed_bin,
-        walle_provider=walle_provider
-    )
+    components = deduce_components_from_args(args, cluster_details)
 
     v = vars(args)
     clear_logs = v.get('clear_logs')
     yav_version = v.get('yav_version')
+
+    nodes = deduce_nodes_from_args(args, configurator.hosts_names, args.ssh_user, args.ssh_key_path)
     slice = handlers.Slice(
         components,
         nodes,
         cluster_details,
-        configurator,
+        kikimr_bin,
+        kikimr_compressed_bin,
         clear_logs,
         yav_version,
         walle_provider,
+        configurator,
     )
     func(slice)
 
+    # used only for configurator and will be removed soon
+    save_raw_cfg = v.get('save_raw_cfg')
+    if save_raw_cfg and configurator:
+        logger.debug("save raw cfg to '%s'", save_raw_cfg)
+        for root, dirs, files in os.walk(temp_dir):
+            for dir in dirs:
+                os.makedirs(os.path.join(save_raw_cfg, dir), 0o755, exist_ok=True)
+
+            for file in files:
+                src = os.path.join(root, file)
+                dst = os.path.join(save_raw_cfg, os.path.relpath(src, temp_dir))
+                with open(src, 'r') as src_f:
+                    with open(dst, 'w') as dst_f:
+                        dst_f.write(src_f.read())
+
     if clear_tmp:
         logger.debug("remove temp dirs '%s'", temp_dir)
         # shutil.rmtree(temp_dir)
@@ -699,6 +748,7 @@ def _run(args):
         parents=[
             direct_nodes_args(),
             cluster_description_args(),
+            yaml_config_path_args(),
             binaries_args(),
             component_args(),
             log_args(),
@@ -709,6 +759,13 @@ def _run(args):
         description="Full installation of the cluster from scratch. "
         "You can use --hosts to specify particular hosts. But it is tricky.",
     )
+    mode.add_argument(
+        "--save-raw-cfg",
+        metavar="DIR",
+        required=False,
+        default="",
+        help="Directory to save all static configuration files generated by configuration.create_static_cfg and configuration.create_dynamic_cfg",
+    )
     mode.set_defaults(handler=_run)
 
 
@@ -722,6 +779,7 @@ def _run(args):
         parents=[
             direct_nodes_args(),
             cluster_description_args(),
+            yaml_config_path_args(),
             binaries_args(),
             component_args(),
             log_args(),
@@ -759,7 +817,14 @@ def _run(args):
 
     mode = modes.add_parser(
         "stop",
-        parents=[direct_nodes_args(), cluster_description_args(), binaries_args(), component_args(), ssh_args()],
+        parents=[
+            direct_nodes_args(),
+            cluster_description_args(),
+            yaml_config_path_args(),
+            binaries_args(),
+            component_args(),
+            ssh_args()
+        ],
         description="Stop ydbd static instances at the nodes. "
                     "If option components specified, try to stop particular component. "
                     "Use --hosts to specify particular hosts."
@@ -773,7 +838,14 @@ def _run(args):
 
     mode = modes.add_parser(
         "start",
-        parents=[direct_nodes_args(), cluster_description_args(), binaries_args(), component_args(), ssh_args()],
+        parents=[
+            direct_nodes_args(),
+            cluster_description_args(),
+            yaml_config_path_args(),
+            binaries_args(),
+            component_args(),
+            ssh_args()
+        ],
         description="Start all ydbd instances at the nodes. "
                     "If option components specified, try to start particular component. "
                     "Otherwise only kikimr-multi-all will be started. "
@@ -791,6 +863,7 @@ def _run(args):
         parents=[
             direct_nodes_args(),
             cluster_description_args(),
+            yaml_config_path_args(),
             binaries_args(),
             component_args(),
             ssh_args(),
@@ -812,6 +885,7 @@ def _run(args):
         parents=[
             direct_nodes_args(),
             cluster_description_args(),
+            yaml_config_path_args(),
             binaries_args(),
             component_args(),
             ssh_args(),
@@ -861,6 +935,24 @@ def _run(args):
     mode.set_defaults(handler=_run)
 
 
+def add_dynconfig_generator(modes):
+    def _run(args):
+        if args.yaml_config:
+            yaml_config = yaml_configurator.YamlConfig(args.yaml_config)
+
+        if args.output_file is not None and args.output_file:
+            with open(args.output_file, "w") as output:
+                output.write(yaml_config.dynamic_simple)
+
+    mode = modes.add_parser(
+        "dynconfig-generator",
+        parents=[yaml_config_path_args(), output_file()],
+        description="Generate a minimalistic dynconfig.yaml for the provided config.yaml"
+    )
+
+    mode.set_defaults(handler=_run)
+
+
 #
 # docker and kube scenarios
 def build_docker_image(build_args, docker_package, build_ydbd, image, force_rebuild):
@@ -1413,6 +1505,7 @@ def main(walle_provider=None):
         add_format_mode(modes, walle_provider)
         add_explain_mode(modes, walle_provider)
         add_sample_config_mode(modes)
+        add_dynconfig_generator(modes)
 
         add_docker_build_mode(modes)
         add_docker_push_mode(modes)
@@ -1429,8 +1522,41 @@ def main(walle_provider=None):
 
         args = parser.parse_args()
         logging.root.setLevel(args.log_level.upper())
-        args.handler(args)
 
+        if not hasattr(args, 'handler'):
+            parser.print_help()
+            return
+
+        if not args.yaml_config:
+            warnings.warn(
+                '''
+
+
+###### WARNING #######
+
+Using cluster.yaml for cluster configuration is deprecated.
+Only the 'domains' section should be filled with database and slot configurations.
+The config.yaml and dynconfig.yaml should be passed as raw files through the --yaml-config and --yaml-dynconfig parameters.
+
+    ydbd_slice install cluster.yaml all --binary /path/to/ydbd --yaml-config /path/to/config.yaml --yaml-dynconfig /path/to/dynconfig.yaml
+
+To save the resulting configuration files from an old cluster.yaml, use the --save-raw-cfg option.
+
+    ydbd_slice install cluster.yaml all --binary /path/to/ydbd --save-raw-cfg /path/to/save
+
+The resulting configuration files will be saved in the /path/to/save directory. You can find config.yaml and dynconfig.yaml in the /path/to/save/kikimr-static directory.
+
+To generate dynconfig.yaml from config.yaml, you can use the dynconfig-generator command.
+
+    ydbd_slice dynconfig-generator --yaml-config /path/to/config.yaml --output-file /path/to/dynconfig.yaml
+
+###### WARNING #######
+
+''',
+                DeprecationWarning
+            )
+
+        args.handler(args)
     except KeyboardInterrupt:
         sys.exit('\nStopped by KeyboardInterrupt.')
     except Terminate:

ydb/tools/ydbd_slice/cluster_description.py

@@ -27,15 +27,15 @@ class ClusterDetails(ClusterDetailsProvider):
     SLOTS_PORTS_START = 31000
     PORTS_SHIFT = 10
 
-    def __init__(self, cluster_description_path, walle_provider):
+    def __init__(self, cluster_description_path, walle_provider, validator=None):
         self.__template = None
         self.__details = None
         self.__databases = None
         self.__dynamic_slots = None
         self._cluster_description_file = cluster_description_path
         self._walle_provider = walle_provider
 
-        super(ClusterDetails, self).__init__(self.template, self._walle_provider, use_new_style_cfg=True)
+        super(ClusterDetails, self).__init__(self.template, self._walle_provider, validator=validator, use_new_style_cfg=True)
 
     @property
     def template(self):
@@ -132,6 +132,10 @@ def template(self):
     def detail(self):
         return self.__cluster_details
 
+    @property
+    def hosts_names(self):
+        return self.detail.hosts_names
+
     @staticmethod
     def _generate_fake_keys():
         content = 'Keys {\n'
@@ -140,6 +144,7 @@ def _generate_fake_keys():
         content += '  Id: "fake-secret"\n'
         content += '  Version: 1\n'
         content += '}\n'
+
         return content
 
     @staticmethod