Added 'FQ_CONNECTOR_ENDPOINT' env variable to local_ydb (ydb-platform#1769)

* Add 'FQ_CONNECTOR_ENDPOINT' env parameter for `local_ydb`

* Handle scheme type in `FQ_CONNECTOR_ENDPOINT` env variable

Author: vitalyisaev2

ydb/public/tools/lib/cmds/__init__.py

@@ -8,7 +8,9 @@
 import string
 import typing  # noqa: F401
 import sys
+from urllib.parse import urlparse
 
+from ydb.library.yql.providers.common.proto.gateways_config_pb2 import TGenericConnectorConfig
 from ydb.tests.library.common import yatest_common
 from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
 from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
@@ -255,6 +257,34 @@ def enable_tls():
     return os.getenv('YDB_GRPC_ENABLE_TLS') == 'true'
 
 
+def generic_connector_config():
+    endpoint = os.getenv("FQ_CONNECTOR_ENDPOINT")
+    if not endpoint:
+        return None
+
+    parsed = urlparse(endpoint)
+    if not parsed.hostname:
+        raise ValueError("Invalid host '{}' in FQ_CONNECTOR_ENDPOINT".format(parsed.hostname))
+
+    if not (1024 <= parsed.port <= 65535):
+        raise ValueError("Invalid port '{}' in FQ_CONNECTOR_ENDPOINT".format(parsed.port))
+
+    valid_schemes = ['grpc', 'grpcs']
+    if parsed.scheme not in valid_schemes:
+        raise ValueError("Invalid schema '{}' in FQ_CONNECTOR_ENDPOINT (possible: {})".format(parsed.schema, valid_schemes))
+
+    cfg = TGenericConnectorConfig()
+    cfg.Endpoint.host = parsed.hostname
+    cfg.Endpoint.port = parsed.port
+
+    if parsed.scheme == 'grpc':
+        cfg.UseSsl = False
+    elif parsed.scheme == 'grpcs':
+        cfg.UseSsl = True
+
+    return cfg
+
+
 def grpc_tls_data_path(arguments):
     default_store = arguments.ydb_working_dir if arguments.ydb_working_dir else None
     return os.getenv('YDB_GRPC_TLS_DATA_PATH', default_store)
@@ -335,6 +365,7 @@ def deploy(arguments):
         default_users=default_users(),
         extra_feature_flags=enable_feature_flags,
         extra_grpc_services=arguments.enabled_grpc_services,
+        generic_connector_config=generic_connector_config(),
         **optionals
     )
 

ydb/public/tools/lib/cmds/ut/test.py

@@ -0,0 +1,26 @@
+import os
+
+from ydb.public.tools.lib.cmds import generic_connector_config
+from ydb.library.yql.providers.common.proto.gateways_config_pb2 import TGenericConnectorConfig
+
+
+def test_kikimr_config_generator_generic_connector_config():
+    os.environ["FQ_CONNECTOR_ENDPOINT"] = "grpc://localhost:50051"
+
+    expected = TGenericConnectorConfig()
+    expected.Endpoint.host = "localhost"
+    expected.Endpoint.port = 50051
+    expected.UseSsl = False
+
+    actual = generic_connector_config()
+    assert actual == expected
+
+    os.environ["FQ_CONNECTOR_ENDPOINT"] = "grpcs://localhost:50051"
+
+    expected = TGenericConnectorConfig()
+    expected.Endpoint.host = "localhost"
+    expected.Endpoint.port = 50051
+    expected.UseSsl = True
+
+    actual = generic_connector_config()
+    assert actual == expected

ydb/public/tools/lib/cmds/ut/ya.make

@@ -0,0 +1,12 @@
+PY3TEST()
+
+PEERDIR(
+    ydb/public/tools/lib/cmds
+    ydb/library/yql/providers/common/proto
+)
+
+TEST_SRCS(
+    test.py
+)
+
+END()

ydb/public/tools/lib/cmds/ya.make

@@ -9,3 +9,5 @@ PEERDIR(
 )
 
 END()
+
+RECURSE_FOR_TESTS(ut)

ydb/public/tools/local_ydb/__main__.py

@@ -9,7 +9,7 @@
 \033[94m
 To deploy the local YDB cluster:
 
-   {prog} deploy --ydb-working-dir /absolute/path/to/working/directory --ydb-binary-path /path/to/kikimr/driver
+  {prog} deploy --ydb-working-dir /absolute/path/to/working/directory --ydb-binary-path /path/to/kikimr/driver
 
 To cleanup the deployed YDB cluster (this includes removal of working directory, all configuration files, disks and so on):
 

ydb/public/tools/local_ydb/ya.make

@@ -3,6 +3,7 @@ PY3_PROGRAM(local_ydb)
 PY_SRCS(__main__.py)
 
 PEERDIR(
+    ydb/library/yql/providers/common/proto
     ydb/public/tools/lib/cmds
 )
 

ydb/tests/library/harness/kikimr_config.py

@@ -159,7 +159,9 @@ def __init__(
             hive_config=None,
             datashard_config=None,
             enforce_user_token_requirement=False,
-            default_user_sid=None
+            default_user_sid=None,
+            pg_compatible_expirement=False,
+            generic_connector_config=None,  # typing.Optional[TGenericConnectorConfig]
     ):
         if extra_feature_flags is None:
             extra_feature_flags = []
@@ -376,6 +378,31 @@ def __init__(
         if default_user_sid:
             self.yaml_config["domains_config"]["security_config"]["default_user_sids"] = [default_user_sid]
 
+        if pg_compatible_expirement:
+            self.yaml_config["table_service_config"]["enable_prepared_ddl"] = True
+            # self.yaml_config["table_service_config"]["enable_ast_cache"] = True
+            # self.yaml_config["table_service_config"]["enable_pg_consts_to_params"] = True
+            self.yaml_config["table_service_config"]["index_auto_choose_mode"] = 'max_used_prefix'
+            self.yaml_config["feature_flags"]['enable_temp_tables'] = True
+            self.yaml_config["feature_flags"]['enable_table_pg_types'] = True
+
+        if generic_connector_config:
+            if "query_service_config" not in self.yaml_config:
+                self.yaml_config["query_service_config"] = {}
+
+            self.yaml_config["query_service_config"]["generic"] = {
+                "connector": {
+                    "endpoint": {
+                        "host": generic_connector_config.Endpoint.host,
+                        "port": generic_connector_config.Endpoint.port,
+                    },
+                    "use_ssl": generic_connector_config.UseSsl
+                }
+            }
+
+            self.yaml_config["feature_flags"]["enable_external_data_sources"] = True
+            self.yaml_config["feature_flags"]["enable_script_execution_operations"] = True
+
     @property
     def pdisks_info(self):
         return self._pdisks_info

ydb/tests/library/ut/kikimr_config.py

@@ -0,0 +1,19 @@
+from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
+
+from ydb.library.yql.providers.common.proto.gateways_config_pb2 import TGenericConnectorConfig
+
+
+def test_kikimr_config_generator_generic_connector_config():
+    generic_connector_config = TGenericConnectorConfig()
+    generic_connector_config.Endpoint.host = "localhost"
+    generic_connector_config.Endpoint.port = 50051
+    generic_connector_config.UseSsl = False
+
+    cfg_gen = KikimrConfigGenerator(generic_connector_config=generic_connector_config)
+    yaml_config = cfg_gen.yaml_config
+
+    assert yaml_config["query_service_config"]["generic"]["connector"]["endpoint"]["host"] == generic_connector_config.Endpoint.host
+    assert yaml_config["query_service_config"]["generic"]["connector"]["endpoint"]["port"] == generic_connector_config.Endpoint.port
+    assert yaml_config["query_service_config"]["generic"]["connector"]["use_ssl"] == generic_connector_config.UseSsl
+    assert yaml_config["feature_flags"]["enable_external_data_sources"] is True
+    assert yaml_config["feature_flags"]["enable_script_execution_operations"] is True

ydb/tests/library/ut/ya.make

@@ -0,0 +1,12 @@
+PY3TEST()
+
+PEERDIR(
+    ydb/tests/library
+    ydb/library/yql/providers/common/proto
+)
+
+TEST_SRCS(
+    kikimr_config.py
+)
+
+END()

ydb/tests/library/ya.make

@@ -95,6 +95,7 @@ PEERDIR(
     library/python/svn_version
     library/python/testing/yatest_common
     ydb/core/protos
+    ydb/library/yql/providers/common/proto
     ydb/public/api/grpc
     ydb/public/api/grpc/draft
     ydb/public/api/protos
@@ -103,3 +104,5 @@ PEERDIR(
 )
 
 END()
+
+RECURSE_FOR_TESTS(ut)