[yql] gateway extension contains only additional settings (#700)

Author: rvu1024

ydb/library/yql/cfg/tests/gateways-experimental.conf

@@ -1,234 +1,5 @@
-Yt {
-    GatewayThreads: 0
-    YtLogLevel: YL_INFO
-
-    DefaultSettings {
-        Name: "InferSchemaMode"
-        Value: "rpc"
-    }
-
-    DefaultSettings {
-        Name: "DQRPCReaderInflight"
-        Value: "10"
-    }
-
-    DefaultSettings {
-        Name: "UseRPCReaderInDq"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "DefaultMaxJobFails"
-        Value: "1"
-    }
-
-    DefaultSettings {
-        Name: "ReleaseTempData"
-        Value: "immediate"
-    }
-
-    DefaultSettings {
-        Name: "NativeYtTypeCompatibility"
-        Value: "all"
-    }
-
-    DefaultSettings {
-        Name: "UseYqlRowSpecCompactForm"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "UseNewPredicateExtraction"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "PruneKeyFilterLambda"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "_UseKeyBoundApi"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "JoinCommonUseMapMultiOut"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "UseAggPhases"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "_EnableWriteReorder"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "_EnableYtPartitioning"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "TableContentLocalExecution"
-        Value: "true"
-    }
-
-    RemoteFilePatterns {
-        Pattern: "yt://([a-zA-Z0-9\\-_]+)/([^&@?]+)$"
-        Cluster: "$1"
-        Path: "$2"
-    }
-
-    RemoteFilePatterns {
-        Pattern: "https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&path=//([^&]+)$"
-        Cluster: "$2"
-        Path: "$3"
-    }
-
-    RemoteFilePatterns {
-        Pattern: "https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?path=//([^&]+)$"
-        Cluster: "$2"
-        Path: "$3"
-    }
-
-    RemoteFilePatterns {
-        Pattern: "https?://([a-zA-Z0-9\\-_]+)\\.yt\\.yandex(\\.net|-team\\.ru)/api/v(2/download|3/read_file)\\?(|.+&)path=//([^&]+)($|&.*)"
-        Cluster: "$1"
-        Path: "$5"
-    }
-}
-
-SqlCore {
-    TranslationFlags: ["FlexibleTypes", "DisableAnsiOptionalAs", "EmitAggApply"]
-}
-
 YqlCore {
-    Flags {
-        Name: "DisableYsonCastToString"
-    }
-    Flags {
-        Name: "_EnableMatchRecognize"
-    }
     Flags {
         Name: "UseBlocks"
     }
 }
-
-Dq {
-    DefaultSettings {
-        Name: "EnableLLVM"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "EnableInsert"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "EnableFullResultWrite"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "_EnablePrecompute"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "_LiteralTimeout"
-        Value: "3600000"
-    }
-
-    DefaultSettings {
-        Name: "_TableTimeout"
-        Value: "3600000"
-    }
-    DefaultSettings {
-        Name: "EnableComputeActor"
-        Value: "1"
-    }
-    DefaultSettings {
-        Name: "ComputeActorType"
-        Value: "async"
-    }
-
-    DefaultSettings {
-        Name: "UseAggPhases"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "HashJoinMode"
-        Value: "grace"
-    }
-
-    DefaultSettings {
-        Name: "UseFastPickleTransport"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "UseOOBTransport"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "UseFinalizeByKey"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "EnableDqReplicate"
-        Value: "true"
-    }
-
-    DefaultSettings {
-        Name: "UseWideChannels"
-        Value: "true"
-    }
-}
-
-Fs {
-    CustomSchemes {
-        Pattern: "sbr:(?://)?(\\d+)"
-        TargetUrl: "https://proxy.sandbox.yandex-team.ru/$1"
-    }
-    CustomSchemes {
-        # backward compatibility rule
-        Pattern: "yt://([a-zA-Z0-9\\-_]+)/(.+)@t=([^&=]+)"
-        TargetUrl: "yt://$1/$2?transaction_id=$3"
-    }
-    CustomSchemes {
-        Pattern: "https?://yt\\.yandex(?:\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&(.+)"
-        TargetUrl: "yt://$1/?$2"
-    }
-    CustomSchemes {
-        Pattern: "https?://yt\\.yandex(?:\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?(.+)"
-        TargetUrl: "yt://$1/?$2"
-    }
-    CustomSchemes {
-        Pattern: "https?://([a-zA-Z0-9\\-_]+)\\.yt\\.yandex(?:\\.net|-team\\.ru)/api/v3/read_file\\?(.+)"
-        TargetUrl: "yt://$1/?$2"
-    }
-    CustomSchemes {
-        Pattern: "https?://a\\.yandex-team\\.ru/(?:arc|svn)/(.+)/arcadia/(.+)\\?rev=r?(\\d+)"
-        TargetUrl: "arc:/$2?rev=$3&branch=$1"
-    }
-    CustomSchemes {
-        Pattern: "https?://a(?:rcanum)?\\.yandex-team\\.ru/arcadia/(.+)\\?rev=r(\\d+)"
-        TargetUrl: "arc:/$1?rev=$2"
-    }
-    CustomSchemes {
-        Pattern: "https?://a(?:rcanum)?\\.yandex-team\\.ru/arcadia/(.+)\\?rev=([^r].*)"
-        TargetUrl: "arc:/$1?hash=$2"
-    }
-    CustomSchemes {
-        Pattern: "arc://(.*)"
-        TargetUrl: "arc:/$1"
-    }
-}

ydb/library/yql/cfg/tests/gateways-forceblocks.conf

@@ -10,6 +10,7 @@
 import tempfile
 import shutil
 
+from google.protobuf import text_format
 from collections import namedtuple, defaultdict, OrderedDict
 from functools import partial
 import codecs
@@ -37,11 +38,6 @@ def get_param(name, default=None):
     return yatest.common.get_param(name, os.environ.get(name) or default)
 
 
-def get_gateway_cfg_suffix():
-    default_suffix = None
-    return get_param('gateway_config_suffix', default_suffix) or ''
-
-
 def do_custom_query_check(res, sql_query):
     custom_check = re.search(r"/\* custom check:(.*)\*/", sql_query)
     if not custom_check:
@@ -54,6 +50,11 @@ def do_custom_query_check(res, sql_query):
     return True
 
 
+def get_gateway_cfg_suffix():
+    default_suffix = None
+    return get_param('gateway_config_suffix', default_suffix) or ''
+
+
 def get_gateway_cfg_filename():
     suffix = get_gateway_cfg_suffix()
     if suffix == '':
@@ -62,6 +63,17 @@ def get_gateway_cfg_filename():
         return 'gateways-' + suffix + '.conf'
 
 
+def merge_default_gateway_cfg(cfg_dir, gateway_config):
+
+    with open(yql_source_path(os.path.join(cfg_dir, 'gateways.conf'))) as f:
+        text_format.Merge(f.read(), gateway_config)
+
+    suffix = get_gateway_cfg_suffix()
+    if suffix:
+        with open(yql_source_path(os.path.join(cfg_dir, 'gateways-' + suffix + '.conf'))) as f:
+            text_format.Merge(f.read(), gateway_config)
+
+
 def find_file(path):
     arcadia_root = '.'
     while '.arcadia.root' not in os.listdir(arcadia_root):

ydb/library/yql/tests/common/test_framework/yql_utils.py

@@ -61,14 +61,11 @@ def __init__(self, udfs_dir=None, prov='yt', use_sql2yql=False, keep_temp=True,
         if gateway_config is not None:
             text_format.Merge(gateway_config, self.gateway_config)
 
-        if cfg_dir is None:
-            cfg_dir = 'ydb/library/yql/cfg/tests'
-        with open(yql_utils.yql_source_path(cfg_dir + '/' + yql_utils.get_gateway_cfg_filename())) as f:
-            text_format.Merge(f.read(), self.gateway_config)
+        yql_utils.merge_default_gateway_cfg(cfg_dir or 'ydb/library/yql/cfg/tests', self.gateway_config)
 
         self.fs_config = file_storage_pb2.TFileStorageConfig()
 
-        with open(yql_utils.yql_source_path(cfg_dir + '/fs.conf')) as f:
+        with open(yql_utils.yql_source_path(os.path.join(cfg_dir or 'ydb/library/yql/cfg/tests', 'fs.conf'))) as f:
             text_format.Merge(f.read(), self.fs_config)
 
         if fs_config is not None: