Merge 7988a93 into 9d6506e

Author: marsaly79

ydb/library/yql/tests/postgresql/make-tests.sh

@@ -2,6 +2,8 @@
 ../../tools/pg-make-test/pg-make-test \
   --srcdir=original/cases \
   --patchdir=patches \
+  --initscriptscfg=testinits.cfg \
+  --initscriptsdir=initscripts \
   --dstdir=cases \
   --runner=../../tools/pgrun/pgrun \
   --splitter="../../tools/pgrun/pgrun split-statements" \

ydb/library/yql/tests/postgresql/testinits.cfg

@@ -0,0 +1,20 @@
+strings: char_tbl varchar_tbl text_tbl
+numerology: int2_tbl int4_tbl float8_tbl
+horology: time_tbl timetz_tbl timestamp_tbl timestamptz_tbl interval_tbl
+expressions: date_tbl
+create_misc: hobbies_r.0 equipment_r.0 onek.0 tenk1.0 tenk2.0 person.0 city.0 slow_emp4000.0 fast_emp4000.0 road.0 ihighway.0 shighway.0 a_star.0 b_star.0 c_star.0 d_star.0 e_star.0 f_star.0
+select: int8_tbl onek person emp student stud_emp onek2
+select_into: int8_tbl onek onek2
+select_distinct: tenk1 person emp student stud_emp tmp
+select_distinct_on: int4_tbl tmp
+subselect: text_tbl int4_tbl int8_tbl onek tenk1 road
+union: char_tbl varchar_tbl text_tbl int4_tbl int8_tbl float8_tbl tenk1
+join: text_tbl int2_tbl int4_tbl int8_tbl float8_tbl onek tenk1 tenk2
+aggregates: varchar_tbl int4_tbl int8_tbl onek tenk1 student aggtest
+arrays: int8_tbl tenk1 array_op_test
+window: int4_tbl tenk1 tenk2
+jsonb: testjsonb
+limit: int8_tbl onek tenk1
+alter_table: onek tenk1 stud_emp
+xml: emp
+

ydb/library/yql/tools/pg-make-test/__main__.py

@@ -19,6 +19,8 @@
 PROGRAM_NAME = "pg-make-test"
 RUNNER = "../pgrun/pgrun"
 SPLITTER = "../pgrun/pgrun split-statements"
+INIT_SCRIPTS_CFG = "testinits.cfg"
+INIT_SCRIPTS_DIR = "initscripts"
 REPORT_FILE = "pg_tests.csv"
 LOGGER = None
 
@@ -76,7 +78,9 @@ class TestCaseBuilder:
     def __init__(self, config):
         self.config = config
 
-    def build(self, sqlfile):
+    def build(self, args):
+        sqlfile, init_scripts = args
+
         is_split_logging = self.config.logfile is not None and self.config.parallel
         if is_split_logging:
             logger = get_logger(
@@ -90,6 +94,9 @@ def build(self, sqlfile):
         splitted_stmts = list(self.split_sql_file(sqlfile))
         stmts_count = len(splitted_stmts)
 
+        if init_scripts:
+            logging.debug("Init scripts: %s", init_scripts)
+
         ressqlfile = self.config.dstdir / sqlfile.name
         resoutfile = ressqlfile.with_suffix('.out')
         reserrfile_base = resoutfile.with_suffix('.err')
@@ -125,7 +132,19 @@ def build(self, sqlfile):
                 test_out_name = Path(tempdir) / "test.out"
                 test_err_name = test_out_name.with_suffix(".err")
 
-                logger.debug("Running %s '%s' -> [%s]", self.config.runner, sqlfile, outfile)
+                if init_scripts:
+                    init_out_name = Path(tempdir) / "init.out"
+                    init_err_name = init_out_name.with_suffix(".err")
+
+                    for init_script in init_scripts:
+                        logger.debug("Running init script %s '%s'", self.config.runner, init_script)
+                        with open(init_script, 'rb') as f, open(init_out_name, 'wb') as fout, open(init_err_name, 'wb') as ferr:
+                            pi = subprocess.run(self.config.runner + ["--datadir", tempdir], stdin=f, stdout=fout, stderr=ferr)
+
+                        if pi.returncode != 0:
+                            logger.warning("%s returned error code %d", self.config.runner, pi.returncode)
+
+                logger.debug("Running test %s '%s' -> [%s]", self.config.runner, sqlfile, outfile)
                 with open(sqlfile, 'rb') as f, open(test_out_name, 'wb') as fout, open(test_err_name, 'wb') as ferr:
                     pi = subprocess.run(self.config.runner + ["--datadir", tempdir], stdin=f, stdout=fout, stderr=ferr)
 
@@ -323,6 +342,47 @@ def load_patches(patchdir):
         if ps is not False:
             yield p.stem, ps
 
+reInitScriptsCfgLine = re.compile(r"^([\w.]+):\s*([\w.]+(?:\s+[\w.]+)*)$")
+
+def load_init_scripts(initscriptscfg, initscriptsdir, tests_set):
+    init_scripts_map = dict()
+
+    if not initscriptscfg.is_file():
+        LOGGER.warning("Init scripts config file is not found: %s", initscriptscfg)
+        return init_scripts_map
+
+    if not initscriptsdir.is_dir():
+        LOGGER.warning("Init scripts directory is not found: %s", initscriptsdir)
+        return init_scripts_map
+
+    scripts = frozenset(s.stem for s in initscriptsdir.glob("*.sql"))
+
+    with open(initscriptscfg, 'r') as cfg:
+        for lineno, line in enumerate(cfg, 1):
+            line = line.strip()
+
+            if not line:
+                continue
+
+            m = reInitScriptsCfgLine.match(line)
+
+            if m is None:
+                LOGGER.warning("Bad line %d in init scripts config %s", lineno, initscriptscfg)
+                continue
+
+            test_name = m[1]
+            if test_name not in tests_set:
+                LOGGER.debug("Skipping init scripts for unknown test case %s", test_name)
+                continue
+
+            deps = [(initscriptsdir / s).with_suffix(".sql") for s in m[2].split() if s in scripts]
+            if not deps:
+                LOGGER.debug("No init scripts are listed for test case %s", test_name)
+                continue
+
+            init_scripts_map[test_name] = deps
+
+    return init_scripts_map
 
 def patch_cases(cases, patches, patchdir):
     for i, sql_full_name in enumerate(cases):
@@ -394,6 +454,22 @@ def patch_cases(cases, patches, patchdir):
     multiple=False,
     type=click.Path(exists=True, file_okay=False, resolve_path=True, path_type=Path),
 )
+@click.option(
+    "--initscriptscfg",
+    help="Config file for tests' init scripts",
+    default=INIT_SCRIPTS_CFG,
+    required=False,
+    multiple=False,
+    type=click.Path(exists=True, dir_okay=False, resolve_path=True, path_type=Path)
+)
+@click.option(
+    "--initscriptsdir",
+    help="Directory with tests' init scripts",
+    default=INIT_SCRIPTS_DIR,
+    required=False,
+    multiple=False,
+    type=click.Path(exists=True, file_okay=False, resolve_path=True, path_type=Path)
+)
 @click.option("--skip", "-s", help="Comma-separated list of testsuits to skip", multiple=False, type=click.STRING)
 @click.option("--runner", help="Test runner", default=RUNNER, required=False, multiple=False, type=click.STRING)
 @click.option(
@@ -420,7 +496,7 @@ def patch_cases(cases, patches, patchdir):
 )
 @click.option("--debug/--no-debug", help="Logs verbosity", default=False, required=False)
 @click.version_option(version=svn_version(), prog_name=PROGRAM_NAME)
-def cli(cases, srcdir, dstdir, patchdir, skip, runner, splitter, report, parallel, logfile, debug):
+def cli(cases, srcdir, dstdir, patchdir, initscriptscfg, initscriptsdir, skip, runner, splitter, report, parallel, logfile, debug):
     setup_logging(logfile, debug)
 
     if skip is not None:
@@ -439,6 +515,9 @@ def cli(cases, srcdir, dstdir, patchdir, skip, runner, splitter, report, paralle
     else:
         cases = [Path(c) if os.path.isabs(c) else config.srcdir / c for c in cases]
 
+    init_scripts = load_init_scripts(initscriptscfg, initscriptsdir, frozenset(c.stem for c in cases))
+    LOGGER.debug("Init scripts: %s", init_scripts)
+
     if config.patchdir is not None:
         patches = dict(load_patches(config.patchdir))
         LOGGER.info("Patches: %s", ", ".join(p for p in patches))
@@ -453,7 +532,7 @@ def cli(cases, srcdir, dstdir, patchdir, skip, runner, splitter, report, paralle
         builder = TestCaseBuilder(config)
         if config.parallel:
             with Pool() as pool:
-                results = list(pool.imap_unordered(builder.build, cases))
+                results = list(pool.imap_unordered(builder.build, [(test_case, init_scripts.get(test_case.stem) or []) for test_case in cases]))
         else:
             results = [builder.build(c) for c in cases]
 

ydb/library/yql/tools/pgrun/pgrun.cpp

@@ -1086,6 +1086,28 @@ void ShowFinalAst(TProgramPtr& program, IOutputStream& stream) {
     PrintExprTo(program, stream);
 }
 
+void FillTablesMapping(const TFsPath& dataDir, THashMap<TString, TString>& tablesMapping) {
+    TVector<TFsPath> children;
+
+    dataDir.List(children);
+
+    Cerr << "Registering pre-existing tables\n";
+    for (const auto& f: children) {
+        if (f.GetExtension() != "attr") {
+            continue;
+        }
+        auto tableName = f.Basename();
+        tableName.resize(tableName.length() - 5);
+
+        if (tableName.EndsWith(".tmp")) {
+            continue;
+        }
+        const auto fullTableName = f.Parent() / tableName;
+        Cerr << "\tyt.plato." << tableName << " -> " << fullTableName << '\n';
+        tablesMapping[TString("yt.plato.") + tableName] = f.Parent() / tableName;
+    }
+}
+
 int Main(int argc, char* argv[])
 {
     using namespace NLastGetopt;
@@ -1132,6 +1154,9 @@ int Main(int argc, char* argv[])
 
     auto yqlNativeServices = NFile::TYtFileServices::Make(funcRegistry.Get(), {}, fileStorage, tempDir.Path(), keepTempFiles);
     auto ytNativeGateway = CreateYtFileGateway(yqlNativeServices, &emulateOutputForMultirun);
+    if (tempDirExists) {
+        FillTablesMapping(tempDir.Path(), yqlNativeServices->GetTablesMapping());
+    }
 
     TVector<TDataProviderInitializer> dataProvidersInit;
     dataProvidersInit.push_back(GetYtNativeDataProviderInitializer(ytNativeGateway));