YQ-2670 switch generic tests to docker compose (#691)

GrigoriyPA · web-flow · commit d4ad98f54706 · 2023-12-26T12:07:32.000+03:00
* Switched KQP generic provider tests to docker-compose

* Fixed retries
diff --git a/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp b/ydb/core/kqp/ut/federated_query/generic/ch_recipe_ut_helpers.cpp
@@ -6,19 +6,19 @@
 namespace NTestUtils {
 
     TString GetChHost() {
-        return GetEnv("RECIPE_CLICKHOUSE_HOST", "localhost");
+        return "localhost";
     }
 
     ui32 GetChPort() {
-        return FromString<ui32>(GetEnv("RECIPE_CLICKHOUSE_NATIVE_PORT", "1234"));
+        return 19000;
     }
 
     TString GetChUser() {
-        return GetEnv("RECIPE_CLICKHOUSE_USER");
+        return "user";
     }
 
     TString GetChPassword() {
-        return GetEnv("RECIPE_CLICKHOUSE_PASSWORD");
+        return "password";
     }
 
     TString GetChDatabase() {
@@ -32,7 +32,20 @@ namespace NTestUtils {
             .SetPort(GetChPort())
             .SetUser(GetChUser())
             .SetPassword(GetChPassword());
-        return NClickHouse::TClient(opt);
+
+        TInstant start = TInstant::Now();
+        ui32 attempt = 0;
+        while ((TInstant::Now() - start).Seconds() < 60) {
+            attempt += 1;
+            try {
+                return NClickHouse::TClient(opt);
+            } catch (const TSystemError& e) {
+                Cerr << "Attempt " << attempt << ": " << e.what() << Endl;
+                Sleep(TDuration::MilliSeconds(100));
+            }
+        }
+
+        throw yexception() << "Failed to connect ClickHouse in " << attempt << " attempt(s)";
     }
 
 } // namespace NTestUtils
diff --git a/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp b/ydb/core/kqp/ut/federated_query/generic/connector_recipe_ut_helpers.cpp
@@ -6,13 +6,11 @@
 namespace NTestUtils {
 
     TString GetConnectorHost() {
-        return GetEnv("YDB_CONNECTOR_RECIPE_GRPC_HOST", "localhost");
+        return "localhost";
     }
 
     ui32 GetConnectorPort() {
-        const TString port = GetEnv("YDB_CONNECTOR_RECIPE_GRPC_PORT");
-        UNIT_ASSERT_C(port, "No connector port specified");
-        return FromString<ui32>(port);
+        return 50051;
     }
 
     std::shared_ptr<NKikimr::NKqp::TKikimrRunner> MakeKikimrRunnerWithConnector() {
diff --git a/ydb/core/kqp/ut/federated_query/generic/docker-compose.yml b/ydb/core/kqp/ut/federated_query/generic/docker-compose.yml
@@ -0,0 +1,25 @@
+version: '3.4'
+services:
+  postgresql:
+    image: postgres:15-bullseye@sha256:3411b9f2e5239cd7867f34fcf22fe964230f7d447a71d63c283e3593d3f84085
+    environment:
+      POSTGRES_DB: db
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: password
+    ports:
+      - 15432:5432
+  clickhouse:
+    image: clickhouse/clickhouse-server:23-alpine@sha256:b078c1cd294632afa2aeba3530e7ba2e568513da23304354f455a25fab575c06
+    environment:
+      CLICKHOUSE_DB: db
+      CLICKHOUSE_USER: user
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+      CLICKHOUSE_PASSWORD: password
+    ports:
+      - 19000:9000
+      - 18123:8123
+  fq-connector-go:
+    image: ghcr.io/ydb-platform/fq-connector-go:v0.0.6-rc.8@sha256:74ebae0530d916c1842a7fddfbddc6c018763a0401f2f627a44e8829692fe41f
+    ports:
+      - 50051:50051
+    network_mode: host
diff --git a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp
@@ -43,7 +43,7 @@ Y_UNIT_TEST_SUITE(KqpGenericPlanTest) {
         {
             const TString sql = fmt::format(
                 R"sql(
-                CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value="");
+                CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value="{pg_password}");
                 CREATE EXTERNAL DATA SOURCE pg_data_source WITH (
                     SOURCE_TYPE="PostgreSQL",
                     LOCATION="{pg_host}:{pg_port}",
@@ -58,6 +58,7 @@ Y_UNIT_TEST_SUITE(KqpGenericPlanTest) {
                 "pg_host"_a = GetPgHost(),
                 "pg_port"_a = GetPgPort(),
                 "pg_user"_a = GetPgUser(),
+                "pg_password"_a = GetPgPassword(),
                 "pg_database"_a = GetPgDatabase());
             auto result = session.ExecuteSchemeQuery(sql).GetValueSync();
             UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
diff --git a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_provider_join_ut.cpp
@@ -71,7 +71,7 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) {
         {
             const TString sql = fmt::format(
                 R"sql(
-                CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value="");
+                CREATE OBJECT pg_password_obj (TYPE SECRET) WITH (value="{pg_password}");
                 CREATE EXTERNAL DATA SOURCE pg_data_source WITH (
                     SOURCE_TYPE="PostgreSQL",
                     LOCATION="{pg_host}:{pg_port}",
@@ -97,6 +97,7 @@ Y_UNIT_TEST_SUITE(FederatedQueryJoin) {
                 "pg_host"_a = GetPgHost(),
                 "pg_port"_a = GetPgPort(),
                 "pg_user"_a = GetPgUser(),
+                "pg_password"_a = GetPgPassword(),
                 "pg_database"_a = GetPgDatabase(),
                 "ch_host"_a = GetChHost(),
                 "ch_port"_a = GetChPort(),
diff --git a/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp b/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.cpp
@@ -10,31 +10,47 @@ using namespace fmt::literals;
 namespace NTestUtils {
 
     TString GetPgHost() {
-        return GetEnv("POSTGRES_RECIPE_HOST", "localhost");
+        return "localhost";
     }
 
     ui32 GetPgPort() {
-        const TString port = GetEnv("POSTGRES_RECIPE_PORT");
-        UNIT_ASSERT_C(port, "No postgresql port specified");
-        return FromString<ui32>(port);
+        return 15432;
     }
 
     TString GetPgUser() {
-        return GetEnv("POSTGRES_RECIPE_USER");
+        return "user";
     }
 
     TString GetPgDatabase() {
-        return GetEnv("POSTGRES_RECIPE_DBNAME");
+        return "db";
+    }
+
+    TString GetPgPassword() {
+        return "password";
     }
 
     pqxx::connection CreatePostgresqlConnection() {
         const TString connectionString = fmt::format(
-            "host={host} port={port} dbname={database} user={user}",
+            "host={host} port={port} dbname={database} user={user} password={password}",
             "host"_a = GetPgHost(),
             "port"_a = GetPgPort(),
             "database"_a = GetPgDatabase(),
-            "user"_a = GetPgUser());
-        return pqxx::connection{connectionString};
+            "user"_a = GetPgUser(),
+            "password"_a = GetPgPassword());
+
+        TInstant start = TInstant::Now();
+        ui32 attempt = 0;
+        while ((TInstant::Now() - start).Seconds() < 60) {
+            attempt += 1;
+            try {
+                return pqxx::connection{connectionString};
+            } catch (const pqxx::broken_connection& e) {
+                Cerr << "Attempt " << attempt << ": " << e.what() << Endl;
+                Sleep(TDuration::MilliSeconds(100));
+            }
+        }
+
+        throw yexception() << "Failed to connect PostgreSQL in " << attempt << " attempt(s)";
     }
 
 } // namespace NTestUtils
diff --git a/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h b/ydb/core/kqp/ut/federated_query/generic/pg_recipe_ut_helpers.h
@@ -9,6 +9,7 @@ namespace NTestUtils {
     ui32 GetPgPort();
     TString GetPgUser();
     TString GetPgDatabase();
+    TString GetPgPassword();
 
     pqxx::connection CreatePostgresqlConnection();
 
diff --git a/ydb/core/kqp/ut/federated_query/generic/ya.make b/ydb/core/kqp/ut/federated_query/generic/ya.make
@@ -1,6 +1,24 @@
 UNITTEST_FOR(ydb/core/kqp)
 
-FORK_SUBTESTS()
+IF (AUTOCHECK) 
+    # Split tests to chunks only when they're running on different machines with distbuild,
+    # otherwise this directive will slow down local test execution.
+    # Look through https://st.yandex-team.ru/DEVTOOLSSUPPORT-39642 for more information.
+    FORK_SUBTESTS()
+
+    # TAG and REQUIREMENTS are copied from: https://docs.yandex-team.ru/devtools/test/environment#docker-compose
+    TAG(
+        ya:external
+        ya:force_sandbox
+        ya:fat
+    )
+
+    REQUIREMENTS(
+        container:4467981730
+        cpu:all
+        dns:dns64
+    )
+ENDIF()
 
 SRCS(
     ch_recipe_ut_helpers.cpp
@@ -20,9 +38,7 @@ PEERDIR(
     ydb/library/yql/sql/pg_dummy
 )
 
-INCLUDE(${ARCADIA_ROOT}/library/recipes/clickhouse/recipe.inc)
-INCLUDE(${ARCADIA_ROOT}/library/recipes/postgresql/recipe.inc)
-INCLUDE(${ARCADIA_ROOT}/ydb/library/yql/providers/generic/connector/recipe/recipe.inc)
+INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc)
 
 YQL_LAST_ABI_VERSION()
 

Original file line number	Diff line number	Diff line change
`@@ -6,13 +6,11 @@`
`6`	`6`	`namespace NTestUtils {`
`7`	`7`
`8`	`8`	`TString GetConnectorHost() {`
`9`		`- return GetEnv("YDB_CONNECTOR_RECIPE_GRPC_HOST", "localhost");`
	`9`	`+ return "localhost";`
`10`	`10`	`}`
`11`	`11`
`12`	`12`	`ui32 GetConnectorPort() {`
`13`		`- const TString port = GetEnv("YDB_CONNECTOR_RECIPE_GRPC_PORT");`
`14`		`- UNIT_ASSERT_C(port, "No connector port specified");`
`15`		`- return FromString<ui32>(port);`
	`13`	`+ return 50051;`
`16`	`14`	`}`
`17`	`15`
`18`	`16`	`std::shared_ptr<NKikimr::NKqp::TKikimrRunner> MakeKikimrRunnerWithConnector() {`