Skip to content

Commit b76e4df

Browse files
evanevanevanevannnnevanevanevanevannnn
committed
limitless solomon read actor draft (ydb-platform#15200)
1 parent 88e21ea commit b76e4df

File tree

87 files changed

+1677
-821
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+1677
-821
lines changed

‎ydb/core/base/events.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ struct TKikimrEvents : TEvents {
184184
ES_INCREMENTAL_RESTORE_SCAN = 4261,
185185
ES_FEATURE_FLAGS = 4262,
186186
ES_PRIORITY_QUEUE = 4263,
187+
ES_SOLOMON_PROVIDER = 4264,
187188
};
188189
};
189190

‎ydb/core/external_sources/external_source_factory.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ IExternalSourceFactory::TPtr CreateExternalSourceFactory(const std::vector<TStri
8282
},
8383
{
8484
ToString(NYql::EDatabaseType::Solomon),
85-
CreateExternalDataSource(TString{NYql::SolomonProviderName}, {"NONE", "TOKEN"}, {}, hostnamePatternsRegEx)
85+
CreateExternalDataSource(TString{NYql::SolomonProviderName}, {"NONE", "TOKEN"}, {"use_ssl", "grpc_port"}, hostnamePatternsRegEx)
8686
}
8787
});
8888
}

‎ydb/core/fq/libs/actors/run_actor.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1984,7 +1984,7 @@ class TRunActor : public NActors::TActorBootstrapped<TRunActor> {
19841984
{
19851985
auto solomonConfig = gatewaysConfig.GetSolomon();
19861986
auto solomonGateway = NYql::CreateSolomonGateway(solomonConfig);
1987-
dataProvidersInit.push_back(GetSolomonDataProviderInitializer(solomonGateway, false));
1987+
dataProvidersInit.push_back(GetSolomonDataProviderInitializer(solomonGateway, Params.CredentialsFactory, false));
19881988
}
19891989

19901990
SessionId = TStringBuilder()

‎ydb/core/fq/libs/init/init.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
#include <ydb/library/yql/providers/pq/async_io/dq_pq_read_actor.h>
4646
#include <ydb/library/yql/providers/pq/async_io/dq_pq_write_actor.h>
4747
#include <ydb/library/yql/providers/pq/gateway/native/yql_pq_gateway.h>
48-
#include <ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h>
48+
#include <ydb/library/yql/providers/solomon/actors/dq_solomon_write_actor.h>
4949
#include <ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.h>
5050

5151

‎ydb/core/fq/libs/init/ya.make

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ PEERDIR(
5252
ydb/library/yql/providers/s3/actors
5353
ydb/library/yql/providers/s3/proto
5454
ydb/library/yql/providers/s3/provider
55-
ydb/library/yql/providers/solomon/async_io
55+
ydb/library/yql/providers/solomon/actors
5656
ydb/library/yql/providers/solomon/gateway
5757
ydb/library/yql/providers/solomon/proto
5858
ydb/library/yql/providers/solomon/provider

‎ydb/core/fq/libs/test_connection/test_monitoring.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
#include <yql/essentials/providers/common/structured_token/yql_token_builder.h>
1010
#include <yql/essentials/utils/url_builder.h>
1111

12-
#include <ydb/library/yql/providers/solomon/async_io/dq_solomon_write_actor.h>
12+
#include <ydb/library/yql/providers/solomon/actors/dq_solomon_write_actor.h>
1313
#include <ydb/library/yql/utils/actors/http_sender_actor.h>
1414

1515
#include <ydb/library/actors/core/actor_bootstrapped.h>

‎ydb/core/fq/libs/test_connection/ya.make

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ PEERDIR(
1818
ydb/core/fq/libs/control_plane_storage
1919
ydb/core/fq/libs/test_connection/events
2020
ydb/library/yql/providers/pq/cm_client
21-
ydb/library/yql/providers/solomon/async_io
21+
ydb/library/yql/providers/solomon/actors
2222
)
2323

2424
YQL_LAST_ABI_VERSION()

‎ydb/core/kqp/compute_actor/kqp_compute_actor.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#include <ydb/library/yql/providers/generic/actors/yql_generic_provider_factories.h>
1313
#include <ydb/library/formats/arrow/protos/ssa.pb.h>
1414
#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
15-
#include <ydb/library/yql/providers/solomon/async_io/dq_solomon_read_actor.h>
15+
#include <ydb/library/yql/providers/solomon/actors/dq_solomon_read_actor.h>
1616

1717

1818
namespace NKikimr {

‎ydb/core/kqp/compute_actor/ya.make

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ PEERDIR(
2727
ydb/library/yql/dq/actors/compute
2828
ydb/library/yql/providers/generic/actors
2929
ydb/library/yql/providers/s3/actors_factory
30-
ydb/library/yql/providers/solomon/async_io
30+
ydb/library/yql/providers/solomon/actors
3131
yql/essentials/public/issue
3232
)
3333

‎ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,9 @@ TString GetOrEmpty(const NYql::TCreateObjectSettings& container, const TString&
8585
"use_tls",
8686
"schema", // managed PG, GP
8787
"service_name", // oracle
88-
"folder_id" // logging
88+
"folder_id", // logging
89+
"use_ssl", // solomon
90+
"grpc_port" // solomon
8991
};
9092

9193
auto& featuresExtractor = settings.GetFeaturesExtractor();

‎ydb/core/kqp/host/kqp_host.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1893,6 +1893,7 @@ class TKqpHost : public IKqpHost {
18931893
solomonState->Gateway = FederatedQuerySetup->SolomonGateway;
18941894
solomonState->DqIntegration = NYql::CreateSolomonDqIntegration(solomonState);
18951895
solomonState->Configuration->Init(FederatedQuerySetup->SolomonGatewayConfig, TypesCtx);
1896+
solomonState->ExecutorPoolId = AppData()->UserPoolId;
18961897

18971898
TypesCtx->AddDataSource(NYql::SolomonProviderName, NYql::CreateSolomonDataSource(solomonState));
18981899
TypesCtx->AddDataSink(NYql::SolomonProviderName, NYql::CreateSolomonDataSink(solomonState));

‎ydb/core/kqp/session_actor/kqp_worker_common.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,16 @@ bool CanCacheQuery(const NKqpProto::TKqpPhyQuery& query) {
181181

182182
for (const auto& stage : tx.GetStages()) {
183183
for (const auto& source : stage.GetSources()) {
184-
// S3 provider stores S3 paths to read in AST, so we can't cache such queries
185-
if (source.HasExternalSource() && source.GetExternalSource().GetType() == "S3Source") {
184+
if (!source.HasExternalSource()) {
185+
continue;
186+
}
187+
const auto& externalSourceType = source.GetExternalSource().GetType();
188+
189+
// S3 provider stores S3 paths to read in AST,
190+
// Solomon provider uses runtime listing,
191+
// so we can't cache such queries
192+
if (externalSourceType == "S3Source" ||
193+
externalSourceType == "SolomonSource") {
186194
return false;
187195
}
188196
}

‎ydb/library/yql/providers/dq/metrics/ya.make

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ SRCS(
1111

1212
PEERDIR(
1313
ydb/library/actors/core
14-
ydb/library/yql/providers/solomon/async_io
14+
ydb/library/yql/providers/solomon/actors
1515
)
1616

1717
YQL_LAST_ABI_VERSION()

‎ydb/library/yql/providers/solomon/async_io/metrics_encoder.cpp renamed to ‎ydb/library/yql/providers/solomon/actors/dq_solomon_actors_util.cpp

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
#include "metrics_encoder.h"
2-
3-
#include <yql/essentials/public/udf/udf_data_type.h>
1+
#include "dq_solomon_actors_util.h"
42

3+
#include <library/cpp/monlib/encode/json/json.h>
54
#include <util/datetime/base.h>
5+
#include <yql/essentials/public/udf/udf_data_type.h>
66

77
using namespace NYql;
88

@@ -96,11 +96,12 @@ void EncodeSensorValue(
9696
Y_ENSURE(false, "Bad type for sensor " << scheme.GetDataTypeId());
9797
}
9898
}
99-
}
99+
100+
} // namespace
100101

101102
namespace NYql::NDq {
102103

103-
TMetricsEncoder::TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool useCloudFormat)
104+
TMetricsEncoder::TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool useCloudFormat)
104105
: Scheme(scheme)
105106
, UseCloudFormat(useCloudFormat)
106107
{
@@ -154,4 +155,4 @@ TString TMetricsEncoder::Encode() {
154155
return res;
155156
}
156157

157-
}
158+
} // namespace NYql::NDq

‎ydb/library/yql/providers/solomon/async_io/metrics_encoder.h renamed to ‎ydb/library/yql/providers/solomon/actors/dq_solomon_actors_util.h

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
#pragma once
22

3-
#include <ydb/library/yql/utils/actors/http_sender.h>
4-
3+
#include <library/cpp/monlib/encode/encoder.h>
4+
#include <ydb/library/yql/providers/solomon/proto/dq_solomon_shard.pb.h>
55
#include <yql/essentials/public/udf/udf_value.h>
66

7-
#include <library/cpp/monlib/encode/json/json.h>
8-
9-
#include <util/stream/str.h>
10-
11-
127
namespace NYql::NDq {
138

9+
struct TDqSolomonReadParams {
10+
NSo::NProto::TDqSolomonSource Source;
11+
};
12+
1413
class TMetricsEncoder {
1514
public:
1615
explicit TMetricsEncoder(const NSo::NProto::TDqSolomonShardScheme& scheme, bool cloudFormat);
@@ -28,4 +27,4 @@ class TMetricsEncoder {
2827
bool UseCloudFormat;
2928
};
3029

31-
}
30+
} // namespace NYql::NDq

0 commit comments

Comments
 (0)