Skip to content

Commit ba6253c

Browse files
shnikdshnikd
authored
Add EnableAntlr4Parser to query replay (#9471)
1 parent 2464d55 commit ba6253c

File tree

8 files changed

+50
-22
lines changed

8 files changed

+50
-22
lines changed

‎ydb/core/kqp/host/kqp_host.cpp

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1230,7 +1230,8 @@ class TKqpHost : public IKqpHost {
12301230
.SetIsEnablePgConstsToParams(SessionCtx->Config().EnablePgConstsToParams)
12311231
.SetQueryParameters(query.ParameterTypes)
12321232
.SetApplicationName(ApplicationName)
1233-
.SetIsEnablePgSyntax(SessionCtx->Config().FeatureFlags.GetEnablePgSyntax());
1233+
.SetIsEnablePgSyntax(SessionCtx->Config().FeatureFlags.GetEnablePgSyntax())
1234+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
12341235
NSQLTranslation::TTranslationSettings effectiveSettings;
12351236
auto astRes = ParseQuery(
12361237
query.Text,
@@ -1298,7 +1299,8 @@ class TKqpHost : public IKqpHost {
12981299
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, query.Text, SessionCtx->Config().BindingsMode, GUCSettings);
12991300
settingsBuilder
13001301
.SetSqlAutoCommit(false)
1301-
.SetUsePgParser(settings.UsePgParser);
1302+
.SetUsePgParser(settings.UsePgParser)
1303+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
13021304
auto compileResult = CompileYqlQuery(query, /* isSql */ true, *ExprCtx, sqlVersion, settingsBuilder);
13031305

13041306
return TSplitResult{
@@ -1392,7 +1394,8 @@ class TKqpHost : public IKqpHost {
13921394
TMaybe<TSqlVersion> sqlVersion;
13931395
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, query.Text, SessionCtx->Config().BindingsMode, GUCSettings);
13941396
settingsBuilder.SetSqlAutoCommit(false)
1395-
.SetUsePgParser(settings.UsePgParser);
1397+
.SetUsePgParser(settings.UsePgParser)
1398+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
13961399
auto compileResult = CompileYqlQuery(query, isSql, ctx, sqlVersion, settingsBuilder);
13971400
if (compileResult.QueryExprs.empty()) {
13981401
return nullptr;
@@ -1452,7 +1455,8 @@ class TKqpHost : public IKqpHost {
14521455

14531456
TMaybe<TSqlVersion> sqlVersion;
14541457
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, query.Text, SessionCtx->Config().BindingsMode, GUCSettings);
1455-
settingsBuilder.SetSqlAutoCommit(false);
1458+
settingsBuilder.SetSqlAutoCommit(false)
1459+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
14561460
auto compileResult = CompileYqlQuery(query, /* isSql */ true, ctx, sqlVersion, settingsBuilder);
14571461
if (compileResult.QueryExprs.empty()) {
14581462
return nullptr;
@@ -1480,7 +1484,8 @@ class TKqpHost : public IKqpHost {
14801484

14811485
TMaybe<TSqlVersion> sqlVersion;
14821486
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, queryAst.Text, SessionCtx->Config().BindingsMode, GUCSettings);
1483-
settingsBuilder.SetSqlAutoCommit(false);
1487+
settingsBuilder.SetSqlAutoCommit(false)
1488+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
14841489
auto compileResult = CompileYqlQuery(queryAst, false, ctx, sqlVersion, settingsBuilder);
14851490
if (compileResult.QueryExprs.empty()) {
14861491
return nullptr;
@@ -1526,7 +1531,8 @@ class TKqpHost : public IKqpHost {
15261531
if (!expr) {
15271532
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, query.Text, SessionCtx->Config().BindingsMode, GUCSettings);
15281533
settingsBuilder.SetSqlAutoCommit(false)
1529-
.SetUsePgParser(settings.UsePgParser);
1534+
.SetUsePgParser(settings.UsePgParser)
1535+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
15301536
auto compileResult = CompileYqlQuery(query, /* isSql */ true, ctx, sqlVersion, settingsBuilder);
15311537
if (compileResult.QueryExprs.empty()) {
15321538
return nullptr;
@@ -1563,7 +1569,8 @@ class TKqpHost : public IKqpHost {
15631569

15641570
TMaybe<TSqlVersion> sqlVersion = 1;
15651571
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, query.Text, SessionCtx->Config().BindingsMode, GUCSettings);
1566-
settingsBuilder.SetSqlAutoCommit(false);
1572+
settingsBuilder.SetSqlAutoCommit(false)
1573+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
15671574
auto compileResult = CompileYqlQuery(query, true, ctx, sqlVersion, settingsBuilder);
15681575
if (compileResult.QueryExprs.empty()) {
15691576
return nullptr;
@@ -1584,7 +1591,8 @@ class TKqpHost : public IKqpHost {
15841591

15851592
TMaybe<TSqlVersion> sqlVersion;
15861593
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, queryAst.Text, SessionCtx->Config().BindingsMode, GUCSettings);
1587-
settingsBuilder.SetSqlAutoCommit(false);
1594+
settingsBuilder.SetSqlAutoCommit(false)
1595+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
15881596
auto compileResult = CompileYqlQuery(queryAst, false, ctx, sqlVersion, settingsBuilder);
15891597
if (compileResult.QueryExprs.empty()) {
15901598
return nullptr;
@@ -1611,7 +1619,8 @@ class TKqpHost : public IKqpHost {
16111619
TMaybe<TSqlVersion> sqlVersion;
16121620
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, script.Text, SessionCtx->Config().BindingsMode, GUCSettings);
16131621
settingsBuilder.SetSqlAutoCommit(true)
1614-
.SetUsePgParser(settings.UsePgParser);
1622+
.SetUsePgParser(settings.UsePgParser)
1623+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
16151624
auto compileResult = CompileYqlQuery(script, true, ctx, sqlVersion, settingsBuilder);
16161625
if (compileResult.QueryExprs.empty()) {
16171626
return nullptr;
@@ -1640,7 +1649,8 @@ class TKqpHost : public IKqpHost {
16401649
TMaybe<TSqlVersion> sqlVersion;
16411650
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, script.Text, SessionCtx->Config().BindingsMode, GUCSettings);
16421651
settingsBuilder.SetSqlAutoCommit(true)
1643-
.SetUsePgParser(settings.UsePgParser);
1652+
.SetUsePgParser(settings.UsePgParser)
1653+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
16441654
auto compileResult = CompileYqlQuery(script, true, ctx, sqlVersion, settingsBuilder);
16451655
if (compileResult.QueryExprs.empty()) {
16461656
return nullptr;
@@ -1664,7 +1674,8 @@ class TKqpHost : public IKqpHost {
16641674

16651675
TMaybe<TSqlVersion> sqlVersion;
16661676
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, script.Text, SessionCtx->Config().BindingsMode, GUCSettings);
1667-
settingsBuilder.SetSqlAutoCommit(true);
1677+
settingsBuilder.SetSqlAutoCommit(true)
1678+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
16681679
auto compileResult = CompileYqlQuery(script, true, ctx, sqlVersion, settingsBuilder);
16691680
if (compileResult.QueryExprs.empty()) {
16701681
return nullptr;
@@ -1692,7 +1703,8 @@ class TKqpHost : public IKqpHost {
16921703

16931704
TMaybe<TSqlVersion> sqlVersion;
16941705
TKqpTranslationSettingsBuilder settingsBuilder(SessionCtx->Query().Type, SessionCtx->Config()._KqpYqlSyntaxVersion.Get().GetRef(), Cluster, script.Text, SessionCtx->Config().BindingsMode, GUCSettings);
1695-
settingsBuilder.SetSqlAutoCommit(true);
1706+
settingsBuilder.SetSqlAutoCommit(true)
1707+
.SetIsEnableAntlr4Parser(SessionCtx->Config().EnableAntlr4Parser);
16961708
auto compileResult = CompileYqlQuery(script, true, ctx, sqlVersion, settingsBuilder);
16971709
if (compileResult.QueryExprs.empty()) {
16981710
return nullptr;

‎ydb/core/kqp/host/kqp_translate.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ NSQLTranslation::TTranslationSettings TKqpTranslationSettingsBuilder::Build(NYql
6464
settings.AutoParametrizeValuesStmt = IsEnablePgConstsToParams;
6565
}
6666

67+
if (!settings.PgParser) {
68+
settings.Antlr4Parser = IsEnableAntlr4Parser;
69+
}
70+
6771
if (QueryType == NYql::EKikimrQueryType::Scan || QueryType == NYql::EKikimrQueryType::Query) {
6872
SqlVersion = SqlVersion ? *SqlVersion : 1;
6973
}

‎ydb/core/kqp/host/kqp_translate.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ class TKqpTranslationSettingsBuilder {
6767
return *this;
6868
}
6969

70+
TKqpTranslationSettingsBuilder& SetIsEnableAntlr4Parser(bool value) {
71+
IsEnableAntlr4Parser = value;
72+
return *this;
73+
}
74+
7075
private:
7176
const NYql::EKikimrQueryType QueryType;
7277
const ui16 KqpYqlSyntaxVersion;
@@ -79,6 +84,7 @@ class TKqpTranslationSettingsBuilder {
7984
bool IsEnableExternalDataSources = false;
8085
bool IsEnablePgConstsToParams = false;
8186
bool IsEnablePgSyntax = false;
87+
bool IsEnableAntlr4Parser = false;
8288
TMaybe<bool> SqlAutoCommit = {};
8389
TGUCSettings::TPtr GUCSettings;
8490
TMaybe<TString> ApplicationName = {};

‎ydb/core/kqp/provider/yql_kikimr_settings.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi
171171
ui32 DefaultCostBasedOptimizationLevel = 4;
172172
bool EnableConstantFolding = true;
173173
ui64 DefaultEnableSpillingNodes = 0;
174+
bool EnableAntlr4Parser = false;
174175

175176
void SetDefaultEnabledSpillingNodes(const TString& node);
176177
ui64 GetEnabledSpillingNodes() const;

‎ydb/tools/query_replay_yt/main.cpp

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ class TQueryReplayMapper
5757
TVector<TString> UdfFiles;
5858
ui32 ActorSystemThreadsCount = 5;
5959
NActors::NLog::EPriority YqlLogPriority = NActors::NLog::EPriority::PRI_ERROR;
60+
bool EnableAntlr4Parser;
6061

6162
public:
6263
static TString GetFailReason(const TQueryReplayEvents::TCheckQueryPlanStatus& status) {
@@ -93,13 +94,14 @@ class TQueryReplayMapper
9394
public:
9495
TQueryReplayMapper() = default;
9596

96-
Y_SAVELOAD_JOB(UdfFiles, ActorSystemThreadsCount, YqlLogPriority);
97+
Y_SAVELOAD_JOB(UdfFiles, ActorSystemThreadsCount, EnableAntlr4Parser, YqlLogPriority);
9798

98-
TQueryReplayMapper(TVector<TString> udfFiles, ui32 actorSystemThreadsCount,
99+
TQueryReplayMapper(TVector<TString> udfFiles, ui32 actorSystemThreadsCount, bool enableAntlr4Parser,
99100
NActors::NLog::EPriority yqlLogPriority = NActors::NLog::EPriority::PRI_ERROR)
100101
: UdfFiles(udfFiles)
101102
, ActorSystemThreadsCount(actorSystemThreadsCount)
102103
, YqlLogPriority(yqlLogPriority)
104+
, EnableAntlr4Parser(enableAntlr4Parser)
103105
{}
104106

105107
void Start(NYT::TTableWriter<NYT::TNode>*) override {
@@ -147,12 +149,12 @@ class TQueryReplayMapper
147149
return nullptr;
148150
}
149151

150-
auto compileActorId = ActorSystem->Register(CreateQueryCompiler(ModuleResolverState, FunctionRegistry.Get(), HttpGateway));
152+
auto compileActorId = ActorSystem->Register(CreateQueryCompiler(ModuleResolverState, FunctionRegistry.Get(), HttpGateway, EnableAntlr4Parser));
151153

152154
auto future = ActorSystem->Ask<TQueryReplayEvents::TEvCompileResponse>(
153155
compileActorId,
154156
THolder(new TQueryReplayEvents::TEvCompileRequest(std::move(json))),
155-
TDuration::Seconds(100));
157+
TDuration::Seconds(300));
156158

157159
return future.ExtractValueSync();
158160
}
@@ -236,7 +238,7 @@ int main(int argc, const char** argv) {
236238
TQueryReplayConfig config;
237239
config.ParseConfig(argc, argv);
238240
if (config.QueryFile) {
239-
auto fakeMapper = TQueryReplayMapper(config.UdfFiles, config.ActorSystemThreadsCount, config.YqlLogLevel);
241+
auto fakeMapper = TQueryReplayMapper(config.UdfFiles, config.ActorSystemThreadsCount, config.EnableAntlr4Parser, config.YqlLogLevel);
240242
fakeMapper.Start(nullptr);
241243
Y_DEFER {
242244
fakeMapper.Finish(nullptr);
@@ -281,7 +283,7 @@ int main(int argc, const char** argv) {
281283
spec.CoreTablePath(config.CoreTablePath);
282284
}
283285

284-
client->Map(spec, new TQueryReplayMapper(config.UdfFiles, config.ActorSystemThreadsCount, config.YqlLogLevel));
286+
client->Map(spec, new TQueryReplayMapper(config.UdfFiles, config.ActorSystemThreadsCount, config.EnableAntlr4Parser, config.YqlLogLevel));
285287

286288
return EXIT_SUCCESS;
287289
}

‎ydb/tools/query_replay_yt/query_compiler.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ class TStaticTableMetadataLoader: public NYql::IKikimrGateway::IKqpTableMetadata
225225
class TReplayCompileActor: public TActorBootstrapped<TReplayCompileActor> {
226226
public:
227227
TReplayCompileActor(TIntrusivePtr<TModuleResolverState> moduleResolverState, const NMiniKQL::IFunctionRegistry* functionRegistry,
228-
NYql::IHTTPGateway::TPtr httpGateway)
228+
NYql::IHTTPGateway::TPtr httpGateway, bool enableAntlr4Parser)
229229
: ModuleResolverState(moduleResolverState)
230230
, KqpSettings()
231231
, Config(MakeIntrusive<TKikimrConfiguration>())
@@ -234,6 +234,7 @@ class TReplayCompileActor: public TActorBootstrapped<TReplayCompileActor> {
234234
{
235235
Config->EnableKqpScanQueryStreamLookup = true;
236236
Config->EnablePreparedDdl = true;
237+
Config->EnableAntlr4Parser = enableAntlr4Parser;
237238
}
238239

239240
void Bootstrap() {
@@ -679,7 +680,7 @@ class TReplayCompileActor: public TActorBootstrapped<TReplayCompileActor> {
679680
};
680681

681682
IActor* CreateQueryCompiler(TIntrusivePtr<TModuleResolverState> moduleResolverState,
682-
const NMiniKQL::IFunctionRegistry* functionRegistry, NYql::IHTTPGateway::TPtr httpGateway)
683+
const NMiniKQL::IFunctionRegistry* functionRegistry, NYql::IHTTPGateway::TPtr httpGateway, bool enableAntlr4Parser)
683684
{
684-
return new TReplayCompileActor(moduleResolverState, functionRegistry, httpGateway);
685+
return new TReplayCompileActor(moduleResolverState, functionRegistry, httpGateway, enableAntlr4Parser);
685686
}

‎ydb/tools/query_replay_yt/query_replay.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ void TQueryReplayConfig::ParseConfig(int argc, const char** argv) {
2222
opts.AddLongOption("udf-file", "UDFS to load").AppendTo(&UdfFiles);
2323
opts.AddLongOption("query", "Single query to replay").StoreResult(&QueryFile);
2424
opts.AddLongOption("log-level", "Yql log level").StoreResult(&YqlLogLevel);
25+
opts.AddLongOption("antlr4-parser", "Use Antlr4 parser").NoArgument().SetFlag(&EnableAntlr4Parser);
2526

2627
NLastGetopt::TOptsParseResult parseResult(&opts, argc, argv);
2728
}

‎ydb/tools/query_replay_yt/query_replay.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ struct TQueryReplayConfig {
2222
TVector<TString> UdfFiles;
2323
TString QueryFile;
2424
NActors::NLog::EPriority YqlLogLevel = NActors::NLog::EPriority::PRI_ERROR;
25+
bool EnableAntlr4Parser = false;
2526

2627
void ParseConfig(int argc, const char** argv);
2728
};
@@ -81,4 +82,4 @@ struct TQueryReplayEvents {
8182
};
8283

8384
NActors::IActor* CreateQueryCompiler(TIntrusivePtr<NKikimr::NKqp::TModuleResolverState> moduleResolverState,
84-
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, std::shared_ptr<NYql::IHTTPGateway> httpGateway);
85+
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, std::shared_ptr<NYql::IHTTPGateway> httpGateway, bool enableAntlr4Parser);

0 commit comments

Comments
 (0)