Skip to content

Commit f7db88f

Browse files
vitstnblinkov
vitstn
authored and
blinkov
committed
split lexers & parsers (#14899)
1 parent ec4afff commit f7db88f

File tree

11 files changed

+124
-8
lines changed

11 files changed

+124
-8
lines changed

‎ydb/core/kqp/host/kqp_translate.cpp

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,15 @@
44
#include <yql/essentials/sql/sql.h>
55
#include <yql/essentials/sql/v0/sql.h>
66
#include <yql/essentials/sql/v1/sql.h>
7+
#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
8+
#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
9+
#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
10+
#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
11+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
12+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
13+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
14+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
15+
716
#include <yql/essentials/parser/pg_wrapper/interface/parser.h>
817
#include <ydb/public/api/protos/ydb_query.pb.h>
918

@@ -332,9 +341,20 @@ TVector<TQueryAst> ParseStatements(const TString& queryText, bool isSql, TMaybe<
332341
NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder) {
333342
TVector<TQueryAst> result;
334343
settingsBuilder.SetSqlVersion(sqlVersion);
344+
NSQLTranslationV1::TLexers lexers;
345+
lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
346+
lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
347+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
348+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
349+
NSQLTranslationV1::TParsers parsers;
350+
parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
351+
parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
352+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
353+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
354+
335355
NSQLTranslation::TTranslators translators(
336356
NSQLTranslationV0::MakeTranslator(),
337-
NSQLTranslationV1::MakeTranslator(),
357+
NSQLTranslationV1::MakeTranslator(lexers, parsers),
338358
NSQLTranslationPG::MakeTranslator()
339359
);
340360

‎ydb/core/kqp/host/ya.make

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ PEERDIR(
2424
yql/essentials/sql
2525
yql/essentials/sql/v0
2626
yql/essentials/sql/v1
27+
yql/essentials/sql/v1/lexer/antlr4
28+
yql/essentials/sql/v1/lexer/antlr4_ansi
29+
yql/essentials/sql/v1/proto_parser/antlr4
30+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
31+
yql/essentials/sql/v1/lexer/antlr3
32+
yql/essentials/sql/v1/lexer/antlr3_ansi
33+
yql/essentials/sql/v1/proto_parser/antlr3
34+
yql/essentials/sql/v1/proto_parser/antlr3_ansi
2735
yql/essentials/parser/pg_wrapper/interface
2836
yql/essentials/core
2937
yql/essentials/providers/common/codec

‎ydb/core/kqp/provider/rewrite_io_utils.cpp

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,14 @@
88
#include <yql/essentials/providers/common/provider/yql_provider_names.h>
99
#include <yql/essentials/sql/sql.h>
1010
#include <yql/essentials/sql/v1/sql.h>
11+
#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
12+
#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
13+
#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
14+
#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
15+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
16+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
17+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
18+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
1119
#include <yql/essentials/utils/log/log.h>
1220

1321
namespace NYql {
@@ -27,9 +35,20 @@ TExprNode::TPtr CompileViewQuery(
2735
translationSettings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW;
2836
NSQLTranslation::Deserialize(viewData.CapturedContext, translationSettings);
2937

38+
NSQLTranslationV1::TLexers lexers;
39+
lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
40+
lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
41+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
42+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
43+
NSQLTranslationV1::TParsers parsers;
44+
parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
45+
parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
46+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
47+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
48+
3049
NSQLTranslation::TTranslators translators(
3150
nullptr,
32-
NSQLTranslationV1::MakeTranslator(),
51+
NSQLTranslationV1::MakeTranslator(lexers, parsers),
3352
nullptr
3453
);
3554

‎ydb/core/kqp/provider/ya.make

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,14 @@ PEERDIR(
5454
yql/essentials/providers/result/provider
5555
yql/essentials/sql
5656
yql/essentials/sql/v1
57+
yql/essentials/sql/v1/lexer/antlr4
58+
yql/essentials/sql/v1/lexer/antlr4_ansi
59+
yql/essentials/sql/v1/proto_parser/antlr4
60+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
61+
yql/essentials/sql/v1/lexer/antlr3
62+
yql/essentials/sql/v1/lexer/antlr3_ansi
63+
yql/essentials/sql/v1/proto_parser/antlr3
64+
yql/essentials/sql/v1/proto_parser/antlr3_ansi
5765
ydb/library/ydb_issue/proto
5866
yql/essentials/public/issue
5967
yql/essentials/utils/log

‎ydb/core/kqp/provider/yql_kikimr_provider_ut.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ TContext CreateDefaultParserContext(NYql::TIssues& issues) {
1616
settings.DefaultCluster = "/Cluster";
1717
settings.AssumeYdbOnClusterWithSlash = true;
1818

19-
return TContext(settings, { /* hints */ }, issues, { /* query */ });
19+
NSQLTranslationV1::TLexers lexers;
20+
NSQLTranslationV1::TParsers parsers;
21+
return TContext(lexers, parsers, settings, { /* hints */ }, issues, { /* query */ });
2022
}
2123

2224
TAstNode* CreateAlterTable(TContext& parserContext, const TString& tableName, const TAlterTableParameters& params) {

‎ydb/library/yql/tools/dqrun/dqrun.cpp

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@
7474
#include <yql/essentials/core/pg_ext/yql_pg_ext.h>
7575
#include <yql/essentials/sql/sql.h>
7676
#include <yql/essentials/sql/v1/sql.h>
77+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
78+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
79+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
80+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
7781
#include <yql/essentials/parser/pg_wrapper/interface/parser.h>
7882
#include <yql/essentials/utils/log/tls_backend.h>
7983
#include <yql/essentials/utils/log/log.h>
@@ -1126,9 +1130,16 @@ int RunMain(int argc, const char* argv[])
11261130
dataProvidersInit.push_back(GetDqDataProviderInitializer(&CreateDqExecTransformer, dqGateway, dqCompFactory, {}, storage));
11271131
}
11281132

1133+
NSQLTranslationV1::TLexers lexers;
1134+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
1135+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
1136+
NSQLTranslationV1::TParsers parsers;
1137+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
1138+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
1139+
11291140
NSQLTranslation::TTranslators translators(
11301141
nullptr,
1131-
NSQLTranslationV1::MakeTranslator(),
1142+
NSQLTranslationV1::MakeTranslator(lexers, parsers),
11321143
NSQLTranslationPG::MakeTranslator()
11331144
);
11341145

‎ydb/library/yql/tools/dqrun/ya.make

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ ENDIF()
4141
yql/essentials/core/services/mounts
4242
yql/essentials/sql
4343
yql/essentials/sql/v1
44+
yql/essentials/sql/v1/lexer/antlr4
45+
yql/essentials/sql/v1/lexer/antlr4_ansi
46+
yql/essentials/sql/v1/proto_parser/antlr4
47+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
4448
ydb/library/yql/dq/actors/input_transforms
4549
ydb/library/yql/dq/comp_nodes
4650
ydb/library/yql/dq/opt

‎ydb/public/lib/ydb_cli/commands/interactive/ya.make

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ PEERDIR(
1111
contrib/restricted/patched/replxx
1212
yql/essentials/parser/lexer_common
1313
yql/essentials/sql/v1/lexer
14+
yql/essentials/sql/v1/lexer/antlr4
15+
yql/essentials/sql/v1/lexer/antlr4_ansi
1416
yql/essentials/sql/settings
1517
yql/essentials/utils
1618
contrib/libs/antlr4_cpp_runtime

‎ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
#include <yql/essentials/public/issue/yql_issue.h>
66
#include <yql/essentials/sql/settings/translation_settings.h>
77
#include <yql/essentials/sql/v1/lexer/lexer.h>
8+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
9+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
810

911
#include <util/charset/utf8.h>
1012
#include <util/string/strip.h>
@@ -26,6 +28,13 @@ namespace NYdb {
2628
using std::regex_constants::ECMAScript;
2729
using std::regex_constants::icase;
2830

31+
NSQLTranslationV1::TLexers MakeLexers() {
32+
NSQLTranslationV1::TLexers lexers;
33+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
34+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
35+
return lexers;
36+
}
37+
2938
constexpr const char* builtinFunctionPattern = ( //
3039
"^("
3140
"abs|aggregate_by|aggregate_list|aggregate_list_distinct|agg_list|agg_list_distinct|"
@@ -133,8 +142,8 @@ namespace NYdb {
133142
: Coloring(color)
134143
, BuiltinFunctionRegex(builtinFunctionPattern, ECMAScript | icase)
135144
, TypeRegex(typePattern, ECMAScript | icase)
136-
, CppLexer(MakeLexer(/* ansi = */ false, /* antlr4 = */ true))
137-
, ANSILexer(MakeLexer(/* ansi = */ true, /* antlr4 = */ true))
145+
, CppLexer(MakeLexer(MakeLexers(), /* ansi = */ false, /* antlr4 = */ true))
146+
, ANSILexer(MakeLexer(MakeLexers(), /* ansi = */ true, /* antlr4 = */ true))
138147
{
139148
}
140149

‎ydb/public/lib/ydb_cli/dump/util/query_utils.cpp

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,14 @@
55
#include <yql/essentials/sql/settings/translation_settings.h>
66
#include <yql/essentials/sql/v1/format/sql_format.h>
77
#include <yql/essentials/sql/v1/proto_parser/proto_parser.h>
8+
#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
9+
#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
10+
#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
11+
#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
12+
#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
13+
#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
14+
#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
15+
#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
816
#include <yql/essentials/public/issue/yql_issue.h>
917

1018
#include <library/cpp/protobuf/util/simple_reflection.h>
@@ -237,9 +245,15 @@ bool SqlToProtoAst(const TString& queryStr, TRule_sql_query& queryProto, NYql::T
237245
return false;
238246
}
239247

248+
NSQLTranslationV1::TParsers parsers;
249+
parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
250+
parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
251+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
252+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
253+
240254
google::protobuf::Arena arena;
241255
const auto* parserProto = NSQLTranslationV1::SqlAST(
242-
queryStr, "query", issues, 0, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, &arena
256+
parsers, queryStr, "query", issues, 0, settings.AnsiLexer, settings.Antlr4Parser, &arena
243257
);
244258
if (!parserProto) {
245259
return false;
@@ -254,7 +268,18 @@ bool Format(const TString& query, TString& formattedQuery, NYql::TIssues& issues
254268
NSQLTranslation::TTranslationSettings settings;
255269
settings.Arena = &arena;
256270

257-
auto formatter = NSQLFormat::MakeSqlFormatter(settings);
271+
NSQLTranslationV1::TLexers lexers;
272+
lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
273+
lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
274+
lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
275+
lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
276+
NSQLTranslationV1::TParsers parsers;
277+
parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
278+
parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
279+
parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
280+
parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
281+
282+
auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings);
258283
return formatter->Format(query, formattedQuery, issues);
259284
}
260285

‎ydb/public/lib/ydb_cli/dump/util/ya.make

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,14 @@ PEERDIR(
1616
yql/essentials/sql/settings
1717
yql/essentials/sql/v1/format
1818
yql/essentials/sql/v1/proto_parser
19+
yql/essentials/sql/v1/lexer/antlr4
20+
yql/essentials/sql/v1/lexer/antlr4_ansi
21+
yql/essentials/sql/v1/proto_parser/antlr4
22+
yql/essentials/sql/v1/proto_parser/antlr4_ansi
23+
yql/essentials/sql/v1/lexer/antlr3
24+
yql/essentials/sql/v1/lexer/antlr3_ansi
25+
yql/essentials/sql/v1/proto_parser/antlr3
26+
yql/essentials/sql/v1/proto_parser/antlr3_ansi
1927
library/cpp/protobuf/util
2028
)
2129

0 commit comments

Comments
 (0)