Skip to content

Commit 072b721

Browse files
vitstnvitstn
authored
Support of functions with lang=sql (#8233)
1 parent d9ff14c commit 072b721

File tree

24 files changed

+653
-160
lines changed

24 files changed

+653
-160
lines changed

‎ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2679,6 +2679,16 @@ TExprNode::TPtr ExpandVersion(const TExprNode::TPtr& node, TExprContext& ctx) {
26792679
.Seal().Build();
26802680
}
26812681

2682+
TExprNode::TPtr ExpandRightOverCons(const TExprNode::TPtr& node, TExprContext& ctx) {
2683+
Y_UNUSED(ctx);
2684+
if (node->Head().IsCallable(ConsName)) {
2685+
YQL_CLOG(DEBUG, CorePeepHole) << "Expand Right! over Cons!";
2686+
return node->Head().TailPtr();
2687+
}
2688+
2689+
return node;
2690+
}
2691+
26822692
TExprNode::TPtr ExpandPartitionsByKeys(const TExprNode::TPtr& node, TExprContext& ctx) {
26832693
YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
26842694
const bool isStream = node->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow ||
@@ -8351,6 +8361,7 @@ struct TPeepHoleRules {
83518361
{"JsonExists", &ExpandJsonExists},
83528362
{"EmptyIterator", &DropDependsOnFromEmptyIterator},
83538363
{"Version", &ExpandVersion},
8364+
{RightName, &ExpandRightOverCons},
83548365
};
83558366

83568367
const TExtPeepHoleOptimizerMap CommonStageExtRules = {

‎ydb/library/yql/core/type_ann/type_ann_pg.cpp

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -368,21 +368,22 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode
368368
const auto procOrType = NPg::LookupProcWithCasts(TString(name), argTypes);
369369
auto children = input->ChildrenList();
370370
if (const auto* procPtr = std::get_if<const NPg::TProcDesc*>(&procOrType)) {
371-
auto idNode = ctx.Expr.NewAtom(input->Pos(), ToString((*procPtr)->ProcId));
371+
const auto& proc = *(*procPtr);
372+
auto idNode = ctx.Expr.NewAtom(input->Pos(), ToString(proc.ProcId));
372373
children.insert(children.begin() + 1, idNode);
373374

374-
const auto& fargTypes = (*procPtr)->ArgTypes;
375+
const auto& fargTypes = proc.ArgTypes;
375376
for (size_t i = 0; i < argTypes.size(); ++i) {
376-
auto targetType = (i >= fargTypes.size()) ? (*procPtr)->VariadicType : fargTypes[i];
377+
auto targetType = (i >= fargTypes.size()) ? proc.VariadicType : fargTypes[i];
377378
if (IsCastRequired(argTypes[i], targetType)) {
378379
children[i+3] = WrapWithPgCast(std::move(children[i+3]), targetType, ctx.Expr);
379380
}
380381
}
381382

382383
if (argTypes.size() < fargTypes.size()) {
383-
YQL_ENSURE(fargTypes.size() - argTypes.size() <= (*procPtr)->DefaultArgs.size());
384+
YQL_ENSURE(fargTypes.size() - argTypes.size() <= proc.DefaultArgs.size());
384385
for (size_t i = argTypes.size(); i < fargTypes.size(); ++i) {
385-
const auto& value = (*procPtr)->DefaultArgs[i + (*procPtr)->DefaultArgs.size() - fargTypes.size()];
386+
const auto& value = proc.DefaultArgs[i + proc.DefaultArgs.size() - fargTypes.size()];
386387
TExprNode::TPtr defNode;
387388
if (!value) {
388389
defNode = ctx.Expr.NewCallable(input->Pos(), "Null", {});
@@ -399,6 +400,27 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode
399400
children.insert(children.end(), defNode);
400401
}
401402
}
403+
404+
if (proc.Lang == NPg::LangSQL) {
405+
if (!proc.ExprNode) {
406+
throw yexception() << "Function " << proc.Name << " has no implementation";
407+
}
408+
409+
// substibute by lambda
410+
YQL_ENSURE(proc.ExprNode->IsLambda());
411+
YQL_ENSURE(proc.ExprNode->Head().ChildrenSize() == fargTypes.size());
412+
TNodeOnNodeOwnedMap deepClones;
413+
YQL_ENSURE(NPg::GetSqlLanguageParser());
414+
auto lambda = ctx.Expr.DeepCopy(*proc.ExprNode, NPg::GetSqlLanguageParser()->GetContext(), deepClones, true, false);
415+
TNodeOnNodeOwnedMap replaces;
416+
for (ui32 i = 0; i < fargTypes.size(); ++i) {
417+
replaces[lambda->Head().Child(i)] = children[i + 3];
418+
}
419+
420+
output = ctx.Expr.ReplaceNodes(lambda->TailPtr(), replaces);
421+
return IGraphTransformer::TStatus::Repeat;
422+
}
423+
402424
output = ctx.Expr.NewCallable(input->Pos(), "PgResolvedCall", std::move(children));
403425
} else if (const auto* typePtr = std::get_if<const NPg::TTypeDesc*>(&procOrType)) {
404426
output = WrapWithPgCast(std::move(children[2]), (*typePtr)->TypeId, ctx.Expr);

‎ydb/library/yql/parser/pg_catalog/catalog.cpp

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -476,10 +476,12 @@ class TProcsParser : public TParser {
476476
} else if (key == "prosrc") {
477477
LastProc.Src = value;
478478
} else if (key == "prolang") {
479-
if (value != "c") {
480-
IsSupported = false;
481-
} else {
479+
if (value == "sql") {
480+
LastProc.Lang = LangSQL;
481+
} else if (value == "c") {
482482
LastProc.Lang = LangC;
483+
} else {
484+
IsSupported = false;
483485
}
484486
} else if (key == "proargtypes") {
485487
TVector<TString> strArgs;
@@ -2272,6 +2274,8 @@ struct TCatalog : public IExtensionSqlBuilder {
22722274

22732275
bool AllowAllFunctions = false;
22742276
THashSet<TString> AllowedProcs;
2277+
2278+
bool SystemFunctionInit = false;
22752279
};
22762280

22772281
mutable TMaybe<TFile> ExportFile;
@@ -3598,6 +3602,49 @@ void AllowFunction(const TString& name) {
35983602
}
35993603
}
36003604

3605+
struct TSqlLanguageParserHolder {
3606+
std::unique_ptr<ISqlLanguageParser> Parser;
3607+
};
3608+
3609+
void SetSqlLanguageParser(std::unique_ptr<ISqlLanguageParser> parser) {
3610+
Singleton<TSqlLanguageParserHolder>()->Parser = std::move(parser);
3611+
}
3612+
3613+
ISqlLanguageParser* GetSqlLanguageParser() {
3614+
return Singleton<TSqlLanguageParserHolder>()->Parser.get();
3615+
}
3616+
3617+
void LoadSystemFunctions(ISystemFunctionsParser& parser) {
3618+
auto& catalog = TCatalog::MutableInstance();
3619+
with_lock (catalog.ExtensionsGuard) {
3620+
Y_ENSURE(!catalog.State->SystemFunctionInit);
3621+
TString data;
3622+
Y_ENSURE(NResource::FindExact("system_functions.sql", &data));
3623+
TVector<TProcDesc> procs;
3624+
parser.Parse(data, procs);
3625+
for (const auto& p : procs) {
3626+
auto procIdPtr = catalog.State->ProcByName.FindPtr(p.Name);
3627+
Y_ENSURE(procIdPtr);
3628+
TProcDesc* foundProc = nullptr;
3629+
for (auto procId : *procIdPtr) {
3630+
auto procPtr = catalog.State->Procs.FindPtr(procId);
3631+
Y_ENSURE(procPtr);
3632+
if (procPtr->ArgTypes == p.ArgTypes && procPtr->VariadicType == p.VariadicType) {
3633+
foundProc = procPtr;
3634+
break;
3635+
}
3636+
}
3637+
3638+
Y_ENSURE(foundProc);
3639+
foundProc->DefaultArgs = p.DefaultArgs;
3640+
foundProc->Src = p.Src;
3641+
foundProc->ExprNode = p.ExprNode;
3642+
}
3643+
3644+
catalog.State->SystemFunctionInit = true;
3645+
}
3646+
}
3647+
36013648
void RegisterExtensions(const TVector<TExtensionDesc>& extensions, bool typesOnly,
36023649
IExtensionSqlParser& parser, IExtensionLoader* loader) {
36033650
if (extensions.size() > MaximumExtensionsCount) {

‎ydb/library/yql/parser/pg_catalog/catalog.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@
88
#include <variant>
99
#include <functional>
1010

11+
struct Node;
12+
13+
namespace NYql {
14+
class TExprNode;
15+
struct TExprContext;
16+
}
17+
1118
namespace NYql::NPg {
1219

1320
constexpr ui32 UnknownOid = 705;
@@ -75,6 +82,7 @@ struct TProcDesc {
7582
ui32 VariadicArgType = 0;
7683
TString VariadicArgName;
7784
TVector<TMaybe<TString>> DefaultArgs;
85+
TExprNode* ExprNode = nullptr;
7886
ui32 ExtensionIndex = 0;
7987
};
8088

@@ -436,6 +444,26 @@ class IExtensionLoader {
436444
virtual void Load(ui32 extensionIndex, const TString& name, const TString& path) = 0;
437445
};
438446

447+
class ISystemFunctionsParser {
448+
public:
449+
virtual ~ISystemFunctionsParser() = default;
450+
virtual void Parse(const TString& sql, TVector<TProcDesc>& procs) const = 0;
451+
};
452+
453+
class ISqlLanguageParser {
454+
public:
455+
virtual ~ISqlLanguageParser() = default;
456+
virtual void Parse(const TString& sql, TProcDesc& proc) = 0;
457+
virtual void ParseNode(const Node* stmt, TProcDesc& proc) = 0;
458+
virtual void Freeze() = 0;
459+
virtual TExprContext& GetContext() = 0;
460+
};
461+
462+
void SetSqlLanguageParser(std::unique_ptr<ISqlLanguageParser> parser);
463+
ISqlLanguageParser* GetSqlLanguageParser();
464+
465+
void LoadSystemFunctions(ISystemFunctionsParser& parser);
466+
439467
// either RegisterExtensions or ImportExtensions should be called at most once, see ClearExtensions as well
440468
void RegisterExtensions(const TVector<TExtensionDesc>& extensions, bool typesOnly,
441469
IExtensionSqlParser& parser, IExtensionLoader* loader);

‎ydb/library/yql/parser/pg_catalog/postgis_procs.h

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@
9090
"_postgis_join_selectivity",
9191
"_postgis_selectivity",
9292
"_postgis_stats",
93+
"postgis_transform_geometry",
9394
"spheroid_in",
9495
"spheroid_out",
9596
"st_3dclosestpoint",
@@ -106,6 +107,7 @@
106107
"st_3dshortestline",
107108
"st_addmeasure",
108109
"st_addpoint",
110+
"st_affine",
109111
"st_angle",
110112
"st_area",
111113
"st_area2d",
@@ -114,6 +116,7 @@
114116
"st_asewkb",
115117
"st_asewkt",
116118
"st_asgeojson",
119+
"_st_asgml",
117120
"st_asgml",
118121
"st_askml",
119122
"st_aslatlontext",
@@ -122,6 +125,7 @@
122125
"st_assvg",
123126
"st_astext",
124127
"st_astwkb",
128+
"_st_asx3d",
125129
"st_azimuth",
126130
"_st_bestsrid",
127131
"st_boundary",
@@ -153,15 +157,15 @@
153157
"st_distance",
154158
"st_distancecpa",
155159
"st_distancespheroid",
156-
"st_dump",
157-
"st_dumppoints",
158-
"st_dumpsegments",
160+
"_st_distancetree",
161+
"_st_distanceuncached",
159162
"st_dwithin",
160163
"st_endpoint",
161164
"st_envelope",
162165
"st_equals",
163166
"st_estimatedextent",
164167
"st_expand",
168+
"st_exteriorring",
165169
"st_filterbym",
166170
"st_flipcoordinates",
167171
"st_force2d",
@@ -184,6 +188,7 @@
184188
"st_geomfromewkb",
185189
"st_geomfromewkt",
186190
"st_geomfromgeojson",
191+
"_st_geomfromgml",
187192
"st_geomfromgml",
188193
"st_geomfromkml",
189194
"st_geomfrommarc21",
@@ -225,13 +230,15 @@
225230
"st_locatealong",
226231
"st_locatebetween",
227232
"st_locatebetweenelevations",
233+
"_st_longestline",
228234
"st_m",
229235
"st_makebox2d",
230236
"st_makeenvelope",
231237
"st_makepoint",
232238
"st_makepointm",
233239
"st_makepolygon",
234240
"st_makevalid",
241+
"_st_maxdistance",
235242
"st_maximuminscribedcircle",
236243
"st_memsize",
237244
"st_minimumboundingcircle",

‎ydb/library/yql/parser/pg_catalog/used_procs.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
"any_out",
3939
"anyrange_in",
4040
"anyrange_out",
41+
"anytextcat",
4142
"any_value_transfn",
4243
"area",
4344
"array_agg_array_combine",
@@ -702,6 +703,7 @@
702703
"int8or",
703704
"int8out",
704705
"int8pl",
706+
"int8pl_inet",
705707
"int8recv",
706708
"int8send",
707709
"int8shl",
@@ -710,6 +712,7 @@
710712
"int8um",
711713
"int8up",
712714
"int8xor",
715+
"integer_pl_date",
713716
"inter_lb",
714717
"internal_in",
715718
"internal_out",
@@ -734,6 +737,11 @@
734737
"interval_ne",
735738
"interval_out",
736739
"interval_pl",
740+
"interval_pl_date",
741+
"interval_pl_time",
742+
"interval_pl_timestamp",
743+
"interval_pl_timestamptz",
744+
"interval_pl_timetz",
737745
"interval_recv",
738746
"interval_send",
739747
"interval_smaller",
@@ -1015,6 +1023,7 @@
10151023
"numeric_mul",
10161024
"numeric_ne",
10171025
"numeric_out",
1026+
"numeric_pl_pg_lsn",
10181027
"numeric_poly_avg",
10191028
"numeric_poly_combine",
10201029
"numeric_poly_deserialize",
@@ -1074,9 +1083,11 @@
10741083
"ordered_set_transition_multi",
10751084
"overlaps",
10761085
"overlay",
1086+
"parse_ident",
10771087
"path",
10781088
"path_add",
10791089
"path_add_pt",
1090+
"path_contain_pt",
10801091
"path_distance",
10811092
"path_div_pt",
10821093
"path_in",
@@ -1312,6 +1323,7 @@
13121323
"sha384",
13131324
"sha512",
13141325
"sign",
1326+
"similar_to_escape",
13151327
"sin",
13161328
"sind",
13171329
"sinh",
@@ -1336,6 +1348,7 @@
13361348
"tand",
13371349
"tanh",
13381350
"text",
1351+
"textanycat",
13391352
"textcat",
13401353
"texteq",
13411354
"texteqname",
@@ -1381,6 +1394,7 @@
13811394
"tidsmaller",
13821395
"time",
13831396
"time_cmp",
1397+
"timedate_pl",
13841398
"time_eq",
13851399
"time_ge",
13861400
"time_gt",
@@ -1465,6 +1479,7 @@
14651479
"timetypmodout",
14661480
"timetz",
14671481
"timetz_cmp",
1482+
"timetzdate_pl",
14681483
"timetz_eq",
14691484
"timetz_ge",
14701485
"timetz_gt",

‎ydb/library/yql/parser/pg_catalog/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_amop.dat pg_amop.dat)
1212
RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_am.dat pg_am.dat)
1313
RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_conversion.dat pg_conversion.dat)
1414
RESOURCE(../pg_wrapper/postgresql/src/include/catalog/pg_language.dat pg_language.dat)
15+
RESOURCE(../pg_wrapper/postgresql/src/backend/catalog/system_functions.sql system_functions.sql)
1516

1617
SRCS(
1718
catalog.cpp

‎ydb/library/yql/parser/pg_wrapper/interface/parser.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,7 @@ namespace NSQLTranslationPG {
1414
NYql::TAstParseResult PGToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TStmtParseInfo* stmtParseInfo = nullptr);
1515
TVector<NYql::TAstParseResult> PGToYqlStatements(const TString& query, const NSQLTranslation::TTranslationSettings& settings, TVector<NYql::TStmtParseInfo>* stmtParseInfo = nullptr);
1616
std::unique_ptr<NYql::NPg::IExtensionSqlParser> CreateExtensionSqlParser();
17+
std::unique_ptr<NYql::NPg::ISystemFunctionsParser> CreateSystemFunctionsParser();
18+
std::unique_ptr<NYql::NPg::ISqlLanguageParser> CreateSqlLanguageParser();
1719

1820
} // NSQLTranslationPG

0 commit comments

Comments
 (0)