Skip to content

Commit 979fcef

Browse files
pavelvelikhovpashandor789
pavelvelikhov
and
pashandor789
authored
[KQP] Fix recursion problem when computing SimplifiedPlan (#9519) (#9631)
Co-authored-by: pilik <[email protected]>
1 parent 16c4b26 commit 979fcef

File tree

2 files changed

+211
-119
lines changed

2 files changed

+211
-119
lines changed

‎ydb/core/kqp/opt/kqp_query_plan.cpp

Lines changed: 184 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -1945,6 +1945,9 @@ TVector<NJson::TJsonValue> RemoveRedundantNodes(NJson::TJsonValue& plan, const T
19451945
}
19461946
}
19471947

1948+
if (!planMap.contains("Node Type")) {
1949+
return {};
1950+
}
19481951
const auto typeName = planMap.at("Node Type").GetStringSafe();
19491952
if (redundantNodes.contains(typeName) || typeName.find("Precompute") != TString::npos) {
19501953
return children;
@@ -1953,167 +1956,235 @@ TVector<NJson::TJsonValue> RemoveRedundantNodes(NJson::TJsonValue& plan, const T
19531956
return {plan};
19541957
}
19551958

1956-
NJson::TJsonValue ReconstructQueryPlanRec(const NJson::TJsonValue& plan,
1957-
int operatorIndex,
1958-
const THashMap<int, NJson::TJsonValue>& planIndex,
1959-
const THashMap<TString, NJson::TJsonValue>& precomputes,
1960-
int& nodeCounter) {
1961-
1962-
int currentNodeId = nodeCounter++;
1963-
1964-
NJson::TJsonValue result;
1965-
result["PlanNodeId"] = currentNodeId;
1966-
1967-
if (plan.GetMapSafe().contains("PlanNodeType")) {
1968-
result["PlanNodeType"] = plan.GetMapSafe().at("PlanNodeType").GetStringSafe();
1969-
}
1959+
struct TQueryPlanReconstructor {
1960+
TQueryPlanReconstructor(
1961+
const THashMap<int, NJson::TJsonValue>& planIndex,
1962+
const THashMap<TString, NJson::TJsonValue>& precomputes
1963+
)
1964+
: PlanIndex(planIndex)
1965+
, Precomputes(precomputes)
1966+
, NodeIDCounter(0)
1967+
, Budget(10'000)
1968+
{}
19701969

1971-
if (plan.GetMapSafe().contains("Stats") && operatorIndex==0) {
1972-
result["Stats"] = plan.GetMapSafe().at("Stats");
1973-
}
1970+
NJson::TJsonValue Reconstruct(
1971+
const NJson::TJsonValue& plan,
1972+
int operatorIndex
1973+
) {
1974+
int currentNodeId = NodeIDCounter++;
19741975

1975-
if (!plan.GetMapSafe().contains("Operators")) {
1976-
NJson::TJsonValue planInputs;
1976+
NJson::TJsonValue result;
1977+
result["PlanNodeId"] = currentNodeId;
19771978

1978-
result["Node Type"] = plan.GetMapSafe().at("Node Type").GetStringSafe();
1979+
if (--Budget <= 0) {
1980+
YQL_CLOG(DEBUG, ProviderKqp) << "Can't build the plan - recursion depth has been exceeded!";
1981+
return result;
1982+
}
19791983

1980-
if (plan.GetMapSafe().contains("CTE Name")) {
1981-
auto precompute = plan.GetMapSafe().at("CTE Name").GetStringSafe();
1982-
if (precomputes.contains(precompute)) {
1983-
planInputs.AppendValue(ReconstructQueryPlanRec(precomputes.at(precompute), 0, planIndex, precomputes, nodeCounter));
1984-
}
1984+
if (plan.GetMapSafe().contains("PlanNodeType")) {
1985+
result["PlanNodeType"] = plan.GetMapSafe().at("PlanNodeType").GetStringSafe();
19851986
}
19861987

1987-
if (!plan.GetMapSafe().contains("Plans")) {
1988-
result["Plans"] = planInputs;
1989-
return result;
1988+
if (plan.GetMapSafe().contains("Stats") && operatorIndex==0) {
1989+
result["Stats"] = plan.GetMapSafe().at("Stats");
19901990
}
19911991

1992-
if (plan.GetMapSafe().at("Node Type").GetStringSafe() == "TableLookup") {
1992+
if (plan.GetMapSafe().at("Node Type") == "TableLookupJoin" && plan.GetMapSafe().contains("Table")) {
1993+
result["Node Type"] = "LookupJoin";
19931994
NJson::TJsonValue newOps;
19941995
NJson::TJsonValue op;
19951996

1996-
op["Name"] = "TableLookup";
1997-
op["Columns"] = plan.GetMapSafe().at("Columns");
1997+
op["Name"] = "LookupJoin";
19981998
op["LookupKeyColumns"] = plan.GetMapSafe().at("LookupKeyColumns");
1999-
op["Table"] = plan.GetMapSafe().at("Table");
1999+
2000+
newOps.AppendValue(std::move(op));
2001+
result["Operators"] = std::move(newOps);
2002+
2003+
NJson::TJsonValue newPlans;
2004+
2005+
NJson::TJsonValue lookupPlan;
2006+
lookupPlan["Node Type"] = "TableLookup";
2007+
lookupPlan["PlanNodeType"] = "TableLookup";
2008+
2009+
NJson::TJsonValue lookupOps;
2010+
NJson::TJsonValue lookupOp;
2011+
2012+
lookupOp["Name"] = "TableLookup";
2013+
lookupOp["Columns"] = plan.GetMapSafe().at("Columns");
2014+
lookupOp["LookupKeyColumns"] = plan.GetMapSafe().at("LookupKeyColumns");
2015+
lookupOp["Table"] = plan.GetMapSafe().at("Table");
20002016

20012017
if (plan.GetMapSafe().contains("E-Cost")) {
2002-
op["E-Cost"] = plan.GetMapSafe().at("E-Cost");
2003-
}
2018+
lookupOp["E-Cost"] = plan.GetMapSafe().at("E-Cost");
2019+
}
20042020
if (plan.GetMapSafe().contains("E-Rows")) {
2005-
op["E-Rows"] = plan.GetMapSafe().at("E-Rows");
2021+
lookupOp["E-Rows"] = plan.GetMapSafe().at("E-Rows");
20062022
}
20072023
if (plan.GetMapSafe().contains("E-Size")) {
2008-
op["E-Size"] = plan.GetMapSafe().at("E-Size");
2024+
lookupOp["E-Size"] = plan.GetMapSafe().at("E-Size");
20092025
}
20102026

2011-
newOps.AppendValue(op);
2027+
lookupOps.AppendValue(std::move(lookupOp));
2028+
lookupPlan["Operators"] = std::move(lookupOps);
2029+
2030+
newPlans.AppendValue(Reconstruct(plan.GetMapSafe().at("Plans").GetArraySafe()[0], 0));
2031+
2032+
newPlans.AppendValue(std::move(lookupPlan));
2033+
2034+
result["Plans"] = std::move(newPlans);
20122035

2013-
result["Operators"] = newOps;
20142036
return result;
20152037
}
20162038

2017-
for (auto p : plan.GetMapSafe().at("Plans").GetArraySafe()) {
2018-
if (!p.GetMapSafe().contains("Operators") && p.GetMapSafe().contains("CTE Name")) {
2019-
auto precompute = p.GetMapSafe().at("CTE Name").GetStringSafe();
2020-
if (precomputes.contains(precompute)) {
2021-
planInputs.AppendValue(ReconstructQueryPlanRec(precomputes.at(precompute), 0, planIndex, precomputes, nodeCounter));
2039+
if (!plan.GetMapSafe().contains("Operators")) {
2040+
NJson::TJsonValue planInputs;
2041+
2042+
result["Node Type"] = plan.GetMapSafe().at("Node Type").GetStringSafe();
2043+
2044+
if (plan.GetMapSafe().contains("CTE Name")) {
2045+
auto precompute = plan.GetMapSafe().at("CTE Name").GetStringSafe();
2046+
if (Precomputes.contains(precompute)) {
2047+
planInputs.AppendValue(Reconstruct(Precomputes.at(precompute), 0));
20222048
}
2023-
} else if (p.GetMapSafe().at("Node Type").GetStringSafe().find("Precompute") == TString::npos) {
2024-
planInputs.AppendValue(ReconstructQueryPlanRec(p, 0, planIndex, precomputes, nodeCounter));
20252049
}
2026-
}
2027-
result["Plans"] = planInputs;
2028-
return result;
2029-
}
20302050

2031-
if (plan.GetMapSafe().contains("CTE Name") && plan.GetMapSafe().at("Node Type").GetStringSafe() == "ConstantExpr") {
2032-
auto precompute = plan.GetMapSafe().at("CTE Name").GetStringSafe();
2033-
if (!precomputes.contains(precompute)) {
2034-
result["Node Type"] = plan.GetMapSafe().at("Node Type");
2051+
if (!plan.GetMapSafe().contains("Plans")) {
2052+
result["Plans"] = std::move(planInputs);
2053+
return result;
2054+
}
2055+
2056+
if (plan.GetMapSafe().at("Node Type").GetStringSafe() == "TableLookup") {
2057+
NJson::TJsonValue newOps;
2058+
NJson::TJsonValue op;
2059+
2060+
op["Name"] = "TableLookup";
2061+
op["Columns"] = plan.GetMapSafe().at("Columns");
2062+
op["LookupKeyColumns"] = plan.GetMapSafe().at("LookupKeyColumns");
2063+
op["Table"] = plan.GetMapSafe().at("Table");
2064+
2065+
if (plan.GetMapSafe().contains("E-Cost")) {
2066+
op["E-Cost"] = plan.GetMapSafe().at("E-Cost");
2067+
}
2068+
if (plan.GetMapSafe().contains("E-Rows")) {
2069+
op["E-Rows"] = plan.GetMapSafe().at("E-Rows");
2070+
}
2071+
if (plan.GetMapSafe().contains("E-Size")) {
2072+
op["E-Size"] = plan.GetMapSafe().at("E-Size");
2073+
}
2074+
2075+
newOps.AppendValue(std::move(op));
2076+
2077+
result["Operators"] = std::move(newOps);
2078+
return result;
2079+
}
2080+
2081+
for (auto p : plan.GetMapSafe().at("Plans").GetArraySafe()) {
2082+
if (!p.GetMapSafe().contains("Operators") && p.GetMapSafe().contains("CTE Name")) {
2083+
auto precompute = p.GetMapSafe().at("CTE Name").GetStringSafe();
2084+
if (Precomputes.contains(precompute)) {
2085+
planInputs.AppendValue(Reconstruct(Precomputes.at(precompute), 0));
2086+
}
2087+
} else if (p.GetMapSafe().at("Node Type").GetStringSafe().find("Precompute") == TString::npos) {
2088+
planInputs.AppendValue(Reconstruct(p, 0));
2089+
}
2090+
}
2091+
result["Plans"] = planInputs;
20352092
return result;
20362093
}
20372094

2038-
return ReconstructQueryPlanRec(precomputes.at(precompute), 0, planIndex, precomputes, nodeCounter);
2039-
}
2095+
if (plan.GetMapSafe().contains("CTE Name") && plan.GetMapSafe().at("Node Type").GetStringSafe() == "ConstantExpr") {
2096+
auto precompute = plan.GetMapSafe().at("CTE Name").GetStringSafe();
2097+
if (!Precomputes.contains(precompute)) {
2098+
result["Node Type"] = plan.GetMapSafe().at("Node Type");
2099+
return result;
2100+
}
20402101

2041-
auto ops = plan.GetMapSafe().at("Operators").GetArraySafe();
2042-
auto op = ops[operatorIndex];
2102+
return Reconstruct(Precomputes.at(precompute), 0);
2103+
}
20432104

2044-
TVector<NJson::TJsonValue> planInputs;
2105+
auto ops = plan.GetMapSafe().at("Operators").GetArraySafe();
2106+
auto op = ops[operatorIndex];
20452107

2046-
auto opName = op.GetMapSafe().at("Name").GetStringSafe();
2108+
TVector<NJson::TJsonValue> planInputs;
20472109

2048-
THashSet<ui32> processedExternalOperators;
2049-
THashSet<ui32> processedInternalOperators;
2050-
for (auto opInput : op.GetMapSafe().at("Inputs").GetArraySafe()) {
2110+
auto opName = op.GetMapSafe().at("Name").GetStringSafe();
20512111

2052-
if (opInput.GetMapSafe().contains("ExternalPlanNodeId")) {
2053-
auto inputPlanKey = opInput.GetMapSafe().at("ExternalPlanNodeId").GetIntegerSafe();
2112+
THashSet<ui32> processedExternalOperators;
2113+
THashSet<ui32> processedInternalOperators;
2114+
for (auto opInput : op.GetMapSafe().at("Inputs").GetArraySafe()) {
20542115

2055-
if (processedExternalOperators.contains(inputPlanKey)) {
2056-
continue;
2057-
}
2058-
processedExternalOperators.insert(inputPlanKey);
2116+
if (opInput.GetMapSafe().contains("ExternalPlanNodeId")) {
2117+
auto inputPlanKey = opInput.GetMapSafe().at("ExternalPlanNodeId").GetIntegerSafe();
20592118

2060-
auto inputPlan = planIndex.at(inputPlanKey);
2061-
planInputs.push_back( ReconstructQueryPlanRec(inputPlan, 0, planIndex, precomputes, nodeCounter));
2062-
} else if (opInput.GetMapSafe().contains("InternalOperatorId")) {
2063-
auto inputPlanId = opInput.GetMapSafe().at("InternalOperatorId").GetIntegerSafe();
2119+
if (processedExternalOperators.contains(inputPlanKey)) {
2120+
continue;
2121+
}
2122+
processedExternalOperators.insert(inputPlanKey);
20642123

2065-
if (processedInternalOperators.contains(inputPlanId)) {
2066-
continue;
2067-
}
2068-
processedInternalOperators.insert(inputPlanId);
2124+
auto inputPlan = PlanIndex.at(inputPlanKey);
2125+
planInputs.push_back( Reconstruct(inputPlan, 0) );
2126+
} else if (opInput.GetMapSafe().contains("InternalOperatorId")) {
2127+
auto inputPlanId = opInput.GetMapSafe().at("InternalOperatorId").GetIntegerSafe();
20692128

2070-
planInputs.push_back( ReconstructQueryPlanRec(plan, inputPlanId, planIndex, precomputes, nodeCounter));
2129+
if (processedInternalOperators.contains(inputPlanId)) {
2130+
continue;
2131+
}
2132+
processedInternalOperators.insert(inputPlanId);
2133+
2134+
planInputs.push_back( Reconstruct(plan, inputPlanId) );
2135+
}
20712136
}
2072-
}
20732137

2074-
if (op.GetMapSafe().contains("Inputs")) {
2075-
op.GetMapSafe().erase("Inputs");
2076-
}
2138+
if (op.GetMapSafe().contains("Inputs")) {
2139+
op.GetMapSafe().erase("Inputs");
2140+
}
20772141

2078-
if (op.GetMapSafe().contains("Input")
2079-
|| op.GetMapSafe().contains("ToFlow")
2080-
|| op.GetMapSafe().contains("Member")
2081-
|| op.GetMapSafe().contains("AssumeSorted")
2082-
|| op.GetMapSafe().contains("Iterator")) {
2142+
if (op.GetMapSafe().contains("Input")
2143+
|| op.GetMapSafe().contains("ToFlow")
2144+
|| op.GetMapSafe().contains("Member")
2145+
|| op.GetMapSafe().contains("AssumeSorted")
2146+
|| op.GetMapSafe().contains("Iterator")) {
20832147

2084-
TString maybePrecompute = "";
2085-
if (op.GetMapSafe().contains("Input")) {
2086-
maybePrecompute = op.GetMapSafe().at("Input").GetStringSafe();
2087-
} else if (op.GetMapSafe().contains("ToFlow")) {
2088-
maybePrecompute = op.GetMapSafe().at("ToFlow").GetStringSafe();
2089-
} else if (op.GetMapSafe().contains("Member")) {
2090-
maybePrecompute = op.GetMapSafe().at("Member").GetStringSafe();
2091-
} else if (op.GetMapSafe().contains("AssumeSorted")) {
2092-
maybePrecompute = op.GetMapSafe().at("AssumeSorted").GetStringSafe();
2093-
} else if (op.GetMapSafe().contains("Iterator")) {
2094-
maybePrecompute = op.GetMapSafe().at("Iterator").GetStringSafe();
2095-
}
2148+
TString maybePrecompute = "";
2149+
if (op.GetMapSafe().contains("Input")) {
2150+
maybePrecompute = op.GetMapSafe().at("Input").GetStringSafe();
2151+
} else if (op.GetMapSafe().contains("ToFlow")) {
2152+
maybePrecompute = op.GetMapSafe().at("ToFlow").GetStringSafe();
2153+
} else if (op.GetMapSafe().contains("Member")) {
2154+
maybePrecompute = op.GetMapSafe().at("Member").GetStringSafe();
2155+
} else if (op.GetMapSafe().contains("AssumeSorted")) {
2156+
maybePrecompute = op.GetMapSafe().at("AssumeSorted").GetStringSafe();
2157+
} else if (op.GetMapSafe().contains("Iterator")) {
2158+
maybePrecompute = op.GetMapSafe().at("Iterator").GetStringSafe();
2159+
}
20962160

2097-
if (precomputes.contains(maybePrecompute) && planInputs.empty()) {
2098-
planInputs.push_back(ReconstructQueryPlanRec(precomputes.at(maybePrecompute), 0, planIndex, precomputes, nodeCounter));
2161+
if (Precomputes.contains(maybePrecompute) && planInputs.empty()) {
2162+
planInputs.push_back(Reconstruct(Precomputes.at(maybePrecompute), 0));
2163+
}
20992164
}
2100-
}
21012165

2102-
result["Node Type"] = opName;
2103-
NJson::TJsonValue newOps;
2104-
newOps.AppendValue(op);
2105-
result["Operators"] = newOps;
2166+
result["Node Type"] = std::move(opName);
2167+
NJson::TJsonValue newOps;
2168+
newOps.AppendValue(std::move(op));
2169+
result["Operators"] = std::move(newOps);
21062170

2107-
if (planInputs.size()){
2108-
NJson::TJsonValue plans;
2109-
for( auto i : planInputs) {
2110-
plans.AppendValue(i);
2171+
if (!planInputs.empty()){
2172+
NJson::TJsonValue plans;
2173+
for(auto&& i : planInputs) {
2174+
plans.AppendValue(std::move(i));
2175+
}
2176+
result["Plans"] = std::move(plans);
21112177
}
2112-
result["Plans"] = plans;
2178+
2179+
return result;
21132180
}
21142181

2115-
return result;
2116-
}
2182+
private:
2183+
const THashMap<int, NJson::TJsonValue>& PlanIndex;
2184+
const THashMap<TString, NJson::TJsonValue>& Precomputes;
2185+
ui32 NodeIDCounter;
2186+
i32 Budget; // Prevent bugs with inf recursion
2187+
};
21172188

21182189
double ComputeCpuTimes(NJson::TJsonValue& plan) {
21192190
double currCpuTime = 0;
@@ -2209,8 +2280,7 @@ NJson::TJsonValue SimplifyQueryPlan(NJson::TJsonValue& plan) {
22092280

22102281
BuildPlanIndex(plan, planIndex, precomputes);
22112282

2212-
int nodeCounter = 0;
2213-
plan = ReconstructQueryPlanRec(plan, 0, planIndex, precomputes, nodeCounter);
2283+
plan = TQueryPlanReconstructor(planIndex, precomputes).Reconstruct(plan, 0);
22142284

22152285
RemoveRedundantNodes(plan, redundantNodes);
22162286
ComputeCpuTimes(plan);

0 commit comments

Comments
 (0)