[CBO] Merge transitive closure + cycles fixes in DPHyp (#10857)

Co-authored-by: yumkam <[email protected]>
Co-authored-by: Pavel Velikhov <[email protected]>
Co-authored-by: Pavel Ivanov <[email protected]>

Author: 4 people

ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp

@@ -36,7 +36,7 @@ TMaybeNode<TKqlKeyInc> GetRightTableKeyPrefix(const TKqlKeyRange& range) {
 /**
  * KQP specific rule to check if a LookupJoin is applicable
 */
-bool IsLookupJoinApplicableDetailed(const std::shared_ptr<NYql::TRelOptimizerNode>& node, const TVector<TString>& joinColumns, const TKqpProviderContext& ctx) {
+bool IsLookupJoinApplicableDetailed(const std::shared_ptr<NYql::TRelOptimizerNode>& node, const TVector<TJoinColumn>& joinColumns, const TKqpProviderContext& ctx) {
 
     auto rel = std::static_pointer_cast<TKqpRelOptimizerNode>(node);
     auto expr = TExprBase(rel->Node);
@@ -45,7 +45,7 @@ bool IsLookupJoinApplicableDetailed(const std::shared_ptr<NYql::TRelOptimizerNod
         return false;
     }
 
-    if (find_if(joinColumns.begin(), joinColumns.end(), [&] (const TString& s) { return node->Stats->KeyColumns->Data[0] == s;}) != joinColumns.end()) {
+    if (std::find_if(joinColumns.begin(), joinColumns.end(), [&] (const TJoinColumn& c) { return node->Stats->KeyColumns->Data[0] == c.AttributeName;}) != joinColumns.end()) {
         return true;
     }
 
@@ -97,8 +97,8 @@ bool IsLookupJoinApplicableDetailed(const std::shared_ptr<NYql::TRelOptimizerNod
         return false;
     }
 
-    if (prefixSize < node->Stats->KeyColumns->Data.size() && (find_if(joinColumns.begin(), joinColumns.end(), [&] (const TString& s) {
-            return node->Stats->KeyColumns->Data[prefixSize] == s;
+    if (prefixSize < node->Stats->KeyColumns->Data.size() && (std::find_if(joinColumns.begin(), joinColumns.end(), [&] (const TJoinColumn& c) {
+            return node->Stats->KeyColumns->Data[prefixSize] == c.AttributeName;
         }) == joinColumns.end())){
             return false;
         }
@@ -108,12 +108,11 @@ bool IsLookupJoinApplicableDetailed(const std::shared_ptr<NYql::TRelOptimizerNod
 
 bool IsLookupJoinApplicable(std::shared_ptr<IBaseOptimizerNode> left, 
     std::shared_ptr<IBaseOptimizerNode> right, 
-    const std::set<std::pair<TJoinColumn, TJoinColumn>>& joinConditions,
-    const TVector<TString>& leftJoinKeys,
-    const TVector<TString>& rightJoinKeys,
+    const TVector<TJoinColumn>& leftJoinKeys,
+    const TVector<TJoinColumn>& rightJoinKeys,
     TKqpProviderContext& ctx
 ) {
-    Y_UNUSED(left, joinConditions, leftJoinKeys);
+    Y_UNUSED(left, leftJoinKeys);
 
     if (!(right->Stats->StorageType == EStorageType::RowStorage)) {
         return false;
@@ -130,7 +129,7 @@ bool IsLookupJoinApplicable(std::shared_ptr<IBaseOptimizerNode> left,
     }
 
     for (auto rightCol : rightJoinKeys) {
-        if (std::find(rightStats->KeyColumns->Data.begin(), rightStats->KeyColumns->Data.end(), rightCol) == rightStats->KeyColumns->Data.end()) {
+        if (find(rightStats->KeyColumns->Data.begin(), rightStats->KeyColumns->Data.end(), rightCol.AttributeName) == rightStats->KeyColumns->Data.end()) {
             return false;
         }
     }
@@ -142,18 +141,17 @@ bool IsLookupJoinApplicable(std::shared_ptr<IBaseOptimizerNode> left,
 
 bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptr<IBaseOptimizerNode>& left, 
     const std::shared_ptr<IBaseOptimizerNode>& right, 
-    const std::set<std::pair<NDq::TJoinColumn, NDq::TJoinColumn>>& joinConditions,
-    const TVector<TString>& leftJoinKeys,
-    const TVector<TString>& rightJoinKeys,
+    const TVector<TJoinColumn>& leftJoinKeys,
+    const TVector<TJoinColumn>& rightJoinKeys,
     EJoinAlgoType joinAlgo,
-    EJoinKind joinKind)  {
+    EJoinKind joinKind) {
 
     switch( joinAlgo ) {
         case EJoinAlgoType::LookupJoin:
             if ((OptLevel != 3) && (left->Stats->Nrows > 1000)) {
                 return false;
             }
-            return IsLookupJoinApplicable(left, right, joinConditions, leftJoinKeys, rightJoinKeys, *this);
+            return IsLookupJoinApplicable(left, right, leftJoinKeys, rightJoinKeys, *this);
 
         case EJoinAlgoType::LookupJoinReverse:
             if (joinKind != EJoinKind::LeftSemi) {
@@ -162,7 +160,7 @@ bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptr<IBaseOptimizerN
             if ((OptLevel != 3) && (right->Stats->Nrows > 1000)) {
                 return false;
             }
-            return IsLookupJoinApplicable(right, left, joinConditions, rightJoinKeys, leftJoinKeys, *this);
+            return IsLookupJoinApplicable(right, left, rightJoinKeys, leftJoinKeys, *this);
 
         case EJoinAlgoType::MapJoin:
             return joinKind != EJoinKind::OuterJoin && joinKind != EJoinKind::Exclusion && right->Stats->ByteSize < 1e6;

ydb/core/kqp/opt/logical/kqp_opt_cbo.h

@@ -25,8 +25,7 @@ struct TKqpProviderContext : public NYql::TBaseProviderContext {
 
     virtual bool IsJoinApplicable(const std::shared_ptr<NYql::IBaseOptimizerNode>& left, 
         const std::shared_ptr<NYql::IBaseOptimizerNode>& right, 
-        const std::set<std::pair<NYql::NDq::TJoinColumn, NYql::NDq::TJoinColumn>>& joinConditions,
-        const TVector<TString>& leftJoinKeys, const TVector<TString>& rightJoinKeys,
+        const TVector<NYql::NDq::TJoinColumn>& leftJoinKeys, const TVector<NYql::NDq::TJoinColumn>& rightJoinKeys,
         NYql::EJoinAlgoType joinAlgo,  NYql::EJoinKind joinKind) override;
 
     virtual double ComputeJoinCost(const NYql::TOptimizerStatistics& leftStats, const NYql::TOptimizerStatistics& rightStats, const double outputRows, const double outputByteSize, NYql::EJoinAlgoType joinAlgo) const override;

ydb/core/kqp/ut/common/kqp_ut_common.cpp

@@ -1463,6 +1463,24 @@ NJson::TJsonValue GetJoinOrder(const TString& deserializedPlan) {
     return GetJoinOrderImpl(optRoot);
 }
 
+NJson::TJsonValue GetJoinOrderFromDetailedJoinOrderImpl(const NJson::TJsonValue& opt) {
+    if (!opt.GetMapSafe().contains("table")) {
+        NJson::TJsonValue res;
+        auto args = opt.GetMapSafe().at("args").GetArraySafe();
+        for (size_t i = 0; i < args.size(); ++i) {
+            res.AppendValue(GetJoinOrderFromDetailedJoinOrderImpl(args[i]));
+        }
+        return res;
+    }
+
+    return opt.GetMapSafe().at("table");
+}
+
+NJson::TJsonValue GetJoinOrderFromDetailedJoinOrder(const TString& deserializedDetailedJoinOrder) {
+    NJson::TJsonValue optRoot;
+    NJson::ReadJsonTree(deserializedDetailedJoinOrder, &optRoot, true);
+    return GetJoinOrderFromDetailedJoinOrderImpl(optRoot);
+}
 
 } // namspace NKqp
 } // namespace NKikimr

ydb/core/kqp/ut/common/kqp_ut_common.h

@@ -350,5 +350,7 @@ NJson::TJsonValue GetDetailedJoinOrder(const TString& deserializedPlan, const TG
 /* Gets tables join order without details : only tables. */
 NJson::TJsonValue GetJoinOrder(const TString& deserializedPlan);
 
+NJson::TJsonValue GetJoinOrderFromDetailedJoinOrder(const TString& deserializedDetailedJoinOrder);
+
 } // namespace NKqp
 } // namespace NKikimr

ydb/core/kqp/ut/join/data/join_order/lookupbug.json

@@ -60,3 +60,4 @@
       }
     ]
 }
+

ydb/core/kqp/ut/join/data/join_order/tpcc.json

@@ -24,3 +24,4 @@
       }
     ]
 }
+