Skip to content

Commit 375629c

Browse files
pashandor789Pavel Ivanov
pashandor789
and
Pavel Ivanov
authored
[CBO] user warning added if cbo didn't work (#11950)
Co-authored-by: Pavel Ivanov <[email protected]>
1 parent 0523407 commit 375629c

10 files changed

+44
-26
lines changed

‎ydb/core/kqp/opt/logical/kqp_opt_log.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -161,10 +161,10 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
161161
}
162162

163163
TMaybeNode<TExprBase> OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) {
164-
auto maxDPccpDPTableSize = Config->MaxDPccpDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPccpDPTableSize);
164+
auto maxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize);
165165
auto optLevel = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel);
166166
auto providerCtx = TKqpProviderContext(KqpCtx, optLevel);
167-
auto opt = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(providerCtx, maxDPccpDPTableSize));
167+
auto opt = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(providerCtx, maxDPhypDPTableSize, ctx));
168168
TExprBase output = DqOptimizeEquiJoinWithCosts(node, ctx, TypesCtx, optLevel,
169169
*opt, [](auto& rels, auto label, auto node, auto stat) {
170170
rels.emplace_back(std::make_shared<TKqpRelOptimizerNode>(TString(label), *stat, node));

‎ydb/core/kqp/provider/yql_kikimr_settings.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ TKikimrConfiguration::TKikimrConfiguration() {
9393
REGISTER_SETTING(*this, EnableSpillingNodes)
9494
.Parser([](const TString& v) { return ParseEnableSpillingNodes(v); });
9595

96-
REGISTER_SETTING(*this, MaxDPccpDPTableSize);
96+
REGISTER_SETTING(*this, MaxDPHypDPTableSize);
9797

9898
REGISTER_SETTING(*this, MaxTasksPerStage);
9999
REGISTER_SETTING(*this, MaxSequentialReadsInFlight);

‎ydb/core/kqp/provider/yql_kikimr_settings.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ struct TKikimrSettings {
6969
NCommon::TConfSetting<bool, false> OptUseFinalizeByKey;
7070
NCommon::TConfSetting<ui32, false> CostBasedOptimizationLevel;
7171

72-
NCommon::TConfSetting<ui32, false> MaxDPccpDPTableSize;
72+
NCommon::TConfSetting<ui32, false> MaxDPHypDPTableSize;
7373

7474

7575
NCommon::TConfSetting<ui32, false> MaxTasksPerStage;

‎ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp

+7-7
Original file line numberDiff line numberDiff line change
@@ -3566,7 +3566,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
35663566
execSettings)
35673567
.ExtractValueSync();
35683568
UNIT_ASSERT(result.IsSuccess());
3569-
UNIT_ASSERT(result.GetIssues().Empty());
3569+
// UNIT_ASSERT(result.GetIssues().Empty());
35703570
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
35713571
"[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]]]");
35723572

@@ -3610,7 +3610,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
36103610
execSettings)
36113611
.ExtractValueSync();
36123612
UNIT_ASSERT(result.IsSuccess());
3613-
UNIT_ASSERT(result.GetIssues().Empty());
3613+
// UNIT_ASSERT(result.GetIssues().Empty());
36143614
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
36153615
"[[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]");
36163616

@@ -3655,7 +3655,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
36553655
execSettings)
36563656
.ExtractValueSync();
36573657
UNIT_ASSERT(result.IsSuccess());
3658-
UNIT_ASSERT(result.GetIssues().Empty());
3658+
// UNIT_ASSERT(result.GetIssues().Empty());
36593659
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
36603660
"[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]];[[\"Table1Primary55\"]]]");
36613661

@@ -3699,7 +3699,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
36993699
execSettings)
37003700
.ExtractValueSync();
37013701
UNIT_ASSERT(result.IsSuccess());
3702-
UNIT_ASSERT(result.GetIssues().Empty());
3702+
// UNIT_ASSERT(result.GetIssues().Empty());
37033703
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
37043704
"[[[\"Table1Primary55\"];[55]];[[\"Table1Primary4\"];[4]];[[\"Table1Primary3\"];[3]]]");
37053705

@@ -3804,7 +3804,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
38043804
execSettings)
38053805
.ExtractValueSync();
38063806
UNIT_ASSERT(result.IsSuccess());
3807-
UNIT_ASSERT(result.GetIssues().Empty());
3807+
// UNIT_ASSERT(result.GetIssues().Empty());
38083808
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
38093809
"[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]]]");
38103810

@@ -3856,7 +3856,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
38563856
execSettings)
38573857
.ExtractValueSync();
38583858
UNIT_ASSERT(result.IsSuccess());
3859-
UNIT_ASSERT(result.GetIssues().Empty());
3859+
// UNIT_ASSERT(result.GetIssues().Empty());
38603860
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
38613861
"[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]];[[\"Table1Primary55\"];#]]");
38623862

@@ -4397,7 +4397,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
43974397
execSettings).ExtractValueSync();
43984398

43994399
UNIT_ASSERT_C(result2.IsSuccess(), result2.GetIssues().ToString());
4400-
UNIT_ASSERT(result2.GetIssues().Empty());
4400+
// UNIT_ASSERT(result2.GetIssues().Empty());
44014401

44024402
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result2.GetResultSet(0)), "[[[\"Payload1\"]]]");
44034403

‎ydb/library/yql/dq/opt/dq_cbo_ut.cpp

+7-4
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,14 @@ Y_UNIT_TEST_SUITE(DQCBO) {
3333

3434
Y_UNIT_TEST(Empty) {
3535
TBaseProviderContext pctx;
36-
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000));
36+
TExprContext dummyCtx;
37+
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000, dummyCtx));
3738
}
3839

3940
Y_UNIT_TEST(JoinSearch2Rels) {
4041
TBaseProviderContext pctx;
41-
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000));
42+
TExprContext dummyCtx;
43+
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000, dummyCtx));
4244

4345
auto rel1 = std::make_shared<TRelOptimizerNode>(
4446
"a",
@@ -80,7 +82,8 @@ Type: ManyManyJoin, Nrows: 2e+10, Ncols: 2, ByteSize: 0, Cost: 2.00112e+10, Sel:
8082

8183
Y_UNIT_TEST(JoinSearch3Rels) {
8284
TBaseProviderContext pctx;
83-
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000));
85+
TExprContext dummyCtx;
86+
std::unique_ptr<IOptimizerNew> optimizer = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(pctx, 100000, dummyCtx));
8487

8588
auto rel1 = std::make_shared<TRelOptimizerNode>("a",
8689
TOptimizerStatistics(BaseTable, 100000, 1, 0, 1000000));
@@ -243,7 +246,7 @@ Y_UNIT_TEST(DqOptimizeEquiJoinWithCostsNative) {
243246
TExprContext ctx;
244247
TBaseProviderContext pctx;
245248
std::function<IOptimizerNew*()> optFactory = [&]() {
246-
return MakeNativeOptimizerNew(pctx, 100000);
249+
return MakeNativeOptimizerNew(pctx, 100000, ctx);
247250
};
248251
_DqOptimizeEquiJoinWithCosts(optFactory, ctx);
249252
}

‎ydb/library/yql/dq/opt/dq_opt_hypergraph_ut.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,9 @@ struct TTestContext : public TBaseProviderContext {
5151
template <typename TProviderContext = TTestContext>
5252
std::shared_ptr<IBaseOptimizerNode> Enumerate(const std::shared_ptr<IBaseOptimizerNode>& root, const TOptimizerHints& hints = {}) {
5353
auto ctx = TProviderContext();
54+
TExprContext dummyCtx;
5455
auto optimizer =
55-
std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(ctx, std::numeric_limits<ui32>::max()));
56+
std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(ctx, std::numeric_limits<ui32>::max(), dummyCtx));
5657

5758
Y_ENSURE(root->Kind == EOptimizerNodeKind::JoinNodeType);
5859
auto res = optimizer->JoinSearch(std::static_pointer_cast<TJoinOptimizerNode>(root), hints);

‎ydb/library/yql/dq/opt/dq_opt_join_cbo_factory.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ namespace NYql::NDq {
99
namespace {
1010
class TDqOptimizerFactory : public IOptimizerFactory {
1111
public:
12-
virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext&, const TNativeSettings& settings) const override {
13-
return IOptimizerNew::TPtr(MakeNativeOptimizerNew(pctx, settings.MaxDPhypDPTableSize));
12+
virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerNative(IProviderContext& pctx, TExprContext& ectx, const TNativeSettings& settings) const override {
13+
return IOptimizerNew::TPtr(MakeNativeOptimizerNew(pctx, settings.MaxDPhypDPTableSize, ectx));
1414
}
1515

1616
virtual IOptimizerNew::TPtr MakeJoinCostBasedOptimizerPG(IProviderContext& pctx, TExprContext& ctx, const TPGSettings& settings) const override {

‎ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp

+20-6
Original file line numberDiff line numberDiff line change
@@ -235,9 +235,10 @@ void ComputeStatistics(const std::shared_ptr<TJoinOptimizerNode>& join, IProvide
235235

236236
class TOptimizerNativeNew: public IOptimizerNew {
237237
public:
238-
TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize)
238+
TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize, TExprContext& exprCtx)
239239
: IOptimizerNew(ctx)
240-
, MaxDPhypTableSize_(maxDPhypDPTableSize)
240+
, MaxDPHypTableSize_(maxDPhypDPTableSize)
241+
, ExprCtx(exprCtx)
241242
{}
242243

243244
std::shared_ptr<TJoinOptimizerNode> JoinSearch(
@@ -272,8 +273,15 @@ class TOptimizerNativeNew: public IOptimizerNew {
272273
TJoinHypergraph<TNodeSet> hypergraph = MakeJoinHypergraph<TNodeSet>(joinTree, hints);
273274
TDPHypSolver<TNodeSet> solver(hypergraph, this->Pctx);
274275

275-
if (solver.CountCC(MaxDPhypTableSize_) >= MaxDPhypTableSize_) {
276+
if (solver.CountCC(MaxDPHypTableSize_) >= MaxDPHypTableSize_) {
276277
YQL_CLOG(TRACE, CoreDq) << "Maximum DPhyp threshold exceeded";
278+
ExprCtx.AddWarning(
279+
YqlIssue(
280+
{}, TIssuesIds::DQ_OPTIMIZE_ERROR,
281+
"Cost Based Optimizer could not be applied to this query: "
282+
"Enumeration is too large, use PRAGMA MaxDPHypDPTableSize='4294967295' to disable the limitation"
283+
)
284+
);
277285
ComputeStatistics(joinTree, this->Pctx);
278286
return joinTree;
279287
}
@@ -304,11 +312,12 @@ class TOptimizerNativeNew: public IOptimizerNew {
304312
}
305313

306314
private:
307-
ui32 MaxDPhypTableSize_;
315+
ui32 MaxDPHypTableSize_;
316+
TExprContext& ExprCtx;
308317
};
309318

310-
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize) {
311-
return new TOptimizerNativeNew(ctx, maxDPhypDPTableSize);
319+
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& pctx, const ui32 maxDPhypDPTableSize, TExprContext& ectx) {
320+
return new TOptimizerNativeNew(pctx, maxDPhypDPTableSize, ectx);
312321
}
313322

314323
TExprBase DqOptimizeEquiJoinWithCosts(
@@ -357,6 +366,11 @@ TExprBase DqOptimizeEquiJoinWithCosts(
357366
// of the EquiJoin and n-1 argument are the parameters to EquiJoin
358367

359368
if (!DqCollectJoinRelationsWithStats(rels, typesCtx, equiJoin, providerCollect)){
369+
ctx.AddWarning(
370+
YqlIssue({}, TIssuesIds::DQ_OPTIMIZE_ERROR,
371+
"Cost Based Optimizer could not be applied to this query: couldn't load statistics"
372+
)
373+
);
360374
return node;
361375
}
362376

‎ydb/library/yql/dq/opt/dq_opt_join_cost_based.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,6 @@ NYql::NNodes::TExprBase DqOptimizeEquiJoinWithCosts(
3838
const TOptimizerHints& hints = {}
3939
);
4040

41-
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPccpDPTableSize);
41+
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPHypDPTableSize, TExprContext& ectx);
4242

4343
} // namespace NYql::NDq

‎ydb/library/yql/providers/dq/common/yql_dq_settings.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ struct TDqSettings {
5757
static constexpr ETaskRunnerStats TaskRunnerStats = ETaskRunnerStats::Basic;
5858
static constexpr ESpillingEngine SpillingEngine = ESpillingEngine::Disable;
5959
static constexpr ui32 CostBasedOptimizationLevel = 4;
60-
static constexpr ui32 MaxDPccpDPTableSize = 40000U;
60+
static constexpr ui32 MaxDPHypDPTableSize = 40000U;
6161
static constexpr ui64 MaxAttachmentsSize = 2_GB;
6262
static constexpr bool SplitStageOnDqReplicate = true;
6363
static constexpr ui64 EnableSpillingNodes = 0;

0 commit comments

Comments
 (0)