Skip to content

[CBO] user warning added if cbo didn't work #11950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Nov 29, 2024
Merged

[CBO] user warning added if cbo didn't work #11950

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
224330a
[CBO] user warning added if cbo didn't work
Nov 25, 2024
b39d5f9
[CBO] user warning added if cbo hadn't worked
Nov 25, 2024
a04d541
fix
Nov 25, 2024
156b603
fix
Nov 25, 2024
652e80a
fix
Nov 25, 2024
07eb425
fix
Nov 25, 2024
628fe06
fix
pashandor789 Nov 26, 2024
0519745
fix
pashandor789 Nov 26, 2024
fe3c9fb
Merge branch 'main' into WARNCBOADDEDD
pashandor789 Nov 26, 2024
fc24ae0
fix
pashandor789 Nov 26, 2024
192cf78
fix
pashandor789 Nov 27, 2024
7859dce
fix
pashandor789 Nov 28, 2024
9147fb4
fix
pashandor789 Nov 28, 2024
d155e5f
fix
pashandor789 Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ydb/core/kqp/opt/logical/kqp_opt_log.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,10 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
}

TMaybeNode<TExprBase> OptimizeEquiJoinWithCosts(TExprBase node, TExprContext& ctx) {
auto maxDPccpDPTableSize = Config->MaxDPccpDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPccpDPTableSize);
auto maxDPhypDPTableSize = Config->MaxDPHypDPTableSize.Get().GetOrElse(TDqSettings::TDefault::MaxDPHypDPTableSize);
auto optLevel = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel);
auto providerCtx = TKqpProviderContext(KqpCtx, optLevel);
auto opt = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(providerCtx, maxDPccpDPTableSize));
auto opt = std::unique_ptr<IOptimizerNew>(MakeNativeOptimizerNew(providerCtx, maxDPhypDPTableSize));
TExprBase output = DqOptimizeEquiJoinWithCosts(node, ctx, TypesCtx, optLevel,
*opt, [](auto& rels, auto label, auto node, auto stat) {
rels.emplace_back(std::make_shared<TKqpRelOptimizerNode>(TString(label), *stat, node));
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/provider/yql_kikimr_settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ TKikimrConfiguration::TKikimrConfiguration() {
REGISTER_SETTING(*this, EnableSpillingNodes)
.Parser([](const TString& v) { return ParseEnableSpillingNodes(v); });

REGISTER_SETTING(*this, MaxDPccpDPTableSize);
REGISTER_SETTING(*this, MaxDPHypDPTableSize);

REGISTER_SETTING(*this, MaxTasksPerStage);
REGISTER_SETTING(*this, MaxSequentialReadsInFlight);
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/provider/yql_kikimr_settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ struct TKikimrSettings {
NCommon::TConfSetting<bool, false> OptUseFinalizeByKey;
NCommon::TConfSetting<ui32, false> CostBasedOptimizationLevel;

NCommon::TConfSetting<ui32, false> MaxDPccpDPTableSize;
NCommon::TConfSetting<ui32, false> MaxDPHypDPTableSize;


NCommon::TConfSetting<ui32, false> MaxTasksPerStage;
Expand Down
6 changes: 3 additions & 3 deletions ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3566,7 +3566,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary3\"]];[[\"Table1Primary4\"]]]");

Expand Down Expand Up @@ -3804,7 +3804,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings)
.ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
UNIT_ASSERT(result.GetIssues().Empty());
// UNIT_ASSERT(result.GetIssues().Empty());
UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)),
"[[[\"Table1Primary3\"];[\"cc\"]];[[\"Table1Primary4\"];[\"dd\"]]]");

Expand Down Expand Up @@ -4397,7 +4397,7 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
execSettings).ExtractValueSync();

UNIT_ASSERT_C(result2.IsSuccess(), result2.GetIssues().ToString());
UNIT_ASSERT(result2.GetIssues().Empty());
// UNIT_ASSERT(result2.GetIssues().Empty());

UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result2.GetResultSet(0)), "[[[\"Payload1\"]]]");

Expand Down
28 changes: 22 additions & 6 deletions ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,10 @@ void ComputeStatistics(const std::shared_ptr<TJoinOptimizerNode>& join, IProvide

class TOptimizerNativeNew: public IOptimizerNew {
public:
TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize)
TOptimizerNativeNew(IProviderContext& ctx, ui32 maxDPhypDPTableSize, TExprContext* exprCtx = nullptr)
: IOptimizerNew(ctx)
, MaxDPhypTableSize_(maxDPhypDPTableSize)
, MaxDPHypTableSize_(maxDPhypDPTableSize)
, ExprCtx(exprCtx)
{}

std::shared_ptr<TJoinOptimizerNode> JoinSearch(
Expand Down Expand Up @@ -272,8 +273,17 @@ class TOptimizerNativeNew: public IOptimizerNew {
TJoinHypergraph<TNodeSet> hypergraph = MakeJoinHypergraph<TNodeSet>(joinTree, hints);
TDPHypSolver<TNodeSet> solver(hypergraph, this->Pctx);

if (solver.CountCC(MaxDPhypTableSize_) >= MaxDPhypTableSize_) {
if (solver.CountCC(MaxDPHypTableSize_) >= MaxDPHypTableSize_) {
YQL_CLOG(TRACE, CoreDq) << "Maximum DPhyp threshold exceeded";
if (ExprCtx) {
ExprCtx->AddWarning(
YqlIssue(
{}, TIssuesIds::DQ_OPTIMIZE_ERROR,
"Cost Based Optimizer could not be applied to this query: "
"Enumeration is too large, use PRAGMA MaxDPHypDPTableSize='4294967295' to disable the limitation"
)
);
}
ComputeStatistics(joinTree, this->Pctx);
return joinTree;
}
Expand Down Expand Up @@ -304,11 +314,12 @@ class TOptimizerNativeNew: public IOptimizerNew {
}

private:
ui32 MaxDPhypTableSize_;
ui32 MaxDPHypTableSize_;
TExprContext* ExprCtx;
};

IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize) {
return new TOptimizerNativeNew(ctx, maxDPhypDPTableSize);
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize, TExprContext* exprCtx) {
return new TOptimizerNativeNew(ctx, maxDPhypDPTableSize, exprCtx);
}

TExprBase DqOptimizeEquiJoinWithCosts(
Expand Down Expand Up @@ -357,6 +368,11 @@ TExprBase DqOptimizeEquiJoinWithCosts(
// of the EquiJoin and n-1 argument are the parameters to EquiJoin

if (!DqCollectJoinRelationsWithStats(rels, typesCtx, equiJoin, providerCollect)){
ctx.AddWarning(
YqlIssue({}, TIssuesIds::DQ_OPTIMIZE_ERROR,
"Cost Based Optimizer could not be applied to this query: couldn't load statistics"
)
);
return node;
}

Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/dq/opt/dq_opt_log.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ IGraphTransformer::TStatus DqWrapIO(const TExprNode::TPtr& input, TExprNode::TPt

NNodes::TExprBase DqExpandMatchRecognize(NNodes::TExprBase node, TExprContext& ctx, TTypeAnnotationContext& typeAnnCtx);

IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPccpDPTableSize);
IOptimizerNew* MakeNativeOptimizerNew(IProviderContext& ctx, const ui32 maxDPhypDPTableSize, TExprContext* exprCtx = nullptr);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Вливаем после https://a.yandex-team.ru/review/7323101/details

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Точнее вот - #12017

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Library Import влился. Вам надо поребейзится на свежий main. В этом месте код поменялся, но кстати ваша задача теперь проще решается - TExprContext в новом API приходит в фабрику оптимизатора

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Да, и я бы предложил передавать TExprContext обязательным аргументом (по ссылке). Во всех местах где эта функция используется, TExprContext& есть под рукой

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

да неудобно
просто оно в тестах еще используется там этот TExprContext надо откуда-то брать

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TExprContext ctx; подойдет


NNodes::TMaybeNode<NNodes::TExprBase> UnorderedOverDqReadWrap(NNodes::TExprBase node, TExprContext& ctx, const std::function<const TParentsMap*()>& getParents, bool enableDqReplicate, TTypeAnnotationContext& typeAnnCtx);

Expand Down
2 changes: 1 addition & 1 deletion ydb/library/yql/providers/dq/common/yql_dq_settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ struct TDqSettings {
static constexpr ETaskRunnerStats TaskRunnerStats = ETaskRunnerStats::Basic;
static constexpr ESpillingEngine SpillingEngine = ESpillingEngine::Disable;
static constexpr ui32 CostBasedOptimizationLevel = 4;
static constexpr ui32 MaxDPccpDPTableSize = 40000U;
static constexpr ui32 MaxDPHypDPTableSize = 40000U;
static constexpr ui64 MaxAttachmentsSize = 2_GB;
static constexpr bool SplitStageOnDqReplicate = true;
static constexpr ui64 EnableSpillingNodes = 0;
Expand Down
Loading