Skip to content

Commit 848fc7f

Browse files
pavelvelikhovpavelvelikhov
committed
Initial commit
1 parent 1c09c5d commit 848fc7f

File tree

9 files changed

+71
-13
lines changed

9 files changed

+71
-13
lines changed

‎ydb/core/kqp/opt/kqp_statistics_transformer.cpp

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ void InferStatisticsForReadTable(const TExprNode::TPtr& input, TTypeAnnotationCo
7979
byteSize,
8080
0.0,
8181
keyColumns,
82-
inputStats->ColumnStatistics);
82+
inputStats->ColumnStatistics,
83+
inputStats->StorageType);
8384

8485
YQL_CLOG(TRACE, CoreDq) << "Infer statistics for read table" << stats->ToString();
8586

@@ -119,6 +120,19 @@ void InferStatisticsForKqpTable(const TExprNode::TPtr& input, TTypeAnnotationCon
119120
}
120121
}
121122

123+
EStorageType storageType = EStorageType::NA;
124+
switch (tableData.Metadata->Kind) {
125+
case EKikimrTableKind::Datashard:
126+
storageType = EStorageType::RowStorage;
127+
break;
128+
case EKikimrTableKind::Olap:
129+
storageType = EStorageType::ColumnStorage;
130+
break;
131+
default:
132+
break;
133+
}
134+
stats->StorageType = storageType;
135+
122136
YQL_CLOG(TRACE, CoreDq) << "Infer statistics for table: " << path.Value() << ": " << stats->ToString();
123137

124138
typeCtx->SetStats(input.Get(), stats);
@@ -151,7 +165,8 @@ void InferStatisticsForSteamLookup(const TExprNode::TPtr& input, TTypeAnnotation
151165
byteSize,
152166
0,
153167
inputStats->KeyColumns,
154-
inputStats->ColumnStatistics);
168+
inputStats->ColumnStatistics,
169+
inputStats->StorageType);
155170

156171
typeCtx->SetStats(input.Get(), res);
157172

@@ -195,7 +210,8 @@ void InferStatisticsForLookupTable(const TExprNode::TPtr& input, TTypeAnnotation
195210
byteSize,
196211
0,
197212
inputStats->KeyColumns,
198-
inputStats->ColumnStatistics));
213+
inputStats->ColumnStatistics,
214+
inputStats->StorageType));
199215
}
200216

201217
/**
@@ -250,7 +266,8 @@ void InferStatisticsForRowsSourceSettings(const TExprNode::TPtr& input, TTypeAnn
250266
byteSize,
251267
cost,
252268
keyColumns,
253-
inputStats->ColumnStatistics));
269+
inputStats->ColumnStatistics,
270+
inputStats->StorageType));
254271
}
255272

256273
/**
@@ -282,7 +299,8 @@ void InferStatisticsForReadTableIndexRanges(const TExprNode::TPtr& input, TTypeA
282299
inputStats->ByteSize,
283300
inputStats->Cost,
284301
indexColumnsPtr,
285-
inputStats->ColumnStatistics);
302+
inputStats->ColumnStatistics,
303+
inputStats->StorageType);
286304

287305
typeCtx->SetStats(input.Get(), stats);
288306

‎ydb/core/kqp/opt/logical/kqp_opt_log.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,14 +166,14 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase {
166166
}
167167

168168
TMaybeNode<TExprBase> RewriteEquiJoin(TExprBase node, TExprContext& ctx) {
169-
bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) == 3;
169+
bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) >= 3;
170170
TExprBase output = DqRewriteEquiJoin(node, KqpCtx.Config->GetHashJoinMode(), useCBO, ctx, TypesCtx, KqpCtx.JoinsCount);
171171
DumpAppliedRule("RewriteEquiJoin", node.Ptr(), output.Ptr(), ctx);
172172
return output;
173173
}
174174

175175
TMaybeNode<TExprBase> JoinToIndexLookup(TExprBase node, TExprContext& ctx) {
176-
bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) == 3;
176+
bool useCBO = Config->CostBasedOptimizationLevel.Get().GetOrElse(Config->DefaultCostBasedOptimizationLevel) >= 3;
177177
TExprBase output = KqpJoinToIndexLookup(node, ctx, KqpCtx, useCBO);
178178
DumpAppliedRule("JoinToIndexLookup", node.Ptr(), output.Ptr(), ctx);
179179
return output;

‎ydb/core/kqp/provider/yql_kikimr_settings.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi
169169
bool EnableOltpSink = false;
170170
NKikimrConfig::TTableServiceConfig_EBlockChannelsMode BlockChannelsMode;
171171
bool EnableSpillingGenericQuery = false;
172-
ui32 DefaultCostBasedOptimizationLevel = 3;
172+
ui32 DefaultCostBasedOptimizationLevel = 4;
173173
bool EnableConstantFolding = true;
174174
ui64 DefaultEnableSpillingNodes = 0;
175175

‎ydb/core/protos/table_service_config.proto

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ message TTableServiceConfig {
298298

299299
optional bool EnableQueryServiceSpilling = 63 [ default = true ];
300300

301-
optional uint32 DefaultCostBasedOptimizationLevel = 64 [default = 3];
301+
optional uint32 DefaultCostBasedOptimizationLevel = 64 [default = 4];
302302

303303
optional bool EnableConstantFolding = 65 [ default = true ];
304304

‎ydb/library/yql/core/yql_statistics.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ TOptimizerStatistics::TOptimizerStatistics(
5151
double cost,
5252
TIntrusivePtr<TKeyColumns> keyColumns,
5353
TIntrusivePtr<TColumnStatMap> columnMap,
54+
EStorageType storageType,
5455
std::unique_ptr<IProviderStatistics> specific)
5556
: Type(type)
5657
, Nrows(nrows)
@@ -59,6 +60,7 @@ TOptimizerStatistics::TOptimizerStatistics(
5960
, Cost(cost)
6061
, KeyColumns(keyColumns)
6162
, ColumnStatistics(columnMap)
63+
, StorageType(storageType)
6264
, Specific(std::move(specific))
6365
{
6466
}

‎ydb/library/yql/core/yql_statistics.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ enum EStatisticsType : ui32 {
1919
ManyManyJoin
2020
};
2121

22+
enum EStorageType : ui32 {
23+
NA,
24+
RowStorage,
25+
ColumnStorage
26+
};
27+
2228
// Providers may subclass this struct to associate specific statistics, useful to
2329
// derive stats for higher-level operators in the plan.
2430
struct IProviderStatistics {
@@ -61,6 +67,7 @@ struct TOptimizerStatistics {
6167
double Selectivity = 1.0;
6268
TIntrusivePtr<TKeyColumns> KeyColumns;
6369
TIntrusivePtr<TColumnStatMap> ColumnStatistics;
70+
EStorageType StorageType = EStorageType::NA;
6471
std::unique_ptr<const IProviderStatistics> Specific;
6572
std::shared_ptr<TVector<TString>> Labels = {};
6673

@@ -75,6 +82,7 @@ struct TOptimizerStatistics {
7582
double cost = 0.0,
7683
TIntrusivePtr<TKeyColumns> keyColumns = {},
7784
TIntrusivePtr<TColumnStatMap> columnMap = {},
85+
EStorageType storageType = EStorageType::NA,
7886
std::unique_ptr<IProviderStatistics> specific = nullptr);
7987

8088
TOptimizerStatistics& operator+=(const TOptimizerStatistics& other);

‎ydb/library/yql/dq/opt/dq_opt_join_cost_based.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,16 @@ TExprBase DqOptimizeEquiJoinWithCosts(
314314

315315
YQL_CLOG(TRACE, CoreDq) << "All statistics for join in place";
316316

317+
bool allRowStorage = std::all_of(
318+
rels.begin(),
319+
rels.end(),
320+
[](std::shared_ptr<TRelOptimizerNode>& r) {return r->Stats->StorageType==EStorageType::RowStorage; });
321+
322+
Y_UNUSED(allRowStorage);
323+
//if (optLevel == 3 && allRowStorage) {
324+
// return node;
325+
//}
326+
317327
equiJoinCounter++;
318328

319329
auto joinTuple = equiJoin.Arg(equiJoin.ArgCount() - 2).Cast<TCoEquiJoinTuple>();

‎ydb/library/yql/dq/opt/dq_opt_stat.cpp

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,9 @@ namespace {
7171
inputStats->Ncols,
7272
inputStats->ByteSize,
7373
inputStats->Cost,
74-
inputStats->KeyColumns);
74+
inputStats->KeyColumns,
75+
inputStats->ColumnStatistics,
76+
inputStats->StorageType);
7577
outputStats->Labels = inputStats->Labels;
7678
return outputStats;
7779
}
@@ -380,7 +382,16 @@ void InferStatisticsForFlatMap(const TExprNode::TPtr& input, TTypeAnnotationCont
380382

381383
double selectivity = TPredicateSelectivityComputer(inputStats).Compute(flatmap.Lambda().Body());
382384

383-
auto outputStats = TOptimizerStatistics(inputStats->Type, inputStats->Nrows * selectivity, inputStats->Ncols, inputStats->ByteSize * selectivity, inputStats->Cost, inputStats->KeyColumns );
385+
auto outputStats = TOptimizerStatistics(
386+
inputStats->Type,
387+
inputStats->Nrows * selectivity,
388+
inputStats->Ncols,
389+
inputStats->ByteSize * selectivity,
390+
inputStats->Cost,
391+
inputStats->KeyColumns,
392+
inputStats->ColumnStatistics,
393+
inputStats->StorageType);
394+
384395
outputStats.Labels = inputStats->Labels;
385396
outputStats.Selectivity *= selectivity;
386397

@@ -424,7 +435,16 @@ void InferStatisticsForFilter(const TExprNode::TPtr& input, TTypeAnnotationConte
424435
auto filterBody = filter.Lambda().Body();
425436
double selectivity = TPredicateSelectivityComputer(inputStats).Compute(filterBody);
426437

427-
auto outputStats = TOptimizerStatistics(inputStats->Type, inputStats->Nrows * selectivity, inputStats->Ncols, inputStats->ByteSize * selectivity, inputStats->Cost, inputStats->KeyColumns);
438+
auto outputStats = TOptimizerStatistics(
439+
inputStats->Type,
440+
inputStats->Nrows * selectivity,
441+
inputStats->Ncols,
442+
inputStats->ByteSize * selectivity,
443+
inputStats->Cost,
444+
inputStats->KeyColumns,
445+
inputStats->ColumnStatistics,
446+
inputStats->StorageType);
447+
428448
outputStats.Selectivity *= selectivity;
429449
outputStats.Labels = inputStats->Labels;
430450

‎ydb/library/yql/providers/dq/common/yql_dq_settings.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ struct TDqSettings {
5656
static constexpr bool ExportStats = false;
5757
static constexpr ETaskRunnerStats TaskRunnerStats = ETaskRunnerStats::Basic;
5858
static constexpr ESpillingEngine SpillingEngine = ESpillingEngine::Disable;
59-
static constexpr ui32 CostBasedOptimizationLevel = 3;
59+
static constexpr ui32 CostBasedOptimizationLevel = 4;
6060
static constexpr ui32 MaxDPccpDPTableSize = 40000U;
6161
static constexpr ui64 MaxAttachmentsSize = 2_GB;
6262
static constexpr bool SplitStageOnDqReplicate = true;

0 commit comments

Comments
 (0)