Skip to content

Commit d64073b

Browse files
ivanmorozov333ivanmorozov333
authored
sort statistics by name (#3144)
1 parent e36030b commit d64073b

File tree

15 files changed

+90
-38
lines changed

15 files changed

+90
-38
lines changed

‎ydb/core/kqp/ut/olap/kqp_olap_ut.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4129,7 +4129,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
41294129
helper.GetStats(stats, true);
41304130
for (auto&& i : stats) {
41314131
AFL_VERIFY(i.ScalarsSize() == 2);
4132-
AFL_VERIFY(i.GetScalars()[1].GetUint32() == 3);
4132+
AFL_VERIFY(i.GetScalars()[0].GetUint32() == 3);
41334133
}
41344134
}
41354135
{
@@ -4151,7 +4151,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
41514151
helper.GetStats(stats, true);
41524152
for (auto&& i : stats) {
41534153
AFL_VERIFY(i.ScalarsSize() == 2);
4154-
AFL_VERIFY(i.GetScalars()[1].GetUint32() == 3);
4154+
AFL_VERIFY(i.GetScalars()[0].GetUint32() == 3);
41554155
}
41564156
}
41574157
}

‎ydb/core/tx/columnshard/columnshard_ut_common.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ struct TTestSchema {
246246
continue;
247247
}
248248
if (NOlap::NStatistics::NMax::TOperator::IsAvailableType(columns[i].GetType())) {
249-
*schema->AddStatistics() = NOlap::NStatistics::TOperatorContainer(std::make_shared<NOlap::NStatistics::NMax::TOperator>(i + 1)).SerializeToProto();
249+
*schema->AddStatistics() = NOlap::NStatistics::TOperatorContainer("MAX::" + columns[i].GetName(), std::make_shared<NOlap::NStatistics::NMax::TOperator>(i + 1)).SerializeToProto();
250250
}
251251
}
252252

‎ydb/core/tx/columnshard/engines/portions/with_blobs.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ bool TPortionInfoWithBlobs::ExtractColumnChunks(const ui32 columnId, std::vector
169169

170170
void TPortionInfoWithBlobs::FillStatistics(const TIndexInfo& index) {
171171
NStatistics::TPortionStorage storage;
172-
for (auto&& i : index.GetStatistics()) {
172+
for (auto&& i : index.GetStatisticsByName()) {
173173
THashMap<ui32, std::vector<std::shared_ptr<IPortionDataChunk>>> data;
174174
for (auto&& entityId : i.second->GetEntityIds()) {
175175
data.emplace(entityId, GetEntityChunks(entityId));
@@ -225,9 +225,9 @@ TPortionInfoWithBlobs TPortionInfoWithBlobs::SyncPortion(TPortionInfoWithBlobs&&
225225
result.GetPortionInfo().MutableMeta().SetTierName(targetTier);
226226

227227
NStatistics::TPortionStorage storage;
228-
for (auto&& i : to->GetIndexInfo().GetStatistics()) {
229-
auto it = from->GetIndexInfo().GetStatistics().find(i.first);
230-
if (it != from->GetIndexInfo().GetStatistics().end()) {
228+
for (auto&& i : to->GetIndexInfo().GetStatisticsByName()) {
229+
auto it = from->GetIndexInfo().GetStatisticsByName().find(i.first);
230+
if (it != from->GetIndexInfo().GetStatisticsByName().end()) {
231231
i.second->CopyData(it->second.GetCursorVerified(), source.PortionInfo.GetMeta().GetStatisticsStorage(), storage);
232232
} else {
233233
i.second->FillStatisticsData(entityChunksNew, storage, to->GetIndexInfo());

‎ydb/core/tx/columnshard/engines/scheme/index_info.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -355,12 +355,14 @@ bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema&
355355
}
356356

357357
{
358-
NStatistics::TPortionStorageCursor cursor;
359358
for (const auto& stat : schema.GetStatistics()) {
360359
NStatistics::TOperatorContainer container;
361360
AFL_VERIFY(container.DeserializeFromProto(stat));
361+
AFL_VERIFY(StatisticsByName.emplace(container.GetName(), std::move(container)).second);
362+
}
363+
NStatistics::TPortionStorageCursor cursor;
364+
for (auto&& [_, container] : StatisticsByName) {
362365
container.SetCursor(cursor);
363-
Statistics.emplace(container->GetIdentifier(), container);
364366
container->ShiftCursor(cursor);
365367
}
366368
}

‎ydb/core/tx/columnshard/engines/scheme/index_info.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo {
4343
THashMap<ui32, TColumnFeatures> ColumnFeatures;
4444
THashMap<ui32, std::shared_ptr<arrow::Field>> ArrowColumnByColumnIdCache;
4545
THashMap<ui32, NIndexes::TIndexMetaContainer> Indexes;
46-
std::map<NStatistics::TIdentifier, NStatistics::TOperatorContainer> Statistics;
46+
std::map<TString, NStatistics::TOperatorContainer> StatisticsByName;
4747
TIndexInfo(const TString& name);
4848
bool SchemeNeedActualization = false;
4949
bool DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema, const std::shared_ptr<IStoragesManager>& operators);
@@ -78,14 +78,15 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo {
7878

7979
std::vector<std::shared_ptr<IPortionDataChunk>> MakeEmptyChunks(const ui32 columnId, const std::vector<ui32>& pages, const TSimpleColumnInfo& columnInfo) const;
8080

81-
const std::map<NStatistics::TIdentifier, NStatistics::TOperatorContainer>& GetStatistics() const {
82-
return Statistics;
81+
const std::map<TString, NStatistics::TOperatorContainer>& GetStatisticsByName() const {
82+
return StatisticsByName;
8383
}
8484

8585
NStatistics::TOperatorContainer GetStatistics(const NStatistics::TIdentifier& id) const {
86-
auto it = Statistics.find(id);
87-
if (it != Statistics.end()) {
88-
return it->second;
86+
for (auto&& i : StatisticsByName) {
87+
if (i.second->GetIdentifier() == id) {
88+
return i.second;
89+
}
8990
}
9091
return NStatistics::TOperatorContainer();
9192
}

‎ydb/core/tx/columnshard/engines/scheme/statistics/abstract/constructor.h

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class IConstructor {
2020
YDB_READONLY(EType, Type, EType::Undefined);
2121
IConstructor() = default;
2222
protected:
23-
virtual TConclusion<TOperatorContainer> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const = 0;
23+
virtual TConclusion<std::shared_ptr<IOperator>> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const = 0;
2424
virtual bool DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) = 0;
2525
virtual void DoSerializeToProto(NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) const = 0;
2626
virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonData) = 0;
@@ -39,8 +39,12 @@ class IConstructor {
3939
return DoDeserializeFromJson(jsonData);
4040
}
4141

42-
TConclusion<TOperatorContainer> CreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
43-
return DoCreateOperator(currentSchema);
42+
TConclusion<TOperatorContainer> CreateOperator(const TString& name, const NSchemeShard::TOlapSchema& currentSchema) const {
43+
auto result = DoCreateOperator(currentSchema);
44+
if (!result) {
45+
return result.GetError();
46+
}
47+
return TOperatorContainer(name, result.DetachResult());
4448
}
4549

4650
TString GetClassName() const {

‎ydb/core/tx/columnshard/engines/scheme/statistics/abstract/operator.h

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,18 @@ class IOperator {
6868

6969
class TOperatorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IOperator> {
7070
private:
71+
YDB_READONLY_DEF(TString, Name);
7172
std::optional<TPortionStorageCursor> Cursor;
7273
using TBase = NBackgroundTasks::TInterfaceProtoContainer<IOperator>;
7374
public:
74-
using TBase::TBase;
75+
TOperatorContainer() = default;
76+
77+
TOperatorContainer(const TString& name, const std::shared_ptr<IOperator>& object)
78+
: TBase(object)
79+
, Name(name)
80+
{
81+
AFL_VERIFY(Name);
82+
}
7583

7684
const TPortionStorageCursor& GetCursorVerified() const {
7785
AFL_VERIFY(Cursor);
@@ -87,6 +95,30 @@ class TOperatorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IOpe
8795
AFL_VERIFY(!!Cursor);
8896
return storage.GetScalarVerified(*Cursor);
8997
}
98+
99+
NKikimrColumnShardStatisticsProto::TOperatorContainer SerializeToProto() const {
100+
NKikimrColumnShardStatisticsProto::TOperatorContainer result = TBase::SerializeToProto();
101+
result.SetName(Name);
102+
AFL_VERIFY(Name);
103+
return result;
104+
}
105+
106+
void SerializeToProto(NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) const {
107+
TBase::SerializeToProto(proto);
108+
proto.SetName(Name);
109+
AFL_VERIFY(Name);
110+
}
111+
112+
bool DeserializeFromProto(const NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) {
113+
Name = proto.GetName();
114+
if (!Name) {
115+
return false;
116+
}
117+
if (!TBase::DeserializeFromProto(proto)) {
118+
return false;
119+
}
120+
return true;
121+
}
90122
};
91123

92124
}

‎ydb/core/tx/columnshard/engines/scheme/statistics/max/constructor.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33

44
namespace NKikimr::NOlap::NStatistics::NMax {
55

6-
NKikimr::TConclusion<TOperatorContainer> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
6+
NKikimr::TConclusion<std::shared_ptr<IOperator>> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
77
auto column = currentSchema.GetColumns().GetByName(ColumnName);
88
if (!TOperator::IsAvailableType(column->GetType())) {
99
return TConclusionStatus::Fail("incorrect type for stat calculation");
1010
}
11-
return TOperatorContainer(std::make_shared<TOperator>(column->GetId()));
11+
return std::make_shared<TOperator>(column->GetId());
1212
}
1313

1414
bool TConstructor::DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) {

‎ydb/core/tx/columnshard/engines/scheme/statistics/max/constructor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ class TConstructor: public IConstructor {
1212
static inline const auto Registrator = TFactory::TRegistrator<TConstructor>(::ToString(EType::Max));
1313
YDB_READONLY(TString, ColumnName, 0);
1414
protected:
15-
virtual TConclusion<TOperatorContainer> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
15+
virtual TConclusion<std::shared_ptr<IOperator>> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
1616
virtual bool DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) override;
1717
virtual void DoSerializeToProto(NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) const override;
1818
virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonData) override;

‎ydb/core/tx/columnshard/engines/scheme/statistics/variability/constructor.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33

44
namespace NKikimr::NOlap::NStatistics::NVariability {
55

6-
NKikimr::TConclusion<TOperatorContainer> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
6+
NKikimr::TConclusion<std::shared_ptr<IOperator>> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
77
auto column = currentSchema.GetColumns().GetByName(ColumnName);
88
if (!TOperator::IsAvailableType(column->GetType())) {
99
return TConclusionStatus::Fail("incorrect type for stat calculation");
1010
}
11-
return TOperatorContainer(std::make_shared<TOperator>(column->GetId()));
11+
return std::make_shared<TOperator>(column->GetId());
1212
}
1313

1414
bool TConstructor::DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) {

‎ydb/core/tx/columnshard/engines/scheme/statistics/variability/constructor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ class TConstructor: public IConstructor {
1212
static inline const auto Registrator = TFactory::TRegistrator<TConstructor>(::ToString(EType::Variability));
1313
YDB_READONLY(TString, ColumnName, 0);
1414
protected:
15-
virtual TConclusion<TOperatorContainer> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
15+
virtual TConclusion<std::shared_ptr<IOperator>> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
1616
virtual bool DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) override;
1717
virtual void DoSerializeToProto(NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) const override;
1818
virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonData) override;

‎ydb/core/tx/columnshard/test_helper/controllers.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,21 @@ class TWaitCompactionController: public NYDBTest::NColumnShard::TController {
2626
virtual bool DoOnStartCompaction(std::shared_ptr<NKikimr::NOlap::TColumnEngineChanges>& changes) override;
2727
virtual bool DoOnWriteIndexComplete(const NKikimr::NOlap::TColumnEngineChanges& changes, const NKikimr::NColumnShard::TColumnShard& /*shard*/) override;
2828
virtual bool DoOnWriteIndexStart(const ui64 /*tabletId*/, const TString& changeClassName) override;
29+
virtual bool NeedForceCompactionBacketsConstruction() const override {
30+
return true;
31+
}
32+
virtual ui64 GetSmallPortionSizeDetector(const ui64 /*def*/) const override {
33+
return 0;
34+
}
35+
virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override {
36+
return TDuration::Zero();
37+
}
38+
virtual TDuration GetLagForCompactionBeforeTierings(const TDuration /*def*/) const override {
39+
return TDuration::Zero();
40+
}
41+
virtual TDuration GetTTLDefaultWaitingDuration(const TDuration /*defaultValue*/) const override {
42+
return TDuration::Seconds(1);
43+
}
2944
public:
3045
ui32 GetFinishedExportsCount() const {
3146
return ExportsFinishedCount.Val();

‎ydb/core/tx/schemeshard/olap/statistics/schema.cpp

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,22 @@
44
namespace NKikimr::NSchemeShard {
55

66
void TOlapStatisticsSchema::SerializeToProto(NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) const {
7-
proto.SetName(Name);
87
Operator.SerializeToProto(proto);
98
}
109

1110
bool TOlapStatisticsSchema::DeserializeFromProto(const NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) {
12-
Name = proto.GetName();
1311
AFL_VERIFY(Operator.DeserializeFromProto(proto))("incorrect_proto", proto.DebugString());
1412
return true;
1513
}
1614

1715
bool TOlapStatisticsSchema::ApplyUpdate(const TOlapSchema& /*currentSchema*/, const TOlapStatisticsUpsert& upsert, IErrorCollector& errors) {
18-
AFL_VERIFY(upsert.GetName() == GetName());
16+
AFL_VERIFY(upsert.GetName() == Operator.GetName());
1917
AFL_VERIFY(!!upsert.GetConstructor());
2018
if (upsert.GetConstructor().GetClassName() != Operator.GetClassName()) {
2119
errors.AddError("different index classes: " + upsert.GetConstructor().GetClassName() + " vs " + Operator.GetClassName());
2220
return false;
2321
}
24-
errors.AddError("cannot modify statistics calculation for " + GetName() + ". not implemented currently.");
22+
errors.AddError("cannot modify statistics calculation for " + Operator.GetName() + ". not implemented currently.");
2523
return false;
2624
}
2725

@@ -33,12 +31,12 @@ bool TOlapStatisticsDescription::ApplyUpdate(const TOlapSchema& currentSchema, c
3331
return false;
3432
}
3533
} else {
36-
auto meta = stat.GetConstructor()->CreateOperator(currentSchema);
34+
auto meta = stat.GetConstructor()->CreateOperator(stat.GetName(), currentSchema);
3735
if (!meta) {
3836
errors.AddError(meta.GetErrorMessage());
3937
return false;
4038
}
41-
TOlapStatisticsSchema object(stat.GetName(), meta.DetachResult());
39+
TOlapStatisticsSchema object(meta.DetachResult());
4240
Y_ABORT_UNLESS(ObjectsByName.emplace(stat.GetName(), std::move(object)).second);
4341
}
4442
}

‎ydb/core/tx/schemeshard/olap/statistics/schema.h

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,14 @@ class TOlapSchema;
77

88
class TOlapStatisticsSchema {
99
private:
10-
YDB_READONLY_DEF(TString, Name);
1110
YDB_READONLY_DEF(NOlap::NStatistics::TOperatorContainer, Operator);
1211
public:
1312
TOlapStatisticsSchema() = default;
1413

15-
TOlapStatisticsSchema(const TString& name, const NOlap::NStatistics::TOperatorContainer& container)
16-
: Name(name)
17-
, Operator(container)
14+
TOlapStatisticsSchema(const NOlap::NStatistics::TOperatorContainer& container)
15+
: Operator(container)
1816
{
19-
17+
AFL_VERIFY(container.GetName());
2018
}
2119

2220
bool ApplyUpdate(const TOlapSchema& currentSchema, const TOlapStatisticsUpsert& upsert, IErrorCollector& errors);

‎ydb/library/conclusion/result.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,13 @@ class TConclusion {
3030
Y_ABORT_UNLESS(IsFail());
3131
}
3232

33-
TConclusion(TResult&& result)
33+
template <class TResultArg>
34+
TConclusion(TResultArg&& result)
3435
: Result(std::move(result)) {
3536
}
3637

37-
TConclusion(const TResult& result)
38+
template <class TResultArg>
39+
TConclusion(const TResultArg& result)
3840
: Result(result) {
3941
}
4042

0 commit comments

Comments
 (0)