From 31fbe8ad26b6f942bc12977d3eb3a7236e3d6e3d Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Fri, 29 Dec 2023 19:58:01 +0300 Subject: [PATCH 1/5] fix statistic calculation --- ydb/core/tx/columnshard/columnshard__stats_scan.cpp | 2 +- ydb/core/tx/columnshard/engines/column_engine_logs.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/columnshard__stats_scan.cpp b/ydb/core/tx/columnshard/columnshard__stats_scan.cpp index af5486faed96..781b60de3b25 100644 --- a/ydb/core/tx/columnshard/columnshard__stats_scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__stats_scan.cpp @@ -77,7 +77,7 @@ void TStatsIterator::AppendStats(const std::vector(*builders[6], r->GetChunkIdx()); NArrow::Append(*builders[7], ReadMetadata->GetColumnNameDef(r->GetColumnId()).value_or("undefined")); NArrow::Append(*builders[8], r->GetColumnId()); - std::string blobIdString = r->BlobRange.ToString(); + std::string blobIdString = r->BlobRange.BlobId.ToString(); NArrow::Append(*builders[9], blobIdString); NArrow::Append(*builders[10], r->BlobRange.Offset); NArrow::Append(*builders[11], r->BlobRange.Size); diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 4f552e4d0626..c88e695947dc 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -197,6 +197,8 @@ class TColumnEngineForLogs : public IColumnEngine { } result.emplace_back(i.second); } + const auto pred = [](const std::shared_ptr& lhs, const std::shared_ptr& rhs) { return lhs->GetPathId() < rhs->GetPathId(); }; + std::sort(result.begin(), result.end(), pred); return result; } From a818bd662e3219811ead155b5bea1f88c9af6e79 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Fri, 29 Dec 2023 20:02:45 +0300 Subject: [PATCH 2/5] fix --- ydb/core/tx/columnshard/columnshard__stats_scan.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/columnshard__stats_scan.cpp b/ydb/core/tx/columnshard/columnshard__stats_scan.cpp index 781b60de3b25..1b6ba0ff6871 100644 --- a/ydb/core/tx/columnshard/columnshard__stats_scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__stats_scan.cpp @@ -77,7 +77,7 @@ void TStatsIterator::AppendStats(const std::vector(*builders[6], r->GetChunkIdx()); NArrow::Append(*builders[7], ReadMetadata->GetColumnNameDef(r->GetColumnId()).value_or("undefined")); NArrow::Append(*builders[8], r->GetColumnId()); - std::string blobIdString = r->BlobRange.BlobId.ToString(); + std::string blobIdString = r->BlobRange.BlobId.ToStringLegacy(); NArrow::Append(*builders[9], blobIdString); NArrow::Append(*builders[10], r->BlobRange.Offset); NArrow::Append(*builders[11], r->BlobRange.Size); From 541868a4a2ae573b0c0df22a4f97affb7a81920c Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Sat, 30 Dec 2023 12:44:07 +0300 Subject: [PATCH 3/5] stats calculation fix --- ydb/core/tx/columnshard/columnshard__scan.cpp | 43 +++++++++++-------- .../columnshard/engines/column_engine_logs.h | 2 - 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp index 2c98457913c6..f9f2568b0c02 100644 --- a/ydb/core/tx/columnshard/columnshard__scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__scan.cpp @@ -1,30 +1,32 @@ -#include "engines/reader/read_context.h" -#include "blobs_reader/events.h" -#include "blobs_reader/read_coordinator.h" -#include "blobs_reader/actor.h" -#include "resource_subscriber/actor.h" - -#include +#include +#include +#include +#include +#include +#include #include -#include #include -#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include +#include #include +#include #include #include #include +#include #include #include -#include -#include + +#include + +#include "blobs_reader/actor.h" +#include "blobs_reader/events.h" +#include "blobs_reader/read_coordinator.h" +#include "engines/reader/read_context.h" +#include "resource_subscriber/actor.h" namespace NKikimr::NColumnShard { @@ -253,7 +255,7 @@ class TColumnShardScan : public TActorBootstrapped, NArrow::IR if (CurrentLastReadKey) { NOlap::NIndexedReader::TSortableBatchPosition pNew(result.GetLastReadKey(), 0, result.GetLastReadKey()->schema()->field_names(), {}, false); NOlap::NIndexedReader::TSortableBatchPosition pOld(CurrentLastReadKey, 0, CurrentLastReadKey->schema()->field_names(), {}, false); - AFL_VERIFY(pOld < pNew); + AFL_VERIFY(pOld < pNew)("old", pOld.DebugJson().GetStringRobust())("new", pNew.DebugJson().GetStringRobust()); } CurrentLastReadKey = result.GetLastReadKey(); @@ -656,6 +658,9 @@ PrepareStatsReadMetadata(ui64 tabletId, const NOlap::TReadDescription& read, con return out; } THashMap> portionsInUse; + const auto predStatSchema = [](const std::shared_ptr& l, const std::shared_ptr& r) { + return std::tuple(l->GetPathId(), l->GetPortionId()) < std::tuple(r->GetPathId(), r->GetPortionId()); + }; for (auto&& filter : read.PKRangesFilter) { const ui64 fromPathId = *filter.GetPredicateFrom().Get(0, 0, 1); const ui64 toPathId = *filter.GetPredicateTo().Get(0, 0, Max()); @@ -671,6 +676,7 @@ PrepareStatsReadMetadata(ui64 tabletId, const NOlap::TReadDescription& read, con } } } + std::sort(out->IndexPortions.begin(), out->IndexPortions.end(), predStatSchema); } else if (read.TableName.EndsWith(NOlap::TIndexInfo::STORE_INDEX_STATS_TABLE)) { auto pathInfos = logsIndex->GetTables(fromPathId, toPathId); for (auto&& pathInfo: pathInfos) { @@ -680,6 +686,7 @@ PrepareStatsReadMetadata(ui64 tabletId, const NOlap::TReadDescription& read, con } } } + std::sort(out->IndexPortions.begin(), out->IndexPortions.end(), predStatSchema); } } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index c88e695947dc..4f552e4d0626 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -197,8 +197,6 @@ class TColumnEngineForLogs : public IColumnEngine { } result.emplace_back(i.second); } - const auto pred = [](const std::shared_ptr& lhs, const std::shared_ptr& rhs) { return lhs->GetPathId() < rhs->GetPathId(); }; - std::sort(result.begin(), result.end(), pred); return result; } From eb200689d2f662d091e9de3c69086f54e3c352c7 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Sat, 30 Dec 2023 13:06:46 +0300 Subject: [PATCH 4/5] fix formatting --- ydb/core/tx/columnshard/columnshard__scan.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp index f9f2568b0c02..43e5d0115d43 100644 --- a/ydb/core/tx/columnshard/columnshard__scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__scan.cpp @@ -1,4 +1,9 @@ -#include +#include "blobs_reader/actor.h" +#include "blobs_reader/events.h" +#include "blobs_reader/read_coordinator.h" +#include "engines/reader/read_context.h" +#include "resource_subscriber/actor.h" + #include #include #include @@ -14,6 +19,8 @@ #include #include #include + +#include #include #include #include @@ -22,12 +29,6 @@ #include -#include "blobs_reader/actor.h" -#include "blobs_reader/events.h" -#include "blobs_reader/read_coordinator.h" -#include "engines/reader/read_context.h" -#include "resource_subscriber/actor.h" - namespace NKikimr::NColumnShard { using namespace NKqp; From a733d8c4d8594060f7e5e0e5aac5c31ca1c084fa Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Sat, 30 Dec 2023 13:58:19 +0300 Subject: [PATCH 5/5] fix pk --- ydb/core/sys_view/common/schema.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index e598f1e0d89c..dc6f9d6196b2 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -401,12 +401,7 @@ struct Schema : NIceDb::Schema { struct Activity : Column<13, NScheme::NTypeIds::Bool> {}; struct TierName : Column<14, NScheme::NTypeIds::Utf8> {}; - using TKey = TableKey< - PathId, - TabletId, - PortionId, - ChunkIdx - >; + using TKey = TableKey; using TColumns = TableColumns< PathId, Kind,