From bbee3d06e59cf9d07265057866d8005e238ddae8 Mon Sep 17 00:00:00 2001 From: Evgeny Zverev Date: Mon, 29 Jul 2024 08:02:42 +0300 Subject: [PATCH] Revert "Delete unsupported special column DELETE_FLAG (#7068)" This reverts commit 1cbae0541be9b49cffc30103439237005da54f97. --- ydb/core/tx/columnshard/common/portion.h | 2 - .../engines/scheme/abstract/index_info.h | 4 +- .../columnshard/engines/scheme/index_info.cpp | 2 - .../normalizer/abstract/abstract.h | 1 - .../normalizer/special_columns/normalizer.cpp | 131 ------------------ .../normalizer/special_columns/normalizer.h | 24 ---- .../normalizer/special_columns/ya.make | 11 -- ydb/core/tx/columnshard/ya.make | 1 - 8 files changed, 1 insertion(+), 175 deletions(-) delete mode 100644 ydb/core/tx/columnshard/normalizer/special_columns/normalizer.cpp delete mode 100644 ydb/core/tx/columnshard/normalizer/special_columns/normalizer.h delete mode 100644 ydb/core/tx/columnshard/normalizer/special_columns/ya.make diff --git a/ydb/core/tx/columnshard/common/portion.h b/ydb/core/tx/columnshard/common/portion.h index d8497d5174a9..ae35cbcfc23f 100644 --- a/ydb/core/tx/columnshard/common/portion.h +++ b/ydb/core/tx/columnshard/common/portion.h @@ -17,10 +17,8 @@ class TSpecialColumns { public: static constexpr const char* SPEC_COL_PLAN_STEP = "_yql_plan_step"; static constexpr const char* SPEC_COL_TX_ID = "_yql_tx_id"; - static constexpr const char* SPEC_COL_DELETE_FLAG = "_yql_delete_flag"; static const ui32 SPEC_COL_PLAN_STEP_INDEX = 0xffffff00; static const ui32 SPEC_COL_TX_ID_INDEX = SPEC_COL_PLAN_STEP_INDEX + 1; - static const ui32 SPEC_COL_DELETE_FLAG_INDEX = SPEC_COL_PLAN_STEP_INDEX + 2; }; } diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h index a13db54a588c..343ad58746b1 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h @@ -9,13 +9,11 @@ class IIndexInfo { public: enum class ESpecialColumn: ui32 { PLAN_STEP = NOlap::NPortion::TSpecialColumns::SPEC_COL_PLAN_STEP_INDEX, - TX_ID = NOlap::NPortion::TSpecialColumns::SPEC_COL_TX_ID_INDEX, - DELETE_FLAG = NOlap::NPortion::TSpecialColumns::SPEC_COL_DELETE_FLAG_INDEX + TX_ID = NOlap::NPortion::TSpecialColumns::SPEC_COL_TX_ID_INDEX }; static constexpr const char* SPEC_COL_PLAN_STEP = NOlap::NPortion::TSpecialColumns::SPEC_COL_PLAN_STEP; static constexpr const char* SPEC_COL_TX_ID = NOlap::NPortion::TSpecialColumns::SPEC_COL_TX_ID; - static constexpr const char* SPEC_COL_DELETE_FLAG = NOlap::NPortion::TSpecialColumns::SPEC_COL_DELETE_FLAG; static const TString STORE_INDEX_STATS_TABLE; static const TString STORE_INDEX_PORTION_STATS_TABLE; static const TString STORE_INDEX_GRANULE_STATS_TABLE; diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index 1aa2ea7a7580..c91f2d0d1370 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -98,8 +98,6 @@ TString TIndexInfo::GetColumnName(ui32 id, bool required) const { return SPEC_COL_PLAN_STEP; } else if (ESpecialColumn(id) == ESpecialColumn::TX_ID) { return SPEC_COL_TX_ID; - } else if (ESpecialColumn(id) == ESpecialColumn::DELETE_FLAG) { - return SPEC_COL_DELETE_FLAG; } else { const auto ci = Columns.find(id); diff --git a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h index 974db9438999..dad21751eb65 100644 --- a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h +++ b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h @@ -53,7 +53,6 @@ namespace NKikimr::NOlap { enum class ENormalizerSequentialId : ui32 { Granules = 1, Chunks, - DeleteUnsupportedSpecialColumns, PortionsCleaner, TablesCleaner, // PortionsMetadata diff --git a/ydb/core/tx/columnshard/normalizer/special_columns/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/special_columns/normalizer.cpp deleted file mode 100644 index 2b83bd54e2e5..000000000000 --- a/ydb/core/tx/columnshard/normalizer/special_columns/normalizer.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "normalizer.h" - -#include - -namespace NKikimr::NOlap { - -namespace { - -constexpr ui32 ColumnIdxToDelete = (ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG; - - -using namespace NColumnShard; - -struct TKey { - ui32 Index; - ui64 Granule; - ui32 ColumnIdx; - ui64 PlanStep; - ui64 TxId; - ui64 Portion; - ui32 Chunk; -}; - -using TKeyBatch = std::vector; - -std::optional> KeysToDelete(NTabletFlatExecutor::TTransactionContext& txc, size_t maxBatchSize) { - NIceDb::TNiceDb db(txc.DB); - if (!Schema::Precharge(db, txc.DB.GetScheme())) { - return std::nullopt; - } - std::vector result; - TKeyBatch currentBatch; - auto rowset = db.Table().Select< - Schema::IndexColumns::Index, - Schema::IndexColumns::Granule, - Schema::IndexColumns::ColumnIdx, - Schema::IndexColumns::PlanStep, - Schema::IndexColumns::TxId, - Schema::IndexColumns::Portion, - Schema::IndexColumns::Chunk - >(); - if (!rowset.IsReady()) { - return std::nullopt; - } - while (!rowset.EndOfSet()) { - if (rowset.GetValue() == ColumnIdxToDelete) { - auto key = TKey { - .Index = rowset.GetValue(), - .Granule = rowset.GetValue(), - .ColumnIdx = rowset.GetValue(), - .PlanStep = rowset.GetValue(), - .Portion = rowset.GetValue(), - .Chunk = rowset.GetValue() - }; - currentBatch.emplace_back(std::move(key)); - if (currentBatch.size() == maxBatchSize) { - result.emplace_back(std::move(currentBatch)); - currentBatch = TKeyBatch{}; - } - } - if (!rowset.Next()) { - return std::nullopt; - } - } - if (!currentBatch.empty()) { - result.emplace_back(std::move(currentBatch)); - } - - return result; -} - -class TChanges : public INormalizerChanges { -public: - TChanges(TKeyBatch&& keys) - : Keys(keys) - {} - bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& /* normController */) const override { - using namespace NColumnShard; - NIceDb::TNiceDb db(txc.DB); - for(const auto& k: Keys) { - db.Table().Key( - k.Index, - k.Granule, - k.ColumnIdx, - k.PlanStep, - k.TxId, - k.Portion, - k.Chunk - ).Delete(); - } - ACFL_INFO("normalizer", "TDeleteUnsupportedSpecialColumnsNormalier")("message", TStringBuilder() << GetSize() << " rows deleted"); - return true; - } - - ui64 GetSize() const override { - return Keys.size(); - } -private: - const TKeyBatch Keys; -}; - -} //namespace - -TConclusion> TDeleteUnsupportedSpecialColumnsNormalier::DoInit(const TNormalizationController& /*controller*/, NTabletFlatExecutor::TTransactionContext& txc) { - using namespace NColumnShard; - NIceDb::TNiceDb db(txc.DB); - const size_t MaxBatchSize = 10000; - auto keysToDelete = KeysToDelete(txc, MaxBatchSize); - if (!keysToDelete) { - return TConclusionStatus::Fail("Not ready"); - } - ACFL_INFO("normalizer", "TDeleteUnsupportedSpecialColumnsNormalier")("message", - TStringBuilder() - << "found " - << std::accumulate(cbegin(*keysToDelete), cend(*keysToDelete), 0, [](size_t a, const TKeyBatch& b){return a + b.size();}) - << " rows to delete grouped in " - << keysToDelete->size() - << " batches" - ); - - std::vector result; - for (auto&& batch: *keysToDelete) { - AFL_VERIFY(!batch.empty()); - result.emplace_back(std::make_shared( - std::make_shared(std::move(std::move(batch))) - )); - } - return result; -} - -} //namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/normalizer/special_columns/normalizer.h b/ydb/core/tx/columnshard/normalizer/special_columns/normalizer.h deleted file mode 100644 index 9cac90b9ead0..000000000000 --- a/ydb/core/tx/columnshard/normalizer/special_columns/normalizer.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include - - -namespace NKikimr::NOlap { - -class TDeleteUnsupportedSpecialColumnsNormalier: public TNormalizationController::INormalizerComponent { - using TThisClass = TDeleteUnsupportedSpecialColumnsNormalier; - static constexpr auto TypeId = ENormalizerSequentialId::DeleteUnsupportedSpecialColumns; - static inline auto Registrator = INormalizerComponent::TFactory::TRegistrator(TypeId); -public: - TDeleteUnsupportedSpecialColumnsNormalier(const TNormalizationController::TInitContext&) - {} - - virtual ENormalizerSequentialId GetType() const override { - return TypeId; - } - - virtual TConclusion> DoInit(const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) override; -}; - -} diff --git a/ydb/core/tx/columnshard/normalizer/special_columns/ya.make b/ydb/core/tx/columnshard/normalizer/special_columns/ya.make deleted file mode 100644 index d44051621eb3..000000000000 --- a/ydb/core/tx/columnshard/normalizer/special_columns/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -LIBRARY() - -SRCS( - GLOBAL normalizer.cpp -) - -PEERDIR( - ydb/core/tx/columnshard/normalizer/abstract -) - -END() diff --git a/ydb/core/tx/columnshard/ya.make b/ydb/core/tx/columnshard/ya.make index d53fcde2e8ef..2c8103b24329 100644 --- a/ydb/core/tx/columnshard/ya.make +++ b/ydb/core/tx/columnshard/ya.make @@ -56,7 +56,6 @@ PEERDIR( ydb/core/tx/columnshard/export ydb/core/tx/columnshard/resource_subscriber ydb/core/tx/columnshard/normalizer/granule - ydb/core/tx/columnshard/normalizer/special_columns ydb/core/tx/columnshard/normalizer/portion ydb/core/tx/columnshard/normalizer/tables ydb/core/tx/columnshard/blobs_action/storages_manager