diff --git a/ydb/core/formats/arrow/arrow_helpers.cpp b/ydb/core/formats/arrow/arrow_helpers.cpp index a790af3015f7..70ed133e6366 100644 --- a/ydb/core/formats/arrow/arrow_helpers.cpp +++ b/ydb/core/formats/arrow/arrow_helpers.cpp @@ -548,7 +548,7 @@ std::shared_ptr DefaultScalar(const std::shared_ptrToString()); return out; } @@ -634,6 +634,19 @@ int ScalarCompare(const std::shared_ptr& x, const std::shared_ptr return ScalarCompare(*x, *y); } +int ScalarCompareNullable(const std::shared_ptr& x, const std::shared_ptr& y) { + if (!x && !!y) { + return -1; + } + if (!!x && !y) { + return 1; + } + if (!x && !y) { + return 0; + } + return ScalarCompare(*x, *y); +} + std::shared_ptr SortBatch(const std::shared_ptr& batch, const std::shared_ptr& sortingKey, const bool andUnique) { auto sortPermutation = MakeSortPermutation(batch, sortingKey, andUnique); diff --git a/ydb/core/formats/arrow/arrow_helpers.h b/ydb/core/formats/arrow/arrow_helpers.h index f6f4fd0c18a0..584803598daf 100644 --- a/ydb/core/formats/arrow/arrow_helpers.h +++ b/ydb/core/formats/arrow/arrow_helpers.h @@ -98,6 +98,7 @@ std::shared_ptr GetScalar(const std::shared_ptr& ar bool IsGoodScalar(const std::shared_ptr& x); int ScalarCompare(const arrow::Scalar& x, const arrow::Scalar& y); int ScalarCompare(const std::shared_ptr& x, const std::shared_ptr& y); +int ScalarCompareNullable(const std::shared_ptr& x, const std::shared_ptr& y); std::partial_ordering ColumnsCompare(const std::vector>& x, const ui32 xRow, const std::vector>& y, const ui32 yRow); bool ScalarLess(const std::shared_ptr& x, const std::shared_ptr& y); bool ScalarLess(const arrow::Scalar& x, const arrow::Scalar& y); diff --git a/ydb/core/formats/arrow/common/accessor.cpp b/ydb/core/formats/arrow/common/accessor.cpp index 9865b2a692f7..775cffa95bab 100644 --- a/ydb/core/formats/arrow/common/accessor.cpp +++ b/ydb/core/formats/arrow/common/accessor.cpp @@ -1,4 +1,5 @@ #include "accessor.h" +#include #include #include #include @@ -94,6 +95,10 @@ class TChunkAccessor { } +std::optional TTrivialArray::DoGetRawSize() const { + return NArrow::GetArrayDataSize(Array); +} + std::partial_ordering IChunkedArray::TCurrentChunkAddress::Compare(const ui64 position, const TCurrentChunkAddress& item, const ui64 itemPosition) const { AFL_VERIFY(StartPosition <= position); AFL_VERIFY(position < FinishPosition); @@ -119,4 +124,12 @@ IChunkedArray::TCurrentChunkAddress TTrivialChunkedArray::DoGetChunk(const std:: return SelectChunk(chunkCurrent, position, accessor); } +std::optional TTrivialChunkedArray::DoGetRawSize() const { + ui64 result = 0; + for (auto&& i : Array->chunks()) { + result += NArrow::GetArrayDataSize(i); + } + return result; +} + } diff --git a/ydb/core/formats/arrow/common/accessor.h b/ydb/core/formats/arrow/common/accessor.h index 3765d726992b..6021f47f5a88 100644 --- a/ydb/core/formats/arrow/common/accessor.h +++ b/ydb/core/formats/arrow/common/accessor.h @@ -84,19 +84,23 @@ class IChunkedArray { YDB_READONLY_DEF(std::shared_ptr, DataType); YDB_READONLY(ui64, RecordsCount, 0); YDB_READONLY(EType, Type, EType::Undefined); + virtual std::optional DoGetRawSize() const = 0; protected: virtual std::shared_ptr DoGetChunkedArray() const = 0; virtual TCurrentChunkAddress DoGetChunk(const std::optional& chunkCurrent, const ui64 position) const = 0; template TCurrentChunkAddress SelectChunk(const std::optional& chunkCurrent, const ui64 position, const TChunkAccessor& accessor) const { - if (!chunkCurrent || position >= chunkCurrent->GetStartPosition() + chunkCurrent->GetLength()) { + if (!chunkCurrent || position >= chunkCurrent->GetStartPosition()) { ui32 startIndex = 0; ui64 idx = 0; if (chunkCurrent) { - AFL_VERIFY(chunkCurrent->GetChunkIndex() + 1 < accessor.GetChunksCount()); - startIndex = chunkCurrent->GetChunkIndex() + 1; - idx = chunkCurrent->GetStartPosition() + chunkCurrent->GetLength(); + if (position < chunkCurrent->GetFinishPosition()) { + return *chunkCurrent; + } + AFL_VERIFY(chunkCurrent->GetChunkIndex() < accessor.GetChunksCount()); + startIndex = chunkCurrent->GetChunkIndex(); + idx = chunkCurrent->GetStartPosition(); } for (ui32 i = startIndex; i < accessor.GetChunksCount(); ++i) { const ui64 nextIdx = idx + accessor.GetChunkLength(i); @@ -105,7 +109,7 @@ class IChunkedArray { } idx = nextIdx; } - } else if (position < chunkCurrent->GetStartPosition()) { + } else { AFL_VERIFY(chunkCurrent->GetChunkIndex() > 0); ui64 idx = chunkCurrent->GetStartPosition(); for (i32 i = chunkCurrent->GetChunkIndex() - 1; i >= 0; --i) { @@ -156,6 +160,10 @@ class IChunkedArray { TString DebugString(const ui32 position) const; }; + std::optional GetRawSize() const { + return DoGetRawSize(); + } + std::shared_ptr GetChunkedArray() const { return DoGetChunkedArray(); } @@ -180,6 +188,8 @@ class TTrivialArray: public IChunkedArray { using TBase = IChunkedArray; const std::shared_ptr Array; protected: + virtual std::optional DoGetRawSize() const override; + virtual TCurrentChunkAddress DoGetChunk(const std::optional& /*chunkCurrent*/, const ui64 /*position*/) const override { return TCurrentChunkAddress(Array, 0, 0); } @@ -204,6 +214,7 @@ class TTrivialChunkedArray: public IChunkedArray { virtual std::shared_ptr DoGetChunkedArray() const override { return Array; } + virtual std::optional DoGetRawSize() const override; public: TTrivialChunkedArray(const std::shared_ptr& data) diff --git a/ydb/core/formats/arrow/common/adapter.h b/ydb/core/formats/arrow/common/adapter.h index 543e78511146..1b368e38de50 100644 --- a/ydb/core/formats/arrow/common/adapter.h +++ b/ydb/core/formats/arrow/common/adapter.h @@ -91,7 +91,7 @@ class TDataBuilderPolicy { return batch; } [[nodiscard]] static std::shared_ptr ApplyArrowFilter(const std::shared_ptr& batch, const std::shared_ptr& filter) { - auto table = batch->BuildTable(); + auto table = batch->BuildTableVerified(); return std::make_shared(TDataBuilderPolicy::ApplyArrowFilter(table, filter)); } [[nodiscard]] static std::shared_ptr GetEmptySame(const std::shared_ptr& batch) { diff --git a/ydb/core/formats/arrow/common/container.cpp b/ydb/core/formats/arrow/common/container.cpp index ccf8dc71fb0c..ad0215737bc8 100644 --- a/ydb/core/formats/arrow/common/container.cpp +++ b/ydb/core/formats/arrow/common/container.cpp @@ -1,50 +1,60 @@ #include "container.h" #include +#include #include namespace NKikimr::NArrow { -NKikimr::TConclusionStatus TGeneralContainer::MergeColumnsStrictly(const TGeneralContainer& container) { - if (RecordsCount != container.RecordsCount) { +TConclusionStatus TGeneralContainer::MergeColumnsStrictly(const TGeneralContainer& container) { + if (!container.RecordsCount) { + return TConclusionStatus::Success(); + } + if (!RecordsCount) { + RecordsCount = container.RecordsCount; + } + if (*RecordsCount != *container.RecordsCount) { return TConclusionStatus::Fail(TStringBuilder() << "inconsistency records count in additional container: " << container.GetSchema()->ToString() << ". expected: " << RecordsCount << ", reality: " << container.GetRecordsCount()); } for (i32 i = 0; i < container.Schema->num_fields(); ++i) { auto addFieldResult = AddField(container.Schema->field(i), container.Columns[i]); - if (!addFieldResult) { + if (addFieldResult.IsFail()) { return addFieldResult; } } return TConclusionStatus::Success(); } -NKikimr::TConclusionStatus TGeneralContainer::AddField(const std::shared_ptr& f, const std::shared_ptr& data) { +TConclusionStatus TGeneralContainer::AddField(const std::shared_ptr& f, const std::shared_ptr& data) { AFL_VERIFY(f); AFL_VERIFY(data); - if (data->GetRecordsCount() != RecordsCount) { + if (RecordsCount && data->GetRecordsCount() != *RecordsCount) { return TConclusionStatus::Fail(TStringBuilder() << "inconsistency records count in new column: " << f->name() << ". expected: " << RecordsCount << ", reality: " << data->GetRecordsCount()); } if (!data->GetDataType()->Equals(f->type())) { return TConclusionStatus::Fail("schema and data type are not equals: " + data->GetDataType()->ToString() + " vs " + f->type()->ToString()); } - if (Schema->GetFieldByName(f->name())) { - return TConclusionStatus::Fail("field name duplication: " + f->name()); - } - auto resultAdd = Schema->AddField(Schema->num_fields(), f); - if (!resultAdd.ok()) { - return TConclusionStatus::Fail("internal schema error on add field: " + resultAdd.status().ToString()); + { + auto conclusion = Schema->AddField(f); + if (conclusion.IsFail()) { + return conclusion; + } } - Schema = *resultAdd; + RecordsCount = data->GetRecordsCount(); Columns.emplace_back(data); return TConclusionStatus::Success(); } -TGeneralContainer::TGeneralContainer(const std::shared_ptr& schema, std::vector>&& columns) - : Schema(schema) - , Columns(std::move(columns)) -{ - AFL_VERIFY(schema); +TConclusionStatus TGeneralContainer::AddField(const std::shared_ptr& f, const std::shared_ptr& data) { + return AddField(f, std::make_shared(data)); +} + +TConclusionStatus TGeneralContainer::AddField(const std::shared_ptr& f, const std::shared_ptr& data) { + return AddField(f, std::make_shared(data)); +} + +void TGeneralContainer::Initialize() { std::optional recordsCount; AFL_VERIFY(Schema->num_fields() == (i32)Columns.size())("schema", Schema->num_fields())("columns", Columns.size()); for (i32 i = 0; i < Schema->num_fields(); ++i) { @@ -58,12 +68,34 @@ TGeneralContainer::TGeneralContainer(const std::shared_ptr& schem } } AFL_VERIFY(recordsCount); + AFL_VERIFY(!RecordsCount || *RecordsCount == *recordsCount); RecordsCount = *recordsCount; } +TGeneralContainer::TGeneralContainer(const std::vector>& fields, std::vector>&& columns) + : Schema(std::make_shared(fields)) + , Columns(std::move(columns)) +{ + Initialize(); +} + +TGeneralContainer::TGeneralContainer(const std::shared_ptr& schema, std::vector>&& columns) + : Schema(std::make_shared(schema)) + , Columns(std::move(columns)) +{ + Initialize(); +} + +TGeneralContainer::TGeneralContainer(const std::shared_ptr& schema, std::vector>&& columns) + : Schema(std::make_shared(schema)) + , Columns(std::move(columns)) +{ + Initialize(); +} + TGeneralContainer::TGeneralContainer(const std::shared_ptr& table) { AFL_VERIFY(table); - Schema = table->schema(); + Schema = std::make_shared(table->schema()); RecordsCount = table->num_rows(); for (auto&& i : table->columns()) { if (i->num_chunks() == 1) { @@ -72,15 +104,17 @@ TGeneralContainer::TGeneralContainer(const std::shared_ptr& table) Columns.emplace_back(std::make_shared(i)); } } + Initialize(); } TGeneralContainer::TGeneralContainer(const std::shared_ptr& table) { AFL_VERIFY(table); - Schema = table->schema(); + Schema = std::make_shared(table->schema()); RecordsCount = table->num_rows(); for (auto&& i : table->columns()) { Columns.emplace_back(std::make_shared(i)); } + Initialize(); } std::shared_ptr TGeneralContainer::GetAccessorByNameVerified(const std::string& fieldId) const { @@ -110,14 +144,78 @@ std::shared_ptr TGeneralContainer::BuildTableOptional(const std::o if (fields.empty()) { return nullptr; } - return arrow::Table::Make(std::make_shared(fields), columns, RecordsCount); + AFL_VERIFY(RecordsCount); + return arrow::Table::Make(std::make_shared(fields), columns, *RecordsCount); } -std::shared_ptr TGeneralContainer::BuildTable(const std::optional>& columnNames /*= {}*/) const { +std::shared_ptr TGeneralContainer::BuildTableVerified(const std::optional>& columnNames /*= {}*/) const { auto result = BuildTableOptional(columnNames); AFL_VERIFY(result); AFL_VERIFY(!columnNames || result->schema()->num_fields() == (i32)columnNames->size()); return result; } +std::shared_ptr TGeneralContainer::GetAccessorByNameOptional(const std::string& fieldId) const { + int idx = Schema->GetFieldIndex(fieldId); + if (idx == -1) { + return nullptr; + } + AFL_VERIFY((ui32)idx < Columns.size())("idx", idx)("count", Columns.size()); + return Columns[idx]; +} + +TConclusionStatus TGeneralContainer::SyncSchemaTo(const std::shared_ptr& schema, const IFieldsConstructor* defaultFieldsConstructor, const bool forceDefaults) { + std::shared_ptr schemaNew = std::make_shared(); + std::vector> columnsNew; + if (!RecordsCount) { + return TConclusionStatus::Fail("original container has not data"); + } + for (auto&& i : schema->fields()) { + const int idx = Schema->GetFieldIndex(i->name()); + if (idx == -1) { + if (!defaultFieldsConstructor) { + return TConclusionStatus::Fail("haven't field for sync: '" + i->name() + "'"); + } else { + schemaNew->AddField(i).Validate(); + auto defConclusion = defaultFieldsConstructor->GetDefaultColumnElementValue(i, forceDefaults); + if (defConclusion.IsFail()) { + return defConclusion; + } + columnsNew.emplace_back(std::make_shared(NArrow::TThreadSimpleArraysCache::Get(i->type(), *defConclusion, *RecordsCount))); + } + } else { + const auto& fOwned = Schema->GetFieldVerified(idx); + if (!fOwned->type()->Equals(i->type())) { + return TConclusionStatus::Fail("different field types for '" + i->name() + "'. Have " + fOwned->type()->ToString() + ", need " + i->type()->ToString()); + } + schemaNew->AddField(fOwned).Validate(); + columnsNew.emplace_back(Columns[idx]); + } + } + std::swap(Schema, schemaNew); + std::swap(columnsNew, Columns); + return TConclusionStatus::Success(); +} + +TString TGeneralContainer::DebugString() const { + TStringBuilder result; + if (RecordsCount) { + result << "records_count=" << *RecordsCount << ";"; + } + result << "schema=" << Schema->ToString() << ";"; + return result; +} + +TConclusion> IFieldsConstructor::GetDefaultColumnElementValue(const std::shared_ptr& field, const bool force) const { + AFL_VERIFY(field); + auto result = DoGetDefaultColumnElementValue(field->name()); + if (result) { + return result; + } + if (force) { + return NArrow::DefaultScalar(field->type()); + } + return TConclusionStatus::Fail("have not default value for column " + field->name()); +} + } diff --git a/ydb/core/formats/arrow/common/container.h b/ydb/core/formats/arrow/common/container.h index 25262d14ff4a..b92871b96c6a 100644 --- a/ydb/core/formats/arrow/common/container.h +++ b/ydb/core/formats/arrow/common/container.h @@ -1,7 +1,10 @@ #pragma once #include "accessor.h" +#include + #include +#include #include #include @@ -12,50 +15,70 @@ namespace NKikimr::NArrow { +class IFieldsConstructor { +private: + virtual std::shared_ptr DoGetDefaultColumnElementValue(const std::string& fieldName) const = 0; +public: + TConclusion> GetDefaultColumnElementValue(const std::shared_ptr& field, const bool force) const; +}; + class TGeneralContainer { private: - YDB_READONLY(ui64, RecordsCount, 0); - YDB_READONLY_DEF(std::shared_ptr, Schema); + YDB_READONLY_DEF(std::optional, RecordsCount); + YDB_READONLY_DEF(std::shared_ptr, Schema); std::vector> Columns; + void Initialize(); public: - TString DebugString() const { - return TStringBuilder() - << "records_count=" << RecordsCount << ";" - << "schema=" << Schema->ToString() << ";" - ; + TString DebugString() const; + + [[nodiscard]] TConclusionStatus SyncSchemaTo(const std::shared_ptr& schema, + const IFieldsConstructor* defaultFieldsConstructor, const bool forceDefaults); + + bool HasColumn(const std::string& name) { + return Schema->HasField(name); + } + + ui64 num_columns() const { + return Columns.size(); } ui64 num_rows() const { - return RecordsCount; + AFL_VERIFY(RecordsCount); + return *RecordsCount; } - std::shared_ptr BuildTable(const std::optional>& columnNames = {}) const; + ui32 GetColumnsCount() const { + return Columns.size(); + } + + const std::shared_ptr& GetColumnVerified(const ui32 idx) const { + AFL_VERIFY(idx < Columns.size()); + return Columns[idx]; + } + + std::shared_ptr BuildTableVerified(const std::optional>& columnNames = {}) const; std::shared_ptr BuildTableOptional(const std::optional>& columnNames = {}) const; std::shared_ptr BuildEmptySame() const; [[nodiscard]] TConclusionStatus MergeColumnsStrictly(const TGeneralContainer& container); [[nodiscard]] TConclusionStatus AddField(const std::shared_ptr& f, const std::shared_ptr& data); + [[nodiscard]] TConclusionStatus AddField(const std::shared_ptr& f, const std::shared_ptr& data); - TGeneralContainer(const std::shared_ptr& table); + [[nodiscard]] TConclusionStatus AddField(const std::shared_ptr& f, const std::shared_ptr& data); + TGeneralContainer() = default; + TGeneralContainer(const std::shared_ptr& table); TGeneralContainer(const std::shared_ptr& table); - TGeneralContainer(const std::shared_ptr& schema, std::vector>&& columns); + TGeneralContainer(const std::shared_ptr& schema, std::vector>&& columns); + TGeneralContainer(const std::vector>& fields, std::vector>&& columns); arrow::Status ValidateFull() const { return arrow::Status::OK(); } - std::shared_ptr GetAccessorByNameOptional(const std::string& fieldId) const { - for (i32 i = 0; i < Schema->num_fields(); ++i) { - if (Schema->field(i)->name() == fieldId) { - return Columns[i]; - } - } - return nullptr; - } - + std::shared_ptr GetAccessorByNameOptional(const std::string& fieldId) const; std::shared_ptr GetAccessorByNameVerified(const std::string& fieldId) const; }; diff --git a/ydb/core/formats/arrow/modifier/schema.cpp b/ydb/core/formats/arrow/modifier/schema.cpp new file mode 100644 index 000000000000..4cf792614802 --- /dev/null +++ b/ydb/core/formats/arrow/modifier/schema.cpp @@ -0,0 +1,69 @@ +#include "schema.h" +#include +#include + +namespace NKikimr::NArrow::NModifier { + +std::shared_ptr TSchema::Finish() { + AFL_VERIFY(!Finished); + Finished = true; + return std::make_shared(Fields); +} + +const std::shared_ptr& TSchema::GetFieldByName(const std::string& name) const { + AFL_VERIFY(!Finished); + auto it = IndexByName.find(name); + if (it == IndexByName.end()) { + return Default>(); + } else { + return Fields[it->second]; + } +} + +TConclusionStatus TSchema::AddField(const std::shared_ptr& f) { + AFL_VERIFY(!Finished); + if (!IndexByName.emplace(f->name(), Fields.size()).second) { + return TConclusionStatus::Fail("field name duplication: " + f->name()); + } + Fields.emplace_back(f); + return TConclusionStatus::Success(); +} + +TString TSchema::ToString() const { + TStringBuilder result; + for (auto&& i : Fields) { + result << i->ToString() << ";"; + } + return result; +} + +const std::shared_ptr& TSchema::field(const ui32 index) const { + AFL_VERIFY(index < Fields.size()); + return Fields[index]; +} + +const std::shared_ptr& TSchema::GetFieldVerified(const ui32 index) const { + AFL_VERIFY(index < Fields.size()); + return Fields[index]; +} + +void TSchema::Initialize(const std::vector>& fields) { + AFL_VERIFY(!Initialized); + Initialized = true; + for (auto&& i : fields) { + IndexByName.emplace(i->name(), Fields.size()); + Fields.emplace_back(i); + } +} + +TSchema::TSchema(const std::shared_ptr& schema) { + AFL_VERIFY(schema); + Initialize(schema->Fields); +} + +TSchema::TSchema(const std::shared_ptr& schema) { + AFL_VERIFY(schema); + Initialize(schema->fields()); +} + +} \ No newline at end of file diff --git a/ydb/core/formats/arrow/modifier/schema.h b/ydb/core/formats/arrow/modifier/schema.h new file mode 100644 index 000000000000..dc663bad9f6a --- /dev/null +++ b/ydb/core/formats/arrow/modifier/schema.h @@ -0,0 +1,55 @@ +#pragma once +#include +#include +#include + +namespace NKikimr::NArrow::NModifier { +class TSchema { +private: + bool Initialized = false; + THashMap IndexByName; + std::vector> Fields; + bool Finished = false; + + void Initialize(const std::vector>& fields); +public: + TSchema() = default; + TSchema(const std::shared_ptr& schema); + + TSchema(const std::shared_ptr& schema); + + TSchema(const std::vector>& fields) { + Initialize(fields); + } + + i32 GetFieldIndex(const std::string& fName) const { + auto it = IndexByName.find(fName); + if (it == IndexByName.end()) { + return -1; + } + return it->second; + } + + const std::vector>& GetFields() const { + return Fields; + } + + TString ToString() const; + + std::shared_ptr Finish(); + [[nodiscard]] TConclusionStatus AddField(const std::shared_ptr& f); + const std::shared_ptr& GetFieldByName(const std::string& name) const; + + bool HasField(const std::string& name) const { + return IndexByName.contains(name); + } + + i32 num_fields() const { + return Fields.size(); + } + + const std::shared_ptr& GetFieldVerified(const ui32 index) const; + + const std::shared_ptr& field(const ui32 index) const; +}; +} \ No newline at end of file diff --git a/ydb/core/formats/arrow/modifier/subset.cpp b/ydb/core/formats/arrow/modifier/subset.cpp new file mode 100644 index 000000000000..1eb9dde2793c --- /dev/null +++ b/ydb/core/formats/arrow/modifier/subset.cpp @@ -0,0 +1,51 @@ +#include "subset.h" +#include + +namespace NKikimr::NArrow { + +TSchemaSubset::TSchemaSubset(const std::set& fieldsIdx, const ui32 fieldsCount) { + AFL_VERIFY(fieldsIdx.size() <= fieldsCount); + AFL_VERIFY(fieldsIdx.size()); + if (fieldsCount == fieldsIdx.size()) { + return; + } + Exclude = (fieldsCount - fieldsIdx.size()) < fieldsIdx.size(); + if (!Exclude) { + FieldIdx = std::vector(fieldsIdx.begin(), fieldsIdx.end()); + } else { + auto it = fieldsIdx.begin(); + for (ui32 i = 0; i < fieldsCount; ++i) { + if (it == fieldsIdx.end() || i < *it) { + FieldIdx.emplace_back(i); + } else if (*it == i) { + ++it; + } else { + AFL_VERIFY(false); + } + } + } +} + +NKikimrArrowSchema::TSchemaSubset TSchemaSubset::SerializeToProto() const { + NKikimrArrowSchema::TSchemaSubset result; + result.MutableList()->SetExclude(Exclude); + for (auto&& i : FieldIdx) { + result.MutableList()->AddFieldsIdx(i); + } + return result; +} + +TConclusionStatus TSchemaSubset::DeserializeFromProto(const NKikimrArrowSchema::TSchemaSubset& proto) { + if (!proto.HasList()) { + return TConclusionStatus::Fail("no schema subset data"); + } + Exclude = proto.GetList().GetExclude(); + std::vector fieldIdx; + for (auto&& i : proto.GetList().GetFieldsIdx()) { + fieldIdx.emplace_back(i); + } + std::swap(fieldIdx, FieldIdx); + return TConclusionStatus::Success(); +} + +} \ No newline at end of file diff --git a/ydb/core/formats/arrow/modifier/subset.h b/ydb/core/formats/arrow/modifier/subset.h new file mode 100644 index 000000000000..a638d8f17343 --- /dev/null +++ b/ydb/core/formats/arrow/modifier/subset.h @@ -0,0 +1,48 @@ +#pragma once +#include +#include +#include + +namespace NKikimr::NArrow { + +class TSchemaSubset { +private: + std::vector FieldIdx; + TString FieldBits; + bool Exclude = false; +public: + TSchemaSubset() = default; + TSchemaSubset(const std::set& fieldsIdx, const ui32 fieldsCount); + + template + std::vector Apply(const std::vector& fullSchema) const { + if (FieldIdx.empty()) { + return fullSchema; + } + std::vector fields; + if (!Exclude) { + for (auto&& i : FieldIdx) { + AFL_VERIFY(i < fullSchema.size()); + fields.emplace_back(fullSchema[i]); + } + } else { + auto it = FieldIdx.begin(); + for (ui32 i = 0; i < fullSchema.size(); ++i) { + if (it == FieldIdx.end() || i < *it) { + AFL_VERIFY(i < fullSchema.size()); + fields.emplace_back(fullSchema[i]); + } else if (i == *it) { + ++it; + } else { + AFL_VERIFY(false); + } + } + } + return fields; + } + + NKikimrArrowSchema::TSchemaSubset SerializeToProto() const; + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrArrowSchema::TSchemaSubset& proto); +}; + +} \ No newline at end of file diff --git a/ydb/core/formats/arrow/modifier/ya.make b/ydb/core/formats/arrow/modifier/ya.make new file mode 100644 index 000000000000..4b2b53f5513a --- /dev/null +++ b/ydb/core/formats/arrow/modifier/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +PEERDIR( + contrib/libs/apache/arrow + ydb/library/conclusion + ydb/core/formats/arrow/switch + ydb/core/formats/arrow/protos + ydb/library/actors/core +) + +SRCS( + schema.cpp + subset.cpp +) + +END() diff --git a/ydb/core/formats/arrow/process_columns.cpp b/ydb/core/formats/arrow/process_columns.cpp index d8795e188055..5fb133f90de5 100644 --- a/ydb/core/formats/arrow/process_columns.cpp +++ b/ydb/core/formats/arrow/process_columns.cpp @@ -1,5 +1,6 @@ #include "process_columns.h" #include "common/adapter.h" +#include "modifier/subset.h" #include @@ -28,16 +29,23 @@ std::shared_ptr ExtractColumnsValidateImpl(const std::shared_ptr template TConclusion> AdaptColumnsImpl(const std::shared_ptr& srcBatch, - const std::shared_ptr& dstSchema) { + const std::shared_ptr& dstSchema, TSchemaSubset* subset) { AFL_VERIFY(srcBatch); AFL_VERIFY(dstSchema); std::vector::TColumn>> columns; columns.reserve(dstSchema->num_fields()); - + std::vector> fields; + fields.reserve(dstSchema->num_fields()); + std::set fieldIdx; + ui32 idx = 0; for (auto& field : dstSchema->fields()) { const int index = srcBatch->schema()->GetFieldIndex(field->name()); if (index > -1) { + if (subset) { + fieldIdx.emplace(idx); + } columns.push_back(srcBatch->column(index)); + fields.emplace_back(field); auto srcField = srcBatch->schema()->field(index); if (field->Equals(srcField)) { AFL_VERIFY(columns.back()->type()->Equals(field->type()))("event", "cannot_use_incoming_batch")("reason", "invalid_column_type")("column", field->name()) @@ -47,14 +55,17 @@ TConclusion> AdaptColumnsImpl(const std::shared_ ("column_type", field->ToString(true))("incoming_type", srcField->ToString(true)); return TConclusionStatus::Fail("incompatible column types"); } - } else { + } else if (!subset) { AFL_ERROR(NKikimrServices::ARROW_HELPER)("event", "not_found_column")("column", field->name()) ("column_type", field->type()->ToString())("columns", JoinSeq(",", srcBatch->schema()->field_names())); return TConclusionStatus::Fail("not found column '" + field->name() + "'"); } + ++idx; } - - return NAdapter::TDataBuilderPolicy::Build(dstSchema, std::move(columns), srcBatch->num_rows()); + if (subset) { + *subset = TSchemaSubset(fieldIdx, dstSchema->num_fields()); + } + return NAdapter::TDataBuilderPolicy::Build(std::make_shared(fields), std::move(columns), srcBatch->num_rows()); } template @@ -114,12 +125,12 @@ std::shared_ptr TColumnOperator::Extract(const std::shared_ptr> TColumnOperator::Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema) { - return AdaptColumnsImpl(incoming, dstSchema); +NKikimr::TConclusion> TColumnOperator::Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema, TSchemaSubset* subset) { + return AdaptColumnsImpl(incoming, dstSchema, subset); } -NKikimr::TConclusion> TColumnOperator::Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema) { - return AdaptColumnsImpl(incoming, dstSchema); +NKikimr::TConclusion> TColumnOperator::Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema, TSchemaSubset* subset) { + return AdaptColumnsImpl(incoming, dstSchema, subset); } NKikimr::TConclusion> TColumnOperator::Reorder(const std::shared_ptr& incoming, const std::vector& columnNames) { diff --git a/ydb/core/formats/arrow/process_columns.h b/ydb/core/formats/arrow/process_columns.h index d07b106231d2..be05e84efb14 100644 --- a/ydb/core/formats/arrow/process_columns.h +++ b/ydb/core/formats/arrow/process_columns.h @@ -5,6 +5,8 @@ namespace NKikimr::NArrow { +class TSchemaSubset; + class TColumnOperator { public: enum class EExtractProblemsPolicy { @@ -36,8 +38,8 @@ class TColumnOperator { std::shared_ptr Extract(const std::shared_ptr& incoming, const std::vector& columnNames); std::shared_ptr Extract(const std::shared_ptr& incoming, const std::vector& columnNames); - TConclusion> Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema); - TConclusion> Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema); + TConclusion> Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema, TSchemaSubset* subset = nullptr); + TConclusion> Adapt(const std::shared_ptr& incoming, const std::shared_ptr& dstSchema, TSchemaSubset* subset = nullptr); TConclusion> Reorder(const std::shared_ptr& incoming, const std::vector& columnNames); TConclusion> Reorder(const std::shared_ptr& incoming, const std::vector& columnNames); diff --git a/ydb/core/formats/arrow/program.cpp b/ydb/core/formats/arrow/program.cpp index eb71d0731c52..e07f76ed3b49 100644 --- a/ydb/core/formats/arrow/program.cpp +++ b/ydb/core/formats/arrow/program.cpp @@ -919,7 +919,7 @@ std::set TProgramStep::GetColumnsInUsage(const bool originalOnly/* } arrow::Result> TProgramStep::BuildFilter(const std::shared_ptr& t) const { - return BuildFilter(t->BuildTable(GetColumnsInUsage(true))); + return BuildFilter(t->BuildTableVerified(GetColumnsInUsage(true))); } arrow::Result> TProgramStep::BuildFilter(const std::shared_ptr& t) const { diff --git a/ydb/core/formats/arrow/protos/fields.proto b/ydb/core/formats/arrow/protos/fields.proto new file mode 100644 index 000000000000..95d7bef08699 --- /dev/null +++ b/ydb/core/formats/arrow/protos/fields.proto @@ -0,0 +1,15 @@ +package NKikimrArrowSchema; + +message TSchemaSubset { + + message TFieldsList { + optional bool Exclude = 1; + repeated uint32 FieldsIdx = 2; + } + + oneof Implementation { + TFieldsList List = 1; + string Bits = 2; + + } +} \ No newline at end of file diff --git a/ydb/core/formats/arrow/protos/ya.make b/ydb/core/formats/arrow/protos/ya.make index f01aa064fbcc..828b0aa0fb77 100644 --- a/ydb/core/formats/arrow/protos/ya.make +++ b/ydb/core/formats/arrow/protos/ya.make @@ -2,6 +2,7 @@ PROTO_LIBRARY() SRCS( ssa.proto + fields.proto ) PEERDIR( diff --git a/ydb/core/formats/arrow/reader/merger.cpp b/ydb/core/formats/arrow/reader/merger.cpp index ddae86c1ed28..a09983971be8 100644 --- a/ydb/core/formats/arrow/reader/merger.cpp +++ b/ydb/core/formats/arrow/reader/merger.cpp @@ -185,13 +185,13 @@ void TMergePartialStream::DrainCurrentPosition(TRecordBatchBuilder* builder, std SortHeap.CleanFinished(); } -std::vector> TMergePartialStream::DrainAllParts(const std::map& positions, +std::vector> TMergePartialStream::DrainAllParts(const TIntervalPositions& positions, const std::vector>& resultFields) { std::vector> result; for (auto&& i : positions) { TRecordBatchBuilder indexesBuilder(resultFields); - DrainCurrentTo(indexesBuilder, i.first, i.second); + DrainCurrentTo(indexesBuilder, i.GetPosition(), i.IsIncludedToLeftInterval()); result.emplace_back(indexesBuilder.Finalize()); if (result.back()->num_rows() == 0) { result.pop_back(); diff --git a/ydb/core/formats/arrow/reader/merger.h b/ydb/core/formats/arrow/reader/merger.h index 196edcd09e3c..972e891fe1fd 100644 --- a/ydb/core/formats/arrow/reader/merger.h +++ b/ydb/core/formats/arrow/reader/merger.h @@ -94,7 +94,7 @@ class TMergePartialStream { std::shared_ptr SingleSourceDrain(const TSortableBatchPosition& readTo, const bool includeFinish, std::optional* lastResultPosition = nullptr); bool DrainCurrentTo(TRecordBatchBuilder& builder, const TSortableBatchPosition& readTo, const bool includeFinish, std::optional* lastResultPosition = nullptr); bool DrainToControlPoint(TRecordBatchBuilder& builder, const bool includeFinish, std::optional* lastResultPosition = nullptr); - std::vector> DrainAllParts(const std::map& positions, + std::vector> DrainAllParts(const TIntervalPositions& positions, const std::vector>& resultFields); }; diff --git a/ydb/core/formats/arrow/reader/position.h b/ydb/core/formats/arrow/reader/position.h index 8a6e15fd79ac..5b8fd35d5bc7 100644 --- a/ydb/core/formats/arrow/reader/position.h +++ b/ydb/core/formats/arrow/reader/position.h @@ -404,6 +404,78 @@ class TSortableBatchPosition { }; +class TIntervalPosition { +private: + TSortableBatchPosition Position; + const bool LeftIntervalInclude; +public: + const TSortableBatchPosition& GetPosition() const { + return Position; + } + bool IsIncludedToLeftInterval() const { + return LeftIntervalInclude; + } + TIntervalPosition(TSortableBatchPosition&& position, const bool leftIntervalInclude) + : Position(std::move(position)) + , LeftIntervalInclude(leftIntervalInclude) { + + } + + TIntervalPosition(const TSortableBatchPosition& position, const bool leftIntervalInclude) + : Position(position) + , LeftIntervalInclude(leftIntervalInclude) { + + } + + bool operator<(const TIntervalPosition& item) const { + std::partial_ordering cmp = Position.Compare(item.Position); + if (cmp == std::partial_ordering::equivalent) { + return (LeftIntervalInclude ? 1 : 0) < (item.LeftIntervalInclude ? 1 : 0); + } + return cmp == std::partial_ordering::less; + } + + NJson::TJsonValue DebugJson() const { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("position", Position.DebugJson()); + result.InsertValue("include", LeftIntervalInclude); + return result; + } +}; + +class TIntervalPositions { +private: + std::vector Positions; +public: + bool IsEmpty() const { + return Positions.empty(); + } + + std::vector::const_iterator begin() const { + return Positions.begin(); + } + + std::vector::const_iterator end() const { + return Positions.end(); + } + + void AddPosition(TSortableBatchPosition&& position, const bool includeLeftInterval) { + TIntervalPosition intervalPosition(std::move(position), includeLeftInterval); + if (Positions.size()) { + AFL_VERIFY(Positions.back() < intervalPosition)("back", Positions.back().DebugJson())("pos", intervalPosition.DebugJson()); + } + Positions.emplace_back(std::move(intervalPosition)); + } + + void AddPosition(const TSortableBatchPosition& position, const bool includeLeftInterval) { + TIntervalPosition intervalPosition(position, includeLeftInterval); + if (Positions.size()) { + AFL_VERIFY(Positions.back() < intervalPosition)("back", Positions.back().DebugJson())("pos", intervalPosition.DebugJson()); + } + Positions.emplace_back(std::move(intervalPosition)); + } +}; + class TRWSortableBatchPosition: public TSortableBatchPosition, public TMoveOnly { private: using TBase = TSortableBatchPosition; diff --git a/ydb/core/formats/arrow/ya.make b/ydb/core/formats/arrow/ya.make index 4615047b39fa..49938a884154 100644 --- a/ydb/core/formats/arrow/ya.make +++ b/ydb/core/formats/arrow/ya.make @@ -12,6 +12,7 @@ PEERDIR( ydb/core/formats/arrow/dictionary ydb/core/formats/arrow/transformer ydb/core/formats/arrow/reader + ydb/core/formats/arrow/modifier ydb/core/formats/arrow/scalar ydb/core/formats/arrow/hash ydb/library/actors/core diff --git a/ydb/core/kqp/ut/olap/helpers/typed_local.h b/ydb/core/kqp/ut/olap/helpers/typed_local.h index 29852b7761da..a72cef64e33e 100644 --- a/ydb/core/kqp/ut/olap/helpers/typed_local.h +++ b/ydb/core/kqp/ut/olap/helpers/typed_local.h @@ -74,7 +74,7 @@ class TTypedLocalHelper: public Tests::NCS::THelper { void GetCount(ui64& count); template - void FillTable(const TFiller& fillPolicy, const ui32 pkKff = 0, const ui32 numRows = 800000) const { + void FillTable(const TFiller& fillPolicy, const double pkKff = 0, const ui32 numRows = 800000) const { std::vector builders; builders.emplace_back(NArrow::NConstruction::TSimpleArrayConstructor>::BuildNotNullable("pk_int", numRows * pkKff)); builders.emplace_back(std::make_shared>("field", fillPolicy)); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp index f9959ede78e3..4f6c90056e89 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp @@ -12,7 +12,7 @@ using namespace NYdb::NTable; Y_UNIT_TEST_SUITE(KqpOlapStats) { constexpr size_t inserted_rows = 1000; constexpr size_t tables_in_store = 1000; - constexpr size_t size_single_table = 13352; + constexpr size_t size_single_table = 13152; const TVector schema = { TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), diff --git a/ydb/core/kqp/ut/olap/sys_view_ut.cpp b/ydb/core/kqp/ut/olap/sys_view_ut.cpp index dc234ac3bffe..8583967214ca 100644 --- a/ydb/core/kqp/ut/olap/sys_view_ut.cpp +++ b/ydb/core/kqp/ut/olap/sys_view_ut.cpp @@ -5,6 +5,7 @@ #include "helpers/get_value.h" #include +#include #include #include @@ -229,7 +230,9 @@ Y_UNIT_TEST_SUITE(KqpOlapSysView) { helper.CreateTestOlapTable(); NArrow::NConstruction::TStringPoolFiller sPool(3, 52); helper.FillTable(sPool, 0, 800000); - csController->WaitCompactions(TDuration::Seconds(10)); + csController->WaitCompactions(TDuration::Seconds(5)); + helper.FillTable(sPool, 0.5, 800000); + csController->WaitCompactions(TDuration::Seconds(5)); helper.GetVolumes(rawBytes1, bytes1, false, {"new_column_ui64"}); AFL_VERIFY(rawBytes1 == 0); @@ -241,9 +244,9 @@ Y_UNIT_TEST_SUITE(KqpOlapSysView) { csController->WaitActualization(TDuration::Seconds(10)); ui64 rawBytes2; ui64 bytes2; - helper.GetVolumes(rawBytes2, bytes2, false, {"new_column_ui64"}); - AFL_VERIFY(rawBytes2 == 6500041)("real", rawBytes2); - AFL_VERIFY(bytes2 == 45360)("b", bytes2); + helper.GetVolumes(rawBytes2, bytes2, false, { "new_column_ui64", NOlap::IIndexInfo::SPEC_COL_DELETE_FLAG }); + AFL_VERIFY(rawBytes2 == 0)("real", rawBytes2); + AFL_VERIFY(bytes2 == 0)("b", bytes2); } } diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 1469403863ae..7ef6fe88bd11 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -3046,6 +3046,13 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); } + { + auto it = client.ExecuteQuery(R"( + UPSERT INTO `/Root/DataShard` (Col3) VALUES ('null'); + )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!it.IsSuccess()); + } + { auto it = client.StreamExecuteQuery(R"( SELECT * FROM `/Root/DataShard` ORDER BY Col1; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 51f51fb2fec3..c782f69d1e72 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1497,7 +1497,6 @@ message TColumnShardConfig { optional bool TTLEnabled = 6 [default = true]; optional bool WritingEnabled = 7 [default = true]; optional uint32 WritingBufferDurationMs = 8 [default = 0]; - optional bool UseChunkedMergeOnCompaction = 9 [default = true]; optional uint64 CompactionMemoryLimit = 10 [default = 536870912]; optional uint64 TieringsMemoryLimit = 11 [default = 536870912]; message TIndexMetadataMemoryLimit { diff --git a/ydb/core/protos/tx_columnshard.proto b/ydb/core/protos/tx_columnshard.proto index 4cfd8c34cb0b..b0339d5aa9fb 100644 --- a/ydb/core/protos/tx_columnshard.proto +++ b/ydb/core/protos/tx_columnshard.proto @@ -4,6 +4,7 @@ import "ydb/core/protos/long_tx_service.proto"; import "ydb/core/protos/statistics.proto"; import "ydb/core/protos/subdomains.proto"; import "ydb/core/protos/tx.proto"; +import "ydb/core/formats/arrow/protos/fields.proto"; package NKikimrTxColumnShard; option java_package = "ru.yandex.kikimr.proto"; @@ -88,6 +89,7 @@ message TLogicalMetadata { optional uint64 DirtyWriteTimeSeconds = 5; optional string SpecialKeysRawData = 6; optional TEvWrite.EModificationType ModificationType = 7; + optional NKikimrArrowSchema.TSchemaSubset SchemaSubset = 8; } message TEvWriteResult { diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 1c8326f12fc3..8e12cf1b8a63 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -18,10 +18,13 @@ bool TTxWrite::InsertOneBlob(TTransactionContext& txc, const NOlap::TWideSeriali const auto& writeMeta = batch.GetAggregation().GetWriteMeta(); meta.SetModificationType(TEnumOperator::SerializeToProto(writeMeta.GetModificationType())); + *meta.MutableSchemaSubset() = batch.GetAggregation().GetSchemaSubset().SerializeToProto(); auto schemeVersion = batch.GetAggregation().GetSchemaVersion(); auto tableSchema = Self->TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetSchemaVerified(schemeVersion); - NOlap::TInsertedData insertData((ui64)writeId, writeMeta.GetTableId(), writeMeta.GetDedupId(), blobRange, meta, tableSchema->GetVersion(), batch->GetData()); + NOlap::TInsertedData insertData((ui64)writeId, writeMeta.GetTableId(), writeMeta.GetDedupId(), blobRange, + meta, tableSchema->GetVersion(), + batch->GetData()); bool ok = Self->InsertTable->Insert(dbTable, std::move(insertData)); if (ok) { Self->UpdateInsertTableCounters(); diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merge_context.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.cpp similarity index 61% rename from ydb/core/tx/columnshard/engines/changes/compaction/merge_context.cpp rename to ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.cpp index 8280e58eec95..be583661ea38 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merge_context.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.cpp @@ -1,4 +1,4 @@ -#include "merge_context.h" +#include "merger.h" namespace NKikimr::NOlap::NCompaction { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.h b/ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.h new file mode 100644 index 000000000000..1e93f0c70979 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.h @@ -0,0 +1,48 @@ +#pragma once +#include +#include + +namespace NKikimr::NOlap::NCompaction { +class IColumnMerger { +private: + bool Started = false; + + virtual std::vector DoExecute( + const NCompaction::TColumnMergeContext& context, const arrow::UInt16Array& pIdxArray, const arrow::UInt32Array& pRecordIdxArray) = 0; + virtual void DoStart(const std::vector>& input) = 0; + +public: + static inline const TString PortionIdFieldName = "$$__portion_id"; + static inline const TString PortionRecordIndexFieldName = "$$__portion_record_idx"; + static inline const std::shared_ptr PortionIdField = + std::make_shared(PortionIdFieldName, std::make_shared()); + static inline const std::shared_ptr PortionRecordIndexField = + std::make_shared(PortionRecordIndexFieldName, std::make_shared()); + + virtual ~IColumnMerger() = default; + + void Start(const std::vector>& input) { + AFL_VERIFY(!Started); + Started = true; + return DoStart(input); + } + + std::vector Execute( + const NCompaction::TColumnMergeContext& context, const std::shared_ptr& remap) { + + auto columnPortionIdx = remap->GetColumnByName(IColumnMerger::PortionIdFieldName); + auto columnPortionRecordIdx = remap->GetColumnByName(IColumnMerger::PortionRecordIndexFieldName); + Y_ABORT_UNLESS(columnPortionIdx && columnPortionRecordIdx); + Y_ABORT_UNLESS(columnPortionIdx->type_id() == arrow::UInt16Type::type_id); + Y_ABORT_UNLESS(columnPortionRecordIdx->type_id() == arrow::UInt32Type::type_id); + const arrow::UInt16Array& pIdxArray = static_cast(*columnPortionIdx); + const arrow::UInt32Array& pRecordIdxArray = static_cast(*columnPortionRecordIdx); + + AFL_VERIFY(remap->num_rows() == pIdxArray.length()); + AFL_VERIFY(remap->num_rows() == pRecordIdxArray.length()); + + return DoExecute(context, pIdxArray, pRecordIdxArray); + } +}; + +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/abstract/ya.make b/ydb/core/tx/columnshard/engines/changes/compaction/abstract/ya.make new file mode 100644 index 000000000000..07be3f70eb68 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/abstract/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + merger.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/engines/changes/compaction/common +) + +END() diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/column_cursor.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/column_cursor.cpp deleted file mode 100644 index cdb81296cf73..000000000000 --- a/ydb/core/tx/columnshard/engines/changes/compaction/column_cursor.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "column_cursor.h" -#include - -namespace NKikimr::NOlap::NCompaction { - -bool TPortionColumnCursor::Fetch(TMergedColumn& column) { - Y_ABORT_UNLESS(ChunkIdx < ColumnChunks.size()); - Y_ABORT_UNLESS(RecordIndexStart); - ui32 currentStartPortionIdx = *RecordIndexStart; - ui32 currentFinishPortionIdx = RecordIndexFinish; -// NActors::TLogContextGuard lg(NActors::TLogContextBuilder::Build()("portion_id", PortionId)); - while (currentStartPortionIdx - ChunkRecordIndexStartPosition >= CurrentChunkRecordsCount) { - if (!NextChunk()) { - return false; - } - } - - ui32 currentStart = currentStartPortionIdx - ChunkRecordIndexStartPosition; - while (currentFinishPortionIdx - ChunkRecordIndexStartPosition >= CurrentChunkRecordsCount) { - const ui32 currentFinish = CurrentChunkRecordsCount; -// if (currentStart == 0 && CurrentColumnChunk) { -// column.AppendBlob(CurrentBlobChunk->GetData(), *CurrentColumnChunk); -// } else { - column.AppendSlice(GetCurrentArray(), currentStart, currentFinish - currentStart); -// } - currentStart = 0; - if (!NextChunk()) { - return false; - } - } - - const ui32 currentFinish = currentFinishPortionIdx - ChunkRecordIndexStartPosition; - if (currentStart < currentFinish) { - Y_ABORT_UNLESS(currentFinish < CurrentChunkRecordsCount); - column.AppendSlice(GetCurrentArray(), currentStart, currentFinish - currentStart); - } - - RecordIndexStart.reset(); - RecordIndexFinish = 0; - return true; -} - -bool TPortionColumnCursor::Next(const ui32 portionRecordIdx, TMergedColumn& column) { - Y_ABORT_UNLESS(ChunkRecordIndexStartPosition <= portionRecordIdx); - if (!RecordIndexStart) { - RecordIndexStart = portionRecordIdx; - RecordIndexFinish = portionRecordIdx + 1; - } else if (RecordIndexFinish == portionRecordIdx) { - RecordIndexFinish = portionRecordIdx + 1; - } else { - Fetch(column); - RecordIndexStart = portionRecordIdx; - RecordIndexFinish = portionRecordIdx + 1; - } - return true; -} - -bool TPortionColumnCursor::NextChunk() { - CurrentArray = nullptr; - if (++ChunkIdx == ColumnChunks.size()) { - return false; - } else { - ChunkRecordIndexStartPosition += CurrentChunkRecordsCount; - CurrentBlobChunk = BlobChunks[ChunkIdx]; - CurrentColumnChunk = ColumnChunks[ChunkIdx]; - CurrentChunkRecordsCount = CurrentBlobChunk->GetRecordsCountVerified(); - return true; - } -} - -const std::shared_ptr& TPortionColumnCursor::GetCurrentArray() { - Y_ABORT_UNLESS(ChunkIdx < ColumnChunks.size()); - Y_ABORT_UNLESS(CurrentBlobChunk); - - if (!CurrentArray) { - auto res = NArrow::TStatusValidator::GetValid(ColumnLoader->Apply(CurrentBlobChunk->GetData())); - AFL_VERIFY(res->num_columns() == 1); - CurrentArray = res->column(0); - } - return CurrentArray; -} - -} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/column_cursor.h b/ydb/core/tx/columnshard/engines/changes/compaction/column_cursor.h deleted file mode 100644 index 493cd6268f6a..000000000000 --- a/ydb/core/tx/columnshard/engines/changes/compaction/column_cursor.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once -#include "merged_column.h" -#include -#include -#include -#include - -namespace NKikimr::NOlap::NCompaction { - -class TPortionColumnCursor { -private: - std::vector> BlobChunks; - std::vector ColumnChunks; - std::optional RecordIndexStart; - YDB_READONLY(ui32, RecordIndexFinish, 0); - ui32 ChunkRecordIndexStartPosition = 0; - ui32 ChunkIdx = 0; - std::shared_ptr CurrentBlobChunk; - const TColumnRecord* CurrentColumnChunk = nullptr; - ui32 CurrentChunkRecordsCount = 0; - std::shared_ptr CurrentArray; - std::shared_ptr ColumnLoader; - const ui64 PortionId; - - const std::shared_ptr& GetCurrentArray(); - - bool NextChunk(); - -public: - ~TPortionColumnCursor() { - AFL_VERIFY(!RecordIndexStart || ChunkIdx == ColumnChunks.size())("chunk", ChunkIdx) - ("size", ColumnChunks.size())("start", RecordIndexStart)("finish", RecordIndexFinish) - ("max", CurrentBlobChunk->GetRecordsCount())("current_start_position", ChunkRecordIndexStartPosition); - } - - bool Next(const ui32 portionRecordIdx, TMergedColumn& column); - - bool Fetch(TMergedColumn& column); - - TPortionColumnCursor(const std::vector>& columnChunks, const std::vector& records, const std::shared_ptr& loader, const ui64 portionId) - : BlobChunks(columnChunks) - , ColumnChunks(records) - , ColumnLoader(loader) - , PortionId(portionId) { - AFL_VERIFY(ColumnLoader); - Y_UNUSED(PortionId); - Y_ABORT_UNLESS(BlobChunks.size()); - Y_ABORT_UNLESS(ColumnChunks.size() == BlobChunks.size()); - CurrentBlobChunk = BlobChunks.front(); - CurrentColumnChunk = ColumnChunks.front(); - CurrentChunkRecordsCount = CurrentBlobChunk->GetRecordsCountVerified(); - } -}; - -} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/common/context.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/common/context.cpp new file mode 100644 index 000000000000..35fbf111c993 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/common/context.cpp @@ -0,0 +1,5 @@ +#include "context.h" + +namespace NKikimr::NOlap::NCompaction { + +} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merge_context.h b/ydb/core/tx/columnshard/engines/changes/compaction/common/context.h similarity index 85% rename from ydb/core/tx/columnshard/engines/changes/compaction/merge_context.h rename to ydb/core/tx/columnshard/engines/changes/compaction/common/context.h index a5da857c2aff..80356224909f 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merge_context.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/common/context.h @@ -23,6 +23,7 @@ class TColumnMergeContext { YDB_READONLY(bool, UseWholeChunksOptimization, true); std::optional ColumnStat; + const TIndexInfo& IndexInfo; public: ISnapshotSchema::TPtr GetSchemaInfo() const { @@ -41,8 +42,9 @@ class TColumnMergeContext { return IndexInfo; } - TColumnMergeContext(const ui32 columnId, const ISnapshotSchema::TPtr& schema, const ui32 portionRowsCountLimit, const ui32 chunkRawBytesLimit, - const std::optional& columnStat) + TColumnMergeContext(const ui32 columnId, const ISnapshotSchema::TPtr& schema, const ui32 portionRowsCountLimit, + const ui32 chunkRawBytesLimit, const std::optional& columnStat, + const NArrow::NSerialization::TSerializerContainer& overrideSerializer) : ColumnId(columnId) , SchemaInfo(schema) , Saver(schema->GetColumnSaver(columnId)) @@ -52,10 +54,12 @@ class TColumnMergeContext { , ChunkRawBytesLimit(chunkRawBytesLimit) , UseWholeChunksOptimization(!schema->GetIndexInfo().GetReplaceKey()->GetFieldByName(ResultField->name())) , ColumnStat(columnStat) - , IndexInfo(schema->GetIndexInfo()) - { + , IndexInfo(schema->GetIndexInfo()) { Y_ABORT_UNLESS(PortionRowsCountLimit); Y_ABORT_UNLESS(ChunkRawBytesLimit); + if (!!overrideSerializer) { + Saver.ResetSerializer(overrideSerializer); + } } }; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/common/result.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/common/result.cpp new file mode 100644 index 000000000000..6482ee301543 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/common/result.cpp @@ -0,0 +1,10 @@ +#include "result.h" +#include + +namespace NKikimr::NOlap::NCompaction { + +TString TColumnPortionResult::DebugString() const { + return TStringBuilder() << "chunks=" << Chunks.size() << ";"; +} + +} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/common/result.h b/ydb/core/tx/columnshard/engines/changes/compaction/common/result.h new file mode 100644 index 000000000000..850e1f6eebe0 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/common/result.h @@ -0,0 +1,25 @@ +#pragma once +#include + +namespace NKikimr::NOlap::NCompaction { + +class TColumnPortionResult { +protected: + std::vector> Chunks; + const ui32 ColumnId; +public: + + TColumnPortionResult(const ui32 columnId) + : ColumnId(columnId) { + + } + + const std::vector>& GetChunks() const { + return Chunks; + } + + TString DebugString() const; + +}; + +} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/common/ya.make b/ydb/core/tx/columnshard/engines/changes/compaction/common/ya.make new file mode 100644 index 000000000000..30667909c931 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/common/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + context.cpp + result.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/engines/scheme +) + +END() diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp new file mode 100644 index 000000000000..856c4107da8b --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp @@ -0,0 +1,162 @@ +#include "merger.h" + +#include "abstract/merger.h" +#include "plain/logic.h" + +#include +#include +#include +#include +#include + +namespace NKikimr::NOlap::NCompaction { + +std::vector TMerger::Execute(const std::shared_ptr& stats, + const NArrow::NMerger::TIntervalPositions& checkPoints, const std::shared_ptr& resultFiltered, const ui64 pathId, + const std::optional shardingActualVersion) { + AFL_VERIFY(Batches.size() == Filters.size()); + std::vector> batchResults; + { + arrow::FieldVector indexFields; + indexFields.emplace_back(IColumnMerger::PortionIdField); + indexFields.emplace_back(IColumnMerger::PortionRecordIndexField); + if (resultFiltered->HasColumnId((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG)) { + IIndexInfo::AddDeleteFields(indexFields); + } + IIndexInfo::AddSnapshotFields(indexFields); + auto dataSchema = std::make_shared(indexFields); + NArrow::NMerger::TMergePartialStream mergeStream( + resultFiltered->GetIndexInfo().GetReplaceKey(), dataSchema, false, IIndexInfo::GetSnapshotColumnNames()); + + ui32 idx = 0; + for (auto&& batch : Batches) { + { + NArrow::NConstruction::IArrayBuilder::TPtr column = + std::make_shared>>( + IColumnMerger::PortionIdFieldName, idx); + batch->AddField(IColumnMerger::PortionIdField, column->BuildArray(batch->num_rows())).Validate(); + } + { + NArrow::NConstruction::IArrayBuilder::TPtr column = + std::make_shared>>( + IColumnMerger::PortionRecordIndexFieldName); + batch->AddField(IColumnMerger::PortionRecordIndexField, column->BuildArray(batch->num_rows())).Validate(); + } + mergeStream.AddSource(batch, Filters[idx]); + ++idx; + } + batchResults = mergeStream.DrainAllParts(checkPoints, indexFields); + } + + std::vector>> chunkGroups; + chunkGroups.resize(batchResults.size()); + for (auto&& columnId : resultFiltered->GetColumnIds()) { + NActors::TLogContextGuard logGuard( + NActors::TLogContextBuilder::Build()("field_name", resultFiltered->GetIndexInfo().GetColumnName(columnId))); + auto columnInfo = stats->GetColumnInfo(columnId); + auto resultField = resultFiltered->GetIndexInfo().GetColumnFieldVerified(columnId); + std::shared_ptr merger = std::make_shared(); + // resultFiltered->BuildColumnMergerVerified(columnId); + + { + std::vector> parts; + for (auto&& p : Batches) { + parts.emplace_back(p->GetColumnVerified(resultFiltered->GetFieldIndex(columnId))); + } + + merger->Start(parts); + } + + std::map> columnChunks; + ui32 batchIdx = 0; + for (auto&& batchResult : batchResults) { + const ui32 portionRecordsCountLimit = + batchResult->num_rows() / (batchResult->num_rows() / NSplitter::TSplitSettings().GetExpectedRecordsCountOnPage() + 1) + 1; + + NArrow::NSerialization::TSerializerContainer externalSaver; + if (OptimizationWritingPackMode) { + if (batchResult->num_rows() < 100) { + externalSaver = NArrow::NSerialization::TSerializerContainer( + std::make_shared(arrow::Compression::type::UNCOMPRESSED)); + } else { + externalSaver = NArrow::NSerialization::TSerializerContainer( + std::make_shared(arrow::Compression::type::LZ4_FRAME)); + } + } + + NCompaction::TColumnMergeContext context(columnId, resultFiltered, portionRecordsCountLimit, + NSplitter::TSplitSettings().GetExpectedUnpackColumnChunkRawSize(), columnInfo, externalSaver); + + chunkGroups[batchIdx][columnId] = merger->Execute(context, batchResult); + ++batchIdx; + } + } + ui32 batchIdx = 0; + + const auto groups = + resultFiltered->GetIndexInfo().GetEntityGroupsByStorageId(IStoragesManager::DefaultStorageId, *SaverContext.GetStoragesManager()); + std::vector result; + for (auto&& columnChunks : chunkGroups) { + auto batchResult = batchResults[batchIdx]; + ++batchIdx; + Y_ABORT_UNLESS(columnChunks.size()); + + for (auto&& i : columnChunks) { + if (i.second.size() != columnChunks.begin()->second.size()) { + for (ui32 p = 0; p < std::min(columnChunks.begin()->second.size(), i.second.size()); ++p) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("p_first", columnChunks.begin()->second[p].DebugString())( + "p", i.second[p].DebugString()); + } + } + AFL_VERIFY(i.second.size() == columnChunks.begin()->second.size())("first", columnChunks.begin()->second.size())( + "current", i.second.size())("first_name", columnChunks.begin()->first)("current_name", i.first); + } + auto columnSnapshotPlanStepIdx = batchResult->GetColumnByName(TIndexInfo::SPEC_COL_PLAN_STEP); + auto columnSnapshotTxIdx = batchResult->GetColumnByName(TIndexInfo::SPEC_COL_TX_ID); + Y_ABORT_UNLESS(columnSnapshotPlanStepIdx); + Y_ABORT_UNLESS(columnSnapshotTxIdx); + Y_ABORT_UNLESS(columnSnapshotPlanStepIdx->type_id() == arrow::UInt64Type::type_id); + Y_ABORT_UNLESS(columnSnapshotTxIdx->type_id() == arrow::UInt64Type::type_id); + + std::vector batchSlices; + std::shared_ptr schemaDetails(new TDefaultSchemaDetails(resultFiltered, stats)); + + for (ui32 i = 0; i < columnChunks.begin()->second.size(); ++i) { + THashMap>> portionColumns; + for (auto&& p : columnChunks) { + portionColumns.emplace(p.first, p.second[i].GetChunks()); + } + batchSlices.emplace_back(portionColumns, schemaDetails, Context.Counters.SplitterCounters); + } + TSimilarPacker slicer(NSplitter::TSplitSettings().GetExpectedPortionSize()); + auto packs = slicer.Split(batchSlices); + + ui32 recordIdx = 0; + for (auto&& i : packs) { + TGeneralSerializedSlice slicePrimary(std::move(i)); + auto dataWithSecondary = resultFiltered->GetIndexInfo() + .AppendIndexes(slicePrimary.GetPortionChunksToHash(), SaverContext.GetStoragesManager()) + .DetachResult(); + TGeneralSerializedSlice slice(dataWithSecondary.GetExternalData(), schemaDetails, Context.Counters.SplitterCounters); + + auto b = batchResult->Slice(recordIdx, slice.GetRecordsCount()); + const ui32 deletionsCount = IIndexInfo::CalcDeletions(b, false); + auto constructor = TWritePortionInfoWithBlobsConstructor::BuildByBlobs(slice.GroupChunksByBlobs(groups), + dataWithSecondary.GetSecondaryInplaceData(), pathId, resultFiltered->GetVersion(), resultFiltered->GetSnapshot(), + SaverContext.GetStoragesManager()); + + NArrow::TFirstLastSpecialKeys primaryKeys(slice.GetFirstLastPKBatch(resultFiltered->GetIndexInfo().GetReplaceKey())); + NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot()); + constructor.GetPortionConstructor().AddMetadata(*resultFiltered, deletionsCount, primaryKeys, snapshotKeys); + constructor.GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId); + if (shardingActualVersion) { + constructor.GetPortionConstructor().SetShardingVersion(*shardingActualVersion); + } + result.emplace_back(std::move(constructor)); + recordIdx += slice.GetRecordsCount(); + } + } + return result; +} + +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merger.h b/ydb/core/tx/columnshard/engines/changes/compaction/merger.h new file mode 100644 index 000000000000..be9beae47584 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/merger.h @@ -0,0 +1,48 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace NKikimr::NOlap::NCompaction { +class TMerger { +private: + YDB_ACCESSOR(bool, OptimizationWritingPackMode, false); + std::vector> Batches; + std::vector> Filters; + const TConstructionContext& Context; + const TSaverContext& SaverContext; + +public: + void AddBatch(const std::shared_ptr& batch, const std::shared_ptr& filter) { + AFL_VERIFY(batch); + Batches.emplace_back(batch); + Filters.emplace_back(filter); + } + + TMerger(const TConstructionContext& context, const TSaverContext& saverContext) + : Context(context) + , SaverContext(saverContext) + { + + } + + TMerger(const TConstructionContext& context, const TSaverContext& saverContext, + std::vector>&& batches, + std::vector>&& filters) + : Batches(std::move(batches)) + , Filters(std::move(filters)) + , Context(context) + , SaverContext(saverContext) { + AFL_VERIFY(Batches.size() == Filters.size()); + } + + std::vector Execute( + const std::shared_ptr& stats, + const NArrow::NMerger::TIntervalPositions& checkPoints, + const std::shared_ptr& resultFiltered, const ui64 pathId, const std::optional shardingActualVersion); +}; +} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_cursor.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_cursor.cpp new file mode 100644 index 000000000000..65412522c879 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_cursor.cpp @@ -0,0 +1,52 @@ +#include "column_cursor.h" +#include + +namespace NKikimr::NOlap::NCompaction { + +bool TPortionColumnCursor::Fetch(TMergedColumn& column) { + Y_ABORT_UNLESS(RecordIndexStart); + if (CurrentChunk && CurrentChunk->GetStartPosition() <= *RecordIndexStart && *RecordIndexStart < CurrentChunk->GetFinishPosition()) { + + } else { + CurrentChunk = BlobChunks->GetChunk(CurrentChunk, *RecordIndexStart); + } + + ui32 currentStart = *RecordIndexStart; + while (RecordIndexFinish >= CurrentChunk->GetFinishPosition()) { + column.AppendSlice( + CurrentChunk->GetArray(), currentStart - CurrentChunk->GetStartPosition(), CurrentChunk->GetFinishPosition() - currentStart); + currentStart = CurrentChunk->GetFinishPosition(); + if (currentStart < BlobChunks->GetRecordsCount()) { + CurrentChunk = BlobChunks->GetChunk(CurrentChunk, currentStart); + } else { + CurrentChunk.reset(); + break; + } + } + + if (currentStart < RecordIndexFinish) { + AFL_VERIFY(CurrentChunk); + Y_ABORT_UNLESS(RecordIndexFinish < CurrentChunk->GetFinishPosition()); + column.AppendSlice(CurrentChunk->GetArray(), currentStart - CurrentChunk->GetStartPosition(), RecordIndexFinish - currentStart); + } + + RecordIndexStart.reset(); + RecordIndexFinish = 0; + return true; +} + +bool TPortionColumnCursor::Next(const ui32 portionRecordIdx, TMergedColumn& column) { + if (!RecordIndexStart) { + RecordIndexStart = portionRecordIdx; + RecordIndexFinish = portionRecordIdx + 1; + } else if (RecordIndexFinish == portionRecordIdx) { + RecordIndexFinish = portionRecordIdx + 1; + } else { + Fetch(column); + RecordIndexStart = portionRecordIdx; + RecordIndexFinish = portionRecordIdx + 1; + } + return true; +} + +} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_cursor.h b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_cursor.h new file mode 100644 index 000000000000..0e54ade2b372 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_cursor.h @@ -0,0 +1,30 @@ +#pragma once +#include "merged_column.h" +#include +#include +#include +#include + +namespace NKikimr::NOlap::NCompaction { + +class TPortionColumnCursor { +private: + std::optional CurrentChunk; + std::shared_ptr BlobChunks; + std::optional RecordIndexStart; + YDB_READONLY(ui32, RecordIndexFinish, 0); +public: + ~TPortionColumnCursor() { + AFL_VERIFY(!RecordIndexStart)("start", RecordIndexStart)("finish", RecordIndexFinish); + } + + bool Next(const ui32 portionRecordIdx, TMergedColumn& column); + + bool Fetch(TMergedColumn& column); + + TPortionColumnCursor(const std::shared_ptr& columnChunks) + : BlobChunks(columnChunks) { + } +}; + +} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/column_portion_chunk.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_portion_chunk.cpp similarity index 97% rename from ydb/core/tx/columnshard/engines/changes/compaction/column_portion_chunk.cpp rename to ydb/core/tx/columnshard/engines/changes/compaction/plain/column_portion_chunk.cpp index 09eed586ac20..1cd921676f01 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/column_portion_chunk.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_portion_chunk.cpp @@ -38,6 +38,7 @@ ui32 TColumnPortion::AppendSlice(const std::shared_ptr& a, const u Y_ABORT_UNLESS(length); Y_ABORT_UNLESS(CurrentPortionRecords < Context.GetPortionRowsCountLimit()); Y_ABORT_UNLESS(startIndex + length <= a->length()); + AFL_VERIFY(Type->id() == a->type_id())("own", Type->ToString())("a", a->type()->ToString()); ui32 i = startIndex; const ui32 packedRecordSize = Context.GetColumnStat() ? Context.GetColumnStat()->GetPackedRecordSize() : 0; for (; i < startIndex + length; ++i) { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/column_portion_chunk.h b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_portion_chunk.h similarity index 68% rename from ydb/core/tx/columnshard/engines/changes/compaction/column_portion_chunk.h rename to ydb/core/tx/columnshard/engines/changes/compaction/plain/column_portion_chunk.h index f1d4cbadd6cf..98fe703f7e1a 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/column_portion_chunk.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/column_portion_chunk.h @@ -1,60 +1,35 @@ #pragma once -#include "merge_context.h" #include -#include +#include +#include +#include #include #include #include -#include #include +#include namespace NKikimr::NOlap::NCompaction { -class TColumnPortionResult { -protected: - std::vector> Chunks; - ui64 CurrentPortionRecords = 0; - const ui32 ColumnId; - ui64 PackedSize = 0; -public: - ui64 GetPackedSize() const { - return PackedSize; - } - - TColumnPortionResult(const ui32 columnId) - : ColumnId(columnId) { - - } - - const std::vector>& GetChunks() const { - return Chunks; - } - - ui64 GetCurrentPortionRecords() const { - return CurrentPortionRecords; - } - - TString DebugString() const { - return TStringBuilder() << "chunks=" << Chunks.size() << ";records=" << CurrentPortionRecords << ";"; - } - -}; - class TColumnPortion: public TColumnPortionResult { private: using TBase = TColumnPortionResult; std::unique_ptr Builder; + std::shared_ptr Type; const TColumnMergeContext& Context; YDB_READONLY(ui64, CurrentChunkRawSize, 0); double PredictedPackedBytes = 0; const TSimpleColumnInfo ColumnInfo; + ui64 PackedSize = 0; + ui64 CurrentPortionRecords = 0; + public: TColumnPortion(const TColumnMergeContext& context) : TBase(context.GetColumnId()) , Context(context) - , ColumnInfo(Context.GetIndexInfo().GetColumnFeaturesVerified(context.GetColumnId())) - { + , ColumnInfo(Context.GetIndexInfo().GetColumnFeaturesVerified(context.GetColumnId())) { Builder = Context.MakeBuilder(); + Type = Builder->type(); } bool IsFullPortion() const { @@ -68,4 +43,4 @@ class TColumnPortion: public TColumnPortionResult { ui32 AppendSlice(const std::shared_ptr& a, const ui32 startIndex, const ui32 length); }; -} +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/plain/logic.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/plain/logic.cpp new file mode 100644 index 000000000000..ac8cb351c572 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/logic.cpp @@ -0,0 +1,33 @@ +#include "logic.h" + +namespace NKikimr::NOlap::NCompaction { + +void TPlainMerger::DoStart(const std::vector>& input) { + for (auto&& p : input) { + Cursors.emplace_back(NCompaction::TPortionColumnCursor(p)); + } +} + +std::vector TPlainMerger::DoExecute( + const NCompaction::TColumnMergeContext& context, const arrow::UInt16Array& pIdxArray, const arrow::UInt32Array& pRecordIdxArray) { + NCompaction::TMergedColumn mColumn(context); + + std::optional predPortionIdx; + for (ui32 idx = 0; idx < pIdxArray.length(); ++idx) { + const ui16 portionIdx = pIdxArray.Value(idx); + const ui32 portionRecordIdx = pRecordIdxArray.Value(idx); + auto& cursor = Cursors[portionIdx]; + cursor.Next(portionRecordIdx, mColumn); + if (predPortionIdx && portionIdx != *predPortionIdx) { + Cursors[*predPortionIdx].Fetch(mColumn); + } + if (idx + 1 == pIdxArray.length()) { + cursor.Fetch(mColumn); + } + predPortionIdx = portionIdx; + } + AFL_VERIFY(pIdxArray.length() == mColumn.GetRecordsCount()); + return mColumn.BuildResult(); +} + +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/plain/logic.h b/ydb/core/tx/columnshard/engines/changes/compaction/plain/logic.h new file mode 100644 index 000000000000..995cd1c33a72 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/logic.h @@ -0,0 +1,19 @@ +#pragma once +#include "column_cursor.h" + +#include +#include + +namespace NKikimr::NOlap::NCompaction { +class TPlainMerger: public IColumnMerger { +private: + std::vector Cursors; + virtual void DoStart(const std::vector>& input) override; + + virtual std::vector DoExecute(const NCompaction::TColumnMergeContext& context, const arrow::UInt16Array& pIdxArray, + const arrow::UInt32Array& pRecordIdxArray) override; + +public: +}; + +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merged_column.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/plain/merged_column.cpp similarity index 100% rename from ydb/core/tx/columnshard/engines/changes/compaction/merged_column.cpp rename to ydb/core/tx/columnshard/engines/changes/compaction/plain/merged_column.cpp diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merged_column.h b/ydb/core/tx/columnshard/engines/changes/compaction/plain/merged_column.h similarity index 85% rename from ydb/core/tx/columnshard/engines/changes/compaction/merged_column.h rename to ydb/core/tx/columnshard/engines/changes/compaction/plain/merged_column.h index f0a90bdd9b44..9dee31b84215 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merged_column.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/merged_column.h @@ -1,6 +1,7 @@ #pragma once #include "column_portion_chunk.h" -#include "merge_context.h" + +#include #include namespace NKikimr::NOlap::NCompaction { @@ -25,4 +26,4 @@ class TMergedColumn { std::vector BuildResult(); }; -} +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/plain/ya.make b/ydb/core/tx/columnshard/engines/changes/compaction/plain/ya.make new file mode 100644 index 000000000000..64de6caea075 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/plain/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + column_cursor.cpp + column_portion_chunk.cpp + merged_column.cpp + logic.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/engines/changes/compaction/common +) + +END() diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/ya.make b/ydb/core/tx/columnshard/engines/changes/compaction/ya.make index aa52c0f9d6a0..c6a7bc101f9a 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/ya.make +++ b/ydb/core/tx/columnshard/engines/changes/compaction/ya.make @@ -1,14 +1,14 @@ LIBRARY() SRCS( - merge_context.cpp - column_cursor.cpp - column_portion_chunk.cpp - merged_column.cpp + merger.cpp ) PEERDIR( ydb/core/tx/tiering + ydb/core/tx/columnshard/engines/changes/compaction/abstract + ydb/core/tx/columnshard/engines/changes/compaction/plain + ydb/core/tx/columnshard/engines/changes/compaction/common ) END() diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 56b5c2e6477d..bfbd1eac5133 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -1,78 +1,39 @@ #include "general_compaction.h" -#include "compaction/column_cursor.h" -#include "compaction/column_portion_chunk.h" -#include "compaction/merge_context.h" -#include "compaction/merged_column.h" #include "counters/general.h" +#include "compaction/merger.h" -#include -#include -#include +#include #include -#include -#include -#include -#include -#include namespace NKikimr::NOlap::NCompaction { -void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches( - TConstructionContext& context, std::vector&& portions) noexcept { - std::vector> batchResults; - auto resultSchema = context.SchemaVersions.GetLastSchema(); - auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId()); - { - auto resultDataSchema = resultSchema->GetIndexInfo().ArrowSchemaWithSpecials(); - NArrow::NMerger::TMergePartialStream mergeStream( - resultSchema->GetIndexInfo().GetReplaceKey(), resultDataSchema, false, IIndexInfo::GetSnapshotColumnNames()); - - THashSet portionsInUsage; - for (auto&& i : portions) { - AFL_VERIFY(portionsInUsage.emplace(i.GetPortionInfo().GetPortionId()).second); - } - - for (auto&& i : portions) { - auto dataSchema = i.GetPortionInfo().GetSchema(context.SchemaVersions); - auto batch = i.GetBatch(dataSchema, *resultSchema); - batch = resultSchema->NormalizeBatch(*dataSchema, batch).DetachResult(); - batch = IIndexInfo::NormalizeDeletionColumn(batch); - Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(batch, resultSchema->GetIndexInfo().GetReplaceKey())); - auto filter = BuildPortionFilter(shardingActual, batch, i.GetPortionInfo(), portionsInUsage, resultSchema); - mergeStream.AddSource(batch, filter); - } - batchResults = mergeStream.DrainAllParts(CheckPoints, resultDataSchema->fields()); - } - Y_ABORT_UNLESS(batchResults.size()); - for (auto&& b : batchResults) { - auto portions = MakeAppendedPortions(b, GranuleMeta->GetPathId(), resultSchema->GetSnapshot(), GranuleMeta.get(), context, {}); - Y_ABORT_UNLESS(portions.size()); - for (auto& portion : portions) { - if (shardingActual) { - portion.GetPortionConstructor().SetShardingVersion(shardingActual->GetSnapshotVersion()); - } - AppendedPortions.emplace_back(std::move(portion)); - } - } -} - std::shared_ptr TGeneralCompactColumnEngineChanges::BuildPortionFilter( - const std::optional& shardingActual, const std::shared_ptr& batch, + const std::optional& shardingActual, const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage, const ISnapshotSchema::TPtr& resultSchema) const { std::shared_ptr filter; if (shardingActual && pInfo.NeedShardingFilter(*shardingActual)) { - filter = shardingActual->GetShardingInfo()->GetFilter(batch); + std::set fieldNames; + for (auto&& i : shardingActual->GetShardingInfo()->GetColumnNames()) { + fieldNames.emplace(i); + } + auto table = batch->BuildTableVerified(fieldNames); + AFL_VERIFY(table); + filter = shardingActual->GetShardingInfo()->GetFilter(table); } NArrow::TColumnFilter filterDeleted = NArrow::TColumnFilter::BuildAllowFilter(); if (pInfo.GetMeta().GetDeletionsCount()) { - auto col = batch->GetColumnByName(TIndexInfo::SPEC_COL_DELETE_FLAG); + auto table = batch->BuildTableVerified(std::set({ TIndexInfo::SPEC_COL_DELETE_FLAG })); + AFL_VERIFY(table); + auto col = table->GetColumnByName(TIndexInfo::SPEC_COL_DELETE_FLAG); AFL_VERIFY(col); AFL_VERIFY(col->type()->id() == arrow::Type::BOOL); - auto bCol = static_pointer_cast(col); - for (ui32 i = 0; i < bCol->length(); ++i) { - filterDeleted.Add(!bCol->GetView(i)); + for (auto&& c : col->chunks()) { + auto bCol = static_pointer_cast(c); + for (ui32 i = 0; i < bCol->length(); ++i) { + filterDeleted.Add(!bCol->GetView(i)); + } } NArrow::TColumnFilter filterCorrection = NArrow::TColumnFilter::BuildDenyFilter(); auto pkSchema = resultSchema->GetIndexInfo().GetReplaceKey(); @@ -117,201 +78,70 @@ std::shared_ptr TGeneralCompactColumnEngineChanges::Build void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks( TConstructionContext& context, std::vector&& portions) noexcept { - static const TString portionIdFieldName = "$$__portion_id"; - static const TString portionRecordIndexFieldName = "$$__portion_record_idx"; - static const std::shared_ptr portionIdField = - std::make_shared(portionIdFieldName, std::make_shared()); - static const std::shared_ptr portionRecordIndexField = - std::make_shared(portionRecordIndexFieldName, std::make_shared()); - auto resultSchema = context.SchemaVersions.GetLastSchema(); auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId()); - std::vector pkFieldNames = resultSchema->GetIndexInfo().GetReplaceKey()->field_names(); - std::set pkFieldNamesSet(pkFieldNames.begin(), pkFieldNames.end()); - for (auto&& i : TIndexInfo::GetSnapshotColumnNames()) { - pkFieldNamesSet.emplace(i); - } - pkFieldNamesSet.emplace(TIndexInfo::SPEC_COL_DELETE_FLAG); - - std::vector> batchResults; + std::shared_ptr stats = std::make_shared(); + std::shared_ptr resultFiltered; + NCompaction::TMerger merger(context, SaverContext); { - arrow::FieldVector indexFields; - indexFields.emplace_back(portionIdField); - indexFields.emplace_back(portionRecordIndexField); - IIndexInfo::AddSpecialFields(indexFields); - auto dataSchema = std::make_shared(indexFields); - NArrow::NMerger::TMergePartialStream mergeStream( - resultSchema->GetIndexInfo().GetReplaceKey(), dataSchema, false, IIndexInfo::GetSnapshotColumnNames()); - THashSet usedPortionIds; - for (auto&& i : portions) { - AFL_VERIFY(usedPortionIds.emplace(i.GetPortionInfo().GetPortionId()).second); + std::set pkColumnIds; + { + auto pkColumnIdsVector = IIndexInfo::AddSnapshotFieldIds(resultSchema->GetIndexInfo().GetPKColumnIds()); + pkColumnIds = std::set(pkColumnIdsVector.begin(), pkColumnIdsVector.end()); } - - ui32 idx = 0; - for (auto&& i : portions) { - auto dataSchema = i.GetPortionInfo().GetSchema(context.SchemaVersions); - auto batch = i.GetBatch(dataSchema, *resultSchema, pkFieldNamesSet); - { - NArrow::NConstruction::IArrayBuilder::TPtr column = - std::make_shared>>( - portionIdFieldName, idx++); - batch = NArrow::TStatusValidator::GetValid( - batch->AddColumn(batch->num_columns(), portionIdField, column->BuildArray(batch->num_rows()))); - } + std::set dataColumnIds; + { { - NArrow::NConstruction::IArrayBuilder::TPtr column = - std::make_shared>>( - portionRecordIndexFieldName); - batch = NArrow::TStatusValidator::GetValid( - batch->AddColumn(batch->num_columns(), portionRecordIndexField, column->BuildArray(batch->num_rows()))); - } - batch = IIndexInfo::NormalizeDeletionColumn(batch); - Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(batch, resultSchema->GetIndexInfo().GetReplaceKey())); - std::shared_ptr filter = - BuildPortionFilter(shardingActual, batch, i.GetPortionInfo(), usedPortionIds, resultSchema); - mergeStream.AddSource(batch, filter); - } - batchResults = mergeStream.DrainAllParts(CheckPoints, indexFields); - } - - std::shared_ptr stats = std::make_shared(); - for (auto&& i : SwitchedPortions) { - stats->Merge(i.GetSerializationStat(*resultSchema)); - } - - std::vector>> chunkGroups; - chunkGroups.resize(batchResults.size()); - for (auto&& columnId : resultSchema->GetIndexInfo().GetColumnIds()) { - NActors::TLogContextGuard logGuard( - NActors::TLogContextBuilder::Build()("field_name", resultSchema->GetIndexInfo().GetColumnName(columnId))); - auto columnInfo = stats->GetColumnInfo(columnId); - auto resultField = resultSchema->GetIndexInfo().GetColumnFieldVerified(columnId); - - std::vector cursors; - for (auto&& p : portions) { - auto dataSchema = p.GetPortionInfo().GetSchema(context.SchemaVersions); - auto loader = dataSchema->GetColumnLoaderOptional(columnId); - std::vector records; - std::vector> chunks; - if (!p.ExtractColumnChunks(columnId, records, chunks)) { - if (!loader) { - loader = resultSchema->GetColumnLoaderVerified(columnId); - } else { - AFL_VERIFY(dataSchema->IsSpecialColumnId(columnId)); + THashMap schemas; + for (auto& portion : SwitchedPortions) { + auto dataSchema = portion.GetSchema(context.SchemaVersions); + schemas.emplace(dataSchema->GetVersion(), dataSchema); } - chunks.emplace_back(std::make_shared(columnId, p.GetPortionInfo().GetRecordsCount(), - p.GetPortionInfo().GetColumnRawBytes({ columnId }), resultField, resultSchema->GetDefaultValueVerified(columnId), - resultSchema->GetColumnSaver(columnId))); - records = { nullptr }; + dataColumnIds = ISnapshotSchema::GetColumnsWithDifferentDefaults(schemas, resultSchema); } - AFL_VERIFY(!!loader); - cursors.emplace_back(TPortionColumnCursor(chunks, records, loader, p.GetPortionInfo().GetPortionId())); - } - - ui32 batchesRecordsCount = 0; - ui32 columnRecordsCount = 0; - std::map> columnChunks; - ui32 batchIdx = 0; - for (auto&& batchResult : batchResults) { - const ui32 portionRecordsCountLimit = - batchResult->num_rows() / (batchResult->num_rows() / NSplitter::TSplitSettings().GetExpectedRecordsCountOnPage() + 1) + 1; - TColumnMergeContext context( - columnId, resultSchema, portionRecordsCountLimit, NSplitter::TSplitSettings().GetExpectedUnpackColumnChunkRawSize(), columnInfo); - TMergedColumn mColumn(context); - - auto columnPortionIdx = batchResult->GetColumnByName(portionIdFieldName); - auto columnPortionRecordIdx = batchResult->GetColumnByName(portionRecordIndexFieldName); - auto columnSnapshotPlanStepIdx = batchResult->GetColumnByName(TIndexInfo::SPEC_COL_PLAN_STEP); - auto columnSnapshotTxIdx = batchResult->GetColumnByName(TIndexInfo::SPEC_COL_TX_ID); - Y_ABORT_UNLESS(columnPortionIdx && columnPortionRecordIdx && columnSnapshotPlanStepIdx && columnSnapshotTxIdx); - Y_ABORT_UNLESS(columnPortionIdx->type_id() == arrow::UInt16Type::type_id); - Y_ABORT_UNLESS(columnPortionRecordIdx->type_id() == arrow::UInt32Type::type_id); - Y_ABORT_UNLESS(columnSnapshotPlanStepIdx->type_id() == arrow::UInt64Type::type_id); - Y_ABORT_UNLESS(columnSnapshotTxIdx->type_id() == arrow::UInt64Type::type_id); - const arrow::UInt16Array& pIdxArray = static_cast(*columnPortionIdx); - const arrow::UInt32Array& pRecordIdxArray = static_cast(*columnPortionRecordIdx); - - AFL_VERIFY(batchResult->num_rows() == pIdxArray.length()); - std::optional predPortionIdx; - for (ui32 idx = 0; idx < pIdxArray.length(); ++idx) { - const ui16 portionIdx = pIdxArray.Value(idx); - const ui32 portionRecordIdx = pRecordIdxArray.Value(idx); - auto& cursor = cursors[portionIdx]; - cursor.Next(portionRecordIdx, mColumn); - if (predPortionIdx && portionIdx != *predPortionIdx) { - cursors[*predPortionIdx].Fetch(mColumn); + for (auto&& i : SwitchedPortions) { + stats->Merge(i.GetSerializationStat(*resultSchema)); + if (dataColumnIds.size() != resultSchema->GetColumnsCount()) { + for (auto id : i.GetColumnIds()) { + if (resultSchema->HasColumnId(id)) { + dataColumnIds.emplace(id); + } + } } - if (idx + 1 == pIdxArray.length()) { - cursor.Fetch(mColumn); - } - predPortionIdx = portionIdx; } - chunkGroups[batchIdx][columnId] = mColumn.BuildResult(); - batchesRecordsCount += batchResult->num_rows(); - columnRecordsCount += mColumn.GetRecordsCount(); - AFL_VERIFY(batchResult->num_rows() == mColumn.GetRecordsCount()); - ++batchIdx; - } - AFL_VERIFY(columnRecordsCount == batchesRecordsCount)("mCount", columnRecordsCount)("bCount", batchesRecordsCount); - } - ui32 batchIdx = 0; - - const auto groups = - resultSchema->GetIndexInfo().GetEntityGroupsByStorageId(IStoragesManager::DefaultStorageId, *SaverContext.GetStoragesManager()); - for (auto&& columnChunks : chunkGroups) { - auto batchResult = batchResults[batchIdx]; - ++batchIdx; - Y_ABORT_UNLESS(columnChunks.size()); - - for (auto&& i : columnChunks) { - if (i.second.size() != columnChunks.begin()->second.size()) { - for (ui32 p = 0; p < std::min(columnChunks.begin()->second.size(), i.second.size()); ++p) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("p_first", columnChunks.begin()->second[p].DebugString())( - "p", i.second[p].DebugString()); - } + AFL_VERIFY(dataColumnIds.size() <= resultSchema->GetColumnsCount()); + if (dataColumnIds.contains((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG)) { + pkColumnIds.emplace((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG); } - AFL_VERIFY(i.second.size() == columnChunks.begin()->second.size())("first", columnChunks.begin()->second.size())( - "current", i.second.size())("first_name", columnChunks.begin()->first)("current_name", i.first); } - std::vector batchSlices; - std::shared_ptr schemaDetails(new TDefaultSchemaDetails(resultSchema, stats)); - - for (ui32 i = 0; i < columnChunks.begin()->second.size(); ++i) { - THashMap>> portionColumns; - for (auto&& p : columnChunks) { - portionColumns.emplace(p.first, p.second[i].GetChunks()); + resultFiltered = std::make_shared(resultSchema, dataColumnIds); + { + auto seqDataColumnIds = dataColumnIds; + for (auto&& i : pkColumnIds) { + AFL_VERIFY(seqDataColumnIds.erase(i)); + } + THashSet usedPortionIds; + for (auto&& i : portions) { + AFL_VERIFY(usedPortionIds.emplace(i.GetPortionInfo().GetPortionId()).second); } - batchSlices.emplace_back(portionColumns, schemaDetails, context.Counters.SplitterCounters); - } - TSimilarPacker slicer(NSplitter::TSplitSettings().GetExpectedPortionSize()); - auto packs = slicer.Split(batchSlices); - - ui32 recordIdx = 0; - for (auto&& i : packs) { - TGeneralSerializedSlice slicePrimary(std::move(i)); - auto dataWithSecondary = - resultSchema->GetIndexInfo().AppendIndexes(slicePrimary.GetPortionChunksToHash(), SaverContext.GetStoragesManager()).DetachResult(); - TGeneralSerializedSlice slice(dataWithSecondary.GetExternalData(), schemaDetails, context.Counters.SplitterCounters); - - auto b = batchResult->Slice(recordIdx, slice.GetRecordsCount()); - const ui32 deletionsCount = IIndexInfo::CalcDeletions(b, true); - auto constructor = TWritePortionInfoWithBlobsConstructor::BuildByBlobs(slice.GroupChunksByBlobs(groups), - dataWithSecondary.GetSecondaryInplaceData(), GranuleMeta->GetPathId(), - resultSchema->GetVersion(), resultSchema->GetSnapshot(), SaverContext.GetStoragesManager()); - NArrow::TFirstLastSpecialKeys primaryKeys(slice.GetFirstLastPKBatch(resultSchema->GetIndexInfo().GetReplaceKey())); - NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot()); - constructor.GetPortionConstructor().AddMetadata(*resultSchema, deletionsCount, primaryKeys, snapshotKeys); - constructor.GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId); - if (shardingActual) { - constructor.GetPortionConstructor().SetShardingVersion(shardingActual->GetSnapshotVersion()); + for (auto&& i : portions) { + auto blobsSchema = i.GetPortionInfo().GetSchema(context.SchemaVersions); + auto batch = i.RestoreBatch(*blobsSchema, *resultFiltered, seqDataColumnIds); + std::shared_ptr filter = + BuildPortionFilter(shardingActual, batch, i.GetPortionInfo(), usedPortionIds, resultFiltered); + merger.AddBatch(batch, filter); } - AppendedPortions.emplace_back(std::move(constructor)); - recordIdx += slice.GetRecordsCount(); } } + + std::optional shardingActualVersion; + if (shardingActual) { + shardingActualVersion = shardingActual->GetSnapshotVersion(); + } + AppendedPortions = merger.Execute(stats, CheckPoints, resultFiltered, GranuleMeta->GetPathId(), shardingActualVersion); } TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept { @@ -337,11 +167,7 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc { std::vector portions = TReadPortionInfoWithBlobs::RestorePortions(SwitchedPortions, Blobs, context.SchemaVersions); - if (!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { - BuildAppendedPortionsByChunks(context, std::move(portions)); - } else { - BuildAppendedPortionsByFullBatches(context, std::move(portions)); - } + BuildAppendedPortionsByChunks(context, std::move(portions)); } if (IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD)) { @@ -386,16 +212,12 @@ NColumnShard::ECumulativeCounters TGeneralCompactColumnEngineChanges::GetCounter } void TGeneralCompactColumnEngineChanges::AddCheckPoint( - const NArrow::NMerger::TSortableBatchPosition& position, const bool include, const bool validationDuplications) { - AFL_VERIFY(CheckPoints.emplace(position, include).second || !validationDuplications); + const NArrow::NMerger::TSortableBatchPosition& position, const bool include) { + CheckPoints.AddPosition(position, include); } std::shared_ptr TGeneralCompactColumnEngineChanges::BuildMemoryPredictor() { - if (!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { - return std::make_shared(); - } else { - return std::make_shared(); - } + return std::make_shared(); } ui64 TGeneralCompactColumnEngineChanges::TMemoryPredictorChunkedPolicy::AddPortion(const TPortionInfo& portionInfo) { diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.h b/ydb/core/tx/columnshard/engines/changes/general_compaction.h index 75dc35317630..ab6f1e18684e 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.h @@ -9,12 +9,11 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { private: using TBase = TCompactColumnEngineChanges; virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; - std::map CheckPoints; - void BuildAppendedPortionsByFullBatches(TConstructionContext& context, std::vector&& portions) noexcept; + NArrow::NMerger::TIntervalPositions CheckPoints; void BuildAppendedPortionsByChunks(TConstructionContext& context, std::vector&& portions) noexcept; std::shared_ptr BuildPortionFilter(const std::optional& shardingActual, - const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage, + const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage, const ISnapshotSchema::TPtr& resultSchema) const; protected: virtual TConclusionStatus DoConstructBlobs(TConstructionContext& context) noexcept override; @@ -64,7 +63,7 @@ class TGeneralCompactColumnEngineChanges: public TCompactColumnEngineChanges { static std::shared_ptr BuildMemoryPredictor(); - void AddCheckPoint(const NArrow::NMerger::TSortableBatchPosition& position, const bool include = true, const bool validationDuplications = true); + void AddCheckPoint(const NArrow::NMerger::TSortableBatchPosition& position, const bool include); virtual TString TypeString() const override { return StaticTypeName(); diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.cpp b/ydb/core/tx/columnshard/engines/changes/indexation.cpp index 6207a8883376..4218409e29d6 100644 --- a/ydb/core/tx/columnshard/engines/changes/indexation.cpp +++ b/ydb/core/tx/columnshard/engines/changes/indexation.cpp @@ -1,12 +1,7 @@ #include "indexation.h" -#include -#include + +#include "compaction/merger.h" #include -#include -#include -#include -#include -#include namespace NKikimr::NOlap { @@ -54,10 +49,10 @@ namespace { class TBatchInfo { private: - YDB_READONLY_DEF(std::shared_ptr, Batch); + YDB_READONLY_DEF(std::shared_ptr, Batch); const NEvWrite::EModificationType ModificationType; public: - TBatchInfo(const std::shared_ptr& batch, const NEvWrite::EModificationType modificationType) + TBatchInfo(const std::shared_ptr& batch, const NEvWrite::EModificationType modificationType) : Batch(batch) , ModificationType(modificationType) { @@ -81,11 +76,19 @@ class TPathData { } + std::vector> GetGeneralContainers() const { + std::vector> result; + for (auto&& i : Batches) { + result.emplace_back(i.GetBatch()); + } + return result; + } + bool HasDeletion() { return HasDeletionFlag; } - void AddBatch(const NOlap::TInsertedData& data, const std::shared_ptr& batch) { + void AddBatch(const NOlap::TInsertedData& data, const std::shared_ptr& batch) { if (data.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete) { HasDeletionFlag = true; } @@ -100,26 +103,6 @@ class TPathData { ShardingInfo = info; } } - - std::shared_ptr Merge(const TIndexInfo& indexInfo) const { - auto fullSchema = indexInfo.ArrowSchemaWithSpecials(); - NArrow::NMerger::TMergePartialStream stream(indexInfo.GetReplaceKey(), fullSchema, false, IIndexInfo::GetSnapshotColumnNames()); - THashMap fieldSizes; - ui64 rowsCount = 0; - for (auto&& batch : Batches) { - auto& forMerge = batch.GetBatch(); - stream.AddSource(forMerge, nullptr); - for (ui32 cIdx = 0; cIdx < (ui32)forMerge->num_columns(); ++cIdx) { - fieldSizes[forMerge->column_name(cIdx)] += NArrow::GetArrayDataSize(forMerge->column(cIdx)); - } - rowsCount += forMerge->num_rows(); - } - - NArrow::NMerger::TRecordBatchBuilder builder(fullSchema->fields(), rowsCount, fieldSizes); - stream.SetPossibleSameVersion(true); - stream.DrainAll(builder); - return builder.Finalize(); - } }; class TPathesData { @@ -131,7 +114,8 @@ class TPathesData { return Data; } - void Add(const NOlap::TInsertedData& inserted, const std::optional& info, const std::shared_ptr& batch) { + void Add(const NOlap::TInsertedData& inserted, const std::optional& info, + const std::shared_ptr& batch) { auto it = Data.find(inserted.PathId); if (it == Data.end()) { it = Data.emplace(inserted.PathId, info).first; @@ -160,61 +144,72 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont Y_ABORT_UNLESS(resultSchema->GetIndexInfo().IsSorted()); TPathesData pathBatches; + std::set usageColumnIds; + { + THashMap schemas; + for (auto& inserted : DataToIndex) { + if (schemas.contains(inserted.GetSchemaVersion())) { + continue; + } + schemas.emplace(inserted.GetSchemaVersion(), context.SchemaVersions.GetSchemaVerified(inserted.GetSchemaVersion())); + } + usageColumnIds = ISnapshotSchema::GetColumnsWithDifferentDefaults(schemas, resultSchema); + } + for (auto& inserted : DataToIndex) { - const TBlobRange& blobRange = inserted.GetBlobRange(); + auto blobSchema = context.SchemaVersions.GetSchemaVerified(inserted.GetSchemaVersion()); + std::vector filteredIds = inserted.GetMeta().GetSchemaSubset().Apply(blobSchema->GetIndexInfo().GetColumnIds(false)); + usageColumnIds.insert(filteredIds.begin(), filteredIds.end()); + if (inserted.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete) { + usageColumnIds.emplace((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG); + } + if (usageColumnIds.size() == resultSchema->GetIndexInfo().GetColumnIds(true).size()) { + break; + } + } + for (auto& inserted : DataToIndex) { + const TBlobRange& blobRange = inserted.GetBlobRange(); auto shardingFilterCommit = context.SchemaVersions.GetShardingInfoOptional(inserted.PathId, inserted.GetSnapshot()); - auto blobSchema = context.SchemaVersions.GetSchemaVerified(inserted.GetSchemaVersion()); - auto& indexInfo = blobSchema->GetIndexInfo(); - Y_ABORT_UNLESS(indexInfo.IsSorted()); - std::shared_ptr batch; + std::shared_ptr batch; { const auto blobData = Blobs.Extract(IStoragesManager::DefaultStorageId, blobRange); - Y_ABORT_UNLESS(blobData.size(), "Blob data not present"); - // Prepare batch - batch = NArrow::DeserializeBatch(blobData, indexInfo.ArrowSchema()); - AFL_VERIFY(batch)("event", "cannot_parse") - ("data_snapshot", TStringBuilder() << inserted.GetSnapshot()) - ("index_snapshot", TStringBuilder() << blobSchema->GetSnapshot()); - ; + auto batchSchema = + std::make_shared(inserted.GetMeta().GetSchemaSubset().Apply(blobSchema->GetIndexInfo().ArrowSchema()->fields())); + batch = std::make_shared(NArrow::DeserializeBatch(blobData, batchSchema)); + } + + IIndexInfo::AddSnapshotColumns(*batch, inserted.GetSnapshot()); + if (usageColumnIds.contains((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG)) { + IIndexInfo::AddDeleteFlagsColumn(*batch, inserted.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete); } + usageColumnIds.insert(IIndexInfo::GetSnapshotColumnIds().begin(), IIndexInfo::GetSnapshotColumnIds().end()); - batch = AddSpecials(batch, indexInfo, inserted); - batch = resultSchema->NormalizeBatch(*blobSchema, batch).DetachResult(); + batch = resultSchema->NormalizeBatch(*blobSchema, batch, usageColumnIds).DetachResult(); pathBatches.Add(inserted, shardingFilterCommit, batch); - Y_DEBUG_ABORT_UNLESS(NArrow::IsSorted(batch, resultSchema->GetIndexInfo().GetReplaceKey())); } Y_ABORT_UNLESS(Blobs.IsEmpty()); - const std::vector comparableColumns = resultSchema->GetIndexInfo().GetReplaceKey()->field_names(); + auto filteredSnapshot = std::make_shared(resultSchema, usageColumnIds); + auto stats = std::make_shared(); + std::vector> filters; for (auto& [pathId, pathInfo] : pathBatches.GetData()) { - auto shardingFilter = context.SchemaVersions.GetShardingInfoActual(pathId); - auto mergedBatch = pathInfo.Merge(resultSchema->GetIndexInfo()); + std::optional shardingVersion; + if (pathInfo.GetShardingInfo()) { + shardingVersion = pathInfo.GetShardingInfo()->GetSnapshotVersion(); + } + auto batches = pathInfo.GetGeneralContainers(); + filters.resize(batches.size()); auto itGranule = PathToGranule.find(pathId); AFL_VERIFY(itGranule != PathToGranule.end()); - std::vector> result = NArrow::NMerger::TRWSortableBatchPosition:: - SplitByBordersInSequentialContainer(mergedBatch, comparableColumns, itGranule->second); - for (auto&& b : result) { - if (!b) { - continue; - } - std::optional externalSaver; - if (b->num_rows() < 100) { - externalSaver = NArrow::NSerialization::TSerializerContainer(std::make_shared(arrow::Compression::type::UNCOMPRESSED)); - } else { - externalSaver = NArrow::NSerialization::TSerializerContainer(std::make_shared(arrow::Compression::type::LZ4_FRAME)); - } - auto portions = MakeAppendedPortions(b, pathId, maxSnapshot, nullptr, context, externalSaver); - Y_ABORT_UNLESS(portions.size()); - for (auto& portion : portions) { - if (pathInfo.GetShardingInfo()) { - portion.GetPortionConstructor().SetShardingVersion(pathInfo.GetShardingInfo()->GetSnapshotVersion()); - } - AppendedPortions.emplace_back(TWritePortionInfoWithBlobsResult(std::move(portion))); - } + NCompaction::TMerger merger(context, SaverContext, std::move(batches), std::move(filters)); + merger.SetOptimizationWritingPackMode(true); + auto localAppended = merger.Execute(stats, itGranule->second, filteredSnapshot, pathId, shardingVersion); + for (auto&& i : localAppended) { + AppendedPortions.emplace_back(std::move(i)); } } @@ -222,13 +217,6 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont return TConclusionStatus::Success(); } -std::shared_ptr TInsertColumnEngineChanges::AddSpecials(const std::shared_ptr& srcBatch, - const TIndexInfo& indexInfo, const TInsertedData& inserted) const { - auto batch = IIndexInfo::AddSnapshotColumns(srcBatch, inserted.GetSnapshot()); - batch = IIndexInfo::AddDeleteFlagsColumn(batch, inserted.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete); - return NArrow::TColumnOperator().Adapt(batch, indexInfo.ArrowSchemaWithSpecials()).DetachResult(); -} - NColumnShard::ECumulativeCounters TInsertColumnEngineChanges::GetCounterIndex(const bool isSuccess) const { return isSuccess ? NColumnShard::COUNTER_INDEXING_SUCCESS : NColumnShard::COUNTER_INDEXING_FAIL; } diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.h b/ydb/core/tx/columnshard/engines/changes/indexation.h index 95befd334c23..d130612b7451 100644 --- a/ydb/core/tx/columnshard/engines/changes/indexation.h +++ b/ydb/core/tx/columnshard/engines/changes/indexation.h @@ -10,8 +10,6 @@ namespace NKikimr::NOlap { class TInsertColumnEngineChanges: public TChangesWithAppend { private: using TBase = TChangesWithAppend; - std::shared_ptr AddSpecials(const std::shared_ptr& srcBatch, - const TIndexInfo& indexInfo, const TInsertedData& inserted) const; std::vector DataToIndex; protected: virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; @@ -34,7 +32,7 @@ class TInsertColumnEngineChanges: public TChangesWithAppend { } public: - THashMap> PathToGranule; // pathId -> positions (sorted by pk) + THashMap PathToGranule; // pathId -> positions (sorted by pk) public: TInsertColumnEngineChanges(std::vector&& dataToIndex, const TSaverContext& saverContext) : TBase(saverContext, NBlobOperations::EConsumer::INDEXATION) diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 4b1d779a6c10..3c4385b62fba 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -102,53 +102,6 @@ void TChangesWithAppend::DoOnAfterCompile() { } } -std::vector TChangesWithAppend::MakeAppendedPortions(const std::shared_ptr batch, - const ui64 pathId, const TSnapshot& snapshot, const TGranuleMeta* granuleMeta, TConstructionContext& context, const std::optional& overrideSaver) const { - Y_ABORT_UNLESS(batch->num_rows()); - - auto resultSchema = context.SchemaVersions.GetSchema(snapshot); - - std::shared_ptr stats = std::make_shared(); - if (granuleMeta) { - stats = granuleMeta->BuildSerializationStats(resultSchema); - } - auto schema = std::make_shared(resultSchema, stats); - if (overrideSaver) { - schema->SetOverrideSerializer(*overrideSaver); - } - std::vector out; - { - std::vector pages = TBatchSerializedSlice::BuildSimpleSlices(batch, NSplitter::TSplitSettings(), context.Counters.SplitterCounters, schema); - std::vector generalPages; - for (auto&& i : pages) { - generalPages.emplace_back(i.GetPortionChunksToHash(), schema, context.Counters.SplitterCounters); - } - - const NSplitter::TEntityGroups groups = resultSchema->GetIndexInfo().GetEntityGroupsByStorageId(IStoragesManager::DefaultStorageId, *SaverContext.GetStoragesManager()); - TSimilarPacker slicer(NSplitter::TSplitSettings().GetExpectedPortionSize()); - auto packs = slicer.Split(generalPages); - - ui32 recordIdx = 0; - for (auto&& i : packs) { - TGeneralSerializedSlice slicePrimary(std::move(i)); - auto dataWithSecondary = - resultSchema->GetIndexInfo().AppendIndexes(slicePrimary.GetPortionChunksToHash(), SaverContext.GetStoragesManager()).DetachResult(); - TGeneralSerializedSlice slice(dataWithSecondary.GetExternalData(), schema, context.Counters.SplitterCounters); - - auto b = batch->Slice(recordIdx, slice.GetRecordsCount()); - auto constructor = TWritePortionInfoWithBlobsConstructor::BuildByBlobs(slice.GroupChunksByBlobs(groups), - dataWithSecondary.GetSecondaryInplaceData(), pathId, resultSchema->GetVersion(), snapshot, SaverContext.GetStoragesManager()); - - constructor.GetPortionConstructor().AddMetadata(*resultSchema, b); - constructor.GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId); - out.emplace_back(std::move(constructor)); - recordIdx += slice.GetRecordsCount(); - } - } - - return out; -} - void TChangesWithAppend::DoStart(NColumnShard::TColumnShard& /*self*/) { } diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.h b/ydb/core/tx/columnshard/engines/changes/with_appended.h index 59fa8227dbab..e35dfbbe4acc 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.h +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.h @@ -17,8 +17,6 @@ class TChangesWithAppend: public TColumnEngineChanges { virtual void DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) override; virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; virtual void DoStart(NColumnShard::TColumnShard& self) override; - std::vector MakeAppendedPortions(const std::shared_ptr batch, const ui64 granule, - const TSnapshot& snapshot, const TGranuleMeta* granuleMeta, TConstructionContext& context, const std::optional& overrideSaver) const; virtual void DoDebugString(TStringOutput& out) const override { out << "remove=" << PortionsToRemove.size() << ";append=" << AppendedPortions.size() << ";"; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 51a2bad07ea9..e7fda23311ae 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -285,7 +285,7 @@ std::shared_ptr TColumnEngineForLogs::StartInsert(st if (changes->PathToGranule.contains(pathId)) { continue; } - changes->PathToGranule[pathId] = GetGranulePtrVerified(pathId)->GetBucketPositions(); + AFL_VERIFY(changes->PathToGranule.emplace(pathId, GetGranulePtrVerified(pathId)->GetBucketPositions()).second); } return changes; diff --git a/ydb/core/tx/columnshard/engines/insert_table/data.cpp b/ydb/core/tx/columnshard/engines/insert_table/data.cpp index 3f27a5ad5e7d..36c17ba89779 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/data.cpp +++ b/ydb/core/tx/columnshard/engines/insert_table/data.cpp @@ -35,16 +35,14 @@ TInsertedData::~TInsertedData() { } TInsertedData::TInsertedData(ui64 planStep, ui64 writeTxId, ui64 pathId, TString dedupId, const TBlobRange& blobRange, - const NKikimrTxColumnShard::TLogicalMetadata& proto, const ui64 schemaVersion, - const std::optional& blobData /*= {}*/) + const NKikimrTxColumnShard::TLogicalMetadata& proto, const ui64 schemaVersion, const std::optional& blobData) : Meta(proto) , BlobRange(blobRange) , PlanStep(planStep) , WriteTxId(writeTxId) , PathId(pathId) , DedupId(dedupId) - , SchemaVersion(schemaVersion) -{ + , SchemaVersion(schemaVersion) { if (blobData) { AFL_VERIFY(blobData->size() == BlobRange.Size); if (Singleton()->Take(blobData->size())) { diff --git a/ydb/core/tx/columnshard/engines/insert_table/data.h b/ydb/core/tx/columnshard/engines/insert_table/data.h index 0317b818073c..2c19a0902390 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/data.h +++ b/ydb/core/tx/columnshard/engines/insert_table/data.h @@ -33,6 +33,7 @@ struct TInsertedData { private: YDB_READONLY(ui64, SchemaVersion, 0); + public: std::optional GetBlobData() const { if (BlobDataGuard) { @@ -55,8 +56,8 @@ struct TInsertedData { TInsertedData(ui64 planStep, ui64 writeTxId, ui64 pathId, TString dedupId, const TBlobRange& blobRange, const NKikimrTxColumnShard::TLogicalMetadata& proto, const ui64 schemaVersion, const std::optional& blobData); - TInsertedData(ui64 writeTxId, ui64 pathId, TString dedupId, const TBlobRange& blobRange, - const NKikimrTxColumnShard::TLogicalMetadata& proto, const ui64 schemaVersion, const std::optional& blobData) + TInsertedData(ui64 writeTxId, ui64 pathId, TString dedupId, const TBlobRange& blobRange, const NKikimrTxColumnShard::TLogicalMetadata& proto, + const ui64 schemaVersion, const std::optional& blobData) : TInsertedData(0, writeTxId, pathId, dedupId, blobRange, proto, schemaVersion, blobData) {} @@ -137,6 +138,8 @@ class TCommittedBlob { YDB_READONLY(bool, IsDelete, false); YDB_READONLY_DEF(std::optional, First); YDB_READONLY_DEF(std::optional, Last); + YDB_READONLY_DEF(NArrow::TSchemaSubset, SchemaSubset); + public: ui64 GetSize() const { return BlobRange.Size; @@ -153,7 +156,7 @@ class TCommittedBlob { } TCommittedBlob(const TBlobRange& blobRange, const TSnapshot& snapshot, const ui64 schemaVersion, const ui64 recordsCount, const std::optional& first, - const std::optional& last, const bool isDelete) + const std::optional& last, const bool isDelete, const NArrow::TSchemaSubset& subset) : BlobRange(blobRange) , CommitSnapshot(snapshot) , SchemaVersion(schemaVersion) @@ -161,6 +164,7 @@ class TCommittedBlob { , IsDelete(isDelete) , First(first) , Last(last) + , SchemaSubset(subset) {} /// It uses trick then we place key with planStep:txId in container and find them later by BlobId only. diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp index 5147fbd02bc8..63e2d0905b99 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp @@ -138,7 +138,7 @@ std::vector TInsertTable::Read(ui64 pathId, const TSnapshot& sna for (auto&& i : ret) { result.emplace_back(TCommittedBlob( i->GetBlobRange(), i->GetSnapshot(), i->GetSchemaVersion(), i->GetMeta().GetNumRows(), i->GetMeta().GetFirstPK(pkSchema), i->GetMeta().GetLastPK(pkSchema) - , i->GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete)); + , i->GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete, i->GetMeta().GetSchemaSubset())); } return result; diff --git a/ydb/core/tx/columnshard/engines/insert_table/meta.h b/ydb/core/tx/columnshard/engines/insert_table/meta.h index 196e4cb667de..085e326d5247 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/meta.h +++ b/ydb/core/tx/columnshard/engines/insert_table/meta.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -14,6 +15,7 @@ class TInsertedDataMeta { YDB_READONLY(ui32, NumRows, 0); YDB_READONLY(ui64, RawBytes, 0); YDB_READONLY(NEvWrite::EModificationType, ModificationType, NEvWrite::EModificationType::Upsert); + YDB_READONLY_DEF(NArrow::TSchemaSubset, SchemaSubset); mutable bool KeysParsed = false; mutable std::optional SpecialKeysParsed; @@ -36,6 +38,9 @@ class TInsertedDataMeta { if (proto.HasModificationType()) { ModificationType = TEnumOperator::DeserializeFromProto(proto.GetModificationType()); } + if (proto.HasSchemaSubset()) { + SchemaSubset.DeserializeFromProto(proto.GetSchemaSubset()).Validate(); + } } std::optional GetFirstPK(const std::shared_ptr& schema) const { diff --git a/ydb/core/tx/columnshard/engines/insert_table/ya.make b/ydb/core/tx/columnshard/engines/insert_table/ya.make index 5f1d92bfb0ee..fd56354b62e6 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/ya.make +++ b/ydb/core/tx/columnshard/engines/insert_table/ya.make @@ -10,6 +10,7 @@ SRCS( PEERDIR( contrib/libs/apache/arrow + ydb/core/formats/arrow/modifier ydb/core/protos ydb/core/formats/arrow ydb/core/tablet_flat diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 492c099ceae7..b89b63e97b5c 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -684,6 +684,24 @@ bool TPortionInfo::NeedShardingFilter(const TGranuleShardingInfo& shardingInfo) return true; } +std::shared_ptr TPortionInfo::TPreparedColumn::AssembleAccessor() const { + Y_ABORT_UNLESS(!Blobs.empty()); + + std::vector> chunks; + chunks.reserve(Blobs.size()); + for (auto& blob : Blobs) { + auto batch = blob.BuildRecordBatch(*Loader); + Y_ABORT_UNLESS(batch); + AFL_VERIFY(batch->num_columns() == 1); + chunks.emplace_back(batch->column(0)); + } + if (chunks.size() > 1) { + return std::make_shared(NArrow::TStatusValidator::GetValid(arrow::ChunkedArray::Make(chunks))); + } else { + return std::make_shared(chunks.front()); + } +} + std::shared_ptr TPortionInfo::TPreparedColumn::AssembleForSeqAccess() const { Y_ABORT_UNLESS(!Blobs.empty()); @@ -692,7 +710,11 @@ std::shared_ptr TPortionInfo::TPreparedColumn::Assembl ui64 recordsCount = 0; for (auto& blob : Blobs) { chunks.push_back(blob.BuildDeserializeChunk(Loader)); - recordsCount += blob.GetExpectedRowsCountVerified(); + if (!!blob.GetData()) { + recordsCount += blob.GetExpectedRowsCountVerified(); + } else { + recordsCount += blob.GetDefaultRowsCount(); + } } return std::make_shared(recordsCount, Loader, std::move(chunks)); @@ -701,16 +723,16 @@ std::shared_ptr TPortionInfo::TPreparedColumn::Assembl std::shared_ptr TPortionInfo::TPreparedColumn::Assemble() const { Y_ABORT_UNLESS(!Blobs.empty()); - std::vector> batches; - batches.reserve(Blobs.size()); + std::vector> chunks; + chunks.reserve(Blobs.size()); for (auto& blob : Blobs) { - batches.push_back(blob.BuildRecordBatch(*Loader)); - Y_ABORT_UNLESS(batches.back()); + auto batch = blob.BuildRecordBatch(*Loader); + Y_ABORT_UNLESS(batch); + Y_ABORT_UNLESS(batch->num_columns() == 1); + chunks.emplace_back(batch->column(0)); } - auto res = arrow::Table::FromRecordBatches(batches); - Y_VERIFY_S(res.ok(), res.status().message()); - return (*res)->column(0); + return NArrow::TStatusValidator::GetValid(arrow::ChunkedArray::Make(chunks)); } TDeserializeChunkedArray::TChunk TPortionInfo::TAssembleBlobInfo::BuildDeserializeChunk(const std::shared_ptr& loader) const { @@ -752,32 +774,23 @@ std::shared_ptr TPortionInfo::TPreparedBatchData::Ass fields.emplace_back(i.GetField()); } - return std::make_shared(std::make_shared(fields), std::move(columns)); + return std::make_shared(fields, std::move(columns)); } -std::shared_ptr TPortionInfo::TPreparedBatchData::AssembleTable(const TAssembleOptions& options) const { - std::vector> columns; +std::shared_ptr TPortionInfo::TPreparedBatchData::AssembleToGeneralContainer( + const std::set& sequentialColumnIds) const { + std::vector> columns; std::vector> fields; for (auto&& i : Columns) { - if (!options.IsAcceptedColumn(i.GetColumnId())) { - continue; - } - std::shared_ptr scalar; - if (options.IsConstantColumn(i.GetColumnId(), scalar)) { - auto type = i.GetField()->type(); - std::shared_ptr arr = NArrow::TThreadSimpleArraysCache::Get(type, scalar, RowsCount); - columns.emplace_back(std::make_shared(arr)); + if (sequentialColumnIds.contains(i.GetColumnId())) { + columns.emplace_back(i.AssembleForSeqAccess()); } else { - columns.emplace_back(i.Assemble()); + columns.emplace_back(i.AssembleAccessor()); } fields.emplace_back(i.GetField()); } - return arrow::Table::Make(std::make_shared(fields), columns); -} - -std::shared_ptr TPortionInfo::TPreparedBatchData::Assemble(const TAssembleOptions& options) const { - return NArrow::ToBatch(AssembleTable(options), true); + return std::make_shared(fields, std::move(columns)); } } diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 4d98f9c15c37..b7b89c2f187d 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -65,6 +65,9 @@ class TDeserializeChunkedArray: public NArrow::NAccessor::IChunkedArray { std::shared_ptr Loader; std::vector Chunks; protected: + virtual std::optional DoGetRawSize() const override { + return {}; + } virtual TCurrentChunkAddress DoGetChunk(const std::optional& chunkCurrent, const ui64 position) const override; virtual std::shared_ptr DoGetChunkedArray() const override { AFL_VERIFY(false); @@ -683,6 +686,7 @@ class TPortionInfo { std::shared_ptr Assemble() const; std::shared_ptr AssembleForSeqAccess() const; + std::shared_ptr AssembleAccessor() const; }; class TPreparedBatchData { @@ -747,8 +751,7 @@ class TPortionInfo { , RowsCount(rowsCount) { } - std::shared_ptr Assemble(const TAssembleOptions& options = {}) const; - std::shared_ptr AssembleTable(const TAssembleOptions& options = {}) const; + std::shared_ptr AssembleToGeneralContainer(const std::set& sequentialColumnIds) const; std::shared_ptr AssembleForSeqAccess() const; }; @@ -799,13 +802,6 @@ class TPortionInfo { TPreparedBatchData PrepareForAssemble(const ISnapshotSchema& dataSchema, const ISnapshotSchema& resultSchema, THashMap& blobsData) const; TPreparedBatchData PrepareForAssemble(const ISnapshotSchema& dataSchema, const ISnapshotSchema& resultSchema, THashMap& blobsData) const; - std::shared_ptr AssembleInBatch(const ISnapshotSchema& dataSchema, const ISnapshotSchema& resultSchema, - THashMap& data) const { - auto batch = PrepareForAssemble(dataSchema, resultSchema, data).Assemble(); - Y_ABORT_UNLESS(batch->Validate().ok()); - return batch; - } - friend IOutputStream& operator << (IOutputStream& out, const TPortionInfo& info) { out << info.DebugString(); return out; diff --git a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp index 69b11dbdab05..14fa9f854dcc 100644 --- a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp +++ b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.cpp @@ -13,34 +13,14 @@ void TReadPortionInfoWithBlobs::RestoreChunk(const std::shared_ptr TReadPortionInfoWithBlobs::GetBatch(const ISnapshotSchema::TPtr& data, const ISnapshotSchema& result, const std::set& columnNames) const { - Y_ABORT_UNLESS(data); - if (columnNames.empty()) { - if (!CachedBatch) { - THashMap blobs; - for (auto&& i : PortionInfo.Records) { - blobs[i.GetAddress()] = GetBlobByAddressVerified(i.ColumnId, i.Chunk); - Y_ABORT_UNLESS(blobs[i.GetAddress()].size() == i.BlobRange.Size); - } - CachedBatch = PortionInfo.AssembleInBatch(*data, result, blobs); - Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(*CachedBatch, result.GetIndexInfo().GetReplaceKey())); - } - return *CachedBatch; - } else if (CachedBatch) { - std::vector columnNamesString; - for (auto&& i : columnNames) { - columnNamesString.emplace_back(i.data(), i.size()); - } - return NArrow::TColumnOperator().VerifyIfAbsent().Extract(*CachedBatch, columnNamesString); - } else { - auto filteredSchema = std::make_shared(data, columnNames); - THashMap blobs; - for (auto&& i : PortionInfo.Records) { - blobs[i.GetAddress()] = GetBlobByAddressVerified(i.ColumnId, i.Chunk); - Y_ABORT_UNLESS(blobs[i.GetAddress()].size() == i.BlobRange.Size); - } - return PortionInfo.AssembleInBatch(*data, *filteredSchema, blobs); +std::shared_ptr TReadPortionInfoWithBlobs::RestoreBatch( + const ISnapshotSchema& data, const ISnapshotSchema& resultSchema, const std::set& seqColumns) const { + THashMap blobs; + for (auto&& i : PortionInfo.Records) { + blobs[i.GetAddress()] = GetBlobByAddressVerified(i.ColumnId, i.Chunk); + Y_ABORT_UNLESS(blobs[i.GetAddress()].size() == i.BlobRange.Size); } + return PortionInfo.PrepareForAssemble(data, resultSchema, blobs).AssembleToGeneralContainer(seqColumns); } NKikimr::NOlap::TReadPortionInfoWithBlobs TReadPortionInfoWithBlobs::RestorePortion(const TPortionInfo& portion, NBlobOperations::NRead::TCompositeReadBlobs& blobs, const TIndexInfo& indexInfo) { @@ -84,21 +64,16 @@ bool TReadPortionInfoWithBlobs::ExtractColumnChunks(const ui32 entityId, std::ve if (records.empty()) { return false; } - std::map> chunksMap; + std::vector> chunksLocal; for (auto it = Chunks.begin(); it != Chunks.end();) { if (it->first.GetEntityId() == entityId) { - chunksMap.emplace(it->first, std::move(it->second)); + AFL_VERIFY(chunksLocal.empty() || chunksLocal.back()->GetChunkAddressVerified() < it->second->GetChunkAddressVerified()); + chunksLocal.emplace_back(std::move(it->second)); it = Chunks.erase(it); } else { ++it; } } - std::vector> chunksLocal; - for (auto&& i : chunksMap) { - Y_ABORT_UNLESS(i.first.GetColumnId() == entityId); - Y_ABORT_UNLESS(i.first.GetChunk() == chunksLocal.size()); - chunksLocal.emplace_back(i.second); - } std::swap(chunksLocal, chunks); return true; } @@ -129,8 +104,6 @@ std::optional TReadPortionInfoWithBlobs::SyncP std::vector> newChunks; if (it != columnChunks.end()) { newChunks = to->GetIndexInfo().ActualizeColumnData(it->second, from->GetIndexInfo(), i); - } else { - newChunks = to->GetIndexInfo().MakeEmptyChunks(i, pageSizes, to->GetIndexInfo().GetColumnFeaturesVerified(i)); } AFL_VERIFY(entityChunksNew.emplace(i, std::move(newChunks)).second); } @@ -143,8 +116,8 @@ std::optional TReadPortionInfoWithBlobs::SyncP TIndexInfo::TSecondaryData secondaryData; secondaryData.MutableExternalData() = entityChunksNew; for (auto&& i : to->GetIndexInfo().GetIndexes()) { - to->GetIndexInfo().AppendIndex(entityChunksNew, i.first, storages, secondaryData).Validate(); - } + to->GetIndexInfo().AppendIndex(entityChunksNew, i.first, storages, secondaryData).Validate(); + } const NSplitter::TEntityGroups groups = to->GetIndexInfo().GetEntityGroupsByStorageId(targetTier, *storages); auto schemaTo = std::make_shared(to, std::make_shared()); diff --git a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h index 42fcd8a52f8a..a9e24eb3c165 100644 --- a/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h +++ b/ydb/core/tx/columnshard/engines/portions/read_with_blobs.h @@ -21,7 +21,6 @@ class TReadPortionInfoWithBlobs: public TBasePortionInfoWithBlobs { void RestoreChunk(const std::shared_ptr& chunk); TPortionInfo PortionInfo; - mutable std::optional> CachedBatch; explicit TReadPortionInfoWithBlobs(TPortionInfo&& portionInfo) : PortionInfo(std::move(portionInfo)) { @@ -39,7 +38,7 @@ class TReadPortionInfoWithBlobs: public TBasePortionInfoWithBlobs { static TReadPortionInfoWithBlobs RestorePortion(const TPortionInfo& portion, NBlobOperations::NRead::TCompositeReadBlobs& blobs, const TIndexInfo& indexInfo); - std::shared_ptr GetBatch(const ISnapshotSchema::TPtr& data, const ISnapshotSchema& result, const std::set& columnNames = {}) const; + std::shared_ptr RestoreBatch(const ISnapshotSchema& data, const ISnapshotSchema& resultSchema, const std::set& seqColumns) const; static std::optional SyncPortion(TReadPortionInfoWithBlobs&& source, const ISnapshotSchema::TPtr& from, const ISnapshotSchema::TPtr& to, const TString& targetTier, const std::shared_ptr& storages, std::shared_ptr counters); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.cpp index 869269fd9d3a..e647c77313e7 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.cpp @@ -5,13 +5,15 @@ namespace NKikimr::NOlap { -void TFetchedData::SyncTableColumns(const std::vector>& fields) { +void TFetchedData::SyncTableColumns(const std::vector>& fields, const ISnapshotSchema& schema) { for (auto&& i : fields) { if (Table->GetSchema()->GetFieldByName(i->name())) { continue; } - Table->AddField(i, std::make_shared( - NArrow::TThreadSimpleArraysCache::GetNull(i->type(), Table->num_rows()))).Validate(); + Table + ->AddField(i, std::make_shared( + NArrow::TThreadSimpleArraysCache::Get(i->type(), schema.GetExternalDefaultValueVerified(i->name()), Table->num_rows()))) + .Validate(); } } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h index 1bd31f77dc0e..adde885f1468 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h @@ -25,7 +25,7 @@ class TFetchedData { } - void SyncTableColumns(const std::vector>& fields); + void SyncTableColumns(const std::vector>& fields, const ISnapshotSchema& schema); std::shared_ptr GetAppliedFilter() const { return UseFilter ? Filter : nullptr; @@ -80,14 +80,10 @@ class TFetchedData { } } - void AddBatch(const std::shared_ptr& batch) { - return AddBatch(arrow::Table::Make(batch->schema(), batch->columns(), batch->num_rows())); - } - void AddBatch(const std::shared_ptr& table) { AFL_VERIFY(table); if (UseFilter) { - AddBatch(table->BuildTable()); + AddBatch(table->BuildTableVerified()); } else { if (!Table) { Table = table; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp index c9863650ff76..7ce1bc9b6595 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp @@ -89,13 +89,14 @@ ui64 TFilterProgramStep::DoPredictRawBytes(const std::shared_ptr& s } TConclusion TPredicateFilter::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { - auto filter = source->GetContext()->GetReadMetadata()->GetPKRangesFilter().BuildFilter(source->GetStageData().GetTable()->BuildTable()); + auto filter = source->GetContext()->GetReadMetadata()->GetPKRangesFilter().BuildFilter(source->GetStageData().GetTable()->BuildTableVerified()); source->MutableStageData().AddFilter(filter); return true; } TConclusion TSnapshotFilter::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { - auto filter = MakeSnapshotFilter(source->GetStageData().GetTable()->BuildTable(), source->GetContext()->GetReadMetadata()->GetRequestSnapshot()); + auto filter = MakeSnapshotFilter( + source->GetStageData().GetTable()->BuildTableVerified(), source->GetContext()->GetReadMetadata()->GetRequestSnapshot()); source->MutableStageData().AddFilter(filter); return true; } @@ -119,7 +120,8 @@ TConclusion TDeletionFilter::DoExecuteInplace(const std::shared_ptr TShardingFilter::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& /*step*/) const { NYDBTest::TControllers::GetColumnShardController()->OnSelectShardingFilter(); - auto filter = source->GetContext()->GetReadMetadata()->GetRequestShardingInfo()->GetShardingInfo()->GetFilter(source->GetStageData().GetTable()->BuildTable()); + auto filter = source->GetContext()->GetReadMetadata()->GetRequestShardingInfo()->GetShardingInfo()->GetFilter( + source->GetStageData().GetTable()->BuildTableVerified()); source->MutableStageData().AddFilter(filter); return true; } @@ -129,7 +131,7 @@ TConclusion TBuildFakeSpec::DoExecuteInplace(const std::shared_ptrfields()) { columns.emplace_back(NArrow::TThreadSimpleArraysCache::GetConst(f->type(), NArrow::DefaultScalar(f->type()), Count)); } - source->MutableStageData().AddBatch(arrow::RecordBatch::Make(TIndexInfo::ArrowSchemaSnapshot(), Count, columns)); + source->MutableStageData().AddBatch(std::make_shared(arrow::RecordBatch::Make(TIndexInfo::ArrowSchemaSnapshot(), Count, columns))); return true; } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp index 6fe0b3992f8e..dfeaec1b7aca 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp @@ -71,7 +71,7 @@ TConclusionStatus TStartMergeTask::DoExecuteImpl() { TMemoryProfileGuard mGuard("SCAN_PROFILE::MERGE::EXCLUSIVE", IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD_SCAN_MEMORY)); auto& container = Sources.begin()->second->GetStageResult().GetBatch(); if (container && container->num_rows()) { - ResultBatch = container->BuildTable(); + ResultBatch = container->BuildTableVerified(); LastPK = Sources.begin()->second->GetLastPK(); ResultBatch = NArrow::TColumnOperator().VerifyIfAbsent().Extract(ResultBatch, Context->GetProgramInputColumns()->GetColumnNamesVector()); Context->GetCommonContext()->GetCounters().OnNoScanInterval(ResultBatch->num_rows()); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index b9742c17b3e7..b5927acc8d1a 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -73,7 +73,7 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, TBlob ++fetchedChunks; } else { defaultBlocks.emplace(c->GetAddress(), - TPortionInfo::TAssembleBlobInfo(c->GetMeta().GetNumRows(), Schema->GetDefaultValueVerified(c->GetColumnId()))); + TPortionInfo::TAssembleBlobInfo(c->GetMeta().GetNumRows(), Schema->GetExternalDefaultValueVerified(c->GetColumnId()))); ++nullChunks; } itFinished = !itFilter.Next(c->GetMeta().GetNumRows()); @@ -187,32 +187,8 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in void TPortionDataSource::DoAssembleColumns(const std::shared_ptr& columns) { auto blobSchema = GetContext()->GetReadMetadata()->GetLoadSchemaVerified(*Portion); - if (SequentialEntityIds.empty()) { - MutableStageData().AddBatch(Portion->PrepareForAssemble(*blobSchema, columns->GetFilteredSchemaVerified(), MutableStageData().MutableBlobs()).AssembleTable()); - } else { - { - auto inMemColumns = columns->GetColumnIds(); - for (auto&& i : SequentialEntityIds) { - inMemColumns.erase(i); - } - if (inMemColumns.size()) { - auto filteredSchema = std::make_shared(columns->GetFilteredSchemaPtrVerified(), inMemColumns); - MutableStageData().AddBatch(Portion->PrepareForAssemble(*blobSchema, *filteredSchema, MutableStageData().MutableBlobs()).AssembleTable()); - } - } - { - std::set scanColumns; - for (auto&& i : columns->GetColumnIds()) { - if (SequentialEntityIds.contains(i)) { - scanColumns.emplace(i); - } - } - if (scanColumns.size()) { - auto filteredSchema = std::make_shared(columns->GetFilteredSchemaPtrVerified(), scanColumns); - MutableStageData().AddBatch(Portion->PrepareForAssemble(*blobSchema, *filteredSchema, MutableStageData().MutableBlobs()).AssembleForSeqAccess()); - } - } - } + MutableStageData().AddBatch(Portion->PrepareForAssemble(*blobSchema, columns->GetFilteredSchemaVerified(), MutableStageData().MutableBlobs()) + .AssembleToGeneralContainer(SequentialEntityIds)); } bool TCommittedDataSource::DoStartFetchingColumns(const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const std::shared_ptr& /*columns*/) { @@ -240,13 +216,14 @@ void TCommittedDataSource::DoAssembleColumns(const std::shared_ptr& AFL_VERIFY(GetStageData().GetBlobs().size() == 1); auto bData = MutableStageData().ExtractBlob(GetStageData().GetBlobs().begin()->first); auto schema = GetContext()->GetReadMetadata()->GetBlobSchema(CommittedBlob.GetSchemaVersion()); - auto batch = NArrow::DeserializeBatch(bData, schema); - AFL_VERIFY(batch)("schema", schema->ToString()); - batch = GetContext()->GetReadMetadata()->GetIndexInfo().AddSnapshotColumns(batch, CommittedBlob.GetSnapshot()); - batch = GetContext()->GetReadMetadata()->GetIndexInfo().AddDeleteFlagsColumn(batch, CommittedBlob.GetIsDelete()); + auto rBatch = NArrow::DeserializeBatch(bData, std::make_shared(CommittedBlob.GetSchemaSubset().Apply(schema->fields()))); + AFL_VERIFY(rBatch)("schema", schema->ToString()); + auto batch = std::make_shared(rBatch); + GetContext()->GetReadMetadata()->GetIndexInfo().AddSnapshotColumns(*batch, CommittedBlob.GetSnapshot()); + GetContext()->GetReadMetadata()->GetIndexInfo().AddDeleteFlagsColumn(*batch, CommittedBlob.GetIsDelete()); MutableStageData().AddBatch(batch); } - MutableStageData().SyncTableColumns(columns->GetSchema()->fields()); + MutableStageData().SyncTableColumns(columns->GetSchema()->fields(), *GetContext()->GetReadMetadata()->GetResultSchema()); } } // namespace NKikimr::NOlap::NReader::NPlain diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp index b019da7e132d..f5473eaab885 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp @@ -14,36 +14,25 @@ std::shared_ptr IIndexInfo::GetColumnLoaderVerifi return result; } -std::shared_ptr IIndexInfo::AddDeleteFlagsColumn(const std::shared_ptr& batch, const bool isDelete) { - Y_ABORT_UNLESS(batch); - i64 numColumns = batch->num_columns(); - i64 numRows = batch->num_rows(); +void IIndexInfo::AddDeleteFlagsColumn(NArrow::TGeneralContainer& batch, const bool isDelete) { + const i64 numRows = batch.num_rows(); - AFL_VERIFY(!batch->GetColumnByName(SPEC_COL_DELETE_FLAG)); - return NArrow::TStatusValidator::GetValid(batch->AddColumn(numColumns, arrow::field(SPEC_COL_DELETE_FLAG, arrow::boolean()), - NArrow::TThreadSimpleArraysCache::GetConst(arrow::boolean(), std::make_shared(isDelete), numRows))); + batch.AddField(arrow::field(SPEC_COL_DELETE_FLAG, arrow::boolean()), + NArrow::TThreadSimpleArraysCache::GetConst(arrow::boolean(), std::make_shared(isDelete), numRows)).Validate(); } -std::shared_ptr IIndexInfo::AddSnapshotColumns(const std::shared_ptr& batch, const TSnapshot& snapshot) { - Y_ABORT_UNLESS(batch); - i64 numColumns = batch->num_columns(); - i64 numRows = batch->num_rows(); +void IIndexInfo::AddSnapshotColumns(NArrow::TGeneralContainer& batch, const TSnapshot& snapshot) { + const i64 numRows = batch.num_rows(); - auto res = batch->AddColumn(numColumns, arrow::field(SPEC_COL_PLAN_STEP, arrow::uint64()), - NArrow::MakeUI64Array(snapshot.GetPlanStep(), numRows)); - Y_ABORT_UNLESS(res.ok()); - res = (*res)->AddColumn(numColumns + 1, arrow::field(SPEC_COL_TX_ID, arrow::uint64()), - NArrow::MakeUI64Array(snapshot.GetTxId(), numRows)); - Y_ABORT_UNLESS(res.ok()); - Y_ABORT_UNLESS((*res)->num_columns() == numColumns + 2); - return *res; + batch.AddField(arrow::field(SPEC_COL_PLAN_STEP, arrow::uint64()), NArrow::MakeUI64Array(snapshot.GetPlanStep(), numRows)).Validate(); + batch.AddField(arrow::field(SPEC_COL_TX_ID, arrow::uint64()), NArrow::MakeUI64Array(snapshot.GetTxId(), numRows)).Validate(); } -std::shared_ptr IIndexInfo::NormalizeDeletionColumn(const std::shared_ptr& batch) { - if (batch->schema()->GetFieldIndex(SPEC_COL_DELETE_FLAG) >= 0) { - return batch; +void IIndexInfo::NormalizeDeletionColumn(NArrow::TGeneralContainer& batch) { + if (batch.HasColumn(SPEC_COL_DELETE_FLAG)) { + return; } - return AddDeleteFlagsColumn(batch, false); + AddDeleteFlagsColumn(batch, false); } std::optional IIndexInfo::GetColumnIdOptional(const std::string& name) const { diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h index ab6cd67a3937..15a9a533ada7 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h @@ -1,6 +1,7 @@ #pragma once #include "loader.h" +#include #include #include @@ -61,12 +62,6 @@ class IIndexInfo { return std::make_shared(std::move(fields)); } - void AddSpecialFieldIds(std::vector& result) const { - result.emplace_back((ui32)ESpecialColumn::PLAN_STEP); - result.emplace_back((ui32)ESpecialColumn::TX_ID); - result.emplace_back((ui32)ESpecialColumn::DELETE_FLAG); - } - static void AddSpecialFields(std::vector>& fields) { AddSnapshotFields(fields); fields.push_back(arrow::field(SPEC_COL_DELETE_FLAG, arrow::boolean())); @@ -82,6 +77,10 @@ class IIndexInfo { fields.push_back(arrow::field(SPEC_COL_TX_ID, arrow::uint64())); } + static void AddDeleteFields(std::vector>& fields) { + fields.push_back(arrow::field(SPEC_COL_DELETE_FLAG, arrow::boolean())); + } + static const std::set& GetSnapshotColumnIdsSet() { static const std::set result = { (ui32)ESpecialColumn::PLAN_STEP, (ui32)ESpecialColumn::TX_ID }; return result; @@ -97,6 +96,21 @@ class IIndexInfo { return result; } + [[nodiscard]] static std::vector AddSpecialFieldIds(const std::vector& baseColumnIds) { + std::vector result = baseColumnIds; + for (auto&& i : GetSystemColumnIds()) { + result.emplace_back(i); + } + return result; + } + + [[nodiscard]] static std::vector AddSnapshotFieldIds(const std::vector& baseColumnIds) { + std::vector result = baseColumnIds; + for (auto&& i : GetSnapshotColumnIds()) { + result.emplace_back(i); + } + return result; + } std::optional GetColumnIdOptional(const std::string& name) const; TString GetColumnName(ui32 id, bool required) const; @@ -106,10 +120,10 @@ class IIndexInfo { virtual std::shared_ptr GetColumnLoaderOptional(const ui32 columnId) const = 0; std::shared_ptr GetColumnLoaderVerified(const ui32 columnId) const; - static std::shared_ptr NormalizeDeletionColumn(const std::shared_ptr& batch); + static void NormalizeDeletionColumn(NArrow::TGeneralContainer& batch); - static std::shared_ptr AddSnapshotColumns(const std::shared_ptr& batch, const TSnapshot& snapshot); - static std::shared_ptr AddDeleteFlagsColumn(const std::shared_ptr& batch, const bool isDelete); + static void AddSnapshotColumns(NArrow::TGeneralContainer& batch, const TSnapshot& snapshot); + static void AddDeleteFlagsColumn(NArrow::TGeneralContainer& batch, const bool isDelete); static ui64 GetSpecialColumnsRecordSize() { return sizeof(ui64) + sizeof(ui64) + sizeof(bool); @@ -146,6 +160,11 @@ class IIndexInfo { || fieldId == (ui32)ESpecialColumn::DELETE_FLAG; } + static bool IsNullableVerified(const ui32 fieldId) { + Y_UNUSED(fieldId); + return false; + } + static ui32 GetSpecialColumnByteWidth(const ui32 field) { Y_ABORT_UNLESS(IsSpecialColumn(field)); return 8; diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index 42dfe2ab2028..5bdfc2838eb9 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -61,15 +61,12 @@ TString TIndexInfo::GetColumnName(ui32 id, bool required) const { return IIndexInfo::GetColumnName(id, required); } -std::vector TIndexInfo::GetColumnIds(const bool withSpecial) const { - std::vector result; - for (auto&& i : Columns) { - result.emplace_back(i.first); - } +const std::vector& TIndexInfo::GetColumnIds(const bool withSpecial) const { if (withSpecial) { - IIndexInfo::AddSpecialFieldIds(result); + return SchemaColumnIdsWithSpecials; + } else { + return SchemaColumnIds; } - return result; } std::vector TIndexInfo::GetColumnNames(const std::vector& ids) const { @@ -160,10 +157,11 @@ void TIndexInfo::SetAllKeys(const std::shared_ptr& operators) /// * apply REPLACE by MergeSort /// * apply PK predicate before REPLACE { + AFL_VERIFY(PKColumnIds.empty()); const auto& primaryKeyNames = NamesOnly(GetPrimaryKeyColumns()); - auto columnIds = GetColumnIds(primaryKeyNames); - AFL_VERIFY(columnIds.size()); - PrimaryKey = MakeArrowSchema(Columns, columnIds); + PKColumnIds = GetColumnIds(primaryKeyNames); + AFL_VERIFY(PKColumnIds.size()); + PrimaryKey = MakeArrowSchema(Columns, PKColumnIds); } for (const auto& [colId, column] : Columns) { @@ -305,17 +303,10 @@ std::optional TIndexInfo::BuildFromProto(const NKikimrSchemeOp::TCol return result; } -std::shared_ptr MakeArrowSchema(const NTable::TScheme::TTableSchema::TColumns& columns, const std::vector& ids, const bool withSpecials) { +std::shared_ptr MakeArrowSchema(const NTable::TScheme::TTableSchema::TColumns& columns, const std::vector& ids) { std::vector> fields; - if (withSpecials) { - IIndexInfo::AddSpecialFields(fields); - } - for (const ui32 id : ids) { - if (TIndexInfo::IsSpecialColumn(id)) { - AFL_VERIFY(withSpecials); - continue; - } + AFL_VERIFY(!TIndexInfo::IsSpecialColumn(id)); auto it = columns.find(id); AFL_VERIFY(it != columns.end()); @@ -332,17 +323,16 @@ std::shared_ptr MakeArrowSchema(const NTable::TScheme::TTableSche void TIndexInfo::InitializeCaches(const std::shared_ptr& operators) { { AFL_VERIFY(!Schema); - std::vector ids; - ids.reserve(Columns.size()); + SchemaColumnIds.reserve(Columns.size()); for (const auto& [id, _] : Columns) { - ids.push_back(id); + SchemaColumnIds.push_back(id); } - // The ids had a set type before so we keep them sorted. - std::sort(ids.begin(), ids.end()); - Schema = MakeArrowSchema(Columns, ids); + std::sort(SchemaColumnIds.begin(), SchemaColumnIds.end()); + Schema = MakeArrowSchema(Columns, SchemaColumnIds); } - SchemaWithSpecials = IIndexInfo::AddSpecialFields(ArrowSchema()); + SchemaWithSpecials = IIndexInfo::AddSpecialFields(Schema); + SchemaColumnIdsWithSpecials = IIndexInfo::AddSpecialFieldIds(SchemaColumnIds); for (auto&& c : Columns) { AFL_VERIFY(ArrowColumnByColumnIdCache.emplace(c.first, GetColumnFieldVerified(c.first)).second); @@ -388,18 +378,16 @@ std::shared_ptr TIndexInfo::Get return CompactionPlannerConstructor; } -std::shared_ptr TIndexInfo::GetColumnDefaultValueVerified(const std::string& colName) const { +std::shared_ptr TIndexInfo::GetColumnExternalDefaultValueVerified(const std::string& colName) const { const ui32 columnId = GetColumnIdVerified(colName); - return GetColumnDefaultValueVerified(columnId); + return GetColumnExternalDefaultValueVerified(columnId); } -std::shared_ptr TIndexInfo::GetColumnDefaultValueVerified(const ui32 columnId) const { - auto& features = GetColumnFeaturesVerified(columnId); - if (features.GetDefaultValue().IsEmpty() && !IsNullableVerified(columnId)) { - return NArrow::DefaultScalar(GetColumnFieldVerified(columnId)->type()); - } else { - return features.GetDefaultValue().GetValue(); +std::shared_ptr TIndexInfo::GetColumnExternalDefaultValueVerified(const ui32 columnId) const { + if (IIndexInfo::IsSpecialColumn(columnId)) { + return IIndexInfo::DefaultColumnValue(columnId); } + return GetColumnFeaturesVerified(columnId).GetDefaultValue().GetValue(); } NKikimr::TConclusionStatus TIndexInfo::AppendIndex(const THashMap>>& originalData, @@ -434,4 +422,12 @@ std::shared_ptr TIndexInfo::GetIndexMax(const ui32 c return nullptr; } +std::vector TIndexInfo::GetEntityIds() const { + auto result = GetColumnIds(true); + for (auto&& i : Indexes) { + result.emplace_back(i.first); + } + return result; +} + } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index 16397100f6d9..622f8c741050 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -54,19 +54,20 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo { std::shared_ptr GetCompactionPlannerConstructor() const; bool IsNullableVerified(const std::string& fName) const { - auto it = Columns.find(GetColumnIdVerified(fName)); - AFL_VERIFY(it != Columns.end()); - return !it->second.NotNull; + return IsNullableVerified(GetColumnIdVerified(fName)); } bool IsNullableVerified(const ui32 colId) const { auto it = Columns.find(colId); - AFL_VERIFY(it != Columns.end()); + if (it == Columns.end()) { + AFL_VERIFY(IIndexInfo::IsSpecialColumn(colId)); + return IIndexInfo::IsNullableVerified(colId); + } return !it->second.NotNull; } - std::shared_ptr GetColumnDefaultValueVerified(const std::string& colName) const; - std::shared_ptr GetColumnDefaultValueVerified(const ui32 colId) const; + std::shared_ptr GetColumnExternalDefaultValueVerified(const std::string& colName) const; + std::shared_ptr GetColumnExternalDefaultValueVerified(const ui32 colId) const; bool GetExternalGuaranteeExclusivePK() const { return ExternalGuaranteeExclusivePK; @@ -250,14 +251,12 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo { /// Returns names of columns defined by the specific ids. std::vector GetColumnNames(const std::vector& ids) const; std::vector GetColumnSTLNames(const std::vector& ids) const; - std::vector GetColumnIds(const bool withSpecial = true) const; - std::vector GetEntityIds() const { - auto result = GetColumnIds(); - for (auto&& i : Indexes) { - result.emplace_back(i.first); - } - return result; + const std::vector& GetColumnIds(const bool withSpecial = true) const; + const std::vector& GetPKColumnIds() const { + AFL_VERIFY(PKColumnIds.size()); + return PKColumnIds; } + std::vector GetEntityIds() const; /// Returns info of columns defined by specific ids. std::vector GetColumns(const std::vector& ids) const; @@ -320,6 +319,9 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo { private: ui64 Version = 0; TString Name; + std::vector SchemaColumnIds; + std::vector SchemaColumnIdsWithSpecials; + std::vector PKColumnIds; std::shared_ptr Schema; std::shared_ptr SchemaWithSpecials; std::shared_ptr PrimaryKey; @@ -328,7 +330,7 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo { NArrow::NSerialization::TSerializerContainer DefaultSerializer = NArrow::NSerialization::TSerializerContainer::GetDefaultSerializer(); }; -std::shared_ptr MakeArrowSchema(const NTable::TScheme::TTableSchema::TColumns& columns, const std::vector& ids, const bool withSpecials = false); +std::shared_ptr MakeArrowSchema(const NTable::TScheme::TTableSchema::TColumns& columns, const std::vector& ids); /// Extracts columns with the specific ids from the schema. std::vector GetColumns(const NTable::TScheme::TTableSchema& tableSchema, const std::vector& ids); diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp index 300952cc3227..e1192f677da5 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp @@ -27,34 +27,31 @@ std::set ISnapshotSchema::GetPkColumnsIds() const { } -TConclusion> ISnapshotSchema::NormalizeBatch(const ISnapshotSchema& dataSchema, const std::shared_ptr batch) const { +TConclusion> ISnapshotSchema::NormalizeBatch( + const ISnapshotSchema& dataSchema, const std::shared_ptr& batch, const std::set& restoreColumnIds) const { if (dataSchema.GetSnapshot() == GetSnapshot()) { return batch; } - Y_ABORT_UNLESS(dataSchema.GetSnapshot() < GetSnapshot()); + AFL_VERIFY(dataSchema.GetSnapshot() < GetSnapshot()); const std::shared_ptr& resultArrowSchema = GetSchema(); - std::vector> newColumns; - newColumns.reserve(resultArrowSchema->num_fields()); + std::shared_ptr result = std::make_shared(); for (size_t i = 0; i < resultArrowSchema->fields().size(); ++i) { auto& resultField = resultArrowSchema->fields()[i]; auto columnId = GetIndexInfo().GetColumnId(resultField->name()); - auto oldColumnIndex = dataSchema.GetFieldIndex(columnId); - if (oldColumnIndex >= 0) { // ColumnExists - auto oldColumnInfo = dataSchema.GetFieldByIndex(oldColumnIndex); - Y_ABORT_UNLESS(oldColumnInfo); - auto columnData = batch->GetColumnByName(oldColumnInfo->name()); - Y_ABORT_UNLESS(columnData); - newColumns.push_back(columnData); - } else { // AddNullColumn - auto conclusion = BuildDefaultBatch({ resultField }, batch->num_rows()); + auto oldField = dataSchema.GetFieldByColumnIdOptional(columnId); + if (oldField) { + auto conclusion = result->AddField(resultField, batch->GetAccessorByNameVerified(oldField->name())); if (conclusion.IsFail()) { return conclusion; } - newColumns.push_back((*conclusion)->column(0)); + } else if (restoreColumnIds.contains(columnId)) { + result->AddField(resultField, + NArrow::TThreadSimpleArraysCache::Get(resultField->type(), GetExternalDefaultValueVerified(columnId), batch->num_rows())) + .Validate(); } } - return arrow::RecordBatch::Make(resultArrowSchema, batch->num_rows(), newColumns); + return result; } TConclusion> ISnapshotSchema::PrepareForModification( @@ -110,11 +107,6 @@ TConclusion> ISnapshotSchema::PrepareForModi Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(batch, GetIndexInfo().GetPrimaryKey())); switch (mType) { - case NEvWrite::EModificationType::Delete: - return AddDefault(batch, true); - case NEvWrite::EModificationType::Replace: - case NEvWrite::EModificationType::Insert: - return AddDefault(batch, false); case NEvWrite::EModificationType::Upsert: { AFL_VERIFY(batch->num_columns() <= dstSchema->num_fields()); if (batch->num_columns() < dstSchema->num_fields()) { @@ -125,13 +117,16 @@ TConclusion> ISnapshotSchema::PrepareForModi if (batch->GetColumnByName(f->name())) { continue; } - if (!GetIndexInfo().GetColumnDefaultValueVerified(f->name())) { + if (!GetIndexInfo().GetColumnExternalDefaultValueVerified(f->name())) { return TConclusionStatus::Fail("empty field for non-default column: '" + f->name() + "'"); } } } return batch; } + case NEvWrite::EModificationType::Delete: + case NEvWrite::EModificationType::Replace: + case NEvWrite::EModificationType::Insert: case NEvWrite::EModificationType::Update: return batch; } @@ -184,48 +179,76 @@ std::vector> ISnapshotSchema::GetAbsentFields(cons return result; } -TConclusion> ISnapshotSchema::BuildDefaultBatch(const std::vector>& fields, const ui32 rowsCount) const { - std::vector> columns; +TConclusionStatus ISnapshotSchema::CheckColumnsDefault(const std::vector>& fields) const { for (auto&& i : fields) { - auto defaultValue = GetDefaultValueVerified(i->name()); + auto defaultValue = GetExternalDefaultValueVerified(i->name()); if (!defaultValue && !GetIndexInfo().IsNullableVerified(i->name())) { return TConclusionStatus::Fail("not nullable field with no default: " + i->name()); } + } + return TConclusionStatus::Success(); +} + +TConclusion> ISnapshotSchema::BuildDefaultBatch( + const std::vector>& fields, const ui32 rowsCount, const bool force) const { + std::vector> columns; + for (auto&& i : fields) { + auto defaultValue = GetExternalDefaultValueVerified(i->name()); + if (!defaultValue && !GetIndexInfo().IsNullableVerified(i->name())) { + if (force) { + defaultValue = NArrow::DefaultScalar(i->type()); + } else { + return TConclusionStatus::Fail("not nullable field with no default: " + i->name()); + } + } columns.emplace_back(NArrow::TThreadSimpleArraysCache::Get(i->type(), defaultValue, rowsCount)); } return arrow::RecordBatch::Make(std::make_shared(fields), rowsCount, columns); } -std::shared_ptr ISnapshotSchema::GetDefaultValueVerified(const std::string& columnName) const { - return GetIndexInfo().GetColumnDefaultValueVerified(columnName); +std::shared_ptr ISnapshotSchema::GetExternalDefaultValueVerified(const std::string& columnName) const { + return GetIndexInfo().GetColumnExternalDefaultValueVerified(columnName); } -std::shared_ptr ISnapshotSchema::GetDefaultValueVerified(const ui32 columnId) const { - return GetIndexInfo().GetColumnDefaultValueVerified(columnId); +std::shared_ptr ISnapshotSchema::GetExternalDefaultValueVerified(const ui32 columnId) const { + return GetIndexInfo().GetColumnExternalDefaultValueVerified(columnId); } -TConclusion> ISnapshotSchema::AddDefault(const std::shared_ptr& batch, const bool force) const { - auto result = batch; - for (auto&& i : GetIndexInfo().ArrowSchema()->fields()) { - if (batch->schema()->GetFieldIndex(i->name()) != -1) { - continue; - } - auto defaultValue = GetDefaultValueVerified(i->name()); - if (!defaultValue && !GetIndexInfo().IsNullableVerified(i->name())) { - if (!force) { - return TConclusionStatus::Fail("not nullable field withno default: " + i->name()); - } else { - defaultValue = NArrow::DefaultScalar(i->type()); +bool ISnapshotSchema::IsSpecialColumnId(const ui32 columnId) const { + return GetIndexInfo().IsSpecialColumn(columnId); +} + +std::set ISnapshotSchema::GetColumnsWithDifferentDefaults( + const THashMap& schemas, const ISnapshotSchema::TPtr& targetSchema) { + std::set result; + if (schemas.size() <= 1) { + return {}; + } + std::map> defaults; + for (auto& [_, blobSchema] : schemas) { + for (auto&& columnId : blobSchema->GetIndexInfo().GetColumnIds(true)) { + if (result.contains(columnId)) { + continue; + } + if (targetSchema && !targetSchema->HasColumnId(columnId)) { + continue; + } + auto def = blobSchema->GetIndexInfo().GetColumnExternalDefaultValueVerified(columnId); + if (!blobSchema->GetIndexInfo().IsNullableVerified(columnId) && !def) { + continue; } + auto it = defaults.find(columnId); + if (it == defaults.end()) { + defaults.emplace(columnId, def); + } else if (NArrow::ScalarCompareNullable(def, it->second) != 0) { + result.emplace(columnId); + } + } + if (targetSchema && result.size() == targetSchema->GetIndexInfo().GetColumnIds(true).size()) { + break; } - std::shared_ptr column = NArrow::TThreadSimpleArraysCache::Get(i->type(), defaultValue, batch->num_rows()); - result = NArrow::TStatusValidator::GetValid(result->AddColumn(result->num_columns(), i->name(), column)); } return result; } -bool ISnapshotSchema::IsSpecialColumnId(const ui32 columnId) const { - return GetIndexInfo().IsSpecialColumn(columnId); -} - } diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h index 91d53230ec38..a2a4cefd9215 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h @@ -1,4 +1,6 @@ #pragma once +#include + #include #include #include @@ -36,17 +38,20 @@ class ISnapshotSchema { std::vector> GetAbsentFields(const std::shared_ptr& existsSchema) const; - std::shared_ptr GetDefaultValueVerified(const std::string& columnName) const; - std::shared_ptr GetDefaultValueVerified(const ui32 columnId) const; - - TConclusion> BuildDefaultBatch(const std::vector>& fields, const ui32 rowsCount) const; - TConclusion> AddDefault(const std::shared_ptr& batch, const bool force) const; + std::shared_ptr GetExternalDefaultValueVerified(const std::string& columnName) const; + std::shared_ptr GetExternalDefaultValueVerified(const ui32 columnId) const; + TConclusion> BuildDefaultBatch( + const std::vector>& fields, const ui32 rowsCount, const bool force) const; + TConclusionStatus CheckColumnsDefault(const std::vector>& fields) const; std::vector GetPKColumnNames() const; virtual std::optional GetColumnIdOptional(const std::string& columnName) const = 0; virtual int GetFieldIndex(const ui32 columnId) const = 0; + bool HasColumnId(const ui32 columnId) const { + return GetFieldIndex(columnId) >= 0; + } ui32 GetColumnId(const std::string& columnName) const; std::shared_ptr GetFieldByIndex(const int index) const; @@ -64,7 +69,10 @@ class ISnapshotSchema { std::set GetPkColumnsIds() const; - [[nodiscard]] TConclusion> NormalizeBatch(const ISnapshotSchema& dataSchema, const std::shared_ptr batch) const; + static std::set GetColumnsWithDifferentDefaults(const THashMap& schemas, const ISnapshotSchema::TPtr& targetSchema); + + [[nodiscard]] TConclusion> NormalizeBatch( + const ISnapshotSchema& dataSchema, const std::shared_ptr& batch, const std::set& restoreColumnIds) const; [[nodiscard]] TConclusion> PrepareForModification( const std::shared_ptr& incomingBatch, const NEvWrite::EModificationType mType) const; }; diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h index e9fa1b41b7c2..1b515d5bb9cf 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h @@ -9,7 +9,7 @@ namespace NKikimr::NOlap { class TFilteredSnapshotSchema: public ISnapshotSchema { ISnapshotSchema::TPtr OriginalSnapshot; std::shared_ptr Schema; - std::set ColumnIds; + YDB_READONLY_DEF(std::set, ColumnIds); protected: virtual TString DoDebugString() const override; public: diff --git a/ydb/core/tx/columnshard/engines/scheme/ya.make b/ydb/core/tx/columnshard/engines/scheme/ya.make index e3d52b2649bd..8e41573bf419 100644 --- a/ydb/core/tx/columnshard/engines/scheme/ya.make +++ b/ydb/core/tx/columnshard/engines/scheme/ya.make @@ -21,7 +21,6 @@ PEERDIR( ydb/core/tx/columnshard/engines/scheme/column ydb/core/tx/columnshard/engines/scheme/defaults ydb/core/tx/columnshard/blobs_action/abstract - ydb/core/tx/columnshard/engines/changes/compaction ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/columnshard/engines/storage/chunks/null_column.h b/ydb/core/tx/columnshard/engines/storage/chunks/null_column.h index 721705fc5e4d..3b2420f0c223 100644 --- a/ydb/core/tx/columnshard/engines/storage/chunks/null_column.h +++ b/ydb/core/tx/columnshard/engines/storage/chunks/null_column.h @@ -11,7 +11,7 @@ class TDefaultChunkPreparation: public IPortionColumnChunk { using TBase = IPortionColumnChunk; const std::shared_ptr DefaultValue; const ui32 RecordsCount; - const ui64 RawBytes; + ui64 RawBytes = 0; TString Data; protected: virtual std::vector> DoInternalSplitImpl(const TColumnSaver& /*saver*/, const std::shared_ptr& /*counters*/, @@ -43,15 +43,16 @@ class TDefaultChunkPreparation: public IPortionColumnChunk { } public: - TDefaultChunkPreparation(const ui32 columnId, const ui32 recordsCount, const ui32 rawBytes, const std::shared_ptr& f, + TDefaultChunkPreparation(const ui32 columnId, const ui32 recordsCount, const std::shared_ptr& f, const std::shared_ptr& defaultValue, const TColumnSaver& saver) : TBase(columnId) , DefaultValue(defaultValue) , RecordsCount(recordsCount) - , RawBytes(rawBytes) { Y_ABORT_UNLESS(RecordsCount); - Data = saver.Apply(NArrow::TThreadSimpleArraysCache::Get(f->type(), defaultValue, RecordsCount), f); + auto arrowData = NArrow::TThreadSimpleArraysCache::Get(f->type(), defaultValue, RecordsCount); + RawBytes = NArrow::GetArrayDataSize(arrowData); + Data = saver.Apply(arrowData, f); SetChunkIdx(0); } }; diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.h b/ydb/core/tx/columnshard/engines/storage/granule/granule.h index 3b3db33a72cf..c8b3e302f1e7 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.h @@ -196,7 +196,7 @@ class TGranuleMeta: TNonCopyable { return OptimizerPlanner->SerializeToJsonVisual(); } - std::vector GetBucketPositions() const { + NArrow::NMerger::TIntervalPositions GetBucketPositions() const { return OptimizerPlanner->GetBucketPositions(); } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h index 21647072eb60..4bd196e552d0 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h @@ -134,7 +134,7 @@ class IOptimizerPlanner { return DoDebugString(); } - virtual std::vector GetBucketPositions() const = 0; + virtual NArrow::NMerger::TIntervalPositions GetBucketPositions() const = 0; bool IsLocked(const std::shared_ptr& dataLocksManager) const { return DoIsLocked(dataLocksManager); } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h index 246b349f2a77..f83183c04e39 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h @@ -899,15 +899,15 @@ class TPortionsBucket: public TMoveOnly { auto result = std::make_shared(granule, portions, saverContext); if (MainPortion) { NArrow::NMerger::TSortableBatchPosition pos(MainPortion->IndexKeyStart().ToBatch(primaryKeysSchema), 0, primaryKeysSchema->field_names(), {}, false); - result->AddCheckPoint(pos, false, false); + result->AddCheckPoint(pos, false); } if (!nextBorder && MainPortion && !forceMergeForTests) { NArrow::NMerger::TSortableBatchPosition pos(MainPortion->IndexKeyEnd().ToBatch(primaryKeysSchema), 0, primaryKeysSchema->field_names(), {}, false); - result->AddCheckPoint(pos, true, false); + result->AddCheckPoint(pos, true); } if (stopPoint) { NArrow::NMerger::TSortableBatchPosition pos(stopPoint->ToBatch(primaryKeysSchema), 0, primaryKeysSchema->field_names(), {}, false); - result->AddCheckPoint(pos, false, false); + result->AddCheckPoint(pos, false); } return result; } @@ -1181,15 +1181,15 @@ class TPortionBuckets { } } - std::vector GetBucketPositions() const { - std::vector result; + NArrow::NMerger::TIntervalPositions GetBucketPositions() const { + NArrow::NMerger::TIntervalPositions result; for (auto&& i : Buckets) { AFL_VERIFY(i.second->GetStartPos()); - result.emplace_back(*i.second->GetStartPos()); + result.AddPosition(*i.second->GetStartPos(), false); } - if (Buckets.size()) { + if (Buckets.size() && Buckets.rbegin()->second->GetPortion()->GetRecordsCount() > 1) { NArrow::NMerger::TSortableBatchPosition pos(Buckets.rbegin()->second->GetPortion()->IndexKeyEnd().ToBatch(PrimaryKeysSchema), 0, PrimaryKeysSchema->field_names(), {}, false); - result.emplace_back(pos); + result.AddPosition(std::move(pos), false); } return result; } @@ -1254,7 +1254,7 @@ class TOptimizerPlanner: public IOptimizerPlanner { public: - virtual std::vector GetBucketPositions() const override { + virtual NArrow::NMerger::TIntervalPositions GetBucketPositions() const override { return Buckets.GetBucketPositions(); } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp index 4fd497984104..5c70d26a38a3 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp @@ -28,7 +28,7 @@ std::shared_ptr TPortionsBucket::BuildOpti auto result = std::make_shared(granule, context.GetPortions(), saverContext); for (auto&& i : context.GetSplitRightOpenIntervalPoints()) { NArrow::NMerger::TSortableBatchPosition pos(i.ToBatch(primaryKeysSchema), 0, primaryKeysSchema->field_names(), {}, false); - result->AddCheckPoint(pos, false, false); + result->AddCheckPoint(pos, false); } return result; } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h index a551fbbb8a1d..56bddb8547fb 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/index.h @@ -220,14 +220,14 @@ class TPortionBuckets { return bucketForOptimization->BuildOptimizationTask(granule, locksManager, PrimaryKeysSchema, StoragesManager); } - std::vector GetBucketPositions() const { - std::vector result; + NArrow::NMerger::TIntervalPositions GetBucketPositions() const { + NArrow::NMerger::TIntervalPositions result; for (auto&& i : Buckets) { if (!i.first.HasValue()) { continue; } NArrow::NMerger::TSortableBatchPosition posStart(i.first.GetValueVerified().ToBatch(PrimaryKeysSchema), 0, PrimaryKeysSchema->field_names(), {}, false); - result.emplace_back(posStart); + result.AddPosition(std::move(posStart), false); } return result; } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/optimizer/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/optimizer/optimizer.h index 4e8595e20f1d..7d756f09deff 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/optimizer/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/optimizer/optimizer.h @@ -64,7 +64,7 @@ class TOptimizerPlanner: public IOptimizerPlanner { } public: - virtual std::vector GetBucketPositions() const override { + virtual NArrow::NMerger::TIntervalPositions GetBucketPositions() const override { return Buckets.GetBucketPositions(); } diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index 5f89ea7ec15c..10a1891c4102 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -500,7 +500,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 txId = 1; auto selectInfo = engine.Select(paths[0], TSnapshot(planStep, txId), NOlap::TPKRangesFilter(false)); UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK[0]->NumChunks(), columnIds.size() + TIndexInfo::GetSystemColumnNames().size()); + UNIT_ASSERT_VALUES_EQUAL(selectInfo->PortionsOrderedPK[0]->NumChunks(), columnIds.size() + TIndexInfo::GetSnapshotColumnIdsSet().size()); } { // select another pathId diff --git a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h index f626e02c6275..22b6ee9b897d 100644 --- a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h +++ b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h @@ -92,6 +92,8 @@ class TWriteAggregation { YDB_ACCESSOR_DEF(std::vector, SplittedBlobs); YDB_READONLY_DEF(TVector, WriteIds); YDB_READONLY_DEF(std::shared_ptr, BlobsAction); + YDB_READONLY_DEF(NArrow::TSchemaSubset, SchemaSubset); + public: const NEvWrite::TWriteMeta& GetWriteMeta() const { return WriteMeta; @@ -110,6 +112,7 @@ class TWriteAggregation { , SchemaVersion(writeData.GetData()->GetSchemaVersion()) , Size(writeData.GetSize()) , BlobsAction(writeData.GetBlobsAction()) + , SchemaSubset(writeData.GetSchemaSubsetVerified()) { for (auto&& s : splittedBlobs) { SplittedBlobs.emplace_back(std::move(s), *this); @@ -120,8 +123,8 @@ class TWriteAggregation { : WriteMeta(writeData.GetWriteMeta()) , SchemaVersion(writeData.GetData()->GetSchemaVersion()) , Size(writeData.GetSize()) - , BlobsAction(writeData.GetBlobsAction()) - { + , BlobsAction(writeData.GetBlobsAction()) { + AFL_VERIFY(!writeData.GetSchemaSubset()); } }; diff --git a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp index 3d94bda84a78..232d3b556e95 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp @@ -41,12 +41,14 @@ TConclusionStatus TBuildBatchesTask::DoExecute(const std::shared_ptr& /*t NConveyor::TInsertServiceOperator::AsyncTaskToExecute(task); return TConclusionStatus::Success(); } else { - auto conclusion = ActualSchema->BuildDefaultBatch(ActualSchema->GetIndexInfo().ArrowSchema()->fields(), 1); + auto insertionConclusion = ActualSchema->CheckColumnsDefault(defaultFields); + auto conclusion = ActualSchema->BuildDefaultBatch(ActualSchema->GetIndexInfo().ArrowSchema()->fields(), 1, true); AFL_VERIFY(!conclusion.IsFail())("error", conclusion.GetErrorMessage()); auto batchDefault = conclusion.DetachResult(); NArrow::NMerger::TSortableBatchPosition pos( batchDefault, 0, batchDefault->schema()->field_names(), batchDefault->schema()->field_names(), false); - merger = std::make_shared(batch, ActualSchema, pos); + merger = std::make_shared( + batch, ActualSchema, insertionConclusion.IsSuccess() ? "" : insertionConclusion.GetErrorMessage(), pos); break; } } @@ -55,7 +57,7 @@ TConclusionStatus TBuildBatchesTask::DoExecute(const std::shared_ptr& /*t break; } case NEvWrite::EModificationType::Update: { - merger = std::make_shared(batch, ActualSchema); + merger = std::make_shared(batch, ActualSchema, ""); break; } case NEvWrite::EModificationType::Replace: diff --git a/ydb/core/tx/columnshard/operations/batch_builder/merger.cpp b/ydb/core/tx/columnshard/operations/batch_builder/merger.cpp index eb156e5c2451..713a397c7a4e 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/merger.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/merger.cpp @@ -60,10 +60,12 @@ NKikimr::TConclusionStatus TUpdateMerger::OnEqualKeys(const NArrow::NMerger::TSo return TConclusionStatus::Success(); } -TUpdateMerger::TUpdateMerger(const std::shared_ptr& incoming, const std::shared_ptr& actualSchema, const std::optional& defaultExists /*= {}*/) +TUpdateMerger::TUpdateMerger(const std::shared_ptr& incoming, const std::shared_ptr& actualSchema, + const TString& insertDenyReason, const std::optional& defaultExists /*= {}*/) : TBase(incoming, actualSchema) , Builder(actualSchema->GetIndexInfo().ArrowSchema()->fields()) , DefaultExists(defaultExists) + , InsertDenyReason(insertDenyReason) { for (auto&& i : actualSchema->GetIndexInfo().ArrowSchema()->field_names()) { auto fIdx = IncomingData->schema()->GetFieldIndex(i); diff --git a/ydb/core/tx/columnshard/operations/batch_builder/merger.h b/ydb/core/tx/columnshard/operations/batch_builder/merger.h index c9c8a986c13e..e503f742b18f 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/merger.h +++ b/ydb/core/tx/columnshard/operations/batch_builder/merger.h @@ -79,8 +79,12 @@ class TUpdateMerger: public IMerger { std::vector> IncomingColumnRemap; std::vector> HasIncomingDataFlags; const std::optional DefaultExists; + const TString InsertDenyReason; virtual TConclusionStatus OnEqualKeys(const NArrow::NMerger::TSortableBatchPosition& exists, const NArrow::NMerger::TSortableBatchPosition& incoming) override; virtual TConclusionStatus OnIncomingOnly(const NArrow::NMerger::TSortableBatchPosition& incoming) override { + if (!!InsertDenyReason) { + return TConclusionStatus::Fail("insertion is impossible: " + InsertDenyReason); + } if (!DefaultExists) { return TConclusionStatus::Success(); } else { @@ -93,7 +97,7 @@ class TUpdateMerger: public IMerger { } TUpdateMerger(const std::shared_ptr& incoming, const std::shared_ptr& actualSchema, - const std::optional& defaultExists = {}); + const TString& insertDenyReason, const std::optional& defaultExists = {}); }; } diff --git a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp index 426c8045fe88..d195c5056470 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp @@ -43,7 +43,8 @@ TConclusionStatus TBuildSlicesTask::DoExecute(const std::shared_ptr& /*ta return TConclusionStatus::Fail("no data in batch"); } const auto& indexSchema = ActualSchema->GetIndexInfo().ArrowSchema(); - auto reorderConclusion = NArrow::TColumnOperator().Reorder(OriginalBatch, indexSchema->field_names()); + NArrow::TSchemaSubset subset; + auto reorderConclusion = NArrow::TColumnOperator().Adapt(OriginalBatch, indexSchema, &subset); if (reorderConclusion.IsFail()) { AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "unadaptable schemas")("index", indexSchema->ToString())("problem", reorderConclusion.GetErrorMessage()); ReplyError("cannot reorder schema: " + reorderConclusion.GetErrorMessage()); @@ -51,18 +52,17 @@ TConclusionStatus TBuildSlicesTask::DoExecute(const std::shared_ptr& /*ta } else { OriginalBatch = reorderConclusion.DetachResult(); } - if (!OriginalBatch->schema()->Equals(indexSchema)) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "unequal schemas")("batch", OriginalBatch->schema()->ToString()) - ("index", indexSchema->ToString()); - ReplyError("unequal schemas"); - return TConclusionStatus::Fail("unequal schemas"); - } + if (OriginalBatch->num_columns() != indexSchema->num_fields()) { + AFL_VERIFY(OriginalBatch->num_columns() < indexSchema->num_fields())("original", OriginalBatch->num_columns())( + "index", indexSchema->num_fields()); + } WriteData.MutableWriteMeta().SetWriteMiddle2StartInstant(TMonotonic::Now()); auto batches = BuildSlices(); WriteData.MutableWriteMeta().SetWriteMiddle3StartInstant(TMonotonic::Now()); if (batches) { auto writeDataPtr = std::make_shared(std::move(WriteData)); + writeDataPtr->SetSchemaSubset(std::move(subset)); auto result = std::make_unique(writeDataPtr, std::move(*batches)); TActorContext::AsActorContext().Send(BufferActorId, result.release()); } else { diff --git a/ydb/core/tx/columnshard/splitter/batch_slice.h b/ydb/core/tx/columnshard/splitter/batch_slice.h index acbb9a0414e4..00a27a3d4739 100644 --- a/ydb/core/tx/columnshard/splitter/batch_slice.h +++ b/ydb/core/tx/columnshard/splitter/batch_slice.h @@ -65,13 +65,13 @@ class TGeneralSerializedSlice { std::shared_ptr Counters; TGeneralSerializedSlice() = default; - const TSplittedEntity& GetEntityDataVerified(const ui32& entityId) const { + const TSplittedEntity& GetEntityDataVerified(const ui32 entityId) const { for (auto&& i : Data) { if (i.GetEntityId() == entityId) { return i; } } - Y_ABORT_UNLESS(false); + AFL_VERIFY(false)("id", entityId); return Data.front(); } bool GroupBlobsImpl(const NSplitter::TGroupFeatures& features, std::vector& blobs); diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h index 127e5464fc12..fe73059a3758 100644 --- a/ydb/core/tx/data_events/write_data.h +++ b/ydb/core/tx/data_events/write_data.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -72,9 +73,15 @@ class TWriteData { YDB_READONLY_DEF(IDataContainer::TPtr, Data); YDB_READONLY_DEF(std::shared_ptr, PrimaryKeySchema); YDB_READONLY_DEF(std::shared_ptr, BlobsAction); + YDB_ACCESSOR_DEF(std::optional, SchemaSubset); public: TWriteData(const TWriteMeta& writeMeta, IDataContainer::TPtr data, const std::shared_ptr& primaryKeySchema, const std::shared_ptr& blobsAction); + const NArrow::TSchemaSubset& GetSchemaSubsetVerified() const { + AFL_VERIFY(SchemaSubset); + return *SchemaSubset; + } + const TWriteMeta& GetWriteMeta() const { return WriteMeta; }