Skip to content

Commit 96881ec

Browse files
ivanmorozov333ivanmorozov333
committed
fixes
1 parent ab9fd33 commit 96881ec

File tree

13 files changed

+35
-40
lines changed

13 files changed

+35
-40
lines changed

‎ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetching.cpp

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -191,23 +191,6 @@ TConclusion<bool> TProgramStepPrepare::DoExecuteInplace(const std::shared_ptr<ID
191191
logic->Start(readActions, context);
192192
AFL_VERIFY(fetchers.emplace(i.GetColumnId(), logic).second)("column_id", i.GetColumnId());
193193
}
194-
auto logicIndexesList = Step->GetIndexesToFetch(source->GetSourceSchema());
195-
for (auto&& i : Step.GetOriginalIndexesToUse()) {
196-
const NIndexes::TIndexMetaContainer indexMeta = source->GetSourceSchema()->GetIndexVerified(i.GetIndexId());
197-
auto chunks = source->GetStageData().GetPortionAccessor().GetIndexChunksPointers(i.GetColumnId());
198-
std::vector<NIndexes::TChunkOriginalData> originalChunks;
199-
for (auto&& c : chunks) {
200-
if (auto range = c->GetBlobRangeOptional()) {
201-
originalChunks.emplace_back(source->GetStageData().GetPortionAccessor().GetPortionInfo().RestoreBlobRange(*range));
202-
} else {
203-
originalChunks.emplace_back(c->GetBlobDataVerified());
204-
}
205-
}
206-
std::shared_ptr<IKernelFetchLogic> logic = indexMeta->BuildFetchTask(
207-
i, originalChunks, *source->GetStageData().GetIndexes(), indexMeta, source->GetContext()->GetCommonContext()->GetStoragesManager());
208-
logic->Start(readActions, context);
209-
AFL_VERIFY(fetchers.emplace(i.GetColumnId(), logic).second)("column_id", i.GetColumnId());
210-
}
211194
if (readActions.IsEmpty()) {
212195
NBlobOperations::NRead::TCompositeReadBlobs blobs;
213196
for (auto&& i : fetchers) {

‎ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::DoGetColumnsFetchingPlan(c
4747
return false;
4848
}
4949
}();
50-
const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false);
50+
const bool useIndexes = (IndexChecker ? source->HasIndexes(NIndexes::TIndexDataAddress::ExtractIndexIds(IndexChecker->GetIndexIds())) : false);
5151
const bool isWholeExclusiveSource = source->GetExclusiveIntervalOnly() && source->IsSourceInMemory();
5252
const bool needSnapshots = GetReadMetadata()->GetRequestSnapshot() < source->GetRecordSnapshotMax() || !isWholeExclusiveSource;
5353
const bool hasDeletions = source->GetHasDeletions();
@@ -87,7 +87,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c
8787

8888
NCommon::TColumnsAccumulator acc(GetMergeColumns(), GetReadMetadata()->GetResultSchema());
8989
if (!!IndexChecker && useIndexes && exclusiveSource) {
90-
result->AddStep(std::make_shared<TIndexBlobsFetchingStep>(std::make_shared<TIndexesSet>(IndexChecker->GetIndexIds())));
90+
result->AddStep(std::make_shared<TIndexBlobsFetchingStep>(std::make_shared<TIndexesSet>(NIndexes::TIndexDataAddress::ExtractIndexIds(IndexChecker->GetIndexIds()))));
9191
result->AddStep(std::make_shared<TApplyIndexStep>(IndexChecker));
9292
}
9393
bool hasFilterSharding = false;

‎ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,8 @@ void TPortionDataSource::DoAbort() {
174174
}
175175

176176
void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) {
177-
THashMap<ui32, std::vector<TString>> indexBlobs;
178-
std::set<ui32> indexIds = indexChecker->GetIndexIds();
177+
THashMap<NIndexes::TIndexDataAddress, std::vector<TString>> indexBlobs;
178+
std::set<ui32> indexIds = NIndexes::TIndexDataAddress::ExtractIndexIds(indexChecker->GetIndexIds());
179179
// NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString());
180180
std::vector<TPortionDataAccessor::TPage> pages = GetStageData().GetPortionAccessor().BuildPages();
181181
NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter();
@@ -185,12 +185,12 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in
185185
continue;
186186
}
187187
if (i->HasBlobData()) {
188-
indexBlobs[i->GetIndexId()].emplace_back(i->GetBlobDataVerified());
188+
indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(i->GetBlobDataVerified());
189189
} else {
190-
indexBlobs[i->GetIndexId()].emplace_back(StageData->ExtractBlob(i->GetAddress()));
190+
indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(StageData->ExtractBlob(i->GetAddress()));
191191
}
192192
}
193-
for (auto&& i : indexIds) {
193+
for (auto&& i : indexChecker->GetIndexIds()) {
194194
if (!indexBlobs.contains(i)) {
195195
return;
196196
}

‎ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::DoGetColumnsFetchingPlan(c
3131
return false;
3232
}
3333
}();
34-
const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false);
34+
35+
const bool useIndexes = (IndexChecker ? source->HasIndexes(NIndexes::TIndexDataAddress::ExtractIndexIds(IndexChecker->GetIndexIds())) : false);
3536
const bool hasDeletions = source->GetHasDeletions();
3637
bool needShardingFilter = false;
3738
if (!!GetReadMetadata()->GetRequestShardingInfo()) {
@@ -62,7 +63,8 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c
6263

6364
NCommon::TColumnsAccumulator acc(GetMergeColumns(), GetReadMetadata()->GetResultSchema());
6465
if (!!IndexChecker && useIndexes) {
65-
result->AddStep(std::make_shared<TIndexBlobsFetchingStep>(IndexChecker->GetIndexIds()));
66+
result->AddStep(std::make_shared<TIndexBlobsFetchingStep>(
67+
std::make_shared<TIndexesSet>(NIndexes::TIndexDataAddress::ExtractIndexIds(IndexChecker->GetIndexIds()))));
6668
result->AddStep(std::make_shared<TApplyIndexStep>(IndexChecker));
6769
}
6870
if (needFilterSharding && !GetShardingColumns()->IsEmpty()) {

‎ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -171,8 +171,8 @@ void TPortionDataSource::DoAbort() {
171171
}
172172

173173
void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) {
174-
THashMap<ui32, std::vector<TString>> indexBlobs;
175-
const std::set<TIndexDataAddress> indexIds = indexChecker->GetIndexIds();
174+
THashMap<NIndexes::TIndexDataAddress, std::vector<TString>> indexBlobs;
175+
const auto indexIds = NIndexes::TIndexDataAddress::ExtractIndexIds(indexChecker->GetIndexIds());
176176
// NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetPortionId());
177177
std::vector<TPortionDataAccessor::TPage> pages = GetStageData().GetPortionAccessor().BuildPages();
178178
NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter();
@@ -182,12 +182,12 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in
182182
continue;
183183
}
184184
if (i->HasBlobData()) {
185-
indexBlobs[i->GetIndexId()].emplace_back(i->GetBlobDataVerified());
185+
indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(i->GetBlobDataVerified());
186186
} else {
187-
indexBlobs[i->GetIndexId()].emplace_back(StageData->ExtractBlob(i->GetAddress()));
187+
indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(StageData->ExtractBlob(i->GetAddress()));
188188
}
189189
}
190-
for (auto&& i : indexIds) {
190+
for (auto&& i : indexChecker->GetIndexIds()) {
191191
if (!indexBlobs.contains(i)) {
192192
GetContext()->GetCommonContext()->GetCounters().OnNotIndexBlobs();
193193
return;

‎ydb/core/tx/columnshard/engines/scheme/indexes/abstract/checker.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@ class TIndexDataAddress {
1616
YDB_READONLY_DEF(std::optional<ui64>, Category);
1717

1818
public:
19+
template <class TContainer>
20+
static std::set<ui32> ExtractIndexIds(const TContainer& addresses) {
21+
std::set<ui32> result;
22+
for (auto&& i : addresses) {
23+
result.emplace(i.GetIndexId());
24+
}
25+
return result;
26+
}
27+
1928
TIndexDataAddress() = default;
2029

2130
explicit TIndexDataAddress(const ui32 indexId)

‎ydb/core/tx/columnshard/engines/scheme/indexes/abstract/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ SRCS(
99
collection.cpp
1010
header.cpp
1111
fetcher.cpp
12+
abstract.cpp
1213
meta.cpp
1314
checker.cpp
1415
GLOBAL composite.cpp

‎ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,11 +366,11 @@ ui32 ISnapshotSchema::GetIndexesCount() const {
366366
return GetIndexInfo().GetIndexes().size();
367367
}
368368

369-
std::shared_ptr<NIndexes::IIndexMeta> ISnapshotSchema::GetIndexVerified(const ui32 indexId) const {
369+
NIndexes::TIndexMetaContainer ISnapshotSchema::GetIndexVerified(const ui32 indexId) const {
370370
return GetIndexInfo().GetIndexVerified(indexId);
371371
}
372372

373-
std::shared_ptr<NIndexes::IIndexMeta> ISnapshotSchema::GetIndexOptional(const ui32 indexId) const {
373+
NIndexes::TIndexMetaContainer ISnapshotSchema::GetIndexOptional(const ui32 indexId) const {
374374
return GetIndexInfo().GetIndexOptional(indexId);
375375
}
376376

‎ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
namespace NKikimr::NOlap {
1515

1616
namespace NIndexes {
17-
class IIndexMeta;
17+
class TIndexMetaContainer;
1818
}
1919

2020
struct TIndexInfo;
@@ -33,8 +33,8 @@ class ISnapshotSchema {
3333
std::shared_ptr<NArrow::NAccessor::TColumnLoader> GetColumnLoaderVerified(const ui32 columnId) const;
3434
std::shared_ptr<NArrow::NAccessor::TColumnLoader> GetColumnLoaderOptional(const std::string& columnName) const;
3535
std::shared_ptr<NArrow::NAccessor::TColumnLoader> GetColumnLoaderVerified(const std::string& columnName) const;
36-
std::shared_ptr<NIndexes::IIndexMeta> GetIndexVerified(const ui32 indexId) const;
37-
std::shared_ptr<NIndexes::IIndexMeta> GetIndexOptional(const ui32 indexId) const;
36+
NIndexes::TIndexMetaContainer GetIndexVerified(const ui32 indexId) const;
37+
NIndexes::TIndexMetaContainer GetIndexOptional(const ui32 indexId) const;
3838

3939
bool IsSpecialColumnId(const ui32 columnId) const;
4040
virtual TColumnIdsView GetColumnIds() const = 0;

‎ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/checker.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class TFilterChecker: public TSimpleIndexChecker {
2222
public:
2323
TFilterChecker() = default;
2424
TFilterChecker(const ui32 indexId, std::set<ui64>&& hashes)
25-
: TBase(indexId)
25+
: TBase(TIndexDataAddress(indexId))
2626
, HashValues(std::move(hashes)) {
2727
}
2828
virtual TString GetClassName() const override {

‎ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class TCountMinSketchChecker: public TSimpleIndexChecker {
2121
public:
2222
TCountMinSketchChecker() = default;
2323
TCountMinSketchChecker(const ui32 indexId)
24-
: TBase(indexId)
24+
: TBase(TIndexDataAddress(indexId))
2525
{}
2626

2727
virtual TString GetClassName() const override {

‎ydb/core/tx/columnshard/engines/storage/indexes/portions/extractor/default.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ THashMap<ui64, ui32> TDefaultDataExtractor::DoGetIndexHitsCount(const std::share
6363
}
6464
{
6565
auto& stats = subColumns->GetOthersData().GetStats();
66-
for (ui32 i = 0; i < stats.GetOthersData(); ++i) {
66+
for (ui32 i = 0; i < stats.GetColumnsCount(); ++i) {
6767
result[NRequest::TOriginalDataAddress::CalcSubColumnHash(stats.GetColumnName(i))] += stats.GetColumnRecordsCount(i);
6868
}
6969
}

‎ydb/core/tx/columnshard/engines/storage/indexes/portions/extractor/sub_column.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ class TSubColumnDataExtractor: public IReadDataExtractor {
4949
return request.GetSubColumnName() == SubColumnName;
5050
}
5151

52-
virtual ui32 DoGetIndexHitsCount(const std::shared_ptr<NArrow::NAccessor::IChunkedArray>& dataArray) const override;
52+
virtual THashMap<ui64, ui32> DoGetIndexHitsCount(const std::shared_ptr<NArrow::NAccessor::IChunkedArray>& dataArray) const override;
5353

5454
public:
5555
virtual TString GetClassName() const override {

0 commit comments

Comments
 (0)