Skip to content

implement sparse data accessor #7055

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jul 28, 2024
Merged

implement sparse data accessor #7055

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
513b0b6
sparsed array accessor
ivanmorozov333 Jul 27, 2024
cf25d27
split used columnIds by pathId
ivanmorozov333 Jul 27, 2024
2b2ba03
fix build
ivanmorozov333 Jul 27, 2024
4b4894b
fix build
ivanmorozov333 Jul 27, 2024
cb52aca
fix build
ivanmorozov333 Jul 27, 2024
6406672
fix test build
ivanmorozov333 Jul 27, 2024
5c0c3d8
fix
ivanmorozov333 Jul 27, 2024
d260336
fixes
ivanmorozov333 Jul 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 19 additions & 49 deletions ydb/core/formats/arrow/common/accessor.cpp → ...mats/arrow/accessor/abstract/accessor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#include <ydb/library/actors/core/log.h>
#include <ydb/core/formats/arrow/permutations.h>
#include <ydb/core/formats/arrow/arrow_helpers.h>
#include <ydb/core/formats/arrow/splitter/simple.h>
#include <ydb/core/formats/arrow/save_load/saver.h>

namespace NKikimr::NArrow::NAccessor {

Expand Down Expand Up @@ -72,64 +74,32 @@ const std::partial_ordering IChunkedArray::TAddress::Compare(const TAddress& ite
return TComparator::TypedCompare<true>(*Array, Position, *item.Array, item.Position);
}

namespace {
class TChunkAccessor {
private:
std::shared_ptr<arrow::ChunkedArray> ChunkedArray;
public:
TChunkAccessor(const std::shared_ptr<arrow::ChunkedArray>& chunkedArray)
: ChunkedArray(chunkedArray)
{

}
ui64 GetChunksCount() const {
return (ui64)ChunkedArray->num_chunks();
}
ui64 GetChunkLength(const ui32 idx) const {
return (ui64)ChunkedArray->chunk(idx)->length();
}
std::shared_ptr<arrow::Array> GetArray(const ui32 idx) const {
return ChunkedArray->chunk(idx);
}
};

}

std::optional<ui64> TTrivialArray::DoGetRawSize() const {
return NArrow::GetArrayDataSize(Array);
TChunkedArraySerialized::TChunkedArraySerialized(const std::shared_ptr<IChunkedArray>& array, const TString& serializedData)
: Array(array)
, SerializedData(serializedData) {
AFL_VERIFY(serializedData);
AFL_VERIFY(Array);
AFL_VERIFY(Array->GetRecordsCount());
}

std::partial_ordering IChunkedArray::TCurrentChunkAddress::Compare(const ui64 position, const TCurrentChunkAddress& item, const ui64 itemPosition) const {
AFL_VERIFY(StartPosition <= position);
AFL_VERIFY(position < FinishPosition);
AFL_VERIFY(item.StartPosition <= itemPosition);
AFL_VERIFY(itemPosition < item.FinishPosition);
return TComparator::TypedCompare<true>(*Array, position - StartPosition, *item.Array, itemPosition - item.StartPosition);
AFL_VERIFY(GetStartPosition() <= position)("pos", position)("start", GetStartPosition());
AFL_VERIFY(position < GetFinishPosition())("pos", position)("finish", GetFinishPosition());
AFL_VERIFY(item.GetStartPosition() <= itemPosition)("start", item.GetStartPosition())("item", itemPosition);
AFL_VERIFY(itemPosition < item.GetFinishPosition())("item", itemPosition)("finish", item.GetFinishPosition());
return TComparator::TypedCompare<true>(*Array, position - GetStartPosition(), *item.Array, itemPosition - item.GetStartPosition());
}

std::shared_ptr<arrow::Array> IChunkedArray::TCurrentChunkAddress::CopyRecord(const ui64 recordIndex) const {
AFL_VERIFY(StartPosition <= recordIndex);
AFL_VERIFY(recordIndex < FinishPosition);
return NArrow::CopyRecords(Array, { recordIndex - StartPosition });
AFL_VERIFY(GetStartPosition() <= recordIndex);
AFL_VERIFY(recordIndex < GetFinishPosition());
return NArrow::CopyRecords(Array, { recordIndex - GetStartPosition() });
}

TString IChunkedArray::TCurrentChunkAddress::DebugString(const ui64 position) const {
AFL_VERIFY(position < FinishPosition);
AFL_VERIFY(StartPosition <= position);
return NArrow::DebugString(Array, position - StartPosition);
}

IChunkedArray::TCurrentChunkAddress TTrivialChunkedArray::DoGetChunk(const std::optional<TCurrentChunkAddress>& chunkCurrent, const ui64 position) const {
TChunkAccessor accessor(Array);
return SelectChunk(chunkCurrent, position, accessor);
}

std::optional<ui64> TTrivialChunkedArray::DoGetRawSize() const {
ui64 result = 0;
for (auto&& i : Array->chunks()) {
result += NArrow::GetArrayDataSize(i);
}
return result;
AFL_VERIFY(position < GetFinishPosition());
AFL_VERIFY(GetStartPosition() <= position);
return NArrow::DebugString(Array, position - GetStartPosition());
}

}
Loading
Loading