implement sparse data accessor (#7055)

Author: ivanmorozov333

ydb/core/formats/arrow/common/accessor.cpp renamed to ydb/core/formats/arrow/accessor/abstract/accessor.cpp

@@ -5,6 +5,8 @@
 #include <ydb/library/actors/core/log.h>
 #include <ydb/core/formats/arrow/permutations.h>
 #include <ydb/core/formats/arrow/arrow_helpers.h>
+#include <ydb/core/formats/arrow/splitter/simple.h>
+#include <ydb/core/formats/arrow/save_load/saver.h>
 
 namespace NKikimr::NArrow::NAccessor {
 
@@ -72,64 +74,32 @@ const std::partial_ordering IChunkedArray::TAddress::Compare(const TAddress& ite
     return TComparator::TypedCompare<true>(*Array, Position, *item.Array, item.Position);
 }
 
-namespace {
-class TChunkAccessor {
-private:
-    std::shared_ptr<arrow::ChunkedArray> ChunkedArray;
-public:
-    TChunkAccessor(const std::shared_ptr<arrow::ChunkedArray>& chunkedArray)
-        : ChunkedArray(chunkedArray)
-    {
-
-    }
-    ui64 GetChunksCount() const {
-        return (ui64)ChunkedArray->num_chunks();
-    }
-    ui64 GetChunkLength(const ui32 idx) const {
-        return (ui64)ChunkedArray->chunk(idx)->length();
-    }
-    std::shared_ptr<arrow::Array> GetArray(const ui32 idx) const {
-        return ChunkedArray->chunk(idx);
-    }
-};
-
-}
-
-std::optional<ui64> TTrivialArray::DoGetRawSize() const {
-    return NArrow::GetArrayDataSize(Array);
+ TChunkedArraySerialized::TChunkedArraySerialized(const std::shared_ptr<IChunkedArray>& array, const TString& serializedData)
+    : Array(array)
+    , SerializedData(serializedData) {
+    AFL_VERIFY(serializedData);
+    AFL_VERIFY(Array);
+    AFL_VERIFY(Array->GetRecordsCount());
 }
 
 std::partial_ordering IChunkedArray::TCurrentChunkAddress::Compare(const ui64 position, const TCurrentChunkAddress& item, const ui64 itemPosition) const {
-    AFL_VERIFY(StartPosition <= position);
-    AFL_VERIFY(position < FinishPosition);
-    AFL_VERIFY(item.StartPosition <= itemPosition);
-    AFL_VERIFY(itemPosition < item.FinishPosition);
-    return TComparator::TypedCompare<true>(*Array, position - StartPosition, *item.Array, itemPosition - item.StartPosition);
+    AFL_VERIFY(GetStartPosition() <= position)("pos", position)("start", GetStartPosition());
+    AFL_VERIFY(position < GetFinishPosition())("pos", position)("finish", GetFinishPosition());
+    AFL_VERIFY(item.GetStartPosition() <= itemPosition)("start", item.GetStartPosition())("item", itemPosition);
+    AFL_VERIFY(itemPosition < item.GetFinishPosition())("item", itemPosition)("finish", item.GetFinishPosition());
+    return TComparator::TypedCompare<true>(*Array, position - GetStartPosition(), *item.Array, itemPosition - item.GetStartPosition());
 }
 
 std::shared_ptr<arrow::Array> IChunkedArray::TCurrentChunkAddress::CopyRecord(const ui64 recordIndex) const {
-    AFL_VERIFY(StartPosition <= recordIndex);
-    AFL_VERIFY(recordIndex < FinishPosition);
-    return NArrow::CopyRecords(Array, { recordIndex - StartPosition });
+    AFL_VERIFY(GetStartPosition() <= recordIndex);
+    AFL_VERIFY(recordIndex < GetFinishPosition());
+    return NArrow::CopyRecords(Array, { recordIndex - GetStartPosition() });
 }
 
 TString IChunkedArray::TCurrentChunkAddress::DebugString(const ui64 position) const {
-    AFL_VERIFY(position < FinishPosition);
-    AFL_VERIFY(StartPosition <= position);
-    return NArrow::DebugString(Array, position - StartPosition);
-}
-
-IChunkedArray::TCurrentChunkAddress TTrivialChunkedArray::DoGetChunk(const std::optional<TCurrentChunkAddress>& chunkCurrent, const ui64 position) const {
-    TChunkAccessor accessor(Array);
-    return SelectChunk(chunkCurrent, position, accessor);
-}
-
-std::optional<ui64> TTrivialChunkedArray::DoGetRawSize() const {
-    ui64 result = 0;
-    for (auto&& i : Array->chunks()) {
-        result += NArrow::GetArrayDataSize(i);
-    }
-    return result;
+    AFL_VERIFY(position < GetFinishPosition());
+    AFL_VERIFY(GetStartPosition() <= position);
+    return NArrow::DebugString(Array, position - GetStartPosition());
 }
 
 }