Skip to content

Commit e693493

Browse files
fix chunks scanner for empty subchunks in chunked array
1 parent 8078a73 commit e693493

File tree

1 file changed

+8
-5
lines changed

1 file changed

+8
-5
lines changed

ydb/core/formats/arrow/arrow_helpers.cpp

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -942,10 +942,13 @@ std::vector<std::shared_ptr<arrow::RecordBatch>> SliceToRecordBatches(const std:
942942
auto it = i->chunks().begin();
943943
ui32 length = (*it)->length();
944944
for (ui32 idx = 0; idx + 1 < positions.size(); ++idx) {
945+
AFL_VERIFY(it != i->chunks().end());
946+
while (!(*it)->length()) {
947+
AFL_VERIFY(++it != i->chunks().end());
948+
}
949+
AFL_VERIFY(positions[idx + 1] - currentPosition <= length)("length", length)("idx+1", positions[idx + 1])("pos", currentPosition);
945950
auto chunk = (*it)->Slice(positions[idx] - currentPosition, positions[idx + 1] - positions[idx]);
946-
AFL_VERIFY_DEBUG(chunk->length() == positions[idx + 1] - positions[idx])("length", chunk->length())(
947-
"delta", positions[idx + 1] - positions[idx]);
948-
AFL_VERIFY_DEBUG(chunk->length())("delta", positions[idx + 1] - positions[idx]);
951+
AFL_VERIFY_DEBUG(chunk->length() == positions[idx + 1] - positions[idx])("length", chunk->length())("expect", positions[idx + 1] - positions[idx]);
949952
if (positions[idx + 1] - currentPosition == length) {
950953
if (++it != i->chunks().end()) {
951954
length = (*it)->length();
@@ -958,8 +961,8 @@ std::vector<std::shared_ptr<arrow::RecordBatch>> SliceToRecordBatches(const std:
958961
std::vector<std::shared_ptr<arrow::RecordBatch>> result;
959962
ui32 count = 0;
960963
for (auto&& i : slicedData) {
961-
AFL_VERIFY_DEBUG(i.size());
962-
AFL_VERIFY_DEBUG(i.front()->length());
964+
AFL_VERIFY(i.size());
965+
AFL_VERIFY(i.front()->length());
963966
result.emplace_back(arrow::RecordBatch::Make(t->schema(), i.front()->length(), i));
964967
count += result.back()->num_rows();
965968
}

0 commit comments

Comments
 (0)