Skip to content

Commit 7b528af

Browse files
Fix chunks scanner (#7913)
1 parent 4372675 commit 7b528af

File tree

1 file changed

+16
-9
lines changed

1 file changed

+16
-9
lines changed

ydb/core/formats/arrow/arrow_helpers.cpp

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -940,16 +940,23 @@ std::vector<std::shared_ptr<arrow::RecordBatch>> SliceToRecordBatches(const std:
940940
for (auto&& i : t->columns()) {
941941
ui32 currentPosition = 0;
942942
auto it = i->chunks().begin();
943-
ui32 length = (*it)->length();
943+
ui32 length = 0;
944+
const auto initializeIt = [&length, &it, &i]() {
945+
for (; it != i->chunks().end() && !(*it)->length(); ++it) {
946+
}
947+
if (it != i->chunks().end()) {
948+
length = (*it)->length();
949+
}
950+
};
951+
initializeIt();
944952
for (ui32 idx = 0; idx + 1 < positions.size(); ++idx) {
953+
AFL_VERIFY(it != i->chunks().end());
954+
AFL_VERIFY(positions[idx + 1] - currentPosition <= length)("length", length)("idx+1", positions[idx + 1])("pos", currentPosition);
945955
auto chunk = (*it)->Slice(positions[idx] - currentPosition, positions[idx + 1] - positions[idx]);
946-
AFL_VERIFY_DEBUG(chunk->length() == positions[idx + 1] - positions[idx])("length", chunk->length())(
947-
"delta", positions[idx + 1] - positions[idx]);
948-
AFL_VERIFY_DEBUG(chunk->length())("delta", positions[idx + 1] - positions[idx]);
956+
AFL_VERIFY_DEBUG(chunk->length() == positions[idx + 1] - positions[idx])("length", chunk->length())("expect", positions[idx + 1] - positions[idx]);
949957
if (positions[idx + 1] - currentPosition == length) {
950-
if (++it != i->chunks().end()) {
951-
length = (*it)->length();
952-
}
958+
++it;
959+
initializeIt();
953960
currentPosition = positions[idx + 1];
954961
}
955962
slicedData[idx].emplace_back(chunk);
@@ -958,8 +965,8 @@ std::vector<std::shared_ptr<arrow::RecordBatch>> SliceToRecordBatches(const std:
958965
std::vector<std::shared_ptr<arrow::RecordBatch>> result;
959966
ui32 count = 0;
960967
for (auto&& i : slicedData) {
961-
AFL_VERIFY_DEBUG(i.size());
962-
AFL_VERIFY_DEBUG(i.front()->length());
968+
AFL_VERIFY(i.size());
969+
AFL_VERIFY(i.front()->length());
963970
result.emplace_back(arrow::RecordBatch::Make(t->schema(), i.front()->length(), i));
964971
count += result.back()->num_rows();
965972
}

0 commit comments

Comments
 (0)