Show RuntimeFeatures on portion_stats sys-view (#7673)

swalrus1 · web-flow · commit ac738f904e8a · 2024-08-16T14:44:00.000+03:00
diff --git a/ydb/core/kqp/ut/olap/helpers/get_value.cpp b/ydb/core/kqp/ut/olap/helpers/get_value.cpp
@@ -36,6 +36,11 @@ void PrintValue(IOutputStream& out, const NYdb::TValue& v) {
             out << value.GetInt64();
             break;
         }
+        case NYdb::EPrimitiveType::Uint8:
+        {
+            out << value.GetUint8();
+            break;
+        }
         case NYdb::EPrimitiveType::Utf8:
         {
             out << value.GetUtf8();
diff --git a/ydb/core/kqp/ut/olap/helpers/typed_local.cpp b/ydb/core/kqp/ut/olap/helpers/typed_local.cpp
@@ -79,7 +79,7 @@ NKikimr::NKqp::TTypedLocalHelper::TDistribution TTypedLocalHelper::GetDistributi
 }
 
 void TTypedLocalHelper::GetVolumes(ui64& rawBytes, ui64& bytes, const bool verbose /*= false*/, const std::vector<TString> columnNames /*= {}*/) {
-    TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_stats` WHERE Activity = true";
+    TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_stats` WHERE Activity == 1";
     if (columnNames.size()) {
         selectQuery += " AND EntityName IN ('" + JoinSeq("','", columnNames) + "')";
     }
@@ -144,7 +144,7 @@ void TTypedLocalHelper::FillPKOnly(const double pkKff /*= 0*/, const ui32 numRow
 }
 
 void TTypedLocalHelper::GetStats(std::vector<NJson::TJsonValue>& stats, const bool verbose /*= false*/) {
-    TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_portion_stats` WHERE Activity = true";
+    TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_portion_stats` WHERE Activity == 1";
     auto tableClient = KikimrRunner.GetTableClient();
     auto rows = ExecuteScanQuery(tableClient, selectQuery, verbose);
     for (auto&& r : rows) {
diff --git a/ydb/core/kqp/ut/olap/sys_view_ut.cpp b/ydb/core/kqp/ut/olap/sys_view_ut.cpp
@@ -453,7 +453,7 @@ Y_UNIT_TEST_SUITE(KqpOlapSysView) {
                 SELECT PathId, Kind, TabletId
                 FROM `/Root/olapStore/.sys/store_primary_index_stats`
                 WHERE
-                    PathId == UInt64("3") AND Activity = true
+                    PathId == UInt64("3") AND Activity == 1
                 GROUP BY TabletId, PathId, Kind
                 ORDER BY TabletId, Kind
             )");
diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h
@@ -418,7 +418,7 @@ struct Schema : NIceDb::Schema {
         struct BlobId : Column<10, NScheme::NTypeIds::Utf8> {};
         struct BlobRangeOffset : Column<11, NScheme::NTypeIds::Uint64> {};
         struct BlobRangeSize : Column<12, NScheme::NTypeIds::Uint64> {};
-        struct Activity : Column<13, NScheme::NTypeIds::Bool> {};
+        struct Activity : Column<13, NScheme::NTypeIds::Uint8> {};
         struct TierName: Column<14, NScheme::NTypeIds::Utf8> {};
         struct EntityType: Column<15, NScheme::NTypeIds::Utf8> {};
 
@@ -528,9 +528,10 @@ struct Schema : NIceDb::Schema {
         struct ColumnBlobBytes: Column<7, NScheme::NTypeIds::Uint64> {};
         struct IndexBlobBytes: Column<8, NScheme::NTypeIds::Uint64> {};
         struct PortionId: Column<9, NScheme::NTypeIds::Uint64> {};
-        struct Activity: Column<10, NScheme::NTypeIds::Bool> {};
+        struct Activity: Column<10, NScheme::NTypeIds::Uint8> {};
         struct TierName: Column<11, NScheme::NTypeIds::Utf8> {};
         struct Stats: Column<12, NScheme::NTypeIds::Utf8> {};
+        struct Optimized: Column<13, NScheme::NTypeIds::Uint8> {};
 
         using TKey = TableKey<PathId, TabletId, PortionId>;
         using TColumns = TableColumns<
@@ -545,7 +546,8 @@ struct Schema : NIceDb::Schema {
             PortionId,
             Activity,
             TierName,
-            Stats
+            Stats,
+            Optimized
         >;
     };
 
diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp
@@ -71,7 +71,7 @@ void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
             }
             NArrow::Append<arrow::UInt64Type>(*builders[10], r->BlobRange.Offset);
             NArrow::Append<arrow::UInt64Type>(*builders[11], r->BlobRange.Size);
-            NArrow::Append<arrow::BooleanType>(*builders[12], activity);
+            NArrow::Append<arrow::UInt8Type>(*builders[12], activity);
 
             NArrow::Append<arrow::StringType>(*builders[13], arrow::util::string_view(lastTierName.data(), lastTierName.size()));
             NArrow::Append<arrow::StringType>(*builders[14], ConstantEntityIsColumnView);
@@ -105,7 +105,7 @@ void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
                 NArrow::Append<arrow::UInt64Type>(*builders[10], 0);
                 NArrow::Append<arrow::UInt64Type>(*builders[11], bData->size());
             }
-            NArrow::Append<arrow::BooleanType>(*builders[12], activity);
+            NArrow::Append<arrow::UInt8Type>(*builders[12], activity);
             const auto tierName = portionSchema->GetIndexInfo().GetEntityStorageId(r->GetIndexId(), portion.GetMeta().GetTierName());
             std::string strTierName(tierName.data(), tierName.size());
             NArrow::Append<arrow::StringType>(*builders[13], strTierName);
diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.cpp
@@ -16,7 +16,7 @@ void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
     NArrow::Append<arrow::UInt64Type>(*builders[6], portion.GetColumnBlobBytes());
     NArrow::Append<arrow::UInt64Type>(*builders[7], portion.GetIndexBlobBytes());
     NArrow::Append<arrow::UInt64Type>(*builders[8], portion.GetPortionId());
-    NArrow::Append<arrow::BooleanType>(*builders[9], !portion.IsRemovedFor(ReadMetadata->GetRequestSnapshot()));
+    NArrow::Append<arrow::UInt8Type>(*builders[9], !portion.IsRemovedFor(ReadMetadata->GetRequestSnapshot()));
 
     auto tierName = portion.GetTierNameDef(NBlobOperations::TGlobal::DefaultStorageId);
     NArrow::Append<arrow::StringType>(*builders[10], arrow::util::string_view(tierName.data(), tierName.size()));
@@ -31,6 +31,8 @@ void TStatsIterator::AppendStats(const std::vector<std::unique_ptr<arrow::ArrayB
     }
     auto statInfo = statReport.GetStringRobust();
     NArrow::Append<arrow::StringType>(*builders[11], arrow::util::string_view(statInfo.data(), statInfo.size()));
+
+    NArrow::Append<arrow::UInt8Type>(*builders[12], portion.HasRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized));
 }
 
 ui32 TStatsIterator::PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const {
diff --git a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/program.cpp b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/program.cpp
@@ -401,12 +401,15 @@ class TNormalForm {
 public:
     TNormalForm() = default;
 
-    bool Add(const NSsa::TAssign& assign) {
+    bool Add(const NSsa::TAssign& assign, const TProgramContainer& program) {
         std::vector<std::shared_ptr<IRequestNode>> argNodes;
         for (auto&& arg : assign.GetArguments()) {
             if (arg.IsGenerated()) {
                 auto it = Nodes.find(arg.GetColumnName());
-                AFL_VERIFY(it != Nodes.end());
+                if (it == Nodes.end()) {
+                    AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "program_arg_is_missing")("program", program.DebugString());
+                    return false;
+                }
                 argNodes.emplace_back(it->second);
             } else {
                 argNodes.emplace_back(std::make_shared<TOriginalColumn>(arg.GetColumnName()));
@@ -443,7 +446,7 @@ std::shared_ptr<TDataForIndexesCheckers> TDataForIndexesCheckers::Build(const TP
     auto fStep = program.GetSteps().front();
     TNormalForm nForm;
     for (auto&& s : fStep->GetAssignes()) {
-        if (!nForm.Add(s)) {
+        if (!nForm.Add(s, program)) {
             return nullptr;
         }
     }
diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp
@@ -2436,7 +2436,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) {
                     ui64 numRows = static_cast<arrow::UInt64Array&>(*rows).Value(i);
                     ui64 numBytes = static_cast<arrow::UInt64Array&>(*bytes).Value(i);
                     ui64 numRawBytes = static_cast<arrow::UInt64Array&>(*rawBytes).Value(i);
-                    bool activity = static_cast<arrow::BooleanArray&>(*activities).Value(i);
+                    bool activity = static_cast<arrow::UInt8Array&>(*activities).Value(i);
                     if (!activity) {
                         continue;
                     }

Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,11 @@ void PrintValue(IOutputStream& out, const NYdb::TValue& v) {`
`36`	`36`	`out << value.GetInt64();`
`37`	`37`	`break;`
`38`	`38`	`}`
	`39`	`+ case NYdb::EPrimitiveType::Uint8:`
	`40`	`+ {`
	`41`	`+ out << value.GetUint8();`
	`42`	`+ break;`
	`43`	`+ }`
`39`	`44`	`case NYdb::EPrimitiveType::Utf8:`
`40`	`45`	`{`
`41`	`46`	`out << value.GetUtf8();`
Original file line number	Diff line number	Diff line change
`@@ -79,7 +79,7 @@ NKikimr::NKqp::TTypedLocalHelper::TDistribution TTypedLocalHelper::GetDistributi`
`79`	`79`	`}`
`80`	`80`
`81`	`81`	`void TTypedLocalHelper::GetVolumes(ui64& rawBytes, ui64& bytes, const bool verbose /= false/, const std::vector<TString> columnNames /= {}/) {`
`82`		- TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_stats` WHERE Activity = true";
	`82`	+ TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_stats` WHERE Activity == 1";
`83`	`83`	`if (columnNames.size()) {`
`84`	`84`	`selectQuery += " AND EntityName IN ('" + JoinSeq("','", columnNames) + "')";`
`85`	`85`	`}`
`@@ -144,7 +144,7 @@ void TTypedLocalHelper::FillPKOnly(const double pkKff /= 0/, const ui32 numRow`
`144`	`144`	`}`
`145`	`145`
`146`	`146`	`void TTypedLocalHelper::GetStats(std::vector<NJson::TJsonValue>& stats, const bool verbose /= false/) {`
`147`		- TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_portion_stats` WHERE Activity = true";
	`147`	+ TString selectQuery = "SELECT * FROM `" + TablePath + "/.sys/primary_index_portion_stats` WHERE Activity == 1";
`148`	`148`	`auto tableClient = KikimrRunner.GetTableClient();`
`149`	`149`	`auto rows = ExecuteScanQuery(tableClient, selectQuery, verbose);`
`150`	`150`	`for (auto&& r : rows) {`
Original file line number	Diff line number	Diff line change
`@@ -2436,7 +2436,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) {`
`2436`	`2436`	`ui64 numRows = static_cast<arrow::UInt64Array&>(*rows).Value(i);`
`2437`	`2437`	`ui64 numBytes = static_cast<arrow::UInt64Array&>(*bytes).Value(i);`
`2438`	`2438`	`ui64 numRawBytes = static_cast<arrow::UInt64Array&>(*rawBytes).Value(i);`
`2439`		`- bool activity = static_cast<arrow::BooleanArray&>(*activities).Value(i);`
	`2439`	`+ bool activity = static_cast<arrow::UInt8Array&>(*activities).Value(i);`
`2440`	`2440`	`if (!activity) {`
`2441`	`2441`	`continue;`
`2442`	`2442`	`}`