Skip to content

Commit 150c1d2

Browse files
author
Konstantin Morozov
committed
up ColumnLimit
1 parent 360da12 commit 150c1d2

File tree

3 files changed

+45
-24
lines changed

3 files changed

+45
-24
lines changed

‎ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp

+22-6
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
1414
constexpr size_t tables_in_store = 1000;
1515
constexpr size_t size_single_table = 13152;
1616

17-
const TVector<TTestHelper::TColumnSchema> schema = {TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8), TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32)};
17+
const TVector<TTestHelper::TColumnSchema> schema = {
18+
TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false),
19+
TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8),
20+
TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32)};
1821

1922
class TOlapStatsController : public NYDBTest::NColumnShard::TController {
2023
public:
@@ -74,7 +77,10 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
7477
testTableStore.SetName("/Root/TableStoreTest").SetPrimaryKey({"id"}).SetSchema(schema);
7578
testHelper.CreateTable(testTableStore);
7679
TTestHelper::TColumnTable testTable;
77-
testTable.SetName("/Root/TableStoreTest/ColumnTableTest").SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema);
80+
testTable.SetName("/Root/TableStoreTest/ColumnTableTest")
81+
.SetPrimaryKey({"id"})
82+
.SetSharding({"id"})
83+
.SetSchema(schema);
7884
testHelper.CreateTable(testTable);
7985

8086
{
@@ -88,7 +94,8 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
8894
Sleep(TDuration::Seconds(1));
8995

9096
auto settings = TDescribeTableSettings().WithTableStatistics(true);
91-
auto describeResult = testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest", settings).GetValueSync();
97+
auto describeResult =
98+
testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest", settings).GetValueSync();
9299

93100
UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());
94101

@@ -115,12 +122,18 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
115122

116123
for (size_t t = 0; t < tables_in_store; t++) {
117124
TTestHelper::TColumnTable testTable;
118-
testTable.SetName("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t)).SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema);
125+
testTable.SetName("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t))
126+
.SetPrimaryKey({"id"})
127+
.SetSharding({"id"})
128+
.SetSchema(schema);
119129
testHelper.CreateTable(testTable);
120130

121131
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
122132
for (size_t i = 0; i < t + inserted_rows; i++) {
123-
tableInserter.AddRow().Add(i + t * tables_in_store).Add("test_res_" + std::to_string(i + t * tables_in_store)).AddNull();
133+
tableInserter.AddRow()
134+
.Add(i + t * tables_in_store)
135+
.Add("test_res_" + std::to_string(i + t * tables_in_store))
136+
.AddNull();
124137
}
125138
testHelper.InsertData(testTable, tableInserter);
126139
}
@@ -129,7 +142,10 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
129142

130143
auto settings = TDescribeTableSettings().WithTableStatistics(true);
131144
for (size_t t = 0; t < tables_in_store; t++) {
132-
auto describeResult = testHelper.GetSession().DescribeTable("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t), settings).GetValueSync();
145+
auto describeResult =
146+
testHelper.GetSession()
147+
.DescribeTable("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t), settings)
148+
.GetValueSync();
133149
UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString());
134150
const auto& description = describeResult.GetTableDescription();
135151

‎ydb/core/tx/columnshard/columnshard.cpp

+7-4
Original file line numberDiff line numberDiff line change
@@ -293,12 +293,14 @@ void TColumnShard::UpdateResourceMetrics(const TActorContext& ctx, const TUsage&
293293
metrics->TryUpdate(ctx);
294294
}
295295

296-
void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats, ::NKikimrTableStats::TTableStats* tabletStats) {
296+
void TColumnShard::ConfigureStats(const NOlap::TColumnEngineStats& indexStats,
297+
::NKikimrTableStats::TTableStats* tabletStats) {
297298
NOlap::TSnapshot lastIndexUpdate = TablesManager.GetPrimaryIndexSafe().LastUpdate();
298299
auto activeIndexStats = indexStats.Active(); // data stats excluding inactive and evicted
299300

300301
if (activeIndexStats.Rows < 0 || activeIndexStats.Bytes < 0) {
301-
LOG_S_WARN("Negative stats counter. Rows: " << activeIndexStats.Rows << " Bytes: " << activeIndexStats.Bytes << TabletID());
302+
LOG_S_WARN("Negative stats counter. Rows: " << activeIndexStats.Rows << " Bytes: " << activeIndexStats.Bytes
303+
<< TabletID());
302304

303305
activeIndexStats.Rows = (activeIndexStats.Rows < 0) ? 0 : activeIndexStats.Rows;
304306
activeIndexStats.Bytes = (activeIndexStats.Bytes < 0) ? 0 : activeIndexStats.Bytes;
@@ -345,7 +347,8 @@ void TColumnShard::FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDa
345347
}
346348
}
347349

348-
void TColumnShard::FillColumnTableStats(const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
350+
void TColumnShard::FillColumnTableStats(const TActorContext& ctx,
351+
std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
349352
if (!TablesManager.HasPrimaryIndex()) {
350353
return;
351354
}
@@ -410,4 +413,4 @@ void TColumnShard::SendPeriodicStats() {
410413
NTabletPipe::SendData(ctx, StatsReportPipe, ev.release());
411414
}
412415

413-
}
416+
} // namespace NKikimr::NColumnShard

‎ydb/core/tx/schemeshard/schemeshard__table_stats.cpp

+16-14
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,9 @@ TPartitionStats TTxStoreTableStats::PrepareStats(const TActorContext& ctx, const
180180
return newStats;
181181
}
182182

183-
bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, const TStatsQueueItem<TEvDataShard::TEvPeriodicTableStats>& item, NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
183+
bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
184+
const TStatsQueueItem<TEvDataShard::TEvPeriodicTableStats>& item,
185+
NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) {
184186
const auto& rec = item.Ev->Get()->Record;
185187
const auto datashardId = TTabletId(rec.GetDatashardId());
186188

@@ -205,19 +207,17 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, const TStatsQ
205207
TShardIdx shardIdx = Self->TabletIdToShardIdx[datashardId];
206208

207209
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
208-
"TTxStoreTableStats.PersistSingleStats: main stats from"
209-
<< " datashardId(TabletID)=" << datashardId
210-
<< " maps to shardIdx: " << shardIdx
211-
<< ", pathId: " << pathId
212-
<< ", pathId map=" << Self->PathsById[pathId]->Name
213-
<< ", is column=" << isColumnTable
214-
<< ", is olap=" << isOlapStore);
210+
"TTxStoreTableStats.PersistSingleStats: main stats from"
211+
<< " datashardId(TabletID)=" << datashardId << " maps to shardIdx: " << shardIdx
212+
<< ", pathId: " << pathId << ", pathId map=" << Self->PathsById[pathId]->Name
213+
<< ", is column=" << isColumnTable << ", is olap=" << isOlapStore);
215214

216215
const TPartitionStats newStats = PrepareStats(ctx, rec);
217216

218217
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
219-
"Add stats from shard with datashardId(TabletID)=" << datashardId << ", pathId " << pathId.LocalPathId
220-
<< ": RowCount " << newStats.RowCount << ", DataSize " << newStats.DataSize);
218+
"Add stats from shard with datashardId(TabletID)=" << datashardId << ", pathId " << pathId.LocalPathId
219+
<< ": RowCount " << newStats.RowCount << ", DataSize "
220+
<< newStats.DataSize);
221221

222222
NIceDb::TNiceDb db(txc.DB);
223223

@@ -266,18 +266,20 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId, const TStatsQ
266266
const TPathId tablePathId = TPathId(TOwnerId(pathId.OwnerId), TLocalPathId(table.GetTableLocalId()));
267267

268268
if (Self->ColumnTables.contains(tablePathId)) {
269-
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "add stats for exists table with pathId=" << tablePathId);
269+
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
270+
"add stats for exists table with pathId=" << tablePathId);
270271

271272
auto columnTable = Self->ColumnTables.TakeVerified(tablePathId);
272273
columnTable->UpdateTableStats(tablePathId, newTableStats);
273274
} else {
274-
LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "failed add stats for table with pathId=" << tablePathId);
275+
LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
276+
"failed add stats for table with pathId=" << tablePathId);
275277
}
276278
}
277279

278280
} else if (isColumnTable) {
279-
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "PersistSingleStats: ColumnTable rec.GetColumnTables() size="
280-
<< rec.GetTables().size());
281+
LOG_INFO_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
282+
"PersistSingleStats: ColumnTable rec.GetColumnTables() size=" << rec.GetTables().size());
281283

282284
auto columnTable = Self->ColumnTables.TakeVerified(pathId);
283285
oldAggrStats = columnTable->GetStats().Aggregated;

0 commit comments

Comments
 (0)