Skip to content

Commit f510b48

Browse files
uzhastikuzhastik
committed
Integrate stable-24-3-9-cs-2 into stable-24-3 (ydb-platform#9675)
1 parent 89ce2f9 commit f510b48

File tree

690 files changed

+23486
-9580
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

690 files changed

+23486
-9580
lines changed

‎.github/config/muted_ya.txt

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDat
1111
ydb/core/kqp/ut/join KqpJoinOrder.Chain65Nodes
1212
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
1313
ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
14-
ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_ResultCountAll_FilterL
15-
ydb/core/kqp/ut/olap KqpOlapWrite.WriteDeleteCleanGC
1614
ydb/core/kqp/ut/pg KqpPg.CreateIndex
1715
ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssueInteractiveTx+withSink
1816
ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssue+withSink
@@ -28,10 +26,6 @@ ydb/core/kqp/ut/federated_query/s3 KqpFederatedQuery.CreateTableAsSelectFromExte
2826
ydb/core/kqp/ut/federated_query/s3 KqpFederatedQuery.CreateTableAsSelectFromExternalTable
2927
ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
3028
ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
31-
ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
32-
ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
33-
ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
34-
ydb/core/kqp/ut/scheme KqpOlapScheme.DropTable
3529
ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
3630
ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
3731
ydb/core/kqp/ut/scheme [14/50]*
@@ -43,6 +37,7 @@ ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
4337
ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
4438
ydb/core/kqp/ut/service [38/50]*
4539
ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
40+
ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
4641
ydb/core/persqueue/ut [37/40] chunk chunk
4742
ydb/core/persqueue/ut [38/40] chunk chunk
4843
ydb/core/persqueue/ut TPQTest.*DirectRead*
@@ -118,4 +113,6 @@ ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
118113
ydb/tests/functional/tenants test_tenants.py.*
119114
ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset*
120115
ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
116+
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
117+
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
121118
ydb/tests/functional/rename [test_rename.py */10] chunk chunk

‎ydb/core/base/events.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ struct TKikimrEvents : TEvents {
179179
ES_TX_BACKGROUND = 4256,
180180
ES_SS_BG_TASKS = 4257,
181181
ES_LIMITER = 4258,
182+
//ES_MEMORY = 4259, NB. exists in main
183+
ES_GROUPED_ALLOCATIONS_MANAGER = 4260,
182184
};
183185
};
184186

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,9 @@
186186
#include <ydb/core/tx/limiter/usage/config.h>
187187
#include <ydb/core/tx/limiter/usage/service.h>
188188

189+
#include <ydb/core/tx/limiter/grouped_memory/usage/config.h>
190+
#include <ydb/core/tx/limiter/grouped_memory/usage/service.h>
191+
189192
#include <ydb/core/backup/controller/tablet.h>
190193

191194
#include <ydb/services/ext_index/common/config.h>
@@ -2180,6 +2183,26 @@ void TKqpServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu
21802183
}
21812184
}
21822185

2186+
TGroupedMemoryLimiterInitializer::TGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig)
2187+
: IKikimrServicesInitializer(runConfig) {
2188+
}
2189+
2190+
void TGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
2191+
NOlap::NGroupedMemoryManager::TConfig serviceConfig;
2192+
Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetGroupedMemoryLimiterConfig()));
2193+
2194+
if (serviceConfig.IsEnabled()) {
2195+
TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets");
2196+
TIntrusivePtr<::NMonitoring::TDynamicCounters> countersGroup = tabletGroup->GetSubgroup("type", "TX_GROUPED_MEMORY_LIMITER");
2197+
2198+
auto service = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::CreateService(serviceConfig, countersGroup);
2199+
2200+
setup->LocalServices.push_back(std::make_pair(
2201+
NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::MakeServiceId(NodeId),
2202+
TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId)));
2203+
}
2204+
}
2205+
21832206
TCompDiskLimiterInitializer::TCompDiskLimiterInitializer(const TKikimrRunConfig& runConfig)
21842207
: IKikimrServicesInitializer(runConfig) {
21852208
}

‎ydb/core/driver_lib/run/kikimr_services_initializers.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,12 @@ class TCompDiskLimiterInitializer: public IKikimrServicesInitializer {
397397
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
398398
};
399399

400+
class TGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer {
401+
public:
402+
TGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig);
403+
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
404+
};
405+
400406
class TCompConveyorInitializer: public IKikimrServicesInitializer {
401407
public:
402408
TCompConveyorInitializer(const TKikimrRunConfig& runConfig);

‎ydb/core/driver_lib/run/run.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1554,6 +1554,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
15541554
sil->AddServiceInitializer(new TCompDiskLimiterInitializer(runConfig));
15551555
}
15561556

1557+
if (serviceMask.EnableGroupedMemoryLimiter) {
1558+
sil->AddServiceInitializer(new TGroupedMemoryLimiterInitializer(runConfig));
1559+
}
1560+
15571561
if (serviceMask.EnableScanConveyor) {
15581562
sil->AddServiceInitializer(new TScanConveyorInitializer(runConfig));
15591563
}

‎ydb/core/driver_lib/run/service_mask.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ union TBasicKikimrServicesMask {
7878
bool EnableDatabaseMetadataCache:1;
7979
bool EnableGraphService:1;
8080
bool EnableCompDiskLimiter:1;
81+
bool EnableGroupedMemoryLimiter:1;
8182
};
8283

8384
struct {

‎ydb/core/driver_lib/run/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ PEERDIR(
111111
ydb/core/tx/coordinator
112112
ydb/core/tx/conveyor/service
113113
ydb/core/tx/limiter/service
114+
ydb/core/tx/limiter/grouped_memory/usage
114115
ydb/core/tx/datashard
115116
ydb/core/tx/long_tx_service
116117
ydb/core/tx/long_tx_service/public

‎ydb/core/formats/arrow/accessor/abstract/constructor.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#include "constructor.h"
2+
#include <ydb/core/formats/arrow/accessor/plain/constructor.h>
3+
4+
namespace NKikimr::NArrow::NAccessor {
5+
6+
TConstructorContainer TConstructorContainer::GetDefaultConstructor() {
7+
static std::shared_ptr<IConstructor> result = std::make_shared<NPlain::TConstructor>();
8+
return result;
9+
}
10+
11+
}

‎ydb/core/formats/arrow/accessor/abstract/constructor.h

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#pragma once
2+
3+
#include <ydb/library/formats/arrow/protos/accessor.pb.h>
4+
#include <ydb/library/formats/arrow/accessor/abstract/accessor.h>
5+
#include <ydb/library/formats/arrow/accessor/common/chunk_data.h>
6+
#include <ydb/services/bg_tasks/abstract/interface.h>
7+
8+
#include <library/cpp/object_factory/object_factory.h>
9+
10+
namespace NKikimr::NArrow::NAccessor {
11+
12+
class IConstructor {
13+
public:
14+
using TFactory = NObjectFactory::TObjectFactory<IConstructor, TString>;
15+
using TProto = NKikimrArrowAccessorProto::TConstructor;
16+
17+
private:
18+
virtual TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> DoConstruct(
19+
const std::shared_ptr<arrow::RecordBatch>& originalData, const TChunkConstructionData& externalInfo) const = 0;
20+
virtual TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> DoConstructDefault(
21+
const TChunkConstructionData& externalInfo) const = 0;
22+
virtual NKikimrArrowAccessorProto::TConstructor DoSerializeToProto() const = 0;
23+
virtual bool DoDeserializeFromProto(const NKikimrArrowAccessorProto::TConstructor& proto) = 0;
24+
virtual std::shared_ptr<arrow::Schema> DoGetExpectedSchema(const std::shared_ptr<arrow::Field>& resultColumn) const = 0;
25+
virtual TString DoDebugString() const {
26+
return "";
27+
}
28+
29+
public:
30+
virtual ~IConstructor() = default;
31+
32+
TString DebugString() const {
33+
return TStringBuilder() << GetClassName() << ":" << DoDebugString();
34+
}
35+
36+
TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> Construct(
37+
const std::shared_ptr<arrow::RecordBatch>& originalData, const TChunkConstructionData& externalInfo) const {
38+
return DoConstruct(originalData, externalInfo);
39+
}
40+
41+
TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> ConstructDefault(const TChunkConstructionData& externalInfo) const {
42+
return DoConstructDefault(externalInfo);
43+
}
44+
45+
bool DeserializeFromProto(const NKikimrArrowAccessorProto::TConstructor& proto) {
46+
return DoDeserializeFromProto(proto);
47+
}
48+
49+
NKikimrArrowAccessorProto::TConstructor SerializeToProto() const {
50+
return DoSerializeToProto();
51+
}
52+
53+
void SerializeToProto(NKikimrArrowAccessorProto::TConstructor& proto) const {
54+
proto = DoSerializeToProto();
55+
}
56+
57+
std::shared_ptr<arrow::Schema> GetExpectedSchema(const std::shared_ptr<arrow::Field>& resultColumn) const {
58+
AFL_VERIFY(resultColumn);
59+
return DoGetExpectedSchema(resultColumn);
60+
}
61+
62+
virtual TString GetClassName() const = 0;
63+
};
64+
65+
class TConstructorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IConstructor> {
66+
private:
67+
using TBase = NBackgroundTasks::TInterfaceProtoContainer<IConstructor>;
68+
69+
public:
70+
using TBase::TBase;
71+
72+
static TConstructorContainer GetDefaultConstructor();
73+
};
74+
75+
} // namespace NKikimr::NArrow::NAccessor

‎ydb/core/formats/arrow/accessor/abstract/request.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#include "request.h"
2+
3+
namespace NKikimr::NArrow::NAccessor {
4+
5+
TConclusionStatus TRequestedConstructorContainer::DeserializeFromRequest(NYql::TFeaturesExtractor& features) {
6+
const std::optional<TString> className = features.Extract("DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME");
7+
if (!className) {
8+
return TConclusionStatus::Success();
9+
}
10+
if (!TBase::Initialize(*className)) {
11+
return TConclusionStatus::Fail("don't know anything about class_name=" + *className);
12+
}
13+
return TBase::GetObjectPtr()->DeserializeFromRequest(features);
14+
}
15+
16+
}

‎ydb/core/formats/arrow/accessor/abstract/request.h

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#pragma once
2+
#include "constructor.h"
3+
4+
#include <ydb/library/formats/arrow/protos/accessor.pb.h>
5+
#include <ydb/services/bg_tasks/abstract/interface.h>
6+
#include <ydb/services/metadata/abstract/request_features.h>
7+
8+
#include <library/cpp/object_factory/object_factory.h>
9+
10+
namespace NKikimr::NArrow::NAccessor {
11+
12+
class IRequestedConstructor {
13+
public:
14+
using TFactory = NObjectFactory::TObjectFactory<IRequestedConstructor, TString>;
15+
using TProto = NKikimrArrowAccessorProto::TRequestedConstructor;
16+
private:
17+
virtual TConclusion<NArrow::NAccessor::TConstructorContainer> DoBuildConstructor() const = 0;
18+
virtual NKikimrArrowAccessorProto::TRequestedConstructor DoSerializeToProto() const = 0;
19+
virtual bool DoDeserializeFromProto(const NKikimrArrowAccessorProto::TRequestedConstructor& proto) = 0;
20+
virtual TConclusionStatus DoDeserializeFromRequest(NYql::TFeaturesExtractor& features) = 0;
21+
22+
public:
23+
virtual ~IRequestedConstructor() = default;
24+
25+
NKikimrArrowAccessorProto::TRequestedConstructor SerializeToProto() const {
26+
return DoSerializeToProto();
27+
}
28+
29+
void SerializeToProto(NKikimrArrowAccessorProto::TRequestedConstructor& proto) const {
30+
proto = DoSerializeToProto();
31+
}
32+
33+
bool DeserializeFromProto(const NKikimrArrowAccessorProto::TRequestedConstructor& proto) {
34+
return DoDeserializeFromProto(proto);
35+
}
36+
37+
TConclusionStatus DeserializeFromRequest(NYql::TFeaturesExtractor& features) {
38+
return DoDeserializeFromRequest(features);
39+
}
40+
41+
TConclusion<TConstructorContainer> BuildConstructor() const {
42+
return DoBuildConstructor();
43+
}
44+
45+
virtual TString GetClassName() const = 0;
46+
};
47+
48+
class TRequestedConstructorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IRequestedConstructor> {
49+
private:
50+
using TBase = NBackgroundTasks::TInterfaceProtoContainer<IRequestedConstructor>;
51+
52+
public:
53+
using TBase::TBase;
54+
TConclusionStatus DeserializeFromRequest(NYql::TFeaturesExtractor& features);
55+
};
56+
57+
} // namespace NKikimr::NArrow::NAccessor

‎ydb/core/formats/arrow/accessor/abstract/ya.make

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
LIBRARY()
2+
3+
PEERDIR(
4+
contrib/libs/apache/arrow
5+
ydb/library/conclusion
6+
ydb/services/metadata/abstract
7+
ydb/library/formats/arrow/accessor/abstract
8+
ydb/library/formats/arrow/accessor/common
9+
ydb/library/formats/arrow/protos
10+
)
11+
12+
SRCS(
13+
constructor.cpp
14+
request.cpp
15+
)
16+
17+
END()

‎ydb/core/formats/arrow/accessor/composite_serial/accessor.cpp

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#include "accessor.h"
2+
3+
namespace NKikimr::NArrow::NAccessor {
4+
5+
namespace {
6+
class TSerializedChunkAccessor {
7+
private:
8+
const std::vector<TDeserializeChunkedArray::TChunk>& Chunks;
9+
const std::shared_ptr<TColumnLoader>& Loader;
10+
std::optional<IChunkedArray::TLocalChunkedArrayAddress>& Result;
11+
12+
public:
13+
TSerializedChunkAccessor(const std::vector<TDeserializeChunkedArray::TChunk>& chunks, const std::shared_ptr<TColumnLoader>& loader,
14+
std::optional<IChunkedArray::TLocalChunkedArrayAddress>& result)
15+
: Chunks(chunks)
16+
, Loader(loader)
17+
, Result(result) {
18+
}
19+
ui64 GetChunksCount() const {
20+
return Chunks.size();
21+
}
22+
ui64 GetChunkLength(const ui32 idx) const {
23+
return Chunks[idx].GetRecordsCount();
24+
}
25+
void OnArray(const ui32 chunkIdx, const ui32 startPosition) const {
26+
Result = IChunkedArray::TLocalChunkedArrayAddress(Chunks[chunkIdx].GetArrayVerified(Loader), startPosition, chunkIdx);
27+
}
28+
};
29+
} // namespace
30+
31+
IChunkedArray::TLocalDataAddress TDeserializeChunkedArray::DoGetLocalData(
32+
const std::optional<TCommonChunkAddress>& /*chunkCurrent*/, const ui64 /*position*/) const {
33+
AFL_VERIFY(false);
34+
return IChunkedArray::TLocalDataAddress(nullptr, 0, 0);
35+
}
36+
37+
IChunkedArray::TLocalChunkedArrayAddress TDeserializeChunkedArray::DoGetLocalChunkedArray(
38+
const std::optional<TCommonChunkAddress>& chunkCurrent, const ui64 position) const {
39+
std::optional<IChunkedArray::TLocalChunkedArrayAddress> result;
40+
TSerializedChunkAccessor accessor(Chunks, Loader, result);
41+
SelectChunk(chunkCurrent, position, accessor);
42+
AFL_VERIFY(result);
43+
return *result;
44+
}
45+
46+
} // namespace NKikimr::NArrow::NAccessor

0 commit comments

Comments
 (0)