Skip to content

Commit e8fd738

Browse files
jepett0jepett0
authored
Merge branch 'stable-24-3' into MemoryLeakInAwsSdk.stable-24-3
2 parents 461d186 + 13c2d99 commit e8fd738

File tree

713 files changed

+25213
-9651
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

713 files changed

+25213
-9651
lines changed

‎.github/config/muted_ya.txt

+7-6
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,15 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
1010
ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
1111
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
1212
ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
13-
ydb/core/kqp/ut/olap KqpOlapAggregations.Aggregation_ResultCountAll_FilterL
14-
ydb/core/kqp/ut/olap KqpOlapWrite.WriteDeleteCleanGC
1513
ydb/core/kqp/ut/pg KqpPg.CreateIndex
14+
ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssueInteractiveTx+withSink
15+
ydb/core/kqp/ut/tx KqpLocksTricky.TestNoLocksIssue+withSink
16+
ydb/core/kqp/ut/tx KqpSnapshotRead.ReadOnlyTxWithIndexCommitsOnConcurrentWrite+withSink
17+
ydb/core/kqp/ut/tx KqpSinkTx.InvalidateOnError
1618
ydb/core/kqp/ut/query KqpLimits.QueryReplySize
1719
ydb/core/kqp/ut/query KqpQuery.QueryTimeout
1820
ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
1921
ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
20-
ydb/core/kqp/ut/scheme KqpOlap.OlapRead_GenericQuerys
21-
ydb/core/kqp/ut/scheme KqpOlap.OlapRead_StreamGenericQuery
22-
ydb/core/kqp/ut/scheme KqpOlap.OlapRead_UsesGenericQueryOnJoinWithDataShardTable
23-
ydb/core/kqp/ut/scheme KqpOlapScheme.DropTable
2422
ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
2523
ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
2624
ydb/core/kqp/ut/scheme [14/50]*
@@ -31,6 +29,7 @@ ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
3129
ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
3230
ydb/core/kqp/ut/service [38/50]*
3331
ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
32+
ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
3433
ydb/core/persqueue/ut [37/40] chunk chunk
3534
ydb/core/persqueue/ut [38/40] chunk chunk
3635
ydb/core/persqueue/ut TPQTest.*DirectRead*
@@ -106,4 +105,6 @@ ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
106105
ydb/tests/functional/tenants test_tenants.py.*
107106
ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset*
108107
ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout
108+
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
109+
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
109110
ydb/tests/functional/rename [test_rename.py */10] chunk chunk

‎ydb/core/base/events.h

+2
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ struct TKikimrEvents : TEvents {
179179
ES_TX_BACKGROUND = 4256,
180180
ES_SS_BG_TASKS = 4257,
181181
ES_LIMITER = 4258,
182+
//ES_MEMORY = 4259, NB. exists in main
183+
ES_GROUPED_ALLOCATIONS_MANAGER = 4260,
182184
};
183185
};
184186

‎ydb/core/driver_lib/run/kikimr_services_initializers.cpp

+23
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,9 @@
186186
#include <ydb/core/tx/limiter/usage/config.h>
187187
#include <ydb/core/tx/limiter/usage/service.h>
188188

189+
#include <ydb/core/tx/limiter/grouped_memory/usage/config.h>
190+
#include <ydb/core/tx/limiter/grouped_memory/usage/service.h>
191+
189192
#include <ydb/core/backup/controller/tablet.h>
190193

191194
#include <ydb/services/ext_index/common/config.h>
@@ -2203,6 +2206,26 @@ void TKqpServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu
22032206
}
22042207
}
22052208

2209+
TGroupedMemoryLimiterInitializer::TGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig)
2210+
: IKikimrServicesInitializer(runConfig) {
2211+
}
2212+
2213+
void TGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
2214+
NOlap::NGroupedMemoryManager::TConfig serviceConfig;
2215+
Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetGroupedMemoryLimiterConfig()));
2216+
2217+
if (serviceConfig.IsEnabled()) {
2218+
TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets");
2219+
TIntrusivePtr<::NMonitoring::TDynamicCounters> countersGroup = tabletGroup->GetSubgroup("type", "TX_GROUPED_MEMORY_LIMITER");
2220+
2221+
auto service = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::CreateService(serviceConfig, countersGroup);
2222+
2223+
setup->LocalServices.push_back(std::make_pair(
2224+
NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::MakeServiceId(NodeId),
2225+
TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId)));
2226+
}
2227+
}
2228+
22062229
TCompDiskLimiterInitializer::TCompDiskLimiterInitializer(const TKikimrRunConfig& runConfig)
22072230
: IKikimrServicesInitializer(runConfig) {
22082231
}

‎ydb/core/driver_lib/run/kikimr_services_initializers.h

+6
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,12 @@ class TCompDiskLimiterInitializer: public IKikimrServicesInitializer {
397397
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
398398
};
399399

400+
class TGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer {
401+
public:
402+
TGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig);
403+
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
404+
};
405+
400406
class TCompConveyorInitializer: public IKikimrServicesInitializer {
401407
public:
402408
TCompConveyorInitializer(const TKikimrRunConfig& runConfig);

‎ydb/core/driver_lib/run/run.cpp

+4
Original file line numberDiff line numberDiff line change
@@ -1554,6 +1554,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
15541554
sil->AddServiceInitializer(new TCompDiskLimiterInitializer(runConfig));
15551555
}
15561556

1557+
if (serviceMask.EnableGroupedMemoryLimiter) {
1558+
sil->AddServiceInitializer(new TGroupedMemoryLimiterInitializer(runConfig));
1559+
}
1560+
15571561
if (serviceMask.EnableScanConveyor) {
15581562
sil->AddServiceInitializer(new TScanConveyorInitializer(runConfig));
15591563
}

‎ydb/core/driver_lib/run/service_mask.h

+1
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ union TBasicKikimrServicesMask {
7878
bool EnableDatabaseMetadataCache:1;
7979
bool EnableGraphService:1;
8080
bool EnableCompDiskLimiter:1;
81+
bool EnableGroupedMemoryLimiter:1;
8182
bool EnableAwsService:1;
8283
};
8384

‎ydb/core/driver_lib/run/ya.make

+1
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ PEERDIR(
121121
ydb/core/tx/coordinator
122122
ydb/core/tx/conveyor/service
123123
ydb/core/tx/limiter/service
124+
ydb/core/tx/limiter/grouped_memory/usage
124125
ydb/core/tx/datashard
125126
ydb/core/tx/long_tx_service
126127
ydb/core/tx/long_tx_service/public

‎ydb/core/formats/arrow/accessor/abstract/constructor.cpp

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#include "constructor.h"
2+
#include <ydb/core/formats/arrow/accessor/plain/constructor.h>
3+
4+
namespace NKikimr::NArrow::NAccessor {
5+
6+
TConstructorContainer TConstructorContainer::GetDefaultConstructor() {
7+
static std::shared_ptr<IConstructor> result = std::make_shared<NPlain::TConstructor>();
8+
return result;
9+
}
10+
11+
}

‎ydb/core/formats/arrow/accessor/abstract/constructor.h

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#pragma once
2+
3+
#include <ydb/library/formats/arrow/protos/accessor.pb.h>
4+
#include <ydb/library/formats/arrow/accessor/abstract/accessor.h>
5+
#include <ydb/library/formats/arrow/accessor/common/chunk_data.h>
6+
#include <ydb/services/bg_tasks/abstract/interface.h>
7+
8+
#include <library/cpp/object_factory/object_factory.h>
9+
10+
namespace NKikimr::NArrow::NAccessor {
11+
12+
class IConstructor {
13+
public:
14+
using TFactory = NObjectFactory::TObjectFactory<IConstructor, TString>;
15+
using TProto = NKikimrArrowAccessorProto::TConstructor;
16+
17+
private:
18+
virtual TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> DoConstruct(
19+
const std::shared_ptr<arrow::RecordBatch>& originalData, const TChunkConstructionData& externalInfo) const = 0;
20+
virtual TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> DoConstructDefault(
21+
const TChunkConstructionData& externalInfo) const = 0;
22+
virtual NKikimrArrowAccessorProto::TConstructor DoSerializeToProto() const = 0;
23+
virtual bool DoDeserializeFromProto(const NKikimrArrowAccessorProto::TConstructor& proto) = 0;
24+
virtual std::shared_ptr<arrow::Schema> DoGetExpectedSchema(const std::shared_ptr<arrow::Field>& resultColumn) const = 0;
25+
virtual TString DoDebugString() const {
26+
return "";
27+
}
28+
29+
public:
30+
virtual ~IConstructor() = default;
31+
32+
TString DebugString() const {
33+
return TStringBuilder() << GetClassName() << ":" << DoDebugString();
34+
}
35+
36+
TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> Construct(
37+
const std::shared_ptr<arrow::RecordBatch>& originalData, const TChunkConstructionData& externalInfo) const {
38+
return DoConstruct(originalData, externalInfo);
39+
}
40+
41+
TConclusion<std::shared_ptr<NArrow::NAccessor::IChunkedArray>> ConstructDefault(const TChunkConstructionData& externalInfo) const {
42+
return DoConstructDefault(externalInfo);
43+
}
44+
45+
bool DeserializeFromProto(const NKikimrArrowAccessorProto::TConstructor& proto) {
46+
return DoDeserializeFromProto(proto);
47+
}
48+
49+
NKikimrArrowAccessorProto::TConstructor SerializeToProto() const {
50+
return DoSerializeToProto();
51+
}
52+
53+
void SerializeToProto(NKikimrArrowAccessorProto::TConstructor& proto) const {
54+
proto = DoSerializeToProto();
55+
}
56+
57+
std::shared_ptr<arrow::Schema> GetExpectedSchema(const std::shared_ptr<arrow::Field>& resultColumn) const {
58+
AFL_VERIFY(resultColumn);
59+
return DoGetExpectedSchema(resultColumn);
60+
}
61+
62+
virtual TString GetClassName() const = 0;
63+
};
64+
65+
class TConstructorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IConstructor> {
66+
private:
67+
using TBase = NBackgroundTasks::TInterfaceProtoContainer<IConstructor>;
68+
69+
public:
70+
using TBase::TBase;
71+
72+
static TConstructorContainer GetDefaultConstructor();
73+
};
74+
75+
} // namespace NKikimr::NArrow::NAccessor

‎ydb/core/formats/arrow/accessor/abstract/request.cpp

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#include "request.h"
2+
3+
namespace NKikimr::NArrow::NAccessor {
4+
5+
TConclusionStatus TRequestedConstructorContainer::DeserializeFromRequest(NYql::TFeaturesExtractor& features) {
6+
const std::optional<TString> className = features.Extract("DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME");
7+
if (!className) {
8+
return TConclusionStatus::Success();
9+
}
10+
if (!TBase::Initialize(*className)) {
11+
return TConclusionStatus::Fail("don't know anything about class_name=" + *className);
12+
}
13+
return TBase::GetObjectPtr()->DeserializeFromRequest(features);
14+
}
15+
16+
}

‎ydb/core/formats/arrow/accessor/abstract/request.h

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#pragma once
2+
#include "constructor.h"
3+
4+
#include <ydb/library/formats/arrow/protos/accessor.pb.h>
5+
#include <ydb/services/bg_tasks/abstract/interface.h>
6+
#include <ydb/services/metadata/abstract/request_features.h>
7+
8+
#include <library/cpp/object_factory/object_factory.h>
9+
10+
namespace NKikimr::NArrow::NAccessor {
11+
12+
class IRequestedConstructor {
13+
public:
14+
using TFactory = NObjectFactory::TObjectFactory<IRequestedConstructor, TString>;
15+
using TProto = NKikimrArrowAccessorProto::TRequestedConstructor;
16+
private:
17+
virtual TConclusion<NArrow::NAccessor::TConstructorContainer> DoBuildConstructor() const = 0;
18+
virtual NKikimrArrowAccessorProto::TRequestedConstructor DoSerializeToProto() const = 0;
19+
virtual bool DoDeserializeFromProto(const NKikimrArrowAccessorProto::TRequestedConstructor& proto) = 0;
20+
virtual TConclusionStatus DoDeserializeFromRequest(NYql::TFeaturesExtractor& features) = 0;
21+
22+
public:
23+
virtual ~IRequestedConstructor() = default;
24+
25+
NKikimrArrowAccessorProto::TRequestedConstructor SerializeToProto() const {
26+
return DoSerializeToProto();
27+
}
28+
29+
void SerializeToProto(NKikimrArrowAccessorProto::TRequestedConstructor& proto) const {
30+
proto = DoSerializeToProto();
31+
}
32+
33+
bool DeserializeFromProto(const NKikimrArrowAccessorProto::TRequestedConstructor& proto) {
34+
return DoDeserializeFromProto(proto);
35+
}
36+
37+
TConclusionStatus DeserializeFromRequest(NYql::TFeaturesExtractor& features) {
38+
return DoDeserializeFromRequest(features);
39+
}
40+
41+
TConclusion<TConstructorContainer> BuildConstructor() const {
42+
return DoBuildConstructor();
43+
}
44+
45+
virtual TString GetClassName() const = 0;
46+
};
47+
48+
class TRequestedConstructorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IRequestedConstructor> {
49+
private:
50+
using TBase = NBackgroundTasks::TInterfaceProtoContainer<IRequestedConstructor>;
51+
52+
public:
53+
using TBase::TBase;
54+
TConclusionStatus DeserializeFromRequest(NYql::TFeaturesExtractor& features);
55+
};
56+
57+
} // namespace NKikimr::NArrow::NAccessor

‎ydb/core/formats/arrow/accessor/abstract/ya.make

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
LIBRARY()
2+
3+
PEERDIR(
4+
contrib/libs/apache/arrow
5+
ydb/library/conclusion
6+
ydb/services/metadata/abstract
7+
ydb/library/formats/arrow/accessor/abstract
8+
ydb/library/formats/arrow/accessor/common
9+
ydb/library/formats/arrow/protos
10+
)
11+
12+
SRCS(
13+
constructor.cpp
14+
request.cpp
15+
)
16+
17+
END()

‎ydb/core/formats/arrow/accessor/composite_serial/accessor.cpp

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#include "accessor.h"
2+
3+
namespace NKikimr::NArrow::NAccessor {
4+
5+
namespace {
6+
class TSerializedChunkAccessor {
7+
private:
8+
const std::vector<TDeserializeChunkedArray::TChunk>& Chunks;
9+
const std::shared_ptr<TColumnLoader>& Loader;
10+
std::optional<IChunkedArray::TLocalChunkedArrayAddress>& Result;
11+
12+
public:
13+
TSerializedChunkAccessor(const std::vector<TDeserializeChunkedArray::TChunk>& chunks, const std::shared_ptr<TColumnLoader>& loader,
14+
std::optional<IChunkedArray::TLocalChunkedArrayAddress>& result)
15+
: Chunks(chunks)
16+
, Loader(loader)
17+
, Result(result) {
18+
}
19+
ui64 GetChunksCount() const {
20+
return Chunks.size();
21+
}
22+
ui64 GetChunkLength(const ui32 idx) const {
23+
return Chunks[idx].GetRecordsCount();
24+
}
25+
void OnArray(const ui32 chunkIdx, const ui32 startPosition) const {
26+
Result = IChunkedArray::TLocalChunkedArrayAddress(Chunks[chunkIdx].GetArrayVerified(Loader), startPosition, chunkIdx);
27+
}
28+
};
29+
} // namespace
30+
31+
IChunkedArray::TLocalDataAddress TDeserializeChunkedArray::DoGetLocalData(
32+
const std::optional<TCommonChunkAddress>& /*chunkCurrent*/, const ui64 /*position*/) const {
33+
AFL_VERIFY(false);
34+
return IChunkedArray::TLocalDataAddress(nullptr, 0, 0);
35+
}
36+
37+
IChunkedArray::TLocalChunkedArrayAddress TDeserializeChunkedArray::DoGetLocalChunkedArray(
38+
const std::optional<TCommonChunkAddress>& chunkCurrent, const ui64 position) const {
39+
std::optional<IChunkedArray::TLocalChunkedArrayAddress> result;
40+
TSerializedChunkAccessor accessor(Chunks, Loader, result);
41+
SelectChunk(chunkCurrent, position, accessor);
42+
AFL_VERIFY(result);
43+
return *result;
44+
}
45+
46+
} // namespace NKikimr::NArrow::NAccessor

0 commit comments

Comments
 (0)