Skip to content

Commit fb4d6bb

Browse files
alexv-smirnovalexv-smirnov
committed
Merge pull request #15324 from ydb-platform/merge-libs-250304-1619
2 parents 23085ba + 6f0fc96 commit fb4d6bb

File tree

16 files changed

+229
-40
lines changed

16 files changed

+229
-40
lines changed

‎library/python/monlib/metric_registry.pxd

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ from library.python.monlib.metric cimport (
88

99

1010
cdef extern from "library/cpp/monlib/metrics/metric_registry.h" namespace "NMonitoring" nogil:
11+
cdef struct TMetricOpts:
12+
bint MemOnly
13+
1114
cdef cppclass TMetricRegistry:
1215
TMetricRegistry() except +
1316
TMetricRegistry(const TLabels&) except +
@@ -19,6 +22,23 @@ cdef extern from "library/cpp/monlib/metrics/metric_registry.h" namespace "NMoni
1922
THistogram* HistogramCounter(const TLabels&, IHistogramCollectorPtr collector) except +
2023
THistogram* HistogramRate(const TLabels&, IHistogramCollectorPtr collector) except +
2124

25+
TGauge* GaugeWithOpts(const TLabels&, TMetricOpts) except +
26+
TIntGauge* IntGaugeWithOpts(const TLabels&, TMetricOpts) except +
27+
TCounter* CounterWithOpts(const TLabels&, TMetricOpts) except +
28+
TRate* RateWithOpts(const TLabels&, TMetricOpts) except +
29+
30+
THistogram* HistogramCounterWithOpts(
31+
const TLabels&,
32+
IHistogramCollectorPtr collector,
33+
TMetricOpts opts
34+
) except +
35+
36+
THistogram* HistogramRateWithOpts(
37+
const TLabels&,
38+
IHistogramCollectorPtr collector,
39+
TMetricOpts opts
40+
) except +
41+
2242
void Reset() except +
2343
void Clear() except +
2444

‎library/python/monlib/metric_registry.pyx

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@ cdef class MetricRegistry:
7373

7474
return native_labels
7575

76+
@staticmethod
77+
cdef TMetricOpts _py_to_native_opts(dict kwargs) except *:
78+
cdef TMetricOpts native_opts = TMetricOpts()
79+
native_opts.MemOnly = kwargs.get('mem_only', False)
80+
return native_opts
81+
7682
@staticmethod
7783
cdef _native_to_py_labels(const TLabels& native_labels):
7884
result = dict()
@@ -94,6 +100,7 @@ cdef class MetricRegistry:
94100

95101
def _histogram(self, labels, is_rate, hist_type, **kwargs):
96102
cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
103+
cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
97104
cdef IHistogramCollectorPtr collector
98105
cdef TVector[double] native_buckets
99106

@@ -117,9 +124,9 @@ cdef class MetricRegistry:
117124

118125
cdef THistogram* native_hist
119126
if is_rate:
120-
native_hist = self.__wrapped.Get().HistogramRate(native_labels, move(collector))
127+
native_hist = self.__wrapped.Get().HistogramRateWithOpts(native_labels, move(collector), native_opts)
121128
else:
122-
native_hist = self.__wrapped.Get().HistogramCounter(native_labels, move(collector))
129+
native_hist = self.__wrapped.Get().HistogramCounterWithOpts(native_labels, move(collector), native_opts)
123130

124131
return Histogram.from_ptr(native_hist)
125132

@@ -135,52 +142,72 @@ cdef class MetricRegistry:
135142

136143
return labels
137144

138-
def gauge(self, labels):
145+
def gauge(self, labels, **kwargs):
139146
"""
140147
Gets a gauge counter or creates a new one in case counter with the specified labels
141148
does not exist
142149
143150
:param labels: A dict of labels which identifies counter
151+
152+
Keyword arguments:
153+
:param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
154+
144155
:returns: Gauge counter
145156
"""
146157
cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
147-
native_gauge = self.__wrapped.Get().Gauge(native_labels)
158+
cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
159+
native_gauge = self.__wrapped.Get().GaugeWithOpts(native_labels, native_opts)
148160
return Gauge.from_ptr(native_gauge)
149161

150-
def int_gauge(self, labels):
162+
def int_gauge(self, labels, **kwargs):
151163
"""
152164
Gets a gauge counter or creates a new one in case counter with the specified labels
153165
does not exist
154166
155167
:param labels: A dict of labels which identifies counter
168+
169+
Keyword arguments:
170+
:param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
171+
156172
:returns: IntGauge counter
157173
"""
158174
cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
159-
native_gauge = self.__wrapped.Get().IntGauge(native_labels)
175+
cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
176+
native_gauge = self.__wrapped.Get().IntGaugeWithOpts(native_labels, native_opts)
160177
return IntGauge.from_ptr(native_gauge)
161178

162-
def counter(self, labels):
179+
def counter(self, labels, **kwargs):
163180
"""
164181
Gets a counter or creates a new one in case counter with the specified labels
165182
does not exist
166183
167184
:param labels: A dict of labels which identifies counter
185+
186+
Keyword arguments:
187+
:param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
188+
168189
:returns: Counter counter
169190
"""
170191
cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
171-
native_counter = self.__wrapped.Get().Counter(native_labels)
192+
cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
193+
native_counter = self.__wrapped.Get().CounterWithOpts(native_labels, native_opts)
172194
return Counter.from_ptr(native_counter)
173195

174-
def rate(self, labels):
196+
def rate(self, labels, **kwargs):
175197
"""
176198
Gets a rate counter or creates a new one in case counter with the specified labels
177199
does not exist
178200
179201
:param labels: A dict of labels which identifies counter
202+
203+
Keyword arguments:
204+
:param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
205+
180206
:returns: Rate counter
181207
"""
182208
cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
183-
native_rate = self.__wrapped.Get().Rate(native_labels)
209+
cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
210+
native_rate = self.__wrapped.Get().RateWithOpts(native_labels, native_opts)
184211
return Rate.from_ptr(native_rate)
185212

186213
def histogram_counter(self, labels, hist_type, **kwargs):
@@ -197,6 +224,7 @@ cdef class MetricRegistry:
197224
:param base: the exponential growth factor for buckets' width (exponential)
198225
:param scale: linear scale for the buckets. Must be >= 1.0 (exponential)
199226
:param start_value: the upper bound of the first bucket (linear)
227+
:param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
200228
201229
:returns: Histogram counter
202230
@@ -236,6 +264,7 @@ cdef class MetricRegistry:
236264
:param base: the exponential growth factor for buckets' width (exponential)
237265
:param scale: linear scale for the buckets. Must be >= 1.0 (exponential)
238266
:param start_value: the upper bound of the first bucket (linear)
267+
:param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
239268
240269
:returns: Histogram counter
241270
@@ -274,4 +303,4 @@ cdef class MetricRegistry:
274303

275304
def remove_metric(self, labels):
276305
cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
277-
self.__wrapped.Get().RemoveMetric(native_labels)
306+
self.__wrapped.Get().RemoveMetric(native_labels)

‎library/python/monlib/ut/py2/test.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,3 +412,51 @@ def test_reset_and_clear_registry():
412412
out = dumps(registry, format="json")
413413
j = json.loads(out)
414414
assert j == {}
415+
416+
417+
def test_mem_only_metrics():
418+
registry = MetricRegistry()
419+
420+
registry.gauge({"some": "gauge"}, mem_only=True)
421+
with pytest.raises(Exception):
422+
registry.gauge({"some": "gauge"})
423+
424+
registry.int_gauge({"some": "int_gauge"}, mem_only=True)
425+
with pytest.raises(Exception):
426+
registry.int_gauge({"some": "int_gauge"})
427+
428+
registry.counter({"some": "counter"}, mem_only=True)
429+
with pytest.raises(Exception):
430+
registry.counter({"some": "counter"})
431+
432+
registry.rate({"some": "rate"}, mem_only=True)
433+
with pytest.raises(Exception):
434+
registry.rate({"some": "rate"})
435+
436+
registry.histogram_counter(
437+
{"some": "histogram_counter"},
438+
HistogramType.Explicit,
439+
mem_only=True,
440+
buckets=[1, 5, 15, 20, 25]
441+
)
442+
with pytest.raises(Exception):
443+
registry.histogram_counter(
444+
{"some": "histogram_counter"},
445+
HistogramType.Explicit,
446+
buckets=[1, 5, 15, 20, 25],
447+
)
448+
449+
registry.histogram_rate(
450+
{"some": "histogram_rate"},
451+
HistogramType.Exponential,
452+
mem_only=True,
453+
bucket_count=5,
454+
base=2
455+
)
456+
with pytest.raises(Exception):
457+
registry.histogram_rate(
458+
{"some": "histogram_rate"},
459+
HistogramType.Exponential,
460+
bucket_count=5,
461+
base=2
462+
)

‎library/python/monlib/ut/py3/test.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,3 +410,51 @@ def test_reset_and_clear_registry():
410410
out = dumps(registry, format="json")
411411
j = json.loads(out)
412412
assert j == {}
413+
414+
415+
def test_mem_only_metrics():
416+
registry = MetricRegistry()
417+
418+
registry.gauge({"some": "gauge"}, mem_only=True)
419+
with pytest.raises(Exception):
420+
registry.gauge({"some": "gauge"})
421+
422+
registry.int_gauge({"some": "int_gauge"}, mem_only=True)
423+
with pytest.raises(Exception):
424+
registry.int_gauge({"some": "int_gauge"})
425+
426+
registry.counter({"some": "counter"}, mem_only=True)
427+
with pytest.raises(Exception):
428+
registry.counter({"some": "counter"})
429+
430+
registry.rate({"some": "rate"}, mem_only=True)
431+
with pytest.raises(Exception):
432+
registry.rate({"some": "rate"})
433+
434+
registry.histogram_counter(
435+
{"some": "histogram_counter"},
436+
HistogramType.Explicit,
437+
mem_only=True,
438+
buckets=[1, 5, 15, 20, 25]
439+
)
440+
with pytest.raises(Exception):
441+
registry.histogram_counter(
442+
{"some": "histogram_counter"},
443+
HistogramType.Explicit,
444+
buckets=[1, 5, 15, 20, 25],
445+
)
446+
447+
registry.histogram_rate(
448+
{"some": "histogram_rate"},
449+
HistogramType.Exponential,
450+
mem_only=True,
451+
bucket_count=5,
452+
base=2
453+
)
454+
with pytest.raises(Exception):
455+
registry.histogram_rate(
456+
{"some": "histogram_rate"},
457+
HistogramType.Exponential,
458+
bucket_count=5,
459+
base=2
460+
)

‎ydb/ci/rightlib.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0ae3f82349eeb4f353c62dd726e4ba06bbc837f9
1+
e34965b5e8228c70bfde422fbe03b40e462530c3

‎ydb/library/yql/dq/transform/yql_common_dq_transform.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ class TCommonDqTaskTransform {
4545
};
4646

4747
TTaskTransformFactory CreateCommonDqTaskTransformFactory() {
48-
return [] (const THashMap<TString, TString>& taskParams, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
49-
Y_UNUSED(taskParams);
48+
return [] (const TTaskTransformArguments& args, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
49+
Y_UNUSED(args);
5050
return TCommonDqTaskTransform(*funcRegistry);
5151
};
5252
}

‎ydb/library/yql/providers/dq/runtime/task_command_executor.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -751,7 +751,7 @@ class TTaskCommandExecutor {
751751
}
752752
settings.OptLLVM = DqConfiguration->OptLLVM.Get().GetOrElse("");
753753

754-
Ctx.FuncProvider = TaskTransformFactory(taskParams, Ctx.FuncRegistry);
754+
Ctx.FuncProvider = TaskTransformFactory({taskParams, settings.ReadRanges}, Ctx.FuncRegistry);
755755

756756
Y_ABORT_UNLESS(!Alloc);
757757
Y_ABORT_UNLESS(FunctionRegistry);

‎ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ class TLocalFactory: public IProxyFactory {
267267
YQL_CLOG(DEBUG, ProviderDq) << message;
268268
};
269269
}
270-
ctx.FuncProvider = TaskTransformFactory(settings.TaskParams, ctx.FuncRegistry);
270+
ctx.FuncProvider = TaskTransformFactory({settings.TaskParams, settings.ReadRanges}, ctx.FuncRegistry);
271271
return MakeDqTaskRunner(alloc, ctx, settings, logger);
272272
}
273273

‎ydb/library/yql/providers/ydb/comp_nodes/yql_ydb_dq_transform.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ class TYdbDqTaskTransform {
5151
};
5252

5353
TTaskTransformFactory CreateYdbDqTaskTransformFactory() {
54-
return [] (const THashMap<TString, TString>& taskParams, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
55-
return TYdbDqTaskTransform(taskParams, *funcRegistry);
54+
return [] (const TTaskTransformArguments& args, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
55+
return TYdbDqTaskTransform(args.TaskParams, *funcRegistry);
5656
};
5757
}
5858

‎yql/essentials/core/dq_integration/transform/yql_dq_task_transform.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
namespace NYql {
44

55
TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories) {
6-
return [factories = std::move(factories)] (const THashMap<TString, TString>& taskParams, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry) -> NKikimr::NMiniKQL::TCallableVisitFuncProvider {
6+
return [factories = std::move(factories)] (const TTaskTransformArguments& args, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry) -> NKikimr::NMiniKQL::TCallableVisitFuncProvider {
77
TVector<NKikimr::NMiniKQL::TCallableVisitFuncProvider> funcProviders;
88
for (auto& factory: factories) {
9-
funcProviders.push_back(factory(taskParams, funcRegistry));
9+
funcProviders.push_back(factory(args, funcRegistry));
1010
}
1111
return [funcProviders = std::move(funcProviders)] (const NKikimr::NMiniKQL::TInternName& name) -> NKikimr::NMiniKQL::TCallableVisitFunc {
1212
for (auto& provider: funcProviders) {

‎yql/essentials/core/dq_integration/transform/yql_dq_task_transform.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@
1010

1111
namespace NYql {
1212

13-
using TTaskTransformFactory = std::function<NKikimr::NMiniKQL::TCallableVisitFuncProvider(const THashMap<TString, TString>&, const NKikimr::NMiniKQL::IFunctionRegistry*)>;
13+
struct TTaskTransformArguments {
14+
THashMap<TString, TString> TaskParams;
15+
TVector<TString> ReadRanges;
16+
};
17+
18+
using TTaskTransformFactory = std::function<NKikimr::NMiniKQL::TCallableVisitFuncProvider(const TTaskTransformArguments&, const NKikimr::NMiniKQL::IFunctionRegistry*)>;
1419

1520
TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories);
1621

‎yql/essentials/tests/s-expressions/minirun/part8/canondata/result.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -578,9 +578,9 @@
578578
],
579579
"test.test[Udf-PythonAvg--Debug]": [
580580
{
581-
"checksum": "3c48becb08f825e3f64de7f909c3d4a7",
581+
"checksum": "d90368f50cd675a0868fa9090bca1a54",
582582
"size": 989,
583-
"uri": "https://{canondata_backend}/1600758/4167b447d68450d04af3e055febcc3a8168a477c/resource.tar.gz#test.test_Udf-PythonAvg--Debug_/opt.yql"
583+
"uri": "https://{canondata_backend}/1936947/488d1ce7f16cf486fb2c04ce444c576f60aa7157/resource.tar.gz#test.test_Udf-PythonAvg--Debug_/opt.yql"
584584
}
585585
],
586586
"test.test[Udf-PythonAvg--Results]": [

‎yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.yqls

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# prepare python udf
55
(let ui32 (DataType 'Uint32))
66
(let dbl (DataType 'Double))
7-
(let rt (ResourceType 'Python2))
7+
(let rt (ResourceType 'Python3))
88

99
(let udfScript (String '@@
1010
class AvgCalc:

‎yql/essentials/udfs/common/python/bindings/py_resource.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ TPyObjectPtr ToPyResource(
5656
const NUdf::TType* type,
5757
const NUdf::TUnboxedValuePod& value)
5858
{
59-
// TODO NILE-43
60-
#if false && UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
59+
60+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
6161
NUdf::TResourceTypeInspector inpector(*ctx->PyCtx->TypeInfoHelper, type);
6262
auto tag = inpector.GetTag();
6363
if (tag == ctx->PyCtx->ResourceTag) {
@@ -80,8 +80,8 @@ NUdf::TUnboxedValue FromPyResource(
8080
const TPyCastContext::TPtr& ctx,
8181
const NUdf::TType* type, PyObject* value)
8282
{
83-
// TODO NILE-43
84-
#if false && UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
83+
84+
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
8585
NUdf::TResourceTypeInspector inpector(*ctx->PyCtx->TypeInfoHelper, type);
8686
auto tag = inpector.GetTag();
8787
if (tag == ctx->PyCtx->ResourceTag) {

0 commit comments

Comments
 (0)