Library import 250304-1619

library/python/monlib/metric_registry.pxd

@@ -8,6 +8,9 @@ from library.python.monlib.metric cimport (
 
 
 cdef extern from "library/cpp/monlib/metrics/metric_registry.h" namespace "NMonitoring" nogil:
+    cdef struct TMetricOpts:
+        bint MemOnly
+
     cdef cppclass TMetricRegistry:
         TMetricRegistry() except +
         TMetricRegistry(const TLabels&) except +
@@ -19,6 +22,23 @@ cdef extern from "library/cpp/monlib/metrics/metric_registry.h" namespace "NMoni
         THistogram* HistogramCounter(const TLabels&, IHistogramCollectorPtr collector) except +
         THistogram* HistogramRate(const TLabels&, IHistogramCollectorPtr collector) except +
 
+        TGauge* GaugeWithOpts(const TLabels&, TMetricOpts) except +
+        TIntGauge* IntGaugeWithOpts(const TLabels&, TMetricOpts) except +
+        TCounter* CounterWithOpts(const TLabels&, TMetricOpts) except +
+        TRate* RateWithOpts(const TLabels&, TMetricOpts) except +
+
+        THistogram* HistogramCounterWithOpts(
+            const TLabels&, 
+            IHistogramCollectorPtr collector,
+            TMetricOpts opts
+        ) except +
+
+        THistogram* HistogramRateWithOpts(
+            const TLabels&, 
+            IHistogramCollectorPtr collector,
+            TMetricOpts opts
+        ) except +
+
         void Reset() except +
         void Clear() except +
 

library/python/monlib/metric_registry.pyx

@@ -73,6 +73,12 @@ cdef class MetricRegistry:
 
         return native_labels
 
+    @staticmethod
+    cdef TMetricOpts _py_to_native_opts(dict kwargs) except *:
+        cdef TMetricOpts native_opts = TMetricOpts()
+        native_opts.MemOnly = kwargs.get('mem_only', False)
+        return native_opts
+
     @staticmethod
     cdef _native_to_py_labels(const TLabels& native_labels):
         result = dict()
@@ -94,6 +100,7 @@ cdef class MetricRegistry:
 
     def _histogram(self, labels, is_rate, hist_type, **kwargs):
         cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
+        cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
         cdef IHistogramCollectorPtr collector
         cdef TVector[double] native_buckets
 
@@ -117,9 +124,9 @@ cdef class MetricRegistry:
 
         cdef THistogram* native_hist
         if is_rate:
-            native_hist = self.__wrapped.Get().HistogramRate(native_labels, move(collector))
+            native_hist = self.__wrapped.Get().HistogramRateWithOpts(native_labels, move(collector), native_opts)
         else:
-            native_hist = self.__wrapped.Get().HistogramCounter(native_labels, move(collector))
+            native_hist = self.__wrapped.Get().HistogramCounterWithOpts(native_labels, move(collector), native_opts)
 
         return Histogram.from_ptr(native_hist)
 
@@ -135,52 +142,72 @@ cdef class MetricRegistry:
 
         return labels
 
-    def gauge(self, labels):
+    def gauge(self, labels, **kwargs):
         """
         Gets a gauge counter or creates a new one in case counter with the specified labels
         does not exist
 
         :param labels: A dict of labels which identifies counter
+
+        Keyword arguments:
+        :param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
+
         :returns: Gauge counter
         """
         cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
-        native_gauge = self.__wrapped.Get().Gauge(native_labels)
+        cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
+        native_gauge = self.__wrapped.Get().GaugeWithOpts(native_labels, native_opts)
         return Gauge.from_ptr(native_gauge)
 
-    def int_gauge(self, labels):
+    def int_gauge(self, labels, **kwargs):
         """
         Gets a gauge counter or creates a new one in case counter with the specified labels
         does not exist
 
         :param labels: A dict of labels which identifies counter
+
+        Keyword arguments:
+        :param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
+
         :returns: IntGauge counter
         """
         cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
-        native_gauge = self.__wrapped.Get().IntGauge(native_labels)
+        cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
+        native_gauge = self.__wrapped.Get().IntGaugeWithOpts(native_labels, native_opts)
         return IntGauge.from_ptr(native_gauge)
 
-    def counter(self, labels):
+    def counter(self, labels, **kwargs):
         """
         Gets a counter or creates a new one in case counter with the specified labels
         does not exist
 
         :param labels: A dict of labels which identifies counter
+
+        Keyword arguments:
+        :param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
+
         :returns: Counter counter
         """
         cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
-        native_counter = self.__wrapped.Get().Counter(native_labels)
+        cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
+        native_counter = self.__wrapped.Get().CounterWithOpts(native_labels, native_opts)
         return Counter.from_ptr(native_counter)
 
-    def rate(self, labels):
+    def rate(self, labels, **kwargs):
         """
         Gets a rate counter or creates a new one in case counter with the specified labels
         does not exist
 
         :param labels: A dict of labels which identifies counter
+
+        Keyword arguments:
+        :param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
+
         :returns: Rate counter
         """
         cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
-        native_rate = self.__wrapped.Get().Rate(native_labels)
+        cdef TMetricOpts native_opts = MetricRegistry._py_to_native_opts(kwargs)
+        native_rate = self.__wrapped.Get().RateWithOpts(native_labels, native_opts)
         return Rate.from_ptr(native_rate)
 
     def histogram_counter(self, labels, hist_type, **kwargs):
@@ -197,6 +224,7 @@ cdef class MetricRegistry:
         :param base: the exponential growth factor for buckets' width (exponential)
         :param scale: linear scale for the buckets. Must be >= 1.0 (exponential)
         :param start_value: the upper bound of the first bucket (linear)
+        :param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
 
         :returns: Histogram counter
 
@@ -236,6 +264,7 @@ cdef class MetricRegistry:
         :param base: the exponential growth factor for buckets' width (exponential)
         :param scale: linear scale for the buckets. Must be >= 1.0 (exponential)
         :param start_value: the upper bound of the first bucket (linear)
+        :param mem_only: flag for memOnly metric (see: https://m.yandex-team.ru/docs/concepts/glossary#memonly)
 
         :returns: Histogram counter
 
@@ -274,4 +303,4 @@ cdef class MetricRegistry:
 
     def remove_metric(self, labels):
         cdef TLabels native_labels = MetricRegistry._py_to_native_labels(labels)
-        self.__wrapped.Get().RemoveMetric(native_labels)
+        self.__wrapped.Get().RemoveMetric(native_labels)

library/python/monlib/ut/py2/test.py

@@ -412,3 +412,51 @@ def test_reset_and_clear_registry():
     out = dumps(registry, format="json")
     j = json.loads(out)
     assert j == {}
+
+
+def test_mem_only_metrics():
+    registry = MetricRegistry()
+
+    registry.gauge({"some": "gauge"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.gauge({"some": "gauge"})
+
+    registry.int_gauge({"some": "int_gauge"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.int_gauge({"some": "int_gauge"})
+
+    registry.counter({"some": "counter"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.counter({"some": "counter"})
+
+    registry.rate({"some": "rate"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.rate({"some": "rate"})
+
+    registry.histogram_counter(
+        {"some": "histogram_counter"},
+        HistogramType.Explicit,
+        mem_only=True,
+        buckets=[1, 5, 15, 20, 25]
+    )
+    with pytest.raises(Exception):
+        registry.histogram_counter(
+            {"some": "histogram_counter"},
+            HistogramType.Explicit,
+            buckets=[1, 5, 15, 20, 25],
+        )
+
+    registry.histogram_rate(
+        {"some": "histogram_rate"},
+        HistogramType.Exponential,
+        mem_only=True,
+        bucket_count=5,
+        base=2
+    )
+    with pytest.raises(Exception):
+        registry.histogram_rate(
+            {"some": "histogram_rate"},
+            HistogramType.Exponential,
+            bucket_count=5,
+            base=2
+        )

library/python/monlib/ut/py3/test.py

@@ -410,3 +410,51 @@ def test_reset_and_clear_registry():
     out = dumps(registry, format="json")
     j = json.loads(out)
     assert j == {}
+
+
+def test_mem_only_metrics():
+    registry = MetricRegistry()
+
+    registry.gauge({"some": "gauge"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.gauge({"some": "gauge"})
+
+    registry.int_gauge({"some": "int_gauge"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.int_gauge({"some": "int_gauge"})
+
+    registry.counter({"some": "counter"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.counter({"some": "counter"})
+
+    registry.rate({"some": "rate"}, mem_only=True)
+    with pytest.raises(Exception):
+        registry.rate({"some": "rate"})
+
+    registry.histogram_counter(
+        {"some": "histogram_counter"},
+        HistogramType.Explicit,
+        mem_only=True,
+        buckets=[1, 5, 15, 20, 25]
+    )
+    with pytest.raises(Exception):
+        registry.histogram_counter(
+            {"some": "histogram_counter"},
+            HistogramType.Explicit,
+            buckets=[1, 5, 15, 20, 25],
+        )
+
+    registry.histogram_rate(
+        {"some": "histogram_rate"},
+        HistogramType.Exponential,
+        mem_only=True,
+        bucket_count=5,
+        base=2
+    )
+    with pytest.raises(Exception):
+        registry.histogram_rate(
+            {"some": "histogram_rate"},
+            HistogramType.Exponential,
+            bucket_count=5,
+            base=2
+        )

ydb/ci/rightlib.txt

@@ -1 +1 @@
-0ae3f82349eeb4f353c62dd726e4ba06bbc837f9
+e34965b5e8228c70bfde422fbe03b40e462530c3

ydb/library/yql/dq/transform/yql_common_dq_transform.cpp

@@ -45,8 +45,8 @@ class TCommonDqTaskTransform {
 };
 
 TTaskTransformFactory CreateCommonDqTaskTransformFactory() {
-    return [] (const THashMap<TString, TString>& taskParams, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
-        Y_UNUSED(taskParams);
+    return [] (const TTaskTransformArguments& args, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
+        Y_UNUSED(args);
         return TCommonDqTaskTransform(*funcRegistry);
     };
 }

ydb/library/yql/providers/dq/runtime/task_command_executor.cpp

@@ -751,7 +751,7 @@ class TTaskCommandExecutor {
             }
             settings.OptLLVM = DqConfiguration->OptLLVM.Get().GetOrElse("");
 
-            Ctx.FuncProvider = TaskTransformFactory(taskParams, Ctx.FuncRegistry);
+            Ctx.FuncProvider = TaskTransformFactory({taskParams, settings.ReadRanges}, Ctx.FuncRegistry);
 
             Y_ABORT_UNLESS(!Alloc);
             Y_ABORT_UNLESS(FunctionRegistry);

ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp

@@ -267,7 +267,7 @@ class TLocalFactory: public IProxyFactory {
                 YQL_CLOG(DEBUG, ProviderDq) << message;
             };
         }
-        ctx.FuncProvider = TaskTransformFactory(settings.TaskParams, ctx.FuncRegistry);
+        ctx.FuncProvider = TaskTransformFactory({settings.TaskParams, settings.ReadRanges}, ctx.FuncRegistry);
         return MakeDqTaskRunner(alloc, ctx, settings, logger);
     }
 

ydb/library/yql/providers/ydb/comp_nodes/yql_ydb_dq_transform.cpp

@@ -51,8 +51,8 @@ class TYdbDqTaskTransform {
 };
 
 TTaskTransformFactory CreateYdbDqTaskTransformFactory() {
-    return [] (const THashMap<TString, TString>& taskParams, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
-        return TYdbDqTaskTransform(taskParams, *funcRegistry);
+    return [] (const TTaskTransformArguments& args, const IFunctionRegistry* funcRegistry) -> TCallableVisitFuncProvider {
+        return TYdbDqTaskTransform(args.TaskParams, *funcRegistry);
     };
 }
 

yql/essentials/core/dq_integration/transform/yql_dq_task_transform.cpp

@@ -3,10 +3,10 @@
 namespace NYql {
 
 TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories) {
-    return [factories = std::move(factories)] (const THashMap<TString, TString>& taskParams, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry) -> NKikimr::NMiniKQL::TCallableVisitFuncProvider {
+    return [factories = std::move(factories)] (const TTaskTransformArguments& args, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry) -> NKikimr::NMiniKQL::TCallableVisitFuncProvider {
         TVector<NKikimr::NMiniKQL::TCallableVisitFuncProvider> funcProviders;
         for (auto& factory: factories) {
-            funcProviders.push_back(factory(taskParams, funcRegistry));
+            funcProviders.push_back(factory(args, funcRegistry));
         }
         return [funcProviders = std::move(funcProviders)] (const NKikimr::NMiniKQL::TInternName& name) -> NKikimr::NMiniKQL::TCallableVisitFunc {
             for (auto& provider: funcProviders) {

yql/essentials/core/dq_integration/transform/yql_dq_task_transform.h

@@ -10,7 +10,12 @@
 
 namespace NYql {
 
-using TTaskTransformFactory = std::function<NKikimr::NMiniKQL::TCallableVisitFuncProvider(const THashMap<TString, TString>&, const NKikimr::NMiniKQL::IFunctionRegistry*)>;
+struct TTaskTransformArguments {
+    THashMap<TString, TString> TaskParams;
+    TVector<TString> ReadRanges;
+};
+
+using TTaskTransformFactory = std::function<NKikimr::NMiniKQL::TCallableVisitFuncProvider(const TTaskTransformArguments&, const NKikimr::NMiniKQL::IFunctionRegistry*)>;
 
 TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories);
 

yql/essentials/tests/s-expressions/minirun/part8/canondata/result.json

@@ -578,9 +578,9 @@
     ],
     "test.test[Udf-PythonAvg--Debug]": [
         {
-            "checksum": "3c48becb08f825e3f64de7f909c3d4a7",
+            "checksum": "d90368f50cd675a0868fa9090bca1a54",
             "size": 989,
-            "uri": "https://{canondata_backend}/1600758/4167b447d68450d04af3e055febcc3a8168a477c/resource.tar.gz#test.test_Udf-PythonAvg--Debug_/opt.yql"
+            "uri": "https://{canondata_backend}/1936947/488d1ce7f16cf486fb2c04ce444c576f60aa7157/resource.tar.gz#test.test_Udf-PythonAvg--Debug_/opt.yql"
         }
     ],
     "test.test[Udf-PythonAvg--Results]": [

yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.yqls

@@ -4,7 +4,7 @@
 # prepare python udf
 (let ui32 (DataType 'Uint32))
 (let dbl (DataType 'Double))
-(let rt (ResourceType 'Python2))
+(let rt (ResourceType 'Python3))
 
 (let udfScript (String '@@
 class AvgCalc:

yql/essentials/udfs/common/python/bindings/py_resource.cpp

@@ -56,8 +56,8 @@ TPyObjectPtr ToPyResource(
         const NUdf::TType* type,
         const NUdf::TUnboxedValuePod& value)
 {
-// TODO NILE-43
-#if false && UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
     NUdf::TResourceTypeInspector inpector(*ctx->PyCtx->TypeInfoHelper, type);
     auto tag = inpector.GetTag();
     if (tag == ctx->PyCtx->ResourceTag) {
@@ -80,8 +80,8 @@ NUdf::TUnboxedValue FromPyResource(
         const TPyCastContext::TPtr& ctx,
         const NUdf::TType* type, PyObject* value)
 {
-// TODO NILE-43
-#if false && UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15)
     NUdf::TResourceTypeInspector inpector(*ctx->PyCtx->TypeInfoHelper, type);
     auto tag = inpector.GetTag();
     if (tag == ctx->PyCtx->ResourceTag) {