clena up

Author: horheynm

src/deepsparse/loggers_v2/root_logger.py

@@ -18,7 +18,10 @@
 from typing import Any, Dict, Optional
 
 from deepsparse.loggers_v2.filters.frequency_filter import FrequencyFilter
-from deepsparse.loggers_v2.filters.pattern import is_match_found
+from deepsparse.loggers_v2.filters.pattern import (
+    is_match_found,
+    unravel_value_as_generator,
+)
 
 from .utils import import_from_registry
 
@@ -43,7 +46,7 @@ def __init__(self, config: Dict, leaf_logger: Dict):
         super().__init__()
         self.config = config
         self.leaf_logger = leaf_logger
-        self.run_args = defaultdict(lambda: defaultdict(list))
+        self.run_args = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
         self.create()
 
     def create(self):
@@ -53,13 +56,15 @@ def create(self):
         Note:
 
         self.run_args = {
-            tag: {                          # tag from config
-                func: [                     # func from config
-                    (freq, leaf_loggers),   # freq from config
-                    (freq, leaf_loggers),
-                    (freq, leaf_loggers),
-                ],
-                func2: [...]
+            tag: {
+                func: {
+                    freq: [
+                        ([loggers], [capture]),
+                        ([loggers2], [capture2]),
+                        ...
+                    ]
+                },
+                func2: {...}
             },
             tag2: {...}
         }
@@ -73,58 +78,115 @@ def create(self):
                 for logger_id in func_arg.get("uses", []):
                     leaf_loggers.append(self.leaf_logger[logger_id])
 
-                self.run_args[tag][func].append(
-                    (func_arg.get("freq", 1), leaf_loggers),
+                self.run_args[tag][func][func_arg.get("freq", 1)].append(
+                    (leaf_loggers, func_arg.get("capture"))
                 )
 
     def log(
-        self, value: Any, log_type: str, tag: Optional[str] = None, *args, **kwargs
+        self,
+        value: Any,
+        log_type: str,
+        tag: str,
+        capture: Optional[str] = None,
+        *args,
+        **kwargs,
     ):
         """
         Send args to the leaf loggers if the given tag, func, freq are accpeted. Need to
-         pass two filters to be accepted.
+         pass three filters to be accepted.
 
-         1. The inputted arg must be found in the config file
+         1. The provided tag must be a subset or regex match with the tags in the
+            root logger config file
          2. The number of calls to the current self.log(...) must be a multiple of
             freq from the config file wrt tag and func
+         3. If capture is speficied in the config file (only for metric log), it must be
+            a subset or have a regex match
 
-        If accepted, log to leaf loggers wrt the leaf loggers that was configured with
-         tag and func
+        If accepted, value=func(value) if func is provided, and pass this value to the
+        leaf loggers
 
         :param value: Any value to log, may be dimentionally reduced by func
         :param log_type: String representing the root logger level
         :param tag: Candidate id that will be used to filter out only the wanted log
+        :param capture: The property or dict key to record if match exists. If set to None
+            no capture filter will be applied even if set in config
 
         """
         for tag_from_config, tag_run_args in self.run_args.items():
             if is_match_found(tag_from_config, tag):
 
-                # key: func_name, value: [(freq, leaf_loggers)]
+                # key: func_name, value: {freq: {...}}
                 for func_from_config, func_execute_args in tag_run_args.items():
 
-                    for execute_arg in func_execute_args:
-                        freq_from_config, leaf_loggers = execute_arg
+                    # key: freq, value = [ ([loggers], [capture]), ... ]
+                    for freq_from_config, execute_args in func_execute_args.items():
 
                         # increment the counter
                         self.inc(tag_from_config, func_from_config)
 
-                        # check if the given tag.func is a multiple of the counter
-                        if self.should_execute_on_frequency(
-                            tag_from_config, func_from_config, freq_from_config
-                        ):
-                            if func_from_config is not None:
-                                func_callable = import_from_registry(func_from_config)
-                                value = func_callable(value)
-
-                            for leaf_logger in leaf_loggers:
-                                leaf_logger.log(
-                                    value=value,
-                                    tag=tag,
-                                    func=func_from_config,
-                                    log_type=log_type,
-                                    *args,
-                                    **kwargs,
-                                )
+                        # execute_arg = ([loggers], [capture])
+                        for execute_arg in execute_args:
+
+                            leaf_loggers, captures_from_config = execute_arg
+
+                            # check if the given tag.func is a multiple of the counter
+                            if self.should_execute_on_frequency(
+                                tag_from_config, func_from_config, freq_from_config
+                            ):
+                                # Capture filter (filter by class prop or dict key)
+                                if capture is None:
+                                    self._apply_func_and_log(
+                                        value=value,
+                                        tag=tag,
+                                        log_type=log_type,
+                                        func_from_config=func_from_config,
+                                        leaf_loggers=leaf_loggers,
+                                        *args,
+                                        **kwargs,
+                                    )
+
+                                else:
+                                    for capture_from_config in captures_from_config:
+                                        if is_match_found(capture_from_config, capture):
+                                            for (
+                                                captured,
+                                                value,
+                                            ) in unravel_value_as_generator(value):
+
+                                                self._apply_func_and_log(
+                                                    value=value,
+                                                    tag=tag,
+                                                    log_type=log_type,
+                                                    func_from_config=func_from_config,
+                                                    leaf_loggers=leaf_loggers,
+                                                    capture=captured,
+                                                    *args,
+                                                    **kwargs,
+                                                )
+
+    def _apply_func_and_log(
+        self,
+        value: Any,
+        log_type: str,
+        tag: str,
+        func_from_config: str,
+        leaf_loggers: list,
+        *args,
+        **kwargs,
+    ):
+        if func_from_config is not None:
+            func_callable = import_from_registry(func_from_config)
+            value = func_callable(value)
+
+        for leaf_logger in leaf_loggers:
+            leaf_logger.log(
+                value=value,
+                tag=tag,
+                func=func_from_config,
+                log_type=log_type,
+                *args,
+                **kwargs,
+            )
 
 
 class SystemLogger(RootLogger):
@@ -159,17 +221,7 @@ class MetricLogger(RootLogger):
     LOG_TYPE = "metric"
 
     def __init__(self, config: Dict, leaf_logger):
-        self.capture = set()
         super().__init__(config, leaf_logger)
 
-    def create(self):
-        super().create()
-        for func_args in self.config.values():
-            for func_arg in func_args:
-                for capture in func_arg["capture"]:
-                    self.capture.add(capture)
-
-    def log(self, capture: str, *args, **kwargs):
-        for pattern in self.capture:
-            if is_match_found(pattern, capture):
-                super().log(log_type=self.LOG_TYPE, capture=capture, *args, **kwargs)
+    def log(self, *args, **kwargs):
+        super().log(log_type=self.LOG_TYPE, *args, **kwargs)

tests/logger_v2/test_root_logger.py

@@ -12,66 +12,65 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
+
 from unittest.mock import Mock
 
 from deepsparse.loggers_v2.root_logger import RootLogger
 
 
-class TestRootLogger(unittest.TestCase):
-    def test_log_method(self):
+def test_log_method():
 
-        mock_leaf_1 = Mock()
-        mock_leaf_1.log = Mock()
+    mock_leaf_1 = Mock()
+    mock_leaf_1.log = Mock()
 
-        mock_leaf_2 = Mock()
-        mock_leaf_2.log = Mock()
+    mock_leaf_2 = Mock()
+    mock_leaf_2.log = Mock()
 
-        mock_leaf_logger = {
-            "logger_id_1": mock_leaf_1,
-            "logger_id_2": mock_leaf_2,
-        }
+    mock_leaf_logger = {
+        "logger_id_1": mock_leaf_1,
+        "logger_id_2": mock_leaf_2,
+    }
 
-        mock_config = {
-            "tag1": [{"func": "identity", "freq": 2, "uses": ["logger_id_1"]}],
-            "tag2": [
-                {"func": "identity", "freq": 3, "uses": ["logger_id_2", "logger_id_1"]}
-            ],
-        }
+    mock_config = {
+        "tag1": [{"func": "identity", "freq": 2, "uses": ["logger_id_1"]}],
+        "tag2": [
+            {"func": "identity", "freq": 3, "uses": ["logger_id_2", "logger_id_1"]}
+        ],
+    }
 
-        root_logger = RootLogger(mock_config, mock_leaf_logger)
+    root_logger = RootLogger(mock_config, mock_leaf_logger)
 
-        root_logger.log("log_value", "log_type", "tag1")
-        assert mock_leaf_1.log.call_count == 0
+    root_logger.log("log_value", "log_type", "tag1")
+    assert mock_leaf_1.log.call_count == 0
 
-        root_logger.log("log_value", "log_type", "tag1")
-        assert mock_leaf_1.log.call_count == 1
+    root_logger.log("log_value", "log_type", "tag1")
+    assert mock_leaf_1.log.call_count == 1
 
-        mock_leaf_logger["logger_id_1"].log.assert_called_with(
-            value="log_value",
-            tag="tag1",
-            func="identity",
-            log_type="log_type",
-        )
+    mock_leaf_logger["logger_id_1"].log.assert_called_with(
+        value="log_value",
+        tag="tag1",
+        func="identity",
+        log_type="log_type",
+    )
 
-        root_logger.log("log_value", "log_type", "tag2")
-        root_logger.log("log_value", "log_type", "tag2")
-        assert mock_leaf_2.log.call_count == 0
+    root_logger.log("log_value", "log_type", "tag2")
+    root_logger.log("log_value", "log_type", "tag2")
+    assert mock_leaf_2.log.call_count == 0
 
-        root_logger.log("log_value", "log_type", "tag2")
-        assert mock_leaf_2.log.call_count == 1
-        assert mock_leaf_1.log.call_count == 2
+    root_logger.log("log_value", "log_type", "tag2")
+    assert mock_leaf_2.log.call_count == 1
+    assert mock_leaf_1.log.call_count == 2
 
-        mock_leaf_logger["logger_id_1"].log.assert_called_with(
-            value="log_value",
-            tag="tag2",
-            func="identity",
-            log_type="log_type",
-        )
+    mock_leaf_logger["logger_id_1"].log.assert_called_with(
+        value="log_value",
+        tag="tag2",
+        func="identity",
+        log_type="log_type",
+    )
 
-        mock_leaf_logger["logger_id_2"].log.assert_called_with(
-            value="log_value",
-            tag="tag2",
-            func="identity",
-            log_type="log_type",
-        )
+    mock_leaf_logger["logger_id_2"].log.assert_called_with(
+        value="log_value",
+        tag="tag2",
+        func="identity",
+        log_type="log_type",
+    )