From 625ecc17fce8efe480d86634dda7ae4893839bbc Mon Sep 17 00:00:00 2001 From: "bogunowicz@arrival.com" Date: Wed, 11 Jan 2023 17:21:13 +0100 Subject: [PATCH 1/7] initial commit --- src/deepsparse/loggers/function_logger.py | 38 +++++++++++++++++++---- src/deepsparse/loggers/helpers.py | 5 +++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/deepsparse/loggers/function_logger.py b/src/deepsparse/loggers/function_logger.py index 9dfeecc88e..ddc69557b8 100644 --- a/src/deepsparse/loggers/function_logger.py +++ b/src/deepsparse/loggers/function_logger.py @@ -15,7 +15,7 @@ """ Implementation of the Function Logger """ -from typing import Any, Callable +from typing import Any, Callable, Dict, Optional, Union from deepsparse.loggers import BaseLogger, MetricCategories from deepsparse.loggers.helpers import NO_MATCH, finalize_identifier, match_and_extract @@ -80,12 +80,38 @@ def log(self, identifier: str, value: Any, category: MetricCategories): if extracted_value != NO_MATCH: if self._function_call_counter % self.frequency == 0: mapped_value = self.function(extracted_value) - self.logger.log( - identifier=finalize_identifier( - identifier, category, self.function_name, remainder - ), - value=mapped_value, + self._log_mapped_value( + mapped_value=mapped_value, + identifier=identifier, category=category, + function_name=self.function_name, + remainder=remainder, ) self._function_call_counter = 0 self._function_call_counter += 1 + + def _log_mapped_value( + self, + mapped_value: Union[int, float, Dict[str, Union[int, float]]], + identifier: str, + category: MetricCategories, + function_name: str, + remainder: Optional[str] = None, + ): + + value_identifiers, mapped_values = [""], [mapped_value] + + if isinstance(mapped_value, dict): + value_identifiers = list(mapped_value.keys()) + mapped_values = list(mapped_value.values()) + + for value_identifier, value in zip(value_identifiers, mapped_values): + identifier = finalize_identifier( + identifier=identifier, + category=category, + function_name=function_name, + value_identifier=value_identifier, + remainder=remainder, + ) + + self.logger.log(identifier=identifier, value=value, category=category) diff --git a/src/deepsparse/loggers/helpers.py b/src/deepsparse/loggers/helpers.py index bf83009cee..f1a5ecd6d1 100644 --- a/src/deepsparse/loggers/helpers.py +++ b/src/deepsparse/loggers/helpers.py @@ -42,6 +42,7 @@ def finalize_identifier( identifier: str, category: MetricCategories, function_name: str, + value_identifier: str, remainder: Optional[str] = None, ) -> str: """ @@ -51,6 +52,8 @@ def finalize_identifier( :param category: The category of the identifier :param function_name: The name of the function applied to the identifier :param remainder: The remainder of the identifier after the matching was applied + :param value_identifier: The identifier that describes the name of the item + computed by the function :return: The final identifier string """ if remainder: @@ -64,6 +67,8 @@ def finalize_identifier( if category == MetricCategories.DATA: # if the category is DATA, add the function name to the identifier identifier += f"__{function_name}" + if value_identifier: + identifier += f"__{value_identifier}" return identifier From 1db5bc44c95bf1921ca08a4896ff61fe76ba9b9b Mon Sep 17 00:00:00 2001 From: "bogunowicz@arrival.com" Date: Thu, 12 Jan 2023 17:15:02 +0100 Subject: [PATCH 2/7] WIP --- proposal.md | 11 +++ .../natural_language_processing/__init__.py | 13 +++ .../natural_language_processing/built_ins.py | 81 +++++++++++++++++++ .../token_classification/__init__.py | 13 +++ .../token_classification/built_ins.py | 57 +++++++++++++ 5 files changed, 175 insertions(+) create mode 100644 proposal.md create mode 100644 src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py create mode 100644 src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py create mode 100644 src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py create mode 100644 src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py diff --git a/proposal.md b/proposal.md new file mode 100644 index 0000000000..0a7b3e9032 --- /dev/null +++ b/proposal.md @@ -0,0 +1,11 @@ +## UI for loggers + pipeline +```python +import Pipeline +pipeline = Pipeline.create(task="...", logger: Union[BaseLogger, str]) # str can be either a path or a yaml config +``` +1. If we decide to pass a loaded logger (from `logger_from_config`), we can use it as is. +This would not be the typical path for the user- we will be missing the pipeline names in target identifiers tho. +2. The preferred way would be to pass a string, we can either load a logger from a yaml config or create a logger from a path. This enables us to call the `logger_from_config` +under the hood and inject the pipeline name into the target identifiers. + +### Loading a preset data configs \ No newline at end of file diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py new file mode 100644 index 0000000000..0c44f887a4 --- /dev/null +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py new file mode 100644 index 0000000000..d179fc81a6 --- /dev/null +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py @@ -0,0 +1,81 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Union + +from pydantic import BaseModel + + +__all__ = ["sequence_length"] + + +def sequence_length(sequence: Union[List[str], str]) -> Union[Dict[str, int], int]: + """ + Returns the length of the sequence + + :param sequence: The sequence whose length is to be returned + :return: The length of the sequence + """ + if isinstance(sequence, str): + return len(sequence) + return {str(string_id): len(string) for string_id, string in enumerate(sequence)} + + +def percent_unknown_tokens(): + pass + + +def answer_found(qa_output: BaseModel) -> bool: + pass + + +def answer_length(qa_output: BaseModel) -> str: + return len(qa_output.answer) + + +def answer_score(qa_output: BaseModel) -> float: + return qa_output.score + + +def percent_zero_labels( + token_classification_output: List[Union[BaseModel, List[BaseModel]]] +): + if isinstance(token_classification_output[0], BaseModel): + return _percent_zero_labels(token_classification_output) + return { + str(result_id): _percent_zero_labels(result) + for result_id, result in enumerate(token_classification_output) + } + + +def mean_score(token_classification_output: List[Union[BaseModel, List[BaseModel]]]): + if isinstance(token_classification_output[0], BaseModel): + return _mean_score(token_classification_output) + return { + str(result_id): _mean_score(result) + for result_id, result in enumerate(token_classification_output) + } + + +def _mean_score(token_classification_output: List[BaseModel]) -> float: + return numpy.mean([result.score for result in token_classification_output]) + + +def _percent_zero_labels(token_classification_output: List[BaseModel]) -> float: + label_zero = "LABEL_0" + all_results = len(token_classification_output) + zero_results = sum( + 1 for result in token_classification_output if result.entity == label_zero + ) + return zero_results / all_results diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py new file mode 100644 index 0000000000..0c44f887a4 --- /dev/null +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py new file mode 100644 index 0000000000..07b21ec36f --- /dev/null +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py @@ -0,0 +1,57 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Union + +import numpy +from pydantic import BaseModel + + +def percent_zero_labels( + token_classification_output: List[Union[BaseModel, List[BaseModel]]] +) -> Union[float, Dict[str, float]]: + """ + Returns the percentage of zero labels in the token classification output + :param token_classification_output: + :return: + """ + + if isinstance(token_classification_output[0], BaseModel): + return _percent_zero_labels(token_classification_output) + return { + str(result_id): _percent_zero_labels(result) + for result_id, result in enumerate(token_classification_output) + } + + +def mean_score(token_classification_output: List[Union[BaseModel, List[BaseModel]]]): + if isinstance(token_classification_output[0], BaseModel): + return _mean_score(token_classification_output) + return { + str(result_id): _mean_score(result) + for result_id, result in enumerate(token_classification_output) + } + + +def _mean_score(token_classification_output: List[BaseModel]) -> float: + return numpy.mean([result.score for result in token_classification_output]) + + +def _percent_zero_labels(token_classification_output: List[BaseModel]) -> float: + label_zero = "LABEL_0" + all_results = len(token_classification_output) + zero_results = sum( + 1 for result in token_classification_output if result.entity == label_zero + ) + return zero_results / all_results From f78dc9ef597d900f04953dbe10cff1fde626d0dc Mon Sep 17 00:00:00 2001 From: "bogunowicz@arrival.com" Date: Mon, 16 Jan 2023 16:04:17 +0100 Subject: [PATCH 3/7] tests passing --- .../loggers/metric_functions/__init__.py | 2 - .../natural_language_processing/__init__.py | 2 +- .../natural_language_processing/built_ins.py | 42 +------------- .../question_answering/__init__.py | 16 ++++- .../question_answering/built_ins.py | 20 +++++-- .../token_classification/__init__.py | 2 +- .../token_classification/built_ins.py | 38 ++++++++---- .../natural_language_processing/__init__.py | 13 +++++ .../natural_language_processing/built_ins.py | 21 ++++++- .../question_answering/__init__.py | 13 +++++ .../question_answering/built_ins.py | 33 +++++++++-- .../token_classification/__init__.py | 13 +++++ .../token_classification/built_ins.py | 58 ++++++++++++++++--- 13 files changed, 196 insertions(+), 77 deletions(-) diff --git a/src/deepsparse/loggers/metric_functions/__init__.py b/src/deepsparse/loggers/metric_functions/__init__.py index 0e5d5ecc30..ef2739def2 100644 --- a/src/deepsparse/loggers/metric_functions/__init__.py +++ b/src/deepsparse/loggers/metric_functions/__init__.py @@ -14,5 +14,3 @@ # flake8: noqa from .built_ins import * -from .computer_vision import * -from .natural_language_processing import * \ No newline at end of file diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py index 48159dd1f6..e06dc921df 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. from .built_ins import * -from .question_answering import * \ No newline at end of file +from .question_answering import * diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py index 8a18796222..5543c64ac7 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py @@ -33,44 +33,4 @@ def sequence_length(sequence: Union[List[str], str]) -> Union[Dict[str, int], in def percent_unknown_tokens(): - pass - - - - - -def answer_score(qa_output: BaseModel) -> float: - return qa_output.score - - -def percent_zero_labels( - token_classification_output: List[Union[BaseModel, List[BaseModel]]] -): - if isinstance(token_classification_output[0], BaseModel): - return _percent_zero_labels(token_classification_output) - return { - str(result_id): _percent_zero_labels(result) - for result_id, result in enumerate(token_classification_output) - } - - -def mean_score(token_classification_output: List[Union[BaseModel, List[BaseModel]]]): - if isinstance(token_classification_output[0], BaseModel): - return _mean_score(token_classification_output) - return { - str(result_id): _mean_score(result) - for result_id, result in enumerate(token_classification_output) - } - - -def _mean_score(token_classification_output: List[BaseModel]) -> float: - return numpy.mean([result.score for result in token_classification_output]) - - -def _percent_zero_labels(token_classification_output: List[BaseModel]) -> float: - label_zero = "LABEL_0" - all_results = len(token_classification_output) - zero_results = sum( - 1 for result in token_classification_output if result.entity == label_zero - ) - return zero_results / all_results + raise NotImplementedError() diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py index b6cc42e289..efd8e91074 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py @@ -1 +1,15 @@ -from .built_ins import * \ No newline at end of file +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .built_ins import * diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index 72563d2f42..65131ac031 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -11,20 +11,32 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from deepsparse.loggers.metric_functions.natural_language_processing import sequence_length +""" +Set of functions for logging metrics from the question answering pipeline +""" +from deepsparse.loggers.metric_functions.natural_language_processing import ( + sequence_length, +) def answer_found(qa_output: "QuestionAnsweringOutput") -> bool: - pass + raise NotImplementedError() + def answer_length(qa_output: "QuestionAnsweringOutput") -> int: """ - Returns the length of the answer + Returns the length of the answer given the QuestionAnsweringOutput + :param qa_output: The output schema of the question answering pipeline :return: The length of the answer """ return sequence_length(qa_output.answer) + def answer_score(qa_output: "QuestionAnsweringOutput") -> float: + """ + Returns the score of the answer given the QuestionAnsweringOutput + :param qa_output: The output schema of the question answering pipeline + :return: The score of the answer + """ return qa_output.score diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py index c9b35a3d1f..efd8e91074 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .built_ins import * \ No newline at end of file +from .built_ins import * diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py index 3ecc29ce34..7e12375454 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py @@ -12,38 +12,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Union, Dict +from typing import Dict, List import numpy -from pydantic import BaseModel def percent_zero_labels( - token_classification_output: "TokenClassificationOutput" + token_classification_output: "TokenClassificationOutput", ) -> Dict[str, float]: """ Returns the percentage of zero labels in the token classification output - :param token_classification_output: - :return: + + :param token_classification_output: the TokenClassificationOutput object + :return: A dictionary where the key is the token sequence index and the + value is the percentage of zero labels in the sequence of tokens """ result = {} - for prediction_idx, prediction in enumerate(token_classification_output.predictions): + for prediction_idx, prediction in enumerate( + token_classification_output.predictions + ): result[str(prediction_idx)] = _percent_zero_labels(prediction) return result -def mean_score(token_classification_output: "TokenClassificationOutput"): +def mean_score( + token_classification_output: "TokenClassificationOutput", +) -> Dict[str, float]: + """ + Returns the mean score of the token classification output + + :param token_classification_output: the TokenClassificationOutput object + :return: A dictionary where the key is the token sequence index and the + value is the mean score of the sequence of tokens + """ result = {} - for prediction_idx, prediction in enumerate(token_classification_output.predictions): + for prediction_idx, prediction in enumerate( + token_classification_output.predictions + ): result[str(prediction_idx)] = _mean_score(prediction) return result -def _mean_score(token_classification_output: List["TokenClassificationResult"]) -> float: +def _mean_score( + token_classification_output: List["TokenClassificationResult"], +) -> float: return numpy.mean([result.score for result in token_classification_output]) -def _percent_zero_labels(token_classification_output: List["TokenClassificationResult"]) -> float: +def _percent_zero_labels( + token_classification_output: List["TokenClassificationResult"], +) -> float: label_zero = "LABEL_0" all_results = len(token_classification_output) zero_results = sum( diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py index e69de29bb2..0c44f887a4 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py index 09ebbd7665..b6aeec7c98 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py @@ -1,12 +1,27 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest from deepsparse.loggers.metric_functions import sequence_length + @pytest.mark.parametrize( "sequence, expected_len", [ ("His palms are sweaty", 20), - (["knees weak","arms are heavy"], {"0": 10, "1": 14}), - ] + (["knees weak", "arms are heavy"], {"0": 10, "1": 14}), + ], ) def test_sequence_length(sequence, expected_len): - assert sequence_length(sequence) == expected_len \ No newline at end of file + assert sequence_length(sequence) == expected_len diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py index e69de29bb2..0c44f887a4 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index 27e935df02..d595768af3 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -1,22 +1,45 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest +from deepsparse.loggers.metric_functions.natural_language_processing import ( + answer_length, + answer_score, +) from deepsparse.transformers.pipelines.question_answering import QuestionAnsweringOutput -from deepsparse.loggers.metric_functions import answer_length, answer_score -output_schema = QuestionAnsweringOutput(answer = "His palms are sweaty", score = 0.69, start = 0, end = 0) + +output_schema = QuestionAnsweringOutput( + answer="His palms are sweaty", score=0.69, start=0, end=0 +) + + @pytest.mark.parametrize( "schema, expected_len", [ (output_schema, 20), - ] + ], ) def test_answer_length(schema, expected_len): assert answer_length(schema) == expected_len + @pytest.mark.parametrize( "schema, expected_score", [ (output_schema, 0.69), - ] + ], ) def test_answer_score(schema, expected_score): - assert answer_score(schema) == expected_score \ No newline at end of file + assert answer_score(schema) == expected_score diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py index e69de29bb2..0c44f887a4 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py index 7376bd426f..01087483c4 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py @@ -1,27 +1,67 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest -from deepsparse.transformers.pipelines.token_classification import TokenClassificationOutput, TokenClassificationResult -from deepsparse.loggers.metric_functions.natural_language_processing.token_classification import percent_zero_labels, mean_score +from deepsparse.loggers.metric_functions.natural_language_processing import ( + mean_score, + percent_zero_labels, +) +from deepsparse.transformers.pipelines.token_classification import ( + TokenClassificationOutput, + TokenClassificationResult, +) + + label_0_result = TokenClassificationResult(entity="LABEL_0", score=0.3, index=0, word=0) label_1_result = TokenClassificationResult(entity="LABEL_1", score=0.6, index=0, word=0) + @pytest.mark.parametrize( "schema, expected_percent", [ - (TokenClassificationOutput(predictions=[[label_1_result, label_0_result], [label_0_result, label_0_result]]), {"0":0.5, "1": 1.0}) - ] + ( + TokenClassificationOutput( + predictions=[ + [label_1_result, label_0_result], + [label_0_result, label_0_result], + ] + ), + {"0": 0.5, "1": 1.0}, + ) + ], ) def test_percent_zero_labels(schema, expected_percent): assert percent_zero_labels(schema) == expected_percent + @pytest.mark.parametrize( "schema, expected_score", [ - (TokenClassificationOutput(predictions=[[label_1_result, label_0_result], [label_0_result, label_0_result]]), {"0":0.45, "1": 0.3}) - ] + ( + TokenClassificationOutput( + predictions=[ + [label_1_result, label_0_result], + [label_0_result, label_0_result], + ] + ), + {"0": 0.45, "1": 0.3}, + ) + ], ) def test_percent_zero_labels(schema, expected_score): result = mean_score(schema) - keys1, values1= set(expected_score.keys()), set(expected_score.values()) - keys2, values2= set(result.keys()), set(result.values()) + keys1, values1 = set(expected_score.keys()), set(expected_score.values()) + keys2, values2 = set(result.keys()), set(result.values()) assert keys1 == keys2 - assert pytest.approx(list(values1)) == list(values2) \ No newline at end of file + assert pytest.approx(list(values1)) == list(values2) From 6b26bdd1189e6a38269bbd53c218a311668da83d Mon Sep 17 00:00:00 2001 From: dbogunowicz <97082108+dbogunowicz@users.noreply.github.com> Date: Mon, 16 Jan 2023 16:05:20 +0100 Subject: [PATCH 4/7] Delete proposal.md --- proposal.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 proposal.md diff --git a/proposal.md b/proposal.md deleted file mode 100644 index 0a7b3e9032..0000000000 --- a/proposal.md +++ /dev/null @@ -1,11 +0,0 @@ -## UI for loggers + pipeline -```python -import Pipeline -pipeline = Pipeline.create(task="...", logger: Union[BaseLogger, str]) # str can be either a path or a yaml config -``` -1. If we decide to pass a loaded logger (from `logger_from_config`), we can use it as is. -This would not be the typical path for the user- we will be missing the pipeline names in target identifiers tho. -2. The preferred way would be to pass a string, we can either load a logger from a yaml config or create a logger from a path. This enables us to call the `logger_from_config` -under the hood and inject the pipeline name into the target identifiers. - -### Loading a preset data configs \ No newline at end of file From 3fb247c9ac39af851a81742962b86b2a3667cd97 Mon Sep 17 00:00:00 2001 From: "bogunowicz@arrival.com" Date: Mon, 16 Jan 2023 16:19:25 +0100 Subject: [PATCH 5/7] tests green --- src/deepsparse/loggers/helpers.py | 5 ----- .../loggers/metric_functions/__init__.py | 2 ++ .../natural_language_processing/__init__.py | 3 ++- .../natural_language_processing/built_ins.py | 8 ++++---- .../question_answering/__init__.py | 2 +- .../question_answering/built_ins.py | 11 ++++++++--- .../token_classification/__init__.py | 2 +- .../token_classification/built_ins.py | 15 ++++++++++----- .../question_answering/built_ins.py | 5 +---- .../token_classification/built_ins.py | 7 ++----- 10 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/deepsparse/loggers/helpers.py b/src/deepsparse/loggers/helpers.py index 1ce5c51ea3..8fdad639c3 100644 --- a/src/deepsparse/loggers/helpers.py +++ b/src/deepsparse/loggers/helpers.py @@ -42,7 +42,6 @@ def finalize_identifier( identifier: str, category: MetricCategories, function_name: str, - value_identifier: str, remainder: Optional[str] = None, ) -> str: """ @@ -52,8 +51,6 @@ def finalize_identifier( :param category: The category of the identifier :param function_name: The name of the function applied to the identifier :param remainder: The remainder of the identifier after the matching was applied - :param value_identifier: The identifier that describes the name of the item - computed by the function :return: The final identifier string """ if remainder: @@ -67,8 +64,6 @@ def finalize_identifier( if category == MetricCategories.DATA: # if the category is DATA, add the function name to the identifier identifier += f"__{function_name}" - if value_identifier: - identifier += f"__{value_identifier}" return identifier diff --git a/src/deepsparse/loggers/metric_functions/__init__.py b/src/deepsparse/loggers/metric_functions/__init__.py index ef2739def2..dcef8ad217 100644 --- a/src/deepsparse/loggers/metric_functions/__init__.py +++ b/src/deepsparse/loggers/metric_functions/__init__.py @@ -14,3 +14,5 @@ # flake8: noqa from .built_ins import * +from .computer_vision import * +from .natural_language_processing import * diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py index e06dc921df..39fb7a4439 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/__init__.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# flake8: noqa from .built_ins import * from .question_answering import * +from .token_classification import * diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py index 5543c64ac7..0d8e865a5b 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +""" +Set of functions for logging metrics from the natural language processing pipelines +""" from typing import Dict, List, Union -from pydantic import BaseModel - -__all__ = ["sequence_length"] +__all__ = ["sequence_length", "percent_unknown_tokens"] def sequence_length(sequence: Union[List[str], str]) -> Union[Dict[str, int], int]: diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py index efd8e91074..cd9cecb8bc 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/__init__.py @@ -11,5 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# flake8: noqa from .built_ins import * diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index 65131ac031..c7b8c942b4 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -14,16 +14,20 @@ """ Set of functions for logging metrics from the question answering pipeline """ + from deepsparse.loggers.metric_functions.natural_language_processing import ( sequence_length, ) -def answer_found(qa_output: "QuestionAnsweringOutput") -> bool: +__all__ = ["answer_found", "answer_length", "answer_score"] + + +def answer_found(qa_output: "QuestionAnsweringOutput") -> bool: # noqa: F821 raise NotImplementedError() -def answer_length(qa_output: "QuestionAnsweringOutput") -> int: +def answer_length(qa_output: "QuestionAnsweringOutput") -> int: # noqa: F821 """ Returns the length of the answer given the QuestionAnsweringOutput @@ -33,9 +37,10 @@ def answer_length(qa_output: "QuestionAnsweringOutput") -> int: return sequence_length(qa_output.answer) -def answer_score(qa_output: "QuestionAnsweringOutput") -> float: +def answer_score(qa_output: "QuestionAnsweringOutput") -> float: # noqa: F821 """ Returns the score of the answer given the QuestionAnsweringOutput + :param qa_output: The output schema of the question answering pipeline :return: The score of the answer """ diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py index efd8e91074..cd9cecb8bc 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/__init__.py @@ -11,5 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# flake8: noqa from .built_ins import * diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py index 7e12375454..c3d607c1f9 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py @@ -11,14 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +""" +Set of functions for logging metrics from the token classification pipeline +""" from typing import Dict, List import numpy +__all__ = ["mean_score", "percent_zero_labels"] + + def percent_zero_labels( - token_classification_output: "TokenClassificationOutput", + token_classification_output: "TokenClassificationOutput", # noqa: F821 ) -> Dict[str, float]: """ Returns the percentage of zero labels in the token classification output @@ -36,7 +41,7 @@ def percent_zero_labels( def mean_score( - token_classification_output: "TokenClassificationOutput", + token_classification_output: "TokenClassificationOutput", # noqa: F821 ) -> Dict[str, float]: """ Returns the mean score of the token classification output @@ -54,13 +59,13 @@ def mean_score( def _mean_score( - token_classification_output: List["TokenClassificationResult"], + token_classification_output: List["TokenClassificationResult"], # noqa: F821 ) -> float: return numpy.mean([result.score for result in token_classification_output]) def _percent_zero_labels( - token_classification_output: List["TokenClassificationResult"], + token_classification_output: List["TokenClassificationResult"], # noqa: F821 ) -> float: label_zero = "LABEL_0" all_results = len(token_classification_output) diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index d595768af3..6fa469866f 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -13,10 +13,7 @@ # limitations under the License. import pytest -from deepsparse.loggers.metric_functions.natural_language_processing import ( - answer_length, - answer_score, -) +from deepsparse.loggers.metric_functions import answer_length, answer_score from deepsparse.transformers.pipelines.question_answering import QuestionAnsweringOutput diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py index 01087483c4..5253574c1a 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py @@ -13,10 +13,7 @@ # limitations under the License. import pytest -from deepsparse.loggers.metric_functions.natural_language_processing import ( - mean_score, - percent_zero_labels, -) +from deepsparse.loggers.metric_functions import mean_score, percent_zero_labels from deepsparse.transformers.pipelines.token_classification import ( TokenClassificationOutput, TokenClassificationResult, @@ -59,7 +56,7 @@ def test_percent_zero_labels(schema, expected_percent): ) ], ) -def test_percent_zero_labels(schema, expected_score): +def test_mean_score(schema, expected_score): result = mean_score(schema) keys1, values1 = set(expected_score.keys()), set(expected_score.values()) keys2, values2 = set(result.keys()), set(result.values()) From 82ebc714c94a0b3e667f35582d7405262d0d76cc Mon Sep 17 00:00:00 2001 From: "bogunowicz@arrival.com" Date: Tue, 17 Jan 2023 10:02:36 +0100 Subject: [PATCH 6/7] ready for reviews --- .../question_answering/built_ins.py | 7 ++++++- .../question_answering/built_ins.py | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index c7b8c942b4..c10b4ac1f8 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -24,7 +24,12 @@ def answer_found(qa_output: "QuestionAnsweringOutput") -> bool: # noqa: F821 - raise NotImplementedError() + """ + Returns whether an answer was found given the QuestionAnsweringOutput + :param qa_output: The output schema of the question answering pipeline + :return: True if an answer was found, False otherwise + """ + return not qa_output.answer == "empty" def answer_length(qa_output: "QuestionAnsweringOutput") -> int: # noqa: F821 diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index 6fa469866f..3c1dcd1d6f 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -13,7 +13,11 @@ # limitations under the License. import pytest -from deepsparse.loggers.metric_functions import answer_length, answer_score +from deepsparse.loggers.metric_functions import ( + answer_found, + answer_length, + answer_score, +) from deepsparse.transformers.pipelines.question_answering import QuestionAnsweringOutput @@ -40,3 +44,14 @@ def test_answer_length(schema, expected_len): ) def test_answer_score(schema, expected_score): assert answer_score(schema) == expected_score + + +@pytest.mark.parametrize( + "schema, expected_bool", + [ + (output_schema, True), + (QuestionAnsweringOutput(answer="empty", score=0.69, start=0, end=0), False), + ], +) +def test_answer_found(schema, expected_bool): + assert answer_found(schema) == expected_bool From 6028f1fde9e336886bea742b225d9316d25150ab Mon Sep 17 00:00:00 2001 From: "bogunowicz@arrival.com" Date: Wed, 18 Jan 2023 19:07:07 +0100 Subject: [PATCH 7/7] corrections --- .../natural_language_processing/built_ins.py | 4 ++-- .../question_answering/built_ins.py | 6 ++++-- .../natural_language_processing/built_ins.py | 10 ++++++---- .../question_answering/built_ins.py | 6 ++++-- .../token_classification/built_ins.py | 5 ++++- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py index 0d8e865a5b..f65b6b2026 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py @@ -17,10 +17,10 @@ from typing import Dict, List, Union -__all__ = ["sequence_length", "percent_unknown_tokens"] +__all__ = ["string_length", "percent_unknown_tokens"] -def sequence_length(sequence: Union[List[str], str]) -> Union[Dict[str, int], int]: +def string_length(sequence: Union[List[str], str]) -> Union[Dict[str, int], int]: """ Returns the length of the sequence diff --git a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index c10b4ac1f8..37c1edc74e 100644 --- a/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -16,7 +16,7 @@ """ from deepsparse.loggers.metric_functions.natural_language_processing import ( - sequence_length, + string_length, ) @@ -39,7 +39,9 @@ def answer_length(qa_output: "QuestionAnsweringOutput") -> int: # noqa: F821 :param qa_output: The output schema of the question answering pipeline :return: The length of the answer """ - return sequence_length(qa_output.answer) + if qa_output.answer == "empty": + return 0 + return string_length(qa_output.answer) def answer_score(qa_output: "QuestionAnsweringOutput") -> float: # noqa: F821 diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py index b6aeec7c98..7102735501 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py @@ -13,15 +13,17 @@ # limitations under the License. import pytest -from deepsparse.loggers.metric_functions import sequence_length +from deepsparse.loggers.metric_functions.natural_language_processing import ( + string_length, +) @pytest.mark.parametrize( - "sequence, expected_len", + "string, expected_len", [ ("His palms are sweaty", 20), (["knees weak", "arms are heavy"], {"0": 10, "1": 14}), ], ) -def test_sequence_length(sequence, expected_len): - assert sequence_length(sequence) == expected_len +def test_string_length(string, expected_len): + assert string_length(string) == expected_len diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py index 3c1dcd1d6f..46e44a8ba2 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py @@ -13,7 +13,7 @@ # limitations under the License. import pytest -from deepsparse.loggers.metric_functions import ( +from deepsparse.loggers.metric_functions.natural_language_processing import ( answer_found, answer_length, answer_score, @@ -24,12 +24,14 @@ output_schema = QuestionAnsweringOutput( answer="His palms are sweaty", score=0.69, start=0, end=0 ) +empty_schema = QuestionAnsweringOutput(answer="empty", score=0.69, start=0, end=0) @pytest.mark.parametrize( "schema, expected_len", [ (output_schema, 20), + (empty_schema, 0), ], ) def test_answer_length(schema, expected_len): @@ -50,7 +52,7 @@ def test_answer_score(schema, expected_score): "schema, expected_bool", [ (output_schema, True), - (QuestionAnsweringOutput(answer="empty", score=0.69, start=0, end=0), False), + (empty_schema, False), ], ) def test_answer_found(schema, expected_bool): diff --git a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py index 5253574c1a..34f1061167 100644 --- a/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py +++ b/tests/deepsparse/loggers/metric_functions/natural_language_processing/token_classification/built_ins.py @@ -13,7 +13,10 @@ # limitations under the License. import pytest -from deepsparse.loggers.metric_functions import mean_score, percent_zero_labels +from deepsparse.loggers.metric_functions.natural_language_processing import ( + mean_score, + percent_zero_labels, +) from deepsparse.transformers.pipelines.token_classification import ( TokenClassificationOutput, TokenClassificationResult,