From 202810d867cabdd7cad9934905c2d740997e283e Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Tue, 3 Oct 2023 12:00:46 -0400 Subject: [PATCH 1/2] create new timer if none --- src/deepsparse/transformers/engines/nl_decoder_engine.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/deepsparse/transformers/engines/nl_decoder_engine.py b/src/deepsparse/transformers/engines/nl_decoder_engine.py index 5ec5001c2f..dd23eb3227 100644 --- a/src/deepsparse/transformers/engines/nl_decoder_engine.py +++ b/src/deepsparse/transformers/engines/nl_decoder_engine.py @@ -185,7 +185,14 @@ def __call__( :return: The generated token and corresponding logits """ + timer = self.timer_manager.current + if not timer: + with self.timer_manager.new_timer_context( + total_inference=False + ) as new_timer: + timer = new_timer + if kv_cache: # if model has kv cache enabled, we need # to add the kv cache state to the input From 436fe8eee1424e38067b024efb86195c8beecd05 Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Tue, 3 Oct 2023 15:50:27 -0400 Subject: [PATCH 2/2] move timer check/creation to timer.py --- .../transformers/engines/nl_decoder_engine.py | 9 +-------- src/deepsparse/utils/timer.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/deepsparse/transformers/engines/nl_decoder_engine.py b/src/deepsparse/transformers/engines/nl_decoder_engine.py index dd23eb3227..9c93616053 100644 --- a/src/deepsparse/transformers/engines/nl_decoder_engine.py +++ b/src/deepsparse/transformers/engines/nl_decoder_engine.py @@ -185,14 +185,7 @@ def __call__( :return: The generated token and corresponding logits """ - - timer = self.timer_manager.current - if not timer: - with self.timer_manager.new_timer_context( - total_inference=False - ) as new_timer: - timer = new_timer - + timer = self.timer_manager.current_or_new() if kv_cache: # if model has kv cache enabled, we need # to add the kv cache state to the input diff --git a/src/deepsparse/utils/timer.py b/src/deepsparse/utils/timer.py index 56a3452b6e..47dfd05dcb 100644 --- a/src/deepsparse/utils/timer.py +++ b/src/deepsparse/utils/timer.py @@ -338,6 +338,16 @@ def all_times(self) -> Dict[str, List[float]]: return all_times + def current_or_new(self) -> StagedTimer: + """ + Return the current timer if there is one, otherwise return a new one. + """ + if self.current: + return self.current + else: + with self.new_timer_context(total_inference=False) as timer: + return timer + def clear(self): for t in self._timers: t.clear()