Skip to content

Commit 5d022a7

Browse files
authored
[TextGeneration][Bug] Fix timer (#1294)
* create new timer if none * move timer check/creation to timer.py
1 parent 93ba723 commit 5d022a7

File tree

2 files changed

+11
-1
lines changed

2 files changed

+11
-1
lines changed

src/deepsparse/transformers/engines/nl_decoder_engine.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ def __call__(
185185
186186
:return: The generated token and corresponding logits
187187
"""
188-
timer = self.timer_manager.current
188+
timer = self.timer_manager.current_or_new()
189189
if kv_cache:
190190
# if model has kv cache enabled, we need
191191
# to add the kv cache state to the input

src/deepsparse/utils/timer.py

+10
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,16 @@ def all_times(self) -> Dict[str, List[float]]:
338338

339339
return all_times
340340

341+
def current_or_new(self) -> StagedTimer:
342+
"""
343+
Return the current timer if there is one, otherwise return a new one.
344+
"""
345+
if self.current:
346+
return self.current
347+
else:
348+
with self.new_timer_context(total_inference=False) as timer:
349+
return timer
350+
341351
def clear(self):
342352
for t in self._timers:
343353
t.clear()

0 commit comments

Comments
 (0)