Add trust_remote_code to tokenizer construction (#1176)

mgoin · web-flow · commit de4fb43dedba · 2023-08-09T11:21:10.000-04:00
* Add `trust_remote_code` to tokenizer construction

* Style
diff --git a/src/deepsparse/transformers/pipelines/pipeline.py b/src/deepsparse/transformers/pipelines/pipeline.py
@@ -122,7 +122,9 @@ def setup_onnx_file_path(self) -> str:
             finetuning_task=self.task if hasattr(self, "task") else None,
         )
         self.tokenizer = AutoTokenizer.from_pretrained(
-            tokenizer_path, model_max_length=self.sequence_length
+            tokenizer_path,
+            trust_remote_code=self._trust_remote_code,
+            model_max_length=self.sequence_length,
         )
         self.config_path = os.path.join(config_path, "config.json")
         self.tokenizer_config_path = os.path.join(tokenizer_path, "tokenizer.json")

Original file line number	Diff line number	Diff line change
`@@ -122,7 +122,9 @@ def setup_onnx_file_path(self) -> str:`
`122`	`122`	`finetuning_task=self.task if hasattr(self, "task") else None,`
`123`	`123`	`)`
`124`	`124`	`self.tokenizer = AutoTokenizer.from_pretrained(`
`125`		`- tokenizer_path, model_max_length=self.sequence_length`
	`125`	`+ tokenizer_path,`
	`126`	`+ trust_remote_code=self._trust_remote_code,`
	`127`	`+ model_max_length=self.sequence_length,`
`126`	`128`	`)`
`127`	`129`	`self.config_path = os.path.join(config_path, "config.json")`
`128`	`130`	`self.tokenizer_config_path = os.path.join(tokenizer_path, "tokenizer.json")`