vocab : fix double-eos check (ggml-org#11273)

ggerganov · tinglou · commit f68e8fa1c3f3 · 2025-02-13T23:04:06.000+08:00
ggml-ci
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
@@ -439,7 +439,7 @@ struct llm_tokenizer_bpe_session {
                 "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
                 "Are you sure this is what you want?\n", __FUNCTION__);
         }
-        if (vocab.get_add_bos() && output.size() >= 2 && *(output.end()-2) == vocab.token_eos()) {
+        if (vocab.get_add_eos() && output.size() >= 2 && *(output.end()-2) == vocab.token_eos()) {
             LLAMA_LOG_WARN(
                 "%s: Added a EOS token to the prompt as specified by the model but the prompt "
                 "also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "

Original file line number	Diff line number	Diff line change
`@@ -439,7 +439,7 @@ struct llm_tokenizer_bpe_session {`
`439`	`439`	`"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "`
`440`	`440`	`"Are you sure this is what you want?\n", __FUNCTION__);`
`441`	`441`	`}`
`442`		`- if (vocab.get_add_bos() && output.size() >= 2 && *(output.end()-2) == vocab.token_eos()) {`
	`442`	`+ if (vocab.get_add_eos() && output.size() >= 2 && *(output.end()-2) == vocab.token_eos()) {`
`443`	`443`	`LLAMA_LOG_WARN(`
`444`	`444`	`"%s: Added a EOS token to the prompt as specified by the model but the prompt "`
`445`	`445`	`"also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "`