llama : fix llama_token_is_prefix

ggerganov · ggerganov · commit 63840020afbf · 2024-10-11T10:51:19.000+03:00
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
@@ -1865,14 +1865,14 @@ bool llama_token_is_prefix_impl(
     char text_buf_0[128];
     char text_buf_1[128];
 
-    const int32_t len0 = llama_token_to_piece_impl(vocab, token0, text_buf_0, 128, 0, false);
-    const int32_t len1 = llama_token_to_piece_impl(vocab, token1, text_buf_1, 128, 0, false);
+    const int32_t len0 = llama_token_to_piece_impl(vocab, token0, text_buf_0, sizeof(text_buf_0) - 1, 0, false);
+    const int32_t len1 = llama_token_to_piece_impl(vocab, token1, text_buf_1, sizeof(text_buf_1) - 1, 0, false);
 
     if (len0 <= 0 || len1 <= 0) {
         return false;
     }
 
-    return len0 < len1 && memcmp(text_buf_0, text_buf_1, len0) == 0;
+    return len0 <= len1 && memcmp(text_buf_0, text_buf_1, len0) == 0;
 }
 
 int32_t llama_detokenize_impl(

Original file line number	Diff line number	Diff line change
`@@ -1865,14 +1865,14 @@ bool llama_token_is_prefix_impl(`
`1865`	`1865`	`char text_buf_0[128];`
`1866`	`1866`	`char text_buf_1[128];`
`1867`	`1867`
`1868`		`- const int32_t len0 = llama_token_to_piece_impl(vocab, token0, text_buf_0, 128, 0, false);`
`1869`		`- const int32_t len1 = llama_token_to_piece_impl(vocab, token1, text_buf_1, 128, 0, false);`
	`1868`	`+ const int32_t len0 = llama_token_to_piece_impl(vocab, token0, text_buf_0, sizeof(text_buf_0) - 1, 0, false);`
	`1869`	`+ const int32_t len1 = llama_token_to_piece_impl(vocab, token1, text_buf_1, sizeof(text_buf_1) - 1, 0, false);`
`1870`	`1870`
`1871`	`1871`	`if (len0 <= 0 \|\| len1 <= 0) {`
`1872`	`1872`	`return false;`
`1873`	`1873`	`}`
`1874`	`1874`
`1875`		`- return len0 < len1 && memcmp(text_buf_0, text_buf_1, len0) == 0;`
	`1875`	`+ return len0 <= len1 && memcmp(text_buf_0, text_buf_1, len0) == 0;`
`1876`	`1876`	`}`
`1877`	`1877`
`1878`	`1878`	`int32_t llama_detokenize_impl(`