Apply suggestions from code review

jaime-m-p · ggerganov · web-flow · commit 0ae2860faa8d · 2024-05-19T21:13:20.000+02:00
Co-authored-by: Georgi Gerganov &lt;ggerganov@gmail.com&gt;
diff --git a/llama.cpp b/llama.cpp
@@ -12820,13 +12820,13 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                             while (isspace(raw_text[num_whitespaces])) {
                                 num_whitespaces++;
                             }
-                            if(num_whitespaces == raw_text.size()) {
+                            if (num_whitespaces == raw_text.size()) {
                                 continue; // skip if all whitespaces
                             }
                             raw_text = raw_text.substr(num_whitespaces);
                         }
 
-                        if(vocab.add_space_prefix) {
+                        if (vocab.add_space_prefix) {
                             if (!output.size() || is_prev_special) {  // prefix with space if first token
                                 raw_text = " " + raw_text;
                             }

Original file line number	Diff line number	Diff line change
`@@ -12820,13 +12820,13 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &`
`12820`	`12820`	`while (isspace(raw_text[num_whitespaces])) {`
`12821`	`12821`	`num_whitespaces++;`
`12822`	`12822`	`}`
`12823`		`- if(num_whitespaces == raw_text.size()) {`
	`12823`	`+ if (num_whitespaces == raw_text.size()) {`
`12824`	`12824`	`continue; // skip if all whitespaces`
`12825`	`12825`	`}`
`12826`	`12826`	`raw_text = raw_text.substr(num_whitespaces);`
`12827`	`12827`	`}`
`12828`	`12828`
`12829`		`- if(vocab.add_space_prefix) {`
	`12829`	`+ if (vocab.add_space_prefix) {`
`12830`	`12830`	`if (!output.size() \|\| is_prev_special) { // prefix with space if first token`
`12831`	`12831`	`raw_text = " " + raw_text;`
`12832`	`12832`	`}`