[ML] fix NLP tokenization never_split handling around punctuation (#82982)

When multiple characters in a row might be part of the never_split we erroneously tokenized them.

This commit handles this scenario so now `[[UNK]` is now tokenized as `[`, `[UNK]`

Author: benwtrent

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BasicTokenizer.java

@@ -128,7 +128,13 @@ private List<DelimitedToken> mergeNeverSplitTokens(String originalText, List<Del
                     matchingTokens = new ArrayList<>();
                     current = neverSplitTokenTrieRoot;
                 }
-                mergedTokens.add(token);
+                childNode = current.getChild(token.getToken());
+                if (childNode == null) {
+                    mergedTokens.add(token);
+                } else {
+                    matchingTokens.add(token);
+                    current = childNode;
+                }
             } else if (childNode.isLeaf()) {
                 matchingTokens.add(token);
                 DelimitedToken mergedToken = DelimitedToken.mergeTokens(matchingTokens);

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BasicTokenizerTests.java

@@ -79,6 +79,11 @@ public void testNeverSplit_GivenNoLowerCase() {
         assertThat(tokenStrings(tokens), contains("Hello", "-", "[UNK]"));
         tokens = tokenizer.tokenize("Hello~[UNK][UNK]");
         assertThat(tokenStrings(tokens), contains("Hello", "~", "[UNK]", "[UNK]"));
+        assertThat(tokenStrings(tokenizer.tokenize("Hello~[[UNK]")), contains("Hello", "~", "[", "[UNK]"));
+        assertThat(tokenStrings(tokenizer.tokenize("Hello~[[[UNK]")), contains("Hello", "~", "[", "[", "[UNK]"));
+        assertThat(tokenStrings(tokenizer.tokenize("Hello~[UNK]]")), contains("Hello", "~", "[UNK]", "]"));
+        assertThat(tokenStrings(tokenizer.tokenize("Hello~[UNK]]]")), contains("Hello", "~", "[UNK]", "]", "]"));
+        assertThat(tokenStrings(tokenizer.tokenize("Hello~[[UNK]]")), contains("Hello", "~", "[", "[UNK]", "]"));
         tokens = tokenizer.tokenize("Hello-[unk]");
         assertThat(tokenStrings(tokens), contains("Hello", "-", "[", "unk", "]"));
     }