Skip to content

Commit 3aa124c

Browse files
jimczijimczi
committed
Add support for inlined user dictionary in the Kuromoji plugin
This change adds a new option called user_dictionary_rules to Kuromoji's tokenizer. It can be used to set additional tokenization rules to the Japanese tokenizer directly in the settings (instead of using a file). This commit also adds a check that no rules are duplicated since this is not allowed in the UserDictionary. Closes elastic#25343
1 parent 5436581 commit 3aa124c

File tree

7 files changed

+123
-15
lines changed

7 files changed

+123
-15
lines changed

‎docs/plugins/analysis-kuromoji.asciidoc

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,39 @@ dictionary to `$ES_HOME/config/userdict_ja.txt`:
9898
東京スカイツリー,東京 スカイツリー,トウキョウ スカイツリー,カスタム名詞
9999
-----------------------
100100

101+
--
102+
103+
You can also inline the rules directly in the tokenizer definition using
104+
the `user_dictionary_rules` option:
105+
106+
[source,js]
107+
--------------------------------------------------
108+
PUT nori_sample
109+
{
110+
"settings": {
111+
"index": {
112+
"analysis": {
113+
"tokenizer": {
114+
"kuromoji_user_dict": {
115+
"type": "kuromoji_tokenizer",
116+
"mode": "extended",
117+
"user_dictionary_rules": ["東京スカイツリー,東京 スカイツリー,トウキョウ スカイツリー,カスタム名詞"]
118+
}
119+
},
120+
"analyzer": {
121+
"my_analyzer": {
122+
"type": "custom",
123+
"tokenizer": "kuromoji_user_dict"
124+
}
125+
}
126+
}
127+
}
128+
}
129+
}
130+
--------------------------------------------------
131+
// CONSOLE
132+
--
133+
101134
`nbest_cost`/`nbest_examples`::
102135
+
103136
--

‎plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,23 @@
2323
import org.apache.lucene.analysis.ja.JapaneseTokenizer;
2424
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
2525
import org.apache.lucene.analysis.ja.dict.UserDictionary;
26+
import org.apache.lucene.analysis.ja.util.CSVUtil;
2627
import org.elasticsearch.ElasticsearchException;
2728
import org.elasticsearch.common.settings.Settings;
2829
import org.elasticsearch.env.Environment;
2930
import org.elasticsearch.index.IndexSettings;
3031

3132
import java.io.IOException;
3233
import java.io.Reader;
34+
import java.io.StringReader;
35+
import java.util.HashSet;
36+
import java.util.List;
37+
import java.util.Set;
3338

3439
public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
3540

36-
private static final String USER_DICT_OPTION = "user_dictionary";
41+
private static final String USER_DICT_PATH_OPTION = "user_dictionary";
42+
private static final String USER_DICT_RULES_OPTION = "user_dictionary_rules";
3743
private static final String NBEST_COST = "nbest_cost";
3844
private static final String NBEST_EXAMPLES = "nbest_examples";
3945

@@ -54,17 +60,32 @@ public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, St
5460
}
5561

5662
public static UserDictionary getUserDictionary(Environment env, Settings settings) {
63+
if (settings.get(USER_DICT_PATH_OPTION) != null && settings.get(USER_DICT_RULES_OPTION) != null) {
64+
throw new IllegalArgumentException("It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" +
65+
" with [" + USER_DICT_RULES_OPTION + "]");
66+
}
5767
try {
58-
final Reader reader = Analysis.getReaderFromFile(env, settings, USER_DICT_OPTION);
59-
if (reader == null) {
68+
List<String> ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, false);
69+
if (ruleList == null || ruleList.isEmpty()) {
6070
return null;
61-
} else {
62-
try {
63-
return UserDictionary.open(reader);
64-
} finally {
65-
reader.close();
71+
}
72+
Set<String> dup = new HashSet<>();
73+
int lineNum = 0;
74+
for (String line : ruleList) {
75+
// ignore comments
76+
if (line.startsWith("#") == false) {
77+
String[] values = CSVUtil.parse(line);
78+
if (dup.add(values[0]) == false) {
79+
throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary at line [" + lineNum + "]");
80+
}
6681
}
82+
++ lineNum;
83+
}
84+
StringBuilder sb = new StringBuilder();
85+
for (String line : ruleList) {
86+
sb.append(line).append(System.lineSeparator());
6787
}
88+
return UserDictionary.open(new StringReader(sb.toString()));
6889
} catch (IOException e) {
6990
throw new ElasticsearchException("failed to load kuromoji user dictionary", e);
7091
}

‎plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.index.analysis;
2121

22+
import org.apache.lucene.analysis.Analyzer;
2223
import org.apache.lucene.analysis.TokenStream;
2324
import org.apache.lucene.analysis.Tokenizer;
2425
import org.apache.lucene.analysis.ja.JapaneseAnalyzer;
@@ -39,6 +40,8 @@
3940
import java.nio.file.Files;
4041
import java.nio.file.Path;
4142

43+
import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
44+
import static org.hamcrest.CoreMatchers.containsString;
4245
import static org.hamcrest.Matchers.equalTo;
4346
import static org.hamcrest.Matchers.greaterThan;
4447
import static org.hamcrest.Matchers.instanceOf;
@@ -307,4 +310,55 @@ public void testNumberFilterFactory() throws Exception {
307310
tokenizer.setReader(new StringReader(source));
308311
assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
309312
}
313+
314+
public void testKuromojiAnalyzerUserDict() throws Exception {
315+
Settings settings = Settings.builder()
316+
.put("index.analysis.analyzer.my_analyzer.type", "kuromoji")
317+
.putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++,c++,w,w", "制限スピード,制限スピード,セイゲンスピード,テスト名詞")
318+
.build();
319+
TestAnalysis analysis = createTestAnalysis(settings);
320+
Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer");
321+
try (TokenStream stream = analyzer.tokenStream("", "制限スピード")) {
322+
assertTokenStreamContents(stream, new String[]{"制限スピード"});
323+
}
324+
325+
try (TokenStream stream = analyzer.tokenStream("", "c++world")) {
326+
assertTokenStreamContents(stream, new String[]{"c++", "world"});
327+
}
328+
}
329+
330+
public void testKuromojiAnalyzerInvalidUserDictOption() throws Exception {
331+
Settings settings = Settings.builder()
332+
.put("index.analysis.analyzer.my_analyzer.type", "kuromoji")
333+
.put("index.analysis.analyzer.my_analyzer.user_dictionary", "user_dict.txt")
334+
.putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++,c++,w,w")
335+
.build();
336+
IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings));
337+
assertThat(exc.getMessage(), containsString("It is not allowed to use [user_dictionary] in conjunction " +
338+
"with [user_dictionary_rules]"));
339+
}
340+
341+
public void testKuromojiAnalyzerDuplicateUserDictRule() throws Exception {
342+
Settings settings = Settings.builder()
343+
.put("index.analysis.analyzer.my_analyzer.type", "kuromoji")
344+
.putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules",
345+
"c++,c++,w,w", "#comment", "制限スピード,制限スピード,セイゲンスピード,テスト名詞", "制限スピード,制限スピード,セイゲンスピード,テスト名詞")
346+
.build();
347+
IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings));
348+
assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [3]"));
349+
}
350+
351+
private TestAnalysis createTestAnalysis(Settings analysisSettings) throws IOException {
352+
InputStream dict = KuromojiAnalysisTests.class.getResourceAsStream("user_dict.txt");
353+
Path home = createTempDir();
354+
Path config = home.resolve("config");
355+
Files.createDirectory(config);
356+
Files.copy(dict, config.resolve("user_dict.txt"));
357+
Settings settings = Settings.builder()
358+
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
359+
.put(Environment.PATH_HOME_SETTING.getKey(), home)
360+
.put(analysisSettings)
361+
.build();
362+
return AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new AnalysisKuromojiPlugin());
363+
}
310364
}

‎plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/10_basic.yml renamed to ‎plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/10_basic.yml

File renamed without changes.

‎plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_nori/20_search.yml renamed to ‎plugins/analysis-kuromoji/src/test/resources/rest-api-spec/test/analysis_kuromoji/20_search.yml

File renamed without changes.

‎plugins/analysis-nori/src/main/java/org/elasticsearch/index/analysis/NoriTokenizerFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting
5151
throw new IllegalArgumentException("It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" +
5252
" with [" + USER_DICT_RULES_OPTION + "]");
5353
}
54-
List<String> ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION);
54+
List<String> ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, true);
5555
StringBuilder sb = new StringBuilder();
5656
if (ruleList == null || ruleList.isEmpty()) {
5757
return null;

‎server/src/main/java/org/elasticsearch/index/analysis/Analysis.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ public static CharArraySet getWordSet(Environment env, Settings settings, String
215215
* If the word list cannot be found at either key.
216216
*/
217217
public static List<String> getWordList(Environment env, Settings settings, String settingPrefix) {
218-
return getWordList(env, settings, settingPrefix + "_path", settingPrefix);
218+
return getWordList(env, settings, settingPrefix + "_path", settingPrefix, true);
219219
}
220220

221221
/**
@@ -225,7 +225,7 @@ public static List<String> getWordList(Environment env, Settings settings, Strin
225225
* @throws IllegalArgumentException
226226
* If the word list cannot be found at either key.
227227
*/
228-
public static List<String> getWordList(Environment env, Settings settings, String settingPath, String settingList) {
228+
public static List<String> getWordList(Environment env, Settings settings, String settingPath, String settingList, boolean removeComments) {
229229
String wordListPath = settings.get(settingPath, null);
230230

231231
if (wordListPath == null) {
@@ -240,7 +240,7 @@ public static List<String> getWordList(Environment env, Settings settings, Strin
240240
final Path path = env.configFile().resolve(wordListPath);
241241

242242
try {
243-
return loadWordList(path, "#");
243+
return loadWordList(path, removeComments);
244244
} catch (CharacterCodingException ex) {
245245
String message = String.format(Locale.ROOT,
246246
"Unsupported character encoding detected while reading %s: %s - files must be UTF-8 encoded",
@@ -252,15 +252,15 @@ public static List<String> getWordList(Environment env, Settings settings, Strin
252252
}
253253
}
254254

255-
private static List<String> loadWordList(Path path, String comment) throws IOException {
255+
private static List<String> loadWordList(Path path, boolean removeComments) throws IOException {
256256
final List<String> result = new ArrayList<>();
257257
try (BufferedReader br = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
258258
String word;
259259
while ((word = br.readLine()) != null) {
260-
if (!Strings.hasText(word)) {
260+
if (Strings.hasText(word) == false) {
261261
continue;
262262
}
263-
if (!word.startsWith(comment)) {
263+
if (removeComments == false || word.startsWith("#") == false) {
264264
result.add(word.trim());
265265
}
266266
}

0 commit comments

Comments
 (0)