Skip to content

Commit 4b366a4

Browse files
author
Christoph Büscher
authored
Make Multiplexer inherit filter chains analysis mode (#50662)
Currently, if an updateable synonym filter is included in a multiplexer filter, it is not reloaded via the _reload_search_analyzers because the multiplexer itself doesn't pass on the analysis mode of the filters it contains, so its not recognized as "updateable" in itself. Instead we can check and merge the AnalysisMode settings of all filters in the multiplexer and use the resulting mode (e.g. search-time only) for the multiplexer itself, thus making any synonym filters contained in it reloadable. This, of course, will also make the analyzers using the multiplexer be usable at search-time only. Closes #50554
1 parent 3631f93 commit 4b366a4

File tree

3 files changed

+137
-45
lines changed

3 files changed

+137
-45
lines changed

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java

+11
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.env.Environment;
3030
import org.elasticsearch.index.IndexSettings;
3131
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
32+
import org.elasticsearch.index.analysis.AnalysisMode;
3233
import org.elasticsearch.index.analysis.CharFilterFactory;
3334
import org.elasticsearch.index.analysis.TokenFilterFactory;
3435
import org.elasticsearch.index.analysis.TokenizerFactory;
@@ -67,12 +68,15 @@ public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory token
6768
if (preserveOriginal) {
6869
filters.add(IDENTITY_FILTER);
6970
}
71+
// also merge and transfer token filter analysis modes with analyzer
72+
AnalysisMode mode = AnalysisMode.ALL;
7073
for (String filter : filterNames) {
7174
String[] parts = Strings.tokenizeToStringArray(filter, ",");
7275
if (parts.length == 1) {
7376
TokenFilterFactory factory = resolveFilterFactory(allFilters, parts[0]);
7477
factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, previousTokenFilters, allFilters);
7578
filters.add(factory);
79+
mode = mode.merge(factory.getAnalysisMode());
7680
} else {
7781
List<TokenFilterFactory> existingChain = new ArrayList<>(previousTokenFilters);
7882
List<TokenFilterFactory> chain = new ArrayList<>();
@@ -81,10 +85,12 @@ public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory token
8185
factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters);
8286
chain.add(factory);
8387
existingChain.add(factory);
88+
mode = mode.merge(factory.getAnalysisMode());
8489
}
8590
filters.add(chainFilters(filter, chain));
8691
}
8792
}
93+
final AnalysisMode analysisMode = mode;
8894

8995
return new TokenFilterFactory() {
9096
@Override
@@ -105,6 +111,11 @@ public TokenStream create(TokenStream tokenStream) {
105111
public TokenFilterFactory getSynonymFilter() {
106112
throw new IllegalArgumentException("Token filter [" + name() + "] cannot be used to parse synonyms");
107113
}
114+
115+
@Override
116+
public AnalysisMode getAnalysisMode() {
117+
return analysisMode;
118+
}
108119
};
109120
}
110121

‎server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,5 +78,5 @@ public String getReadableName() {
7878
* <li>INDEX_TIME.merge(SEARCH_TIME) throws an {@link IllegalStateException}</li>
7979
* </ul>
8080
*/
81-
abstract AnalysisMode merge(AnalysisMode other);
81+
public abstract AnalysisMode merge(AnalysisMode other);
8282
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java

+125-44
Original file line numberDiff line numberDiff line change
@@ -43,35 +43,25 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
4343

4444
public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
4545
String synonymsFileName = "synonyms.txt";
46-
Path configDir = node().getEnvironment().configFile();
47-
if (Files.exists(configDir) == false) {
48-
Files.createDirectory(configDir);
49-
}
50-
Path synonymsFile = configDir.resolve(synonymsFileName);
51-
if (Files.exists(synonymsFile) == false) {
52-
Files.createFile(synonymsFile);
53-
}
54-
try (PrintWriter out = new PrintWriter(
55-
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
56-
out.println("foo, baz");
57-
}
46+
Path synonymsFile = setupSynonymsFile(synonymsFileName, "foo, baz");
5847

5948
final String indexName = "test";
6049
final String synonymAnalyzerName = "synonym_analyzer";
6150
final String synonymGraphAnalyzerName = "synonym_graph_analyzer";
62-
assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder()
63-
.put("index.number_of_shards", 5)
64-
.put("index.number_of_replicas", 0)
65-
.put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "standard")
66-
.putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "lowercase", "synonym_filter")
67-
.put("analysis.analyzer." + synonymGraphAnalyzerName + ".tokenizer", "standard")
68-
.putList("analysis.analyzer." + synonymGraphAnalyzerName + ".filter", "lowercase", "synonym_graph_filter")
69-
.put("analysis.filter.synonym_filter.type", "synonym")
70-
.put("analysis.filter.synonym_filter.updateable", "true")
71-
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
72-
.put("analysis.filter.synonym_graph_filter.type", "synonym_graph")
73-
.put("analysis.filter.synonym_graph_filter.updateable", "true")
74-
.put("analysis.filter.synonym_graph_filter.synonyms_path", synonymsFileName))
51+
assertAcked(client().admin().indices().prepareCreate(indexName)
52+
.setSettings(Settings.builder()
53+
.put("index.number_of_shards", 5)
54+
.put("index.number_of_replicas", 0)
55+
.put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "standard")
56+
.putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "lowercase", "synonym_filter")
57+
.put("analysis.analyzer." + synonymGraphAnalyzerName + ".tokenizer", "standard")
58+
.putList("analysis.analyzer." + synonymGraphAnalyzerName + ".filter", "lowercase", "synonym_graph_filter")
59+
.put("analysis.filter.synonym_filter.type", "synonym")
60+
.put("analysis.filter.synonym_filter.updateable", "true")
61+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
62+
.put("analysis.filter.synonym_graph_filter.type", "synonym_graph")
63+
.put("analysis.filter.synonym_graph_filter.updateable", "true")
64+
.put("analysis.filter.synonym_graph_filter.synonyms_path", synonymsFileName))
7565
.addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=" + synonymAnalyzerName));
7666

7767
client().prepareIndex(indexName).setId("1").setSource("field", "Foo").get();
@@ -84,8 +74,7 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
8474

8575
{
8676
for (String analyzerName : new String[] { synonymAnalyzerName, synonymGraphAnalyzerName }) {
87-
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName)
88-
.get();
77+
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName).get();
8978
assertEquals(2, analyzeResponse.getTokens().size());
9079
Set<String> tokens = new HashSet<>();
9180
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
@@ -109,8 +98,7 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
10998

11099
{
111100
for (String analyzerName : new String[] { synonymAnalyzerName, synonymGraphAnalyzerName }) {
112-
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName)
113-
.get();
101+
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName).get();
114102
assertEquals(3, analyzeResponse.getTokens().size());
115103
Set<String> tokens = new HashSet<>();
116104
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
@@ -126,8 +114,69 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
126114
assertHitCount(response, 1L);
127115
}
128116

117+
public void testSynonymsInMultiplexerUpdateable() throws FileNotFoundException, IOException {
118+
String synonymsFileName = "synonyms.txt";
119+
Path synonymsFile = setupSynonymsFile(synonymsFileName, "foo, baz");
120+
121+
final String indexName = "test";
122+
final String synonymAnalyzerName = "synonym_in_multiplexer_analyzer";
123+
assertAcked(client().admin().indices().prepareCreate(indexName)
124+
.setSettings(Settings.builder()
125+
.put("index.number_of_shards", 5)
126+
.put("index.number_of_replicas", 0)
127+
.put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "whitespace")
128+
.putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "my_multiplexer")
129+
.put("analysis.filter.synonym_filter.type", "synonym")
130+
.put("analysis.filter.synonym_filter.updateable", "true")
131+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
132+
.put("analysis.filter.my_multiplexer.type", "multiplexer")
133+
.putList("analysis.filter.my_multiplexer.filters", "synonym_filter"))
134+
.addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=" + synonymAnalyzerName));
135+
136+
client().prepareIndex(indexName).setId("1").setSource("field", "foo").get();
137+
assertNoFailures(client().admin().indices().prepareRefresh(indexName).execute().actionGet());
138+
139+
SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get();
140+
assertHitCount(response, 1L);
141+
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
142+
assertHitCount(response, 0L);
143+
144+
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(synonymAnalyzerName).get();
145+
assertEquals(2, analyzeResponse.getTokens().size());
146+
final Set<String> tokens = new HashSet<>();
147+
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
148+
assertTrue(tokens.contains("foo"));
149+
assertTrue(tokens.contains("baz"));
150+
151+
// now update synonyms file and trigger reloading
152+
try (PrintWriter out = new PrintWriter(
153+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
154+
out.println("foo, baz, buzz");
155+
}
156+
ReloadAnalyzersResponse reloadResponse = client().execute(ReloadAnalyzerAction.INSTANCE, new ReloadAnalyzersRequest(indexName))
157+
.actionGet();
158+
assertNoFailures(reloadResponse);
159+
Set<String> reloadedAnalyzers = reloadResponse.getReloadDetails().get(indexName).getReloadedAnalyzers();
160+
assertEquals(1, reloadedAnalyzers.size());
161+
assertTrue(reloadedAnalyzers.contains(synonymAnalyzerName));
162+
163+
analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(synonymAnalyzerName).get();
164+
assertEquals(3, analyzeResponse.getTokens().size());
165+
tokens.clear();
166+
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
167+
assertTrue(tokens.contains("foo"));
168+
assertTrue(tokens.contains("baz"));
169+
assertTrue(tokens.contains("buzz"));
170+
171+
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get();
172+
assertHitCount(response, 1L);
173+
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
174+
assertHitCount(response, 1L);
175+
}
176+
129177
public void testUpdateableSynonymsRejectedAtIndexTime() throws FileNotFoundException, IOException {
130178
String synonymsFileName = "synonyms.txt";
179+
setupSynonymsFile(synonymsFileName, "foo, baz");
131180
Path configDir = node().getEnvironment().configFile();
132181
if (Files.exists(configDir) == false) {
133182
Files.createDirectory(configDir);
@@ -143,20 +192,52 @@ public void testUpdateableSynonymsRejectedAtIndexTime() throws FileNotFoundExcep
143192

144193
final String indexName = "test";
145194
final String analyzerName = "my_synonym_analyzer";
146-
MapperException ex = expectThrows(MapperException.class, () -> client().admin().indices().prepareCreate(indexName)
147-
.setSettings(Settings.builder()
148-
.put("index.number_of_shards", 5)
149-
.put("index.number_of_replicas", 0)
150-
.put("analysis.analyzer." + analyzerName + ".tokenizer", "standard")
151-
.putList("analysis.analyzer." + analyzerName + ".filter", "lowercase", "synonym_filter")
152-
.put("analysis.filter.synonym_filter.type", "synonym")
153-
.put("analysis.filter.synonym_filter.updateable", "true")
154-
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName))
155-
.addMapping("_doc", "field", "type=text,analyzer=" + analyzerName).get());
156-
157-
assertEquals(
158-
"Failed to parse mapping: analyzer [my_synonym_analyzer] "
159-
+ "contains filters [synonym_filter] that are not allowed to run in all mode.",
160-
ex.getMessage());
195+
MapperException ex = expectThrows(MapperException.class,
196+
() -> client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder()
197+
.put("index.number_of_shards", 5)
198+
.put("index.number_of_replicas", 0)
199+
.put("analysis.analyzer." + analyzerName + ".tokenizer", "standard")
200+
.putList("analysis.analyzer." + analyzerName + ".filter", "lowercase", "synonym_filter")
201+
.put("analysis.filter.synonym_filter.type", "synonym")
202+
.put("analysis.filter.synonym_filter.updateable", "true")
203+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName))
204+
.addMapping("_doc", "field", "type=text,analyzer=" + analyzerName).get());
205+
206+
assertEquals("Failed to parse mapping: analyzer [my_synonym_analyzer] "
207+
+ "contains filters [synonym_filter] that are not allowed to run in all mode.", ex.getMessage());
208+
209+
// same for synonym filters in multiplexer chain
210+
ex = expectThrows(MapperException.class,
211+
() -> client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder()
212+
.put("index.number_of_shards", 5)
213+
.put("index.number_of_replicas", 0)
214+
.put("analysis.analyzer." + analyzerName + ".tokenizer", "whitespace")
215+
.putList("analysis.analyzer." + analyzerName + ".filter", "my_multiplexer")
216+
.put("analysis.filter.synonym_filter.type", "synonym")
217+
.put("analysis.filter.synonym_filter.updateable", "true")
218+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
219+
.put("analysis.filter.my_multiplexer.type", "multiplexer")
220+
.putList("analysis.filter.my_multiplexer.filters", "synonym_filter"))
221+
.addMapping("_doc", "field", "type=text,analyzer=" + analyzerName).get());
222+
223+
assertEquals("Failed to parse mapping: analyzer [my_synonym_analyzer] "
224+
+ "contains filters [my_multiplexer] that are not allowed to run in all mode.", ex.getMessage());
161225
}
226+
227+
private Path setupSynonymsFile(String synonymsFileName, String content) throws IOException {
228+
Path configDir = node().getEnvironment().configFile();
229+
if (Files.exists(configDir) == false) {
230+
Files.createDirectory(configDir);
231+
}
232+
Path synonymsFile = configDir.resolve(synonymsFileName);
233+
if (Files.exists(synonymsFile) == false) {
234+
Files.createFile(synonymsFile);
235+
}
236+
try (PrintWriter out = new PrintWriter(
237+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
238+
out.println(content);
239+
}
240+
return synonymsFile;
241+
}
242+
162243
}

0 commit comments

Comments
 (0)