Skip to content

Commit 1a8ce87

Browse files
romseygeekromseygeek
authored
Restore use of default search and search_quote analyzers (elastic#65491)
In the refactoring of TextFieldMapper, we lost the ability to define a default search or search_quote analyzer in index settings. This commit restores that ability, and adds some more comprehensive testing. Fixes elastic#65434
1 parent 04f453c commit 1a8ce87

File tree

20 files changed

+285
-91
lines changed

20 files changed

+285
-91
lines changed

‎modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/41_query_string_with_default_analyzer.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
"Test default search analyzer is applied":
3+
- do:
4+
indices.create:
5+
index: test
6+
body:
7+
settings:
8+
index.analysis.analyzer.default.type: simple
9+
index.analysis.analyzer.default_search.type: german
10+
mappings:
11+
properties:
12+
body:
13+
type: text
14+
15+
- do:
16+
index:
17+
index: test
18+
id: 1
19+
body:
20+
body: Ich lese die Bücher
21+
22+
- do:
23+
indices.refresh:
24+
index: [ test ]
25+
26+
- do:
27+
search:
28+
index: test
29+
q: "body:Bücher"
30+
31+
- match: { hits.total.value: 0 }
32+
33+
- do:
34+
search:
35+
index: test
36+
q: "body:Bücher"
37+
analyzer: simple
38+
39+
- match: { hits.total.value: 1 }

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.apache.lucene.util.automaton.Operations;
4949
import org.elasticsearch.common.collect.Iterators;
5050
import org.elasticsearch.index.analysis.AnalyzerScope;
51+
import org.elasticsearch.index.analysis.IndexAnalyzers;
5152
import org.elasticsearch.index.analysis.NamedAnalyzer;
5253
import org.elasticsearch.index.query.QueryShardContext;
5354
import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -61,7 +62,6 @@
6162
import java.util.List;
6263
import java.util.Map;
6364
import java.util.Objects;
64-
import java.util.function.Supplier;
6565

6666
import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps;
6767

@@ -93,7 +93,7 @@ public static class Defaults {
9393
}
9494

9595
public static final TypeParser PARSER
96-
= new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
96+
= new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers()));
9797

9898
private static Builder builder(FieldMapper in) {
9999
return ((SearchAsYouTypeFieldMapper)in).builder;
@@ -134,9 +134,9 @@ public static class Builder extends FieldMapper.Builder {
134134

135135
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
136136

137-
public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
137+
public Builder(String name, IndexAnalyzers indexAnalyzers) {
138138
super(name);
139-
this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers);
139+
this.analyzers = new TextParams.Analyzers(indexAnalyzers, m -> builder(m).analyzers);
140140
}
141141

142142
@Override
@@ -592,7 +592,7 @@ protected String contentType() {
592592
}
593593

594594
public FieldMapper.Builder getMergeBuilder() {
595-
return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
595+
return new Builder(simpleName(), builder.analyzers.indexAnalyzers).init(this);
596596
}
597597

598598
public static String getShingleFieldName(String parentField, int shingleSize) {

‎plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.lucene.index.IndexOptions;
3434
import org.elasticsearch.ElasticsearchParseException;
3535
import org.elasticsearch.index.analysis.AnalyzerScope;
36+
import org.elasticsearch.index.analysis.IndexAnalyzers;
3637
import org.elasticsearch.index.analysis.NamedAnalyzer;
3738
import org.elasticsearch.index.mapper.ContentPath;
3839
import org.elasticsearch.index.mapper.FieldMapper;
@@ -54,7 +55,6 @@
5455
import java.util.Map;
5556
import java.util.Objects;
5657
import java.util.concurrent.atomic.AtomicInteger;
57-
import java.util.function.Supplier;
5858
import java.util.regex.Matcher;
5959
import java.util.regex.Pattern;
6060

@@ -96,9 +96,9 @@ public static class Builder extends FieldMapper.Builder {
9696

9797
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
9898

99-
public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
99+
public Builder(String name, IndexAnalyzers indexAnalyzers) {
100100
super(name);
101-
this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers);
101+
this.analyzers = new TextParams.Analyzers(indexAnalyzers, m -> builder(m).analyzers);
102102
}
103103

104104
@Override
@@ -115,12 +115,11 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, ContentPath c
115115
similarity.get(),
116116
wrapAnalyzer(analyzers.getSearchAnalyzer()),
117117
wrapAnalyzer(analyzers.getSearchQuoteAnalyzer()));
118-
AnnotatedTextFieldType ft = new AnnotatedTextFieldType(
118+
return new AnnotatedTextFieldType(
119119
buildFullName(contentPath),
120120
store.getValue(),
121121
tsi,
122122
meta.getValue());
123-
return ft;
124123
}
125124

126125
@Override
@@ -141,7 +140,7 @@ name, fieldType, buildFieldType(fieldType, contentPath),
141140
}
142141
}
143142

144-
public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
143+
public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers()));
145144

146145
/**
147146
* Parses markdown-like syntax into plain text and AnnotationTokens with offsets for
@@ -546,6 +545,6 @@ protected String contentType() {
546545

547546
@Override
548547
public FieldMapper.Builder getMergeBuilder() {
549-
return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
548+
return new Builder(simpleName(), builder.analyzers.indexAnalyzers).init(this);
550549
}
551550
}

‎plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.lucene.analysis.standard.StandardAnalyzer;
2323
import org.apache.lucene.queries.intervals.Intervals;
2424
import org.apache.lucene.queries.intervals.IntervalsSource;
25-
import org.elasticsearch.common.lucene.Lucene;
2625
import org.elasticsearch.index.analysis.AnalyzerScope;
2726
import org.elasticsearch.index.analysis.NamedAnalyzer;
2827
import org.elasticsearch.index.mapper.ContentPath;
@@ -43,7 +42,7 @@ public void testIntervals() throws IOException {
4342
}
4443

4544
public void testFetchSourceValue() throws IOException {
46-
MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", () -> Lucene.STANDARD_ANALYZER)
45+
MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", createDefaultIndexAnalyzers())
4746
.build(new ContentPath())
4847
.fieldType();
4948

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -693,8 +693,7 @@ private static Mapper.Builder createBuilderFromDynamicValue(final ParseContext c
693693

694694
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
695695
if (builder == null) {
696-
builder = new TextFieldMapper.Builder(currentFieldName,
697-
() -> context.indexAnalyzers().getDefaultIndexAnalyzer())
696+
builder = new TextFieldMapper.Builder(currentFieldName, context.indexAnalyzers())
698697
.addMultiField(new KeywordFieldMapper.Builder("keyword").ignoreAbove(256));
699698
}
700699
return builder;

‎server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.elasticsearch.common.xcontent.XContentBuilder;
6565
import org.elasticsearch.common.xcontent.support.XContentMapValues;
6666
import org.elasticsearch.index.analysis.AnalyzerScope;
67+
import org.elasticsearch.index.analysis.IndexAnalyzers;
6768
import org.elasticsearch.index.analysis.NamedAnalyzer;
6869
import org.elasticsearch.index.fielddata.IndexFieldData;
6970
import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
@@ -266,14 +267,14 @@ public static class Builder extends FieldMapper.Builder {
266267

267268
final TextParams.Analyzers analyzers;
268269

269-
public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
270-
this(name, Version.CURRENT, defaultAnalyzer);
270+
public Builder(String name, IndexAnalyzers indexAnalyzers) {
271+
this(name, Version.CURRENT, indexAnalyzers);
271272
}
272273

273-
public Builder(String name, Version indexCreatedVersion, Supplier<NamedAnalyzer> defaultAnalyzer) {
274+
public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
274275
super(name);
275276
this.indexCreatedVersion = indexCreatedVersion;
276-
this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers);
277+
this.analyzers = new TextParams.Analyzers(indexAnalyzers, m -> builder(m).analyzers);
277278
}
278279

279280
public Builder index(boolean index) {
@@ -421,7 +422,7 @@ public TextFieldMapper build(ContentPath contentPath) {
421422
}
422423

423424
public static final TypeParser PARSER
424-
= new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
425+
= new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
425426

426427
private static class PhraseWrappedAnalyzer extends AnalyzerWrapper {
427428

@@ -807,7 +808,7 @@ protected TextFieldMapper(String simpleName, FieldType fieldType,
807808

808809
@Override
809810
public FieldMapper.Builder getMergeBuilder() {
810-
return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
811+
return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
811812
}
812813

813814
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/TextParams.java

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import org.apache.lucene.document.FieldType;
2323
import org.apache.lucene.index.IndexOptions;
2424
import org.elasticsearch.index.analysis.AnalysisMode;
25+
import org.elasticsearch.index.analysis.AnalysisRegistry;
26+
import org.elasticsearch.index.analysis.IndexAnalyzers;
2527
import org.elasticsearch.index.analysis.NamedAnalyzer;
2628
import org.elasticsearch.index.mapper.FieldMapper.Parameter;
2729
import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -44,22 +46,35 @@ public static final class Analyzers {
4446
public final Parameter<NamedAnalyzer> searchAnalyzer;
4547
public final Parameter<NamedAnalyzer> searchQuoteAnalyzer;
4648
public final Parameter<Integer> positionIncrementGap;
49+
public final IndexAnalyzers indexAnalyzers;
4750

48-
public Analyzers(Supplier<NamedAnalyzer> defaultAnalyzer,
51+
public Analyzers(IndexAnalyzers indexAnalyzers,
4952
Function<FieldMapper, Analyzers> analyzerInitFunction) {
5053
this.indexAnalyzer = Parameter.analyzerParam("analyzer", false,
51-
m -> analyzerInitFunction.apply(m).indexAnalyzer.get(), defaultAnalyzer)
54+
m -> analyzerInitFunction.apply(m).indexAnalyzer.get(), indexAnalyzers::getDefaultIndexAnalyzer)
5255
.setSerializerCheck((id, ic, a) -> id || ic ||
5356
Objects.equals(a, getSearchAnalyzer()) == false || Objects.equals(a, getSearchQuoteAnalyzer()) == false)
5457
.setValidator(a -> a.checkAllowedInMode(AnalysisMode.INDEX_TIME));
5558
this.searchAnalyzer
5659
= Parameter.analyzerParam("search_analyzer", true,
57-
m -> m.fieldType().getTextSearchInfo().getSearchAnalyzer(), indexAnalyzer::getValue)
60+
m -> m.fieldType().getTextSearchInfo().getSearchAnalyzer(), () -> {
61+
NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME);
62+
if (defaultAnalyzer != null) {
63+
return defaultAnalyzer;
64+
}
65+
return indexAnalyzer.get();
66+
})
5867
.setSerializerCheck((id, ic, a) -> id || ic || Objects.equals(a, getSearchQuoteAnalyzer()) == false)
5968
.setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
6069
this.searchQuoteAnalyzer
6170
= Parameter.analyzerParam("search_quote_analyzer", true,
62-
m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), searchAnalyzer::getValue)
71+
m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), () -> {
72+
NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME);
73+
if (defaultAnalyzer != null) {
74+
return defaultAnalyzer;
75+
}
76+
return searchAnalyzer.get();
77+
})
6378
.setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
6479
this.positionIncrementGap = Parameter.intParam("position_increment_gap", false,
6580
m -> analyzerInitFunction.apply(m).positionIncrementGap.get(), POSITION_INCREMENT_GAP_USE_ANALYZER)
@@ -68,6 +83,7 @@ public Analyzers(Supplier<NamedAnalyzer> defaultAnalyzer,
6883
throw new MapperParsingException("[position_increment_gap] must be positive, got [" + v + "]");
6984
}
7085
});
86+
this.indexAnalyzers = indexAnalyzers;
7187
}
7288

7389
public NamedAnalyzer getIndexAnalyzer() {

‎server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap
374374
return fieldMapping;
375375
} else if (mapUnmappedFieldAsString) {
376376
TextFieldMapper.Builder builder
377-
= new TextFieldMapper.Builder(name, () -> mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer());
377+
= new TextFieldMapper.Builder(name, mapperService.getIndexAnalyzers());
378378
return builder.build(new ContentPath(1)).fieldType();
379379
} else {
380380
throw new QueryShardException(this, "No field mapping can be found for the field with name [{}]", name);

‎server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.apache.lucene.search.IndexSearcher;
3232
import org.apache.lucene.search.Query;
3333
import org.apache.lucene.store.ByteBuffersDirectory;
34-
import org.elasticsearch.common.lucene.Lucene;
3534
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
3635
import org.elasticsearch.common.settings.Settings;
3736
import org.elasticsearch.index.IndexService;
@@ -100,7 +99,7 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
10099
if (docValues) {
101100
fieldType = new KeywordFieldMapper.Builder(fieldName).build(contentPath).fieldType();
102101
} else {
103-
fieldType = new TextFieldMapper.Builder(fieldName, () -> Lucene.STANDARD_ANALYZER)
102+
fieldType = new TextFieldMapper.Builder(fieldName, createDefaultIndexAnalyzers())
104103
.fielddata(true).build(contentPath).fieldType();
105104
}
106105
} else if (type.equals("float")) {

‎server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.lucene.document.StringField;
2424
import org.apache.lucene.index.LeafReaderContext;
2525
import org.apache.lucene.index.SortedSetDocValues;
26-
import org.elasticsearch.common.lucene.Lucene;
2726
import org.elasticsearch.index.mapper.ContentPath;
2827
import org.elasticsearch.index.mapper.MappedFieldType;
2928
import org.elasticsearch.index.mapper.TextFieldMapper;
@@ -64,7 +63,7 @@ public void testFilterByFrequency() throws Exception {
6463

6564
{
6665
indexService.clearCaches(false, true);
67-
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", () -> Lucene.STANDARD_ANALYZER)
66+
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers())
6867
.fielddata(true)
6968
.fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0)
7069
.build(contentPath).fieldType();
@@ -79,7 +78,7 @@ public void testFilterByFrequency() throws Exception {
7978
}
8079
{
8180
indexService.clearCaches(false, true);
82-
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", () -> Lucene.STANDARD_ANALYZER)
81+
MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers())
8382
.fielddata(true)
8483
.fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, 201, 100)
8584
.build(contentPath).fieldType();
@@ -94,7 +93,7 @@ public void testFilterByFrequency() throws Exception {
9493

9594
{
9695
indexService.clearCaches(false, true);// test # docs with value
97-
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", () -> Lucene.STANDARD_ANALYZER)
96+
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers())
9897
.fielddata(true)
9998
.fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
10099
.build(contentPath).fieldType();
@@ -110,7 +109,7 @@ public void testFilterByFrequency() throws Exception {
110109

111110
{
112111
indexService.clearCaches(false, true);
113-
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", () -> Lucene.STANDARD_ANALYZER)
112+
MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers())
114113
.fielddata(true)
115114
.fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
116115
.build(contentPath).fieldType();
@@ -127,7 +126,7 @@ public void testFilterByFrequency() throws Exception {
127126
}
128127

129128
@Override
130-
public void testEmpty() throws Exception {
129+
public void testEmpty() {
131130
assumeTrue("No need to test empty usage here", false);
132131
}
133132
}

‎server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.apache.lucene.store.ByteBuffersDirectory;
3232
import org.apache.lucene.util.Accountable;
3333
import org.apache.lucene.util.SetOnce;
34-
import org.elasticsearch.common.lucene.Lucene;
3534
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
3635
import org.elasticsearch.common.settings.Settings;
3736
import org.elasticsearch.index.IndexService;
@@ -142,9 +141,9 @@ public void testClearField() throws Exception {
142141

143142
final ContentPath contentPath = new ContentPath(1);
144143
final MappedFieldType mapper1
145-
= new TextFieldMapper.Builder("field_1", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(contentPath).fieldType();
144+
= new TextFieldMapper.Builder("field_1", createDefaultIndexAnalyzers()).fielddata(true).build(contentPath).fieldType();
146145
final MappedFieldType mapper2
147-
= new TextFieldMapper.Builder("field_2", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(contentPath).fieldType();
146+
= new TextFieldMapper.Builder("field_2", createDefaultIndexAnalyzers()).fielddata(true).build(contentPath).fieldType();
148147
final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
149148
Document doc = new Document();
150149
doc.add(new StringField("field_1", "thisisastring", Store.NO));
@@ -208,7 +207,7 @@ public void testFieldDataCacheListener() throws Exception {
208207

209208
final ContentPath contentPath = new ContentPath(1);
210209
final MappedFieldType mapper1
211-
= new TextFieldMapper.Builder("s", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(contentPath).fieldType();
210+
= new TextFieldMapper.Builder("s", createDefaultIndexAnalyzers()).fielddata(true).build(contentPath).fieldType();
212211
final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
213212
Document doc = new Document();
214213
doc.add(new StringField("s", "thisisastring", Store.NO));

0 commit comments

Comments
 (0)