Skip to content

Commit 4ff1b1d

Browse files
jasontedorjasontedor
committed
Introduce private settings (#33327)
This commit introduces the formal notion of a private setting. This enables us to register some settings that we had previously not registered as fully-fledged settings to avoid them being exposed via APIs such as the create index API. For example, we had hacks in the codebase to allow index.version.created to be passed around inside of settings objects, but was not registered as a setting so that if a user tried to use the setting on any API then they would get an exception. This prevented users from setting index.version.created on index creation, or updating it via the index settings API. By introducing private settings, we can continue to reject these attempts, yet now we can represent these settings as actual settings. In this change, we register index.version.created as an actual setting. We do not cutover all settings that we had been treating as private in this pull request, it is already quite large due to moving some tests around to account for the fact that some tests need to be able to set the index.version.created. This can be done in a follow-up change.
1 parent 4da9f18 commit 4ff1b1d

File tree

116 files changed

+4416
-2481
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

116 files changed

+4416
-2481
lines changed

‎buildSrc/src/main/resources/checkstyle_suppressions.xml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -570,7 +570,6 @@
570570
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperMergeTests.java" checks="LineLength" />
571571
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParserTests.java" checks="LineLength" />
572572
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicMappingTests.java" checks="LineLength" />
573-
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalFieldMapperTests.java" checks="LineLength" />
574573
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMapper.java" checks="LineLength" />
575574
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMetadataMapper.java" checks="LineLength" />
576575
<suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapperTests.java" checks="LineLength" />

‎modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ protected boolean legacy() {
7777
return false;
7878
}
7979

80+
@Override
81+
protected boolean forbidPrivateIndexSettings() {
82+
return legacy() == false;
83+
}
84+
8085
protected IndexRequestBuilder createIndexRequest(String index, String type, String id, String parentId, Object... fields) {
8186
Map<String, Object> source = new HashMap<>();
8287
for (int i = 0; i < fields.length; i += 2) {

‎modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,12 @@
1919

2020
package org.elasticsearch.index.reindex;
2121

22-
import org.elasticsearch.Version;
2322
import org.elasticsearch.action.admin.indices.alias.Alias;
2423
import org.elasticsearch.action.index.IndexRequestBuilder;
2524
import org.elasticsearch.common.settings.Settings;
2625
import org.elasticsearch.index.IndexNotFoundException;
2726
import org.elasticsearch.index.query.QueryBuilders;
28-
import org.elasticsearch.plugins.Plugin;
2927
import org.elasticsearch.search.sort.SortOrder;
30-
import org.elasticsearch.test.InternalSettingsPlugin;
3128

3229
import java.util.ArrayList;
3330
import java.util.Collection;
@@ -46,12 +43,6 @@
4643
import static org.hamcrest.Matchers.hasSize;
4744

4845
public class DeleteByQueryBasicTests extends ReindexTestCase {
49-
@Override
50-
protected Collection<Class<? extends Plugin>> nodePlugins() {
51-
List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
52-
plugins.add(InternalSettingsPlugin.class);
53-
return plugins;
54-
}
5546

5647
public void testBasics() throws Exception {
5748
indexRandom(true,
@@ -307,25 +298,4 @@ public void testMultipleSources() throws Exception {
307298

308299
}
309300

310-
/**
311-
* Test delete by query support for filtering by type. This entire feature
312-
* can and should be removed when we drop support for types index with
313-
* multiple types from core.
314-
*/
315-
public void testFilterByType() throws Exception {
316-
assertAcked(client().admin().indices().prepareCreate("test")
317-
.setSettings(Settings.builder().put("index.version.created", Version.V_5_6_0.id))); // allows for multiple types
318-
indexRandom(true,
319-
client().prepareIndex("test", "test1", "1").setSource("foo", "a"),
320-
client().prepareIndex("test", "test2", "2").setSource("foo", "a"),
321-
client().prepareIndex("test", "test2", "3").setSource("foo", "b"));
322-
323-
assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);
324-
325-
// Deletes doc of the type "type2" that also matches foo:a
326-
DeleteByQueryRequestBuilder builder = deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true);
327-
builder.source().setTypes("test2");
328-
assertThat(builder.get(), matcher().deleted(1));
329-
assertHitCount(client().prepareSearch("test").setSize(0).get(), 2);
330-
}
331301
}

‎modules/reindex/src/test/java/org/elasticsearch/index/reindex/LegacyDeleteByQueryBasicTests.java

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.index.reindex;
21+
22+
import org.elasticsearch.Version;
23+
import org.elasticsearch.common.settings.Settings;
24+
25+
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
26+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
27+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
28+
29+
public class LegacyDeleteByQueryBasicTests extends ReindexTestCase {
30+
31+
@Override
32+
protected boolean forbidPrivateIndexSettings() {
33+
return false;
34+
}
35+
36+
/**
37+
* Test delete by query support for filtering by type. This entire feature
38+
* can and should be removed when we drop support for types index with
39+
* multiple types from core.
40+
*/
41+
public void testFilterByType() throws Exception {
42+
assertAcked(client().admin().indices().prepareCreate("test")
43+
.setSettings(Settings.builder().put("index.version.created", Version.V_5_6_0.id))); // allows for multiple types
44+
indexRandom(true,
45+
client().prepareIndex("test", "test1", "1").setSource("foo", "a"),
46+
client().prepareIndex("test", "test2", "2").setSource("foo", "a"),
47+
client().prepareIndex("test", "test2", "3").setSource("foo", "b"));
48+
49+
assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);
50+
51+
// Deletes doc of the type "type2" that also matches foo:a
52+
DeleteByQueryRequestBuilder builder = deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true);
53+
builder.source().setTypes("test2");
54+
assertThat(builder.get(), matcher().deleted(1));
55+
assertHitCount(client().prepareSearch("test").setSize(0).get(), 2);
56+
}
57+
58+
}

‎modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.elasticsearch.join.ParentJoinPlugin;
2828
import org.elasticsearch.plugins.Plugin;
2929
import org.elasticsearch.script.MockScriptPlugin;
30-
import org.elasticsearch.test.InternalSettingsPlugin;
3130

3231
import java.util.ArrayList;
3332
import java.util.Collection;
@@ -65,7 +64,6 @@ protected boolean ignoreExternalCluster() {
6564
protected Collection<Class<? extends Plugin>> nodePlugins() {
6665
final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
6766
plugins.add(ParentJoinPlugin.class);
68-
plugins.add(InternalSettingsPlugin.class);
6967
plugins.add(CustomScriptPlugin.class);
7068
return Collections.unmodifiableList(plugins);
7169
}
@@ -75,6 +73,11 @@ protected Collection<Class<? extends Plugin>> transportClientPlugins() {
7573
return nodePlugins();
7674
}
7775

76+
@Override
77+
protected boolean forbidPrivateIndexSettings() {
78+
return false;
79+
}
80+
7881
public void testParentChild() throws Exception {
7982
createParentChildIndex("source");
8083
createParentChildIndex("dest");

‎plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java

Lines changed: 6 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
*/
1919
package org.elasticsearch.index.mapper;
2020

21-
import static org.hamcrest.Matchers.equalTo;
22-
2321
import com.ibm.icu.text.Collator;
2422
import com.ibm.icu.text.RawCollationKey;
2523
import com.ibm.icu.util.ULocale;
@@ -28,36 +26,36 @@
2826
import org.apache.lucene.index.IndexableField;
2927
import org.apache.lucene.index.IndexableFieldType;
3028
import org.apache.lucene.util.BytesRef;
31-
import org.elasticsearch.Version;
3229
import org.elasticsearch.common.Strings;
3330
import org.elasticsearch.common.bytes.BytesReference;
3431
import org.elasticsearch.common.compress.CompressedXContent;
35-
import org.elasticsearch.common.settings.Settings;
3632
import org.elasticsearch.common.xcontent.XContentFactory;
3733
import org.elasticsearch.common.xcontent.XContentType;
3834
import org.elasticsearch.index.IndexService;
3935
import org.elasticsearch.index.mapper.MapperService.MergeReason;
4036
import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
4137
import org.elasticsearch.plugins.Plugin;
4238
import org.elasticsearch.test.ESSingleNodeTestCase;
43-
import org.elasticsearch.test.InternalSettingsPlugin;
4439
import org.junit.Before;
4540

4641
import java.io.IOException;
4742
import java.util.Arrays;
4843
import java.util.Collection;
44+
import java.util.Collections;
45+
46+
import static org.hamcrest.Matchers.equalTo;
4947

5048
public class ICUCollationKeywordFieldMapperTests extends ESSingleNodeTestCase {
5149

5250
private static final String FIELD_TYPE = "icu_collation_keyword";
5351

5452
@Override
5553
protected Collection<Class<? extends Plugin>> getPlugins() {
56-
return Arrays.asList(AnalysisICUPlugin.class, InternalSettingsPlugin.class);
54+
return Collections.singletonList(AnalysisICUPlugin.class);
5755
}
5856

59-
IndexService indexService;
60-
DocumentMapperParser parser;
57+
private IndexService indexService;
58+
private DocumentMapperParser parser;
6159

6260
@Before
6361
public void setup() {
@@ -106,51 +104,6 @@ public void testDefaults() throws Exception {
106104
assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
107105
}
108106

109-
public void testBackCompat() throws Exception {
110-
indexService = createIndex("oldindex", Settings.builder().put("index.version.created", Version.V_5_5_0).build());
111-
parser = indexService.mapperService().documentMapperParser();
112-
113-
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
114-
.startObject("_all").field("enabled", false).endObject()
115-
.startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()
116-
.endObject().endObject());
117-
118-
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
119-
120-
assertEquals(mapping, mapper.mappingSource().toString());
121-
122-
ParsedDocument doc = mapper.parse(SourceToParse.source("oldindex", "type", "1", BytesReference
123-
.bytes(XContentFactory.jsonBuilder()
124-
.startObject()
125-
.field("field", "1234")
126-
.endObject()),
127-
XContentType.JSON));
128-
129-
IndexableField[] fields = doc.rootDoc().getFields("field");
130-
assertEquals(2, fields.length);
131-
132-
Collator collator = Collator.getInstance(ULocale.ROOT);
133-
RawCollationKey key = collator.getRawCollationKey("1234", null);
134-
BytesRef expected = new BytesRef(key.bytes, 0, key.size);
135-
136-
assertEquals(expected, fields[0].binaryValue());
137-
IndexableFieldType fieldType = fields[0].fieldType();
138-
assertThat(fieldType.omitNorms(), equalTo(true));
139-
assertFalse(fieldType.tokenized());
140-
assertFalse(fieldType.stored());
141-
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
142-
assertThat(fieldType.storeTermVectors(), equalTo(false));
143-
assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
144-
assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
145-
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
146-
assertEquals(DocValuesType.NONE, fieldType.docValuesType());
147-
148-
assertEquals(expected, fields[1].binaryValue());
149-
fieldType = fields[1].fieldType();
150-
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
151-
assertEquals(DocValuesType.SORTED, fieldType.docValuesType());
152-
}
153-
154107
public void testNullValue() throws IOException {
155108
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
156109
.startObject("properties").startObject("field").field("type", FIELD_TYPE).endObject().endObject()

‎plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/LegacyICUCollationKeywordFieldMapperTests.java

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.index.mapper;
21+
22+
import com.ibm.icu.text.Collator;
23+
import com.ibm.icu.text.RawCollationKey;
24+
import com.ibm.icu.util.ULocale;
25+
import org.apache.lucene.index.DocValuesType;
26+
import org.apache.lucene.index.IndexOptions;
27+
import org.apache.lucene.index.IndexableField;
28+
import org.apache.lucene.index.IndexableFieldType;
29+
import org.apache.lucene.util.BytesRef;
30+
import org.elasticsearch.Version;
31+
import org.elasticsearch.common.Strings;
32+
import org.elasticsearch.common.bytes.BytesReference;
33+
import org.elasticsearch.common.compress.CompressedXContent;
34+
import org.elasticsearch.common.settings.Settings;
35+
import org.elasticsearch.common.xcontent.XContentFactory;
36+
import org.elasticsearch.common.xcontent.XContentType;
37+
import org.elasticsearch.index.IndexService;
38+
import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin;
39+
import org.elasticsearch.plugins.Plugin;
40+
import org.elasticsearch.test.ESSingleNodeTestCase;
41+
42+
import java.util.Collection;
43+
import java.util.Collections;
44+
45+
import static org.hamcrest.Matchers.equalTo;
46+
47+
public class LegacyICUCollationKeywordFieldMapperTests extends ESSingleNodeTestCase {
48+
49+
@Override
50+
protected Collection<Class<? extends Plugin>> getPlugins() {
51+
return Collections.singletonList(AnalysisICUPlugin.class);
52+
}
53+
54+
@Override
55+
protected boolean forbidPrivateIndexSettings() {
56+
return false;
57+
}
58+
59+
public void testBackCompat() throws Exception {
60+
final IndexService indexService = createIndex("oldindex", Settings.builder().put("index.version.created", Version.V_5_5_0).build());
61+
final DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
62+
63+
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
64+
.startObject("_all").field("enabled", false).endObject()
65+
.startObject("properties").startObject("field").field("type", "icu_collation_keyword").endObject().endObject()
66+
.endObject().endObject());
67+
68+
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
69+
70+
assertEquals(mapping, mapper.mappingSource().toString());
71+
72+
ParsedDocument doc = mapper.parse(SourceToParse.source("oldindex", "type", "1", BytesReference
73+
.bytes(XContentFactory.jsonBuilder()
74+
.startObject()
75+
.field("field", "1234")
76+
.endObject()),
77+
XContentType.JSON));
78+
79+
IndexableField[] fields = doc.rootDoc().getFields("field");
80+
assertEquals(2, fields.length);
81+
82+
Collator collator = Collator.getInstance(ULocale.ROOT);
83+
RawCollationKey key = collator.getRawCollationKey("1234", null);
84+
BytesRef expected = new BytesRef(key.bytes, 0, key.size);
85+
86+
assertEquals(expected, fields[0].binaryValue());
87+
IndexableFieldType fieldType = fields[0].fieldType();
88+
assertThat(fieldType.omitNorms(), equalTo(true));
89+
assertFalse(fieldType.tokenized());
90+
assertFalse(fieldType.stored());
91+
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
92+
assertThat(fieldType.storeTermVectors(), equalTo(false));
93+
assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
94+
assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
95+
assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
96+
assertEquals(DocValuesType.NONE, fieldType.docValuesType());
97+
98+
assertEquals(expected, fields[1].binaryValue());
99+
fieldType = fields[1].fieldType();
100+
assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
101+
assertEquals(DocValuesType.SORTED, fieldType.docValuesType());
102+
}
103+
104+
}

0 commit comments

Comments
 (0)