Skip to content

Commit c2deb28

Browse files
jimczijimczi
authored
Add a cluster setting to disallow loading fielddata on _id field (#49166)
This change adds a dynamic cluster setting named `indices.id_field_data.enabled`. When set to `false` any attempt to load the fielddata for the `_id` field will fail with an exception. The default value in this change is set to `false` in order to prevent fielddata usage on this field for future versions but it will be set to `true` when backporting to 7x. When the setting is set to true (manually or by default in 7x) the loading will also issue a deprecation warning since we want to disallow fielddata entirely when #26472 is implemented. Closes #43599
1 parent 35355e2 commit c2deb28

File tree

36 files changed

+478
-248
lines changed

36 files changed

+478
-248
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

+19-18
Original file line numberDiff line numberDiff line change
@@ -114,28 +114,28 @@ public class SearchIT extends ESRestHighLevelClientTestCase {
114114
public void indexDocuments() throws IOException {
115115
{
116116
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1");
117-
doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}");
117+
doc1.setJsonEntity("{\"type\":\"type1\", \"id\":1, \"num\":10, \"num2\":50}");
118118
client().performRequest(doc1);
119119
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/_doc/2");
120-
doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}");
120+
doc2.setJsonEntity("{\"type\":\"type1\", \"id\":2, \"num\":20, \"num2\":40}");
121121
client().performRequest(doc2);
122122
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/_doc/3");
123-
doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}");
123+
doc3.setJsonEntity("{\"type\":\"type1\", \"id\":3, \"num\":50, \"num2\":35}");
124124
client().performRequest(doc3);
125125
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/_doc/4");
126-
doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
126+
doc4.setJsonEntity("{\"type\":\"type2\", \"id\":4, \"num\":100, \"num2\":10}");
127127
client().performRequest(doc4);
128128
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/_doc/5");
129-
doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
129+
doc5.setJsonEntity("{\"type\":\"type2\", \"id\":5, \"num\":100, \"num2\":10}");
130130
client().performRequest(doc5);
131131
}
132132

133133
{
134134
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1");
135-
doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}");
135+
doc1.setJsonEntity("{\"id\":1, \"field\":\"value1\", \"rating\": 7}");
136136
client().performRequest(doc1);
137137
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/2");
138-
doc2.setJsonEntity("{\"field\":\"value2\"}");
138+
doc2.setJsonEntity("{\"id\":2, \"field\":\"value2\"}");
139139
client().performRequest(doc2);
140140
}
141141

@@ -153,19 +153,19 @@ public void indexDocuments() throws IOException {
153153
"}");
154154
client().performRequest(create);
155155
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3");
156-
doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}");
156+
doc3.setJsonEntity("{\"id\":3, \"field\":\"value1\", \"rating\": \"good\"}");
157157
client().performRequest(doc3);
158158
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4");
159-
doc4.setJsonEntity("{\"field\":\"value2\"}");
159+
doc4.setJsonEntity("{\"id\":4, \"field\":\"value2\"}");
160160
client().performRequest(doc4);
161161
}
162162

163163
{
164164
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5");
165-
doc5.setJsonEntity("{\"field\":\"value1\"}");
165+
doc5.setJsonEntity("{\"id\":5, \"field\":\"value1\"}");
166166
client().performRequest(doc5);
167167
Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/6");
168-
doc6.setJsonEntity("{\"field\":\"value2\"}");
168+
doc6.setJsonEntity("{\"id\":6, \"field\":\"value2\"}");
169169
client().performRequest(doc6);
170170
}
171171

@@ -188,7 +188,7 @@ public void indexDocuments() throws IOException {
188188
"}");
189189
client().performRequest(create);
190190
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/_doc/1");
191-
doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}");
191+
doc1.setJsonEntity("{\"id\":1, \"field1\":\"value1\", \"field2\":\"value2\"}");
192192
client().performRequest(doc1);
193193

194194
Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases");
@@ -225,7 +225,7 @@ public void testSearchNoQuery() throws IOException {
225225
assertEquals(1.0f, searchHit.getScore(), 0);
226226
assertEquals(-1L, searchHit.getVersion());
227227
assertNotNull(searchHit.getSourceAsMap());
228-
assertEquals(3, searchHit.getSourceAsMap().size());
228+
assertEquals(4, searchHit.getSourceAsMap().size());
229229
assertTrue(searchHit.getSourceAsMap().containsKey("type"));
230230
assertTrue(searchHit.getSourceAsMap().containsKey("num"));
231231
assertTrue(searchHit.getSourceAsMap().containsKey("num2"));
@@ -249,7 +249,7 @@ public void testSearchMatchQuery() throws IOException {
249249
assertThat(searchHit.getScore(), greaterThan(0f));
250250
assertEquals(-1L, searchHit.getVersion());
251251
assertNotNull(searchHit.getSourceAsMap());
252-
assertEquals(3, searchHit.getSourceAsMap().size());
252+
assertEquals(4, searchHit.getSourceAsMap().size());
253253
assertEquals("type1", searchHit.getSourceAsMap().get("type"));
254254
assertEquals(50, searchHit.getSourceAsMap().get("num2"));
255255
}
@@ -705,13 +705,13 @@ public void testSearchScroll() throws Exception {
705705
public void testMultiSearch() throws Exception {
706706
MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
707707
SearchRequest searchRequest1 = new SearchRequest("index1");
708-
searchRequest1.source().sort("_id", SortOrder.ASC);
708+
searchRequest1.source().sort("id", SortOrder.ASC);
709709
multiSearchRequest.add(searchRequest1);
710710
SearchRequest searchRequest2 = new SearchRequest("index2");
711-
searchRequest2.source().sort("_id", SortOrder.ASC);
711+
searchRequest2.source().sort("id", SortOrder.ASC);
712712
multiSearchRequest.add(searchRequest2);
713713
SearchRequest searchRequest3 = new SearchRequest("index3");
714-
searchRequest3.source().sort("_id", SortOrder.ASC);
714+
searchRequest3.source().sort("id", SortOrder.ASC);
715715
multiSearchRequest.add(searchRequest3);
716716

717717
MultiSearchResponse multiSearchResponse =
@@ -1198,7 +1198,8 @@ public void testExplainWithFetchSource() throws IOException {
11981198
assertTrue(explainResponse.hasExplanation());
11991199
assertThat(explainResponse.getExplanation().getValue(), equalTo(1.0f));
12001200
assertTrue(explainResponse.getGetResult().isExists());
1201-
assertThat(explainResponse.getGetResult().getSource(), equalTo(Collections.singletonMap("field1", "value1")));
1201+
assertEquals(2, explainResponse.getGetResult().getSource().size());
1202+
assertThat(explainResponse.getGetResult().getSource().get("field1"), equalTo("value1"));
12021203
}
12031204
}
12041205

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java

+12-6
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ public void testSearch() throws Exception {
169169

170170
// tag::search-source-sorting
171171
sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1>
172-
sourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); // <2>
172+
sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); // <2>
173173
// end::search-source-sorting
174174

175175
// tag::search-source-filtering-off
@@ -1251,6 +1251,9 @@ private void indexSearchTestData() throws IOException {
12511251
CreateIndexRequest authorsRequest = new CreateIndexRequest("authors")
12521252
.mapping(XContentFactory.jsonBuilder().startObject()
12531253
.startObject("properties")
1254+
.startObject("id")
1255+
.field("type", "keyword")
1256+
.endObject()
12541257
.startObject("user")
12551258
.field("type", "keyword")
12561259
.field("doc_values", "false")
@@ -1263,6 +1266,9 @@ private void indexSearchTestData() throws IOException {
12631266
CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors")
12641267
.mapping(XContentFactory.jsonBuilder().startObject()
12651268
.startObject("properties")
1269+
.startObject("id")
1270+
.field("type", "keyword")
1271+
.endObject()
12661272
.startObject("user")
12671273
.field("type", "keyword")
12681274
.field("store", "true")
@@ -1274,19 +1280,19 @@ private void indexSearchTestData() throws IOException {
12741280

12751281
BulkRequest bulkRequest = new BulkRequest();
12761282
bulkRequest.add(new IndexRequest("posts").id("1")
1277-
.source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
1283+
.source(XContentType.JSON, "id", 1, "title", "In which order are my Elasticsearch queries executed?", "user",
12781284
Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
12791285
bulkRequest.add(new IndexRequest("posts").id("2")
1280-
.source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
1286+
.source(XContentType.JSON, "id", 2, "title", "Current status and upcoming changes in Elasticsearch", "user",
12811287
Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
12821288
bulkRequest.add(new IndexRequest("posts").id("3")
1283-
.source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
1289+
.source(XContentType.JSON, "id", 3, "title", "The Future of Federated Search in Elasticsearch", "user",
12841290
Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
12851291

12861292
bulkRequest.add(new IndexRequest("authors").id("1")
1287-
.source(XContentType.JSON, "user", "kimchy"));
1293+
.source(XContentType.JSON, "id", 1, "user", "kimchy"));
12881294
bulkRequest.add(new IndexRequest("contributors").id("1")
1289-
.source(XContentType.JSON, "user", "tanguy"));
1295+
.source(XContentType.JSON, "id", 1, "user", "tanguy"));
12901296

12911297

12921298
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

‎docs/reference/mapping/types/parent-join.asciidoc

+14-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ PUT my_index
1616
{
1717
"mappings": {
1818
"properties": {
19+
"my_id": {
20+
"type": "keyword"
21+
},
1922
"my_join_field": { <1>
2023
"type": "join",
2124
"relations": {
@@ -38,6 +41,7 @@ For instance the following example creates two `parent` documents in the `questi
3841
--------------------------------------------------
3942
PUT my_index/_doc/1?refresh
4043
{
44+
"my_id": "1",
4145
"text": "This is a question",
4246
"my_join_field": {
4347
"name": "question" <1>
@@ -46,6 +50,7 @@ PUT my_index/_doc/1?refresh
4650
4751
PUT my_index/_doc/2?refresh
4852
{
53+
"my_id": "2",
4954
"text": "This is another question",
5055
"my_join_field": {
5156
"name": "question"
@@ -63,12 +68,14 @@ as a shortcut instead of encapsulating it in the normal object notation:
6368
--------------------------------------------------
6469
PUT my_index/_doc/1?refresh
6570
{
71+
"my_id": "1",
6672
"text": "This is a question",
6773
"my_join_field": "question" <1>
6874
}
6975
7076
PUT my_index/_doc/2?refresh
7177
{
78+
"my_id": "2",
7279
"text": "This is another question",
7380
"my_join_field": "question"
7481
}
@@ -89,6 +96,7 @@ For instance the following example shows how to index two `child` documents:
8996
--------------------------------------------------
9097
PUT my_index/_doc/3?routing=1&refresh <1>
9198
{
99+
"my_id": "3",
92100
"text": "This is an answer",
93101
"my_join_field": {
94102
"name": "answer", <2>
@@ -98,6 +106,7 @@ PUT my_index/_doc/3?routing=1&refresh <1>
98106
99107
PUT my_index/_doc/4?routing=1&refresh
100108
{
109+
"my_id": "4",
101110
"text": "This is another answer",
102111
"my_join_field": {
103112
"name": "answer",
@@ -159,7 +168,7 @@ GET my_index/_search
159168
"query": {
160169
"match_all": {}
161170
},
162-
"sort": ["_id"]
171+
"sort": ["my_id"]
163172
}
164173
--------------------------
165174
// TEST[continued]
@@ -182,6 +191,7 @@ Will return:
182191
"_id": "1",
183192
"_score": null,
184193
"_source": {
194+
"my_id": "1",
185195
"text": "This is a question",
186196
"my_join_field": "question" <1>
187197
},
@@ -194,6 +204,7 @@ Will return:
194204
"_id": "2",
195205
"_score": null,
196206
"_source": {
207+
"my_id": "2",
197208
"text": "This is another question",
198209
"my_join_field": "question" <2>
199210
},
@@ -207,6 +218,7 @@ Will return:
207218
"_score": null,
208219
"_routing": "1",
209220
"_source": {
221+
"my_id": "3",
210222
"text": "This is an answer",
211223
"my_join_field": {
212224
"name": "answer", <3>
@@ -223,6 +235,7 @@ Will return:
223235
"_score": null,
224236
"_routing": "1",
225237
"_source": {
238+
"my_id": "4",
226239
"text": "This is another answer",
227240
"my_join_field": {
228241
"name": "answer",

‎modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java

+22-16
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) {
7979

8080
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
8181
req.setQuery(QueryBuilders.matchAllQuery())
82-
.addSort(SortBuilders.fieldSort("_id")
83-
.order(SortOrder.ASC))
82+
.addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC).unmappedType("long"))
8483
.addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap));
8584
return req;
8685
}
@@ -147,8 +146,10 @@ public void testDateMethods() throws Exception {
147146
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
148147
ensureGreen("test");
149148
indexRandom(true,
150-
client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
151-
client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
149+
client().prepareIndex("test").setId("1")
150+
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
151+
client().prepareIndex("test").setId("2")
152+
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
152153
SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get();
153154
assertEquals(2, rsp.getHits().getTotalHits().value);
154155
SearchHits hits = rsp.getHits();
@@ -175,8 +176,10 @@ public void testDateObjectMethods() throws Exception {
175176
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
176177
ensureGreen("test");
177178
indexRandom(true,
178-
client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
179-
client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
179+
client().prepareIndex("test").setId("1")
180+
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
181+
client().prepareIndex("test").setId("2")
182+
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
180183
SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get();
181184
assertEquals(2, rsp.getHits().getTotalHits().value);
182185
SearchHits hits = rsp.getHits();
@@ -207,15 +210,18 @@ public void testMultiValueMethods() throws Exception {
207210
ensureGreen("test");
208211

209212
Map<String, Object> doc1 = new HashMap<>();
213+
doc1.put("id", 1);
210214
doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d});
211215
doc1.put("double1", new Double[]{1.2d, 2.4d});
212216
doc1.put("double2", 3.0d);
213217

214218
Map<String, Object> doc2 = new HashMap<>();
219+
doc2.put("id", 2);
215220
doc2.put("double0", 5.0d);
216221
doc2.put("double1", 3.0d);
217222

218223
Map<String, Object> doc3 = new HashMap<>();
224+
doc3.put("id", 3);
219225
doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d});
220226
doc3.put("double1", 4.0d);
221227

@@ -319,8 +325,8 @@ public void testSparseField() throws Exception {
319325
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long"));
320326
ensureGreen("test");
321327
indexRandom(true,
322-
client().prepareIndex("test").setId("1").setSource("x", 4),
323-
client().prepareIndex("test").setId("2").setSource("y", 2));
328+
client().prepareIndex("test").setId("1").setSource("id", 1, "x", 4),
329+
client().prepareIndex("test").setId("2").setSource("id", 2, "y", 2));
324330
SearchResponse rsp = buildRequest("doc['x'] + 1").get();
325331
ElasticsearchAssertions.assertSearchResponse(rsp);
326332
SearchHits hits = rsp.getHits();
@@ -348,9 +354,9 @@ public void testParams() throws Exception {
348354
createIndex("test");
349355
ensureGreen("test");
350356
indexRandom(true,
351-
client().prepareIndex("test").setId("1").setSource("x", 10),
352-
client().prepareIndex("test").setId("2").setSource("x", 3),
353-
client().prepareIndex("test").setId("3").setSource("x", 5));
357+
client().prepareIndex("test").setId("1").setSource("id", 1, "x", 10),
358+
client().prepareIndex("test").setId("2").setSource("id", 2, "x", 3),
359+
client().prepareIndex("test").setId("3").setSource("id", 3, "x", 5));
354360
// a = int, b = double, c = long
355361
String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
356362
SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get();
@@ -621,9 +627,9 @@ public void testBoolean() throws Exception {
621627
assertAcked(prepareCreate("test").addMapping("doc", xContentBuilder));
622628
ensureGreen();
623629
indexRandom(true,
624-
client().prepareIndex("test").setId("1").setSource("price", 1.0, "vip", true),
625-
client().prepareIndex("test").setId("2").setSource("price", 2.0, "vip", false),
626-
client().prepareIndex("test").setId("3").setSource("price", 2.0, "vip", false));
630+
client().prepareIndex("test").setId("1").setSource("id", 1, "price", 1.0, "vip", true),
631+
client().prepareIndex("test").setId("2").setSource("id", 2, "price", 2.0, "vip", false),
632+
client().prepareIndex("test").setId("3").setSource("id", 3, "price", 2.0, "vip", false));
627633
// access .value
628634
SearchResponse rsp = buildRequest("doc['vip'].value").get();
629635
assertSearchResponse(rsp);
@@ -652,8 +658,8 @@ public void testFilterScript() throws Exception {
652658
createIndex("test");
653659
ensureGreen("test");
654660
indexRandom(true,
655-
client().prepareIndex("test").setId("1").setSource("foo", 1.0),
656-
client().prepareIndex("test").setId("2").setSource("foo", 0.0));
661+
client().prepareIndex("test").setId("1").setSource("id", 1, "foo", 1.0),
662+
client().prepareIndex("test").setId("2").setSource("id", 2, "foo", 0.0));
657663
SearchRequestBuilder builder = buildRequest("doc['foo'].value");
658664
Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap());
659665
builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script)));

‎modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ public void testParentWithMultipleBuckets() throws Exception {
103103
.setQuery(matchQuery("randomized", false))
104104
.addAggregation(
105105
terms("category").field("category").size(10000).subAggregation(
106-
children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_id", SortOrder.ASC))
106+
children("to_comment", "comment").subAggregation(topHits("top_comments").sort("id", SortOrder.ASC))
107107
)
108108
).get();
109109
assertSearchResponse(searchResponse);

0 commit comments

Comments
 (0)