Skip to content

Commit d6445fa

Browse files
jimczijimczi
committed
Add a cluster setting to disallow loading fielddata on _id field (#49166)
This change adds a dynamic cluster setting named `indices.id_field_data.enabled`. When set to `false` any attempt to load the fielddata for the `_id` field will fail with an exception. The default value in this change is set to `false` in order to prevent fielddata usage on this field for future versions but it will be set to `true` when backporting to 7x. When the setting is set to true (manually or by default in 7x) the loading will also issue a deprecation warning since we want to disallow fielddata entirely when #26472 is implemented. Closes #43599
1 parent b236076 commit d6445fa

File tree

36 files changed

+495
-261
lines changed

36 files changed

+495
-261
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

+19-18
Original file line numberDiff line numberDiff line change
@@ -116,32 +116,32 @@ public void indexDocuments() throws IOException {
116116
{
117117
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/type/1");
118118
doc1.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
119-
doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}");
119+
doc1.setJsonEntity("{\"type\":\"type1\", \"id\":1, \"num\":10, \"num2\":50}");
120120
client().performRequest(doc1);
121121
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/type/2");
122122
doc2.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
123-
doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}");
123+
doc2.setJsonEntity("{\"type\":\"type1\", \"id\":2, \"num\":20, \"num2\":40}");
124124
client().performRequest(doc2);
125125
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/type/3");
126126
doc3.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
127-
doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}");
127+
doc3.setJsonEntity("{\"type\":\"type1\", \"id\":3, \"num\":50, \"num2\":35}");
128128
client().performRequest(doc3);
129129
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/type/4");
130130
doc4.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
131-
doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
131+
doc4.setJsonEntity("{\"type\":\"type2\", \"id\":4, \"num\":100, \"num2\":10}");
132132
client().performRequest(doc4);
133133
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/type/5");
134134
doc5.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
135-
doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}");
135+
doc5.setJsonEntity("{\"type\":\"type2\", \"id\":5, \"num\":100, \"num2\":10}");
136136
client().performRequest(doc5);
137137
}
138138

139139
{
140140
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1");
141-
doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}");
141+
doc1.setJsonEntity("{\"id\":1, \"field\":\"value1\", \"rating\": 7}");
142142
client().performRequest(doc1);
143143
Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/2");
144-
doc2.setJsonEntity("{\"field\":\"value2\"}");
144+
doc2.setJsonEntity("{\"id\":2, \"field\":\"value2\"}");
145145
client().performRequest(doc2);
146146
}
147147

@@ -159,19 +159,19 @@ public void indexDocuments() throws IOException {
159159
"}");
160160
client().performRequest(create);
161161
Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3");
162-
doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}");
162+
doc3.setJsonEntity("{\"id\":3, \"field\":\"value1\", \"rating\": \"good\"}");
163163
client().performRequest(doc3);
164164
Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4");
165-
doc4.setJsonEntity("{\"field\":\"value2\"}");
165+
doc4.setJsonEntity("{\"id\":4, \"field\":\"value2\"}");
166166
client().performRequest(doc4);
167167
}
168168

169169
{
170170
Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5");
171-
doc5.setJsonEntity("{\"field\":\"value1\"}");
171+
doc5.setJsonEntity("{\"id\":5, \"field\":\"value1\"}");
172172
client().performRequest(doc5);
173173
Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/6");
174-
doc6.setJsonEntity("{\"field\":\"value2\"}");
174+
doc6.setJsonEntity("{\"id\":6, \"field\":\"value2\"}");
175175
client().performRequest(doc6);
176176
}
177177

@@ -194,7 +194,7 @@ public void indexDocuments() throws IOException {
194194
"}");
195195
client().performRequest(create);
196196
Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/_doc/1");
197-
doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}");
197+
doc1.setJsonEntity("{\"id\":1, \"field1\":\"value1\", \"field2\":\"value2\"}");
198198
client().performRequest(doc1);
199199

200200
Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases");
@@ -232,7 +232,7 @@ public void testSearchNoQuery() throws IOException {
232232
assertEquals(1.0f, searchHit.getScore(), 0);
233233
assertEquals(-1L, searchHit.getVersion());
234234
assertNotNull(searchHit.getSourceAsMap());
235-
assertEquals(3, searchHit.getSourceAsMap().size());
235+
assertEquals(4, searchHit.getSourceAsMap().size());
236236
assertTrue(searchHit.getSourceAsMap().containsKey("type"));
237237
assertTrue(searchHit.getSourceAsMap().containsKey("num"));
238238
assertTrue(searchHit.getSourceAsMap().containsKey("num2"));
@@ -257,7 +257,7 @@ public void testSearchMatchQuery() throws IOException {
257257
assertThat(searchHit.getScore(), greaterThan(0f));
258258
assertEquals(-1L, searchHit.getVersion());
259259
assertNotNull(searchHit.getSourceAsMap());
260-
assertEquals(3, searchHit.getSourceAsMap().size());
260+
assertEquals(4, searchHit.getSourceAsMap().size());
261261
assertEquals("type1", searchHit.getSourceAsMap().get("type"));
262262
assertEquals(50, searchHit.getSourceAsMap().get("num2"));
263263
}
@@ -713,13 +713,13 @@ public void testSearchScroll() throws Exception {
713713
public void testMultiSearch() throws Exception {
714714
MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
715715
SearchRequest searchRequest1 = new SearchRequest("index1");
716-
searchRequest1.source().sort("_id", SortOrder.ASC);
716+
searchRequest1.source().sort("id", SortOrder.ASC);
717717
multiSearchRequest.add(searchRequest1);
718718
SearchRequest searchRequest2 = new SearchRequest("index2");
719-
searchRequest2.source().sort("_id", SortOrder.ASC);
719+
searchRequest2.source().sort("id", SortOrder.ASC);
720720
multiSearchRequest.add(searchRequest2);
721721
SearchRequest searchRequest3 = new SearchRequest("index3");
722-
searchRequest3.source().sort("_id", SortOrder.ASC);
722+
searchRequest3.source().sort("id", SortOrder.ASC);
723723
multiSearchRequest.add(searchRequest3);
724724

725725
MultiSearchResponse multiSearchResponse =
@@ -1211,7 +1211,8 @@ public void testExplainWithFetchSource() throws IOException {
12111211
assertTrue(explainResponse.hasExplanation());
12121212
assertThat(explainResponse.getExplanation().getValue(), equalTo(1.0f));
12131213
assertTrue(explainResponse.getGetResult().isExists());
1214-
assertThat(explainResponse.getGetResult().getSource(), equalTo(Collections.singletonMap("field1", "value1")));
1214+
assertEquals(2, explainResponse.getGetResult().getSource().size());
1215+
assertThat(explainResponse.getGetResult().getSource().get("field1"), equalTo("value1"));
12151216
}
12161217
}
12171218

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java

+12-6
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ public void testSearch() throws Exception {
169169

170170
// tag::search-source-sorting
171171
sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1>
172-
sourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); // <2>
172+
sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); // <2>
173173
// end::search-source-sorting
174174

175175
// tag::search-source-filtering-off
@@ -1251,6 +1251,9 @@ private void indexSearchTestData() throws IOException {
12511251
CreateIndexRequest authorsRequest = new CreateIndexRequest("authors")
12521252
.mapping(XContentFactory.jsonBuilder().startObject()
12531253
.startObject("properties")
1254+
.startObject("id")
1255+
.field("type", "keyword")
1256+
.endObject()
12541257
.startObject("user")
12551258
.field("type", "keyword")
12561259
.field("doc_values", "false")
@@ -1263,6 +1266,9 @@ private void indexSearchTestData() throws IOException {
12631266
CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors")
12641267
.mapping(XContentFactory.jsonBuilder().startObject()
12651268
.startObject("properties")
1269+
.startObject("id")
1270+
.field("type", "keyword")
1271+
.endObject()
12661272
.startObject("user")
12671273
.field("type", "keyword")
12681274
.field("store", "true")
@@ -1274,19 +1280,19 @@ private void indexSearchTestData() throws IOException {
12741280

12751281
BulkRequest bulkRequest = new BulkRequest();
12761282
bulkRequest.add(new IndexRequest("posts").id("1")
1277-
.source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user",
1283+
.source(XContentType.JSON, "id", 1, "title", "In which order are my Elasticsearch queries executed?", "user",
12781284
Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value")));
12791285
bulkRequest.add(new IndexRequest("posts").id("2")
1280-
.source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user",
1286+
.source(XContentType.JSON, "id", 2, "title", "Current status and upcoming changes in Elasticsearch", "user",
12811287
Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value")));
12821288
bulkRequest.add(new IndexRequest("posts").id("3")
1283-
.source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user",
1289+
.source(XContentType.JSON, "id", 3, "title", "The Future of Federated Search in Elasticsearch", "user",
12841290
Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value")));
12851291

12861292
bulkRequest.add(new IndexRequest("authors").id("1")
1287-
.source(XContentType.JSON, "user", "kimchy"));
1293+
.source(XContentType.JSON, "id", 1, "user", "kimchy"));
12881294
bulkRequest.add(new IndexRequest("contributors").id("1")
1289-
.source(XContentType.JSON, "user", "tanguy"));
1295+
.source(XContentType.JSON, "id", 1, "user", "tanguy"));
12901296

12911297

12921298
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

‎docs/reference/mapping/types/parent-join.asciidoc

+14-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ PUT my_index
1616
{
1717
"mappings": {
1818
"properties": {
19+
"my_id": {
20+
"type": "keyword"
21+
},
1922
"my_join_field": { <1>
2023
"type": "join",
2124
"relations": {
@@ -38,6 +41,7 @@ For instance the following example creates two `parent` documents in the `questi
3841
--------------------------------------------------
3942
PUT my_index/_doc/1?refresh
4043
{
44+
"my_id": "1",
4145
"text": "This is a question",
4246
"my_join_field": {
4347
"name": "question" <1>
@@ -46,6 +50,7 @@ PUT my_index/_doc/1?refresh
4650
4751
PUT my_index/_doc/2?refresh
4852
{
53+
"my_id": "2",
4954
"text": "This is another question",
5055
"my_join_field": {
5156
"name": "question"
@@ -63,12 +68,14 @@ as a shortcut instead of encapsulating it in the normal object notation:
6368
--------------------------------------------------
6469
PUT my_index/_doc/1?refresh
6570
{
71+
"my_id": "1",
6672
"text": "This is a question",
6773
"my_join_field": "question" <1>
6874
}
6975
7076
PUT my_index/_doc/2?refresh
7177
{
78+
"my_id": "2",
7279
"text": "This is another question",
7380
"my_join_field": "question"
7481
}
@@ -89,6 +96,7 @@ For instance the following example shows how to index two `child` documents:
8996
--------------------------------------------------
9097
PUT my_index/_doc/3?routing=1&refresh <1>
9198
{
99+
"my_id": "3",
92100
"text": "This is an answer",
93101
"my_join_field": {
94102
"name": "answer", <2>
@@ -98,6 +106,7 @@ PUT my_index/_doc/3?routing=1&refresh <1>
98106
99107
PUT my_index/_doc/4?routing=1&refresh
100108
{
109+
"my_id": "4",
101110
"text": "This is another answer",
102111
"my_join_field": {
103112
"name": "answer",
@@ -159,7 +168,7 @@ GET my_index/_search
159168
"query": {
160169
"match_all": {}
161170
},
162-
"sort": ["_id"]
171+
"sort": ["my_id"]
163172
}
164173
--------------------------
165174
// TEST[continued]
@@ -183,6 +192,7 @@ Will return:
183192
"_id": "1",
184193
"_score": null,
185194
"_source": {
195+
"my_id": "1",
186196
"text": "This is a question",
187197
"my_join_field": "question" <1>
188198
},
@@ -196,6 +206,7 @@ Will return:
196206
"_id": "2",
197207
"_score": null,
198208
"_source": {
209+
"my_id": "2",
199210
"text": "This is another question",
200211
"my_join_field": "question" <2>
201212
},
@@ -210,6 +221,7 @@ Will return:
210221
"_score": null,
211222
"_routing": "1",
212223
"_source": {
224+
"my_id": "3",
213225
"text": "This is an answer",
214226
"my_join_field": {
215227
"name": "answer", <3>
@@ -227,6 +239,7 @@ Will return:
227239
"_score": null,
228240
"_routing": "1",
229241
"_source": {
242+
"my_id": "4",
230243
"text": "This is another answer",
231244
"my_join_field": {
232245
"name": "answer",

‎modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java

+22-16
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) {
7979

8080
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
8181
req.setQuery(QueryBuilders.matchAllQuery())
82-
.addSort(SortBuilders.fieldSort("_id")
83-
.order(SortOrder.ASC))
82+
.addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC).unmappedType("long"))
8483
.addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap));
8584
return req;
8685
}
@@ -147,8 +146,10 @@ public void testDateMethods() throws Exception {
147146
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
148147
ensureGreen("test");
149148
indexRandom(true,
150-
client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
151-
client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
149+
client().prepareIndex("test", "doc", "1")
150+
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
151+
client().prepareIndex("test", "doc", "2")
152+
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
152153
SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get();
153154
assertEquals(2, rsp.getHits().getTotalHits().value);
154155
SearchHits hits = rsp.getHits();
@@ -175,8 +176,10 @@ public void testDateObjectMethods() throws Exception {
175176
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
176177
ensureGreen("test");
177178
indexRandom(true,
178-
client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
179-
client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
179+
client().prepareIndex("test", "doc", "1")
180+
.setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
181+
client().prepareIndex("test", "doc", "2")
182+
.setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
180183
SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get();
181184
assertEquals(2, rsp.getHits().getTotalHits().value);
182185
SearchHits hits = rsp.getHits();
@@ -207,15 +210,18 @@ public void testMultiValueMethods() throws Exception {
207210
ensureGreen("test");
208211

209212
Map<String, Object> doc1 = new HashMap<>();
213+
doc1.put("id", 1);
210214
doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d});
211215
doc1.put("double1", new Double[]{1.2d, 2.4d});
212216
doc1.put("double2", 3.0d);
213217

214218
Map<String, Object> doc2 = new HashMap<>();
219+
doc2.put("id", 2);
215220
doc2.put("double0", 5.0d);
216221
doc2.put("double1", 3.0d);
217222

218223
Map<String, Object> doc3 = new HashMap<>();
224+
doc3.put("id", 3);
219225
doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d});
220226
doc3.put("double1", 4.0d);
221227

@@ -319,8 +325,8 @@ public void testSparseField() throws Exception {
319325
ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long"));
320326
ensureGreen("test");
321327
indexRandom(true,
322-
client().prepareIndex("test", "doc", "1").setSource("x", 4),
323-
client().prepareIndex("test", "doc", "2").setSource("y", 2));
328+
client().prepareIndex("test", "doc", "1").setSource("id", 1, "x", 4),
329+
client().prepareIndex("test", "doc","2").setSource("id", 2, "y", 2));
324330
SearchResponse rsp = buildRequest("doc['x'] + 1").get();
325331
ElasticsearchAssertions.assertSearchResponse(rsp);
326332
SearchHits hits = rsp.getHits();
@@ -348,9 +354,9 @@ public void testParams() throws Exception {
348354
createIndex("test");
349355
ensureGreen("test");
350356
indexRandom(true,
351-
client().prepareIndex("test", "doc", "1").setSource("x", 10),
352-
client().prepareIndex("test", "doc", "2").setSource("x", 3),
353-
client().prepareIndex("test", "doc", "3").setSource("x", 5));
357+
client().prepareIndex("test", "doc", "1").setSource("id", 1, "x", 10),
358+
client().prepareIndex("test", "doc", "2").setSource("id", 2, "x", 3),
359+
client().prepareIndex("test", "doc", "3").setSource("id", 3, "x", 5));
354360
// a = int, b = double, c = long
355361
String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)";
356362
SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get();
@@ -622,9 +628,9 @@ public void testBoolean() throws Exception {
622628
assertAcked(prepareCreate("test").addMapping("doc", xContentBuilder));
623629
ensureGreen();
624630
indexRandom(true,
625-
client().prepareIndex("test", "doc", "1").setSource("price", 1.0, "vip", true),
626-
client().prepareIndex("test", "doc", "2").setSource("price", 2.0, "vip", false),
627-
client().prepareIndex("test", "doc", "3").setSource("price", 2.0, "vip", false));
631+
client().prepareIndex("test", "doc", "1").setSource("id", 1, "price", 1.0, "vip", true),
632+
client().prepareIndex("test", "doc", "2").setSource("id", 2, "price", 2.0, "vip", false),
633+
client().prepareIndex("test", "doc", "3").setSource("id", 3, "price", 2.0, "vip", false));
628634
// access .value
629635
SearchResponse rsp = buildRequest("doc['vip'].value").get();
630636
assertSearchResponse(rsp);
@@ -653,8 +659,8 @@ public void testFilterScript() throws Exception {
653659
createIndex("test");
654660
ensureGreen("test");
655661
indexRandom(true,
656-
client().prepareIndex("test", "doc", "1").setSource("foo", 1.0),
657-
client().prepareIndex("test", "doc", "2").setSource("foo", 0.0));
662+
client().prepareIndex("test", "doc", "1").setSource("id", 1, "foo", 1.0),
663+
client().prepareIndex("test", "doc", "2").setSource("id", 2, "foo", 0.0));
658664
SearchRequestBuilder builder = buildRequest("doc['foo'].value");
659665
Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap());
660666
builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script)));

0 commit comments

Comments
 (0)