Skip to content

Commit 42c8300

Browse files
dnhatndnhatn
authored
Add BWC test for field-caps (#84455) (#84989)
Relates #83494
1 parent 107e68b commit 42c8300

File tree

2 files changed

+275
-0
lines changed

2 files changed

+275
-0
lines changed

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.upgrades;
10+
11+
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
12+
import org.elasticsearch.client.Request;
13+
import org.elasticsearch.common.Strings;
14+
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.index.query.QueryBuilder;
16+
import org.elasticsearch.index.query.QueryBuilders;
17+
import org.elasticsearch.xcontent.json.JsonXContent;
18+
import org.junit.Before;
19+
20+
import java.util.List;
21+
22+
import static org.hamcrest.Matchers.contains;
23+
import static org.hamcrest.Matchers.equalTo;
24+
25+
/**
26+
* Since ES 8.2, field-caps internal responses are shared between indices that have the same index mapping hash to
27+
* reduce the transport message size between nodes and clusters, and the memory usage to hold these internal responses.
28+
* As the optimization is applied for only field-caps requests without index-filter and nodes on 8.2 or later,
29+
* these BWC tests verify these combinations of field-caps requests: (old|new|mixed indices) and (with|without index filter)
30+
*/
31+
public class FieldCapsIT extends AbstractRollingTestCase {
32+
private static boolean indicesCreated = false;
33+
34+
@Before
35+
public void setupIndices() throws Exception {
36+
if (indicesCreated) {
37+
return;
38+
}
39+
indicesCreated = true;
40+
final String redMapping = """
41+
"properties": {
42+
"red_field": { "type": "keyword" },
43+
"yellow_field": { "type": "integer" },
44+
"blue_field": { "type": "keyword" },
45+
"timestamp": {"type": "date"}
46+
}
47+
""";
48+
final String greenMapping = """
49+
"properties": {
50+
"green_field": { "type": "keyword" },
51+
"yellow_field": { "type": "long" },
52+
"blue_field": { "type": "keyword" },
53+
"timestamp": {"type": "date"}
54+
}
55+
""";
56+
if (CLUSTER_TYPE == ClusterType.OLD) {
57+
createIndex("old_red_1", Settings.EMPTY, redMapping);
58+
createIndex("old_red_2", Settings.EMPTY, redMapping);
59+
createIndex("old_red_empty", Settings.EMPTY, redMapping);
60+
createIndex("old_green_1", Settings.EMPTY, greenMapping);
61+
createIndex("old_green_2", Settings.EMPTY, greenMapping);
62+
createIndex("old_green_empty", Settings.EMPTY, greenMapping);
63+
for (String index : List.of("old_red_1", "old_red_2", "old_green_1", "old_green_2")) {
64+
final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1");
65+
indexRequest.addParameter("refresh", "true");
66+
indexRequest.setJsonEntity(
67+
Strings.toString(JsonXContent.contentBuilder().startObject().field("timestamp", "2020-01-01").endObject())
68+
);
69+
assertOK(client().performRequest(indexRequest));
70+
}
71+
} else if (CLUSTER_TYPE == ClusterType.MIXED && FIRST_MIXED_ROUND) {
72+
createIndex("new_red_1", Settings.EMPTY, redMapping);
73+
createIndex("new_red_2", Settings.EMPTY, redMapping);
74+
createIndex("new_red_empty", Settings.EMPTY, redMapping);
75+
createIndex("new_green_1", Settings.EMPTY, greenMapping);
76+
createIndex("new_green_2", Settings.EMPTY, greenMapping);
77+
createIndex("new_green_empty", Settings.EMPTY, greenMapping);
78+
for (String index : List.of("new_red_1", "new_red_2", "new_green_1", "new_green_2")) {
79+
final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/1");
80+
indexRequest.addParameter("refresh", "true");
81+
indexRequest.setJsonEntity(
82+
Strings.toString(JsonXContent.contentBuilder().startObject().field("timestamp", "2020-10-10").endObject())
83+
);
84+
assertOK(client().performRequest(indexRequest));
85+
}
86+
}
87+
}
88+
89+
public void testOldIndicesOnly() throws Exception {
90+
{
91+
FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), null);
92+
assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2", "old_red_empty" }));
93+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
94+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
95+
assertThat(resp.getField("yellow_field").keySet(), contains("integer"));
96+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
97+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
98+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
99+
}
100+
{
101+
FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), null);
102+
assertThat(
103+
resp.getIndices(),
104+
equalTo(new String[] { "old_green_1", "old_green_2", "old_green_empty", "old_red_1", "old_red_2", "old_red_empty" })
105+
);
106+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
107+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
108+
assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long"));
109+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
110+
assertTrue(resp.getField("yellow_field").get("long").isSearchable());
111+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
112+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
113+
}
114+
}
115+
116+
public void testOldIndicesWithIndexFilter() throws Exception {
117+
final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
118+
{
119+
FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), indexFilter);
120+
assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2" }));
121+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
122+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
123+
assertThat(resp.getField("yellow_field").keySet(), contains("integer"));
124+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
125+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
126+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
127+
}
128+
{
129+
FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), indexFilter);
130+
assertThat(resp.getIndices(), equalTo(new String[] { "old_green_1", "old_green_2", "old_red_1", "old_red_2" }));
131+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
132+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
133+
assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long"));
134+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
135+
assertTrue(resp.getField("yellow_field").get("long").isSearchable());
136+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
137+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
138+
}
139+
}
140+
141+
public void testNewIndicesOnly() throws Exception {
142+
assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
143+
{
144+
FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null);
145+
assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" }));
146+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
147+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
148+
assertThat(resp.getField("yellow_field").keySet(), contains("integer"));
149+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
150+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
151+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
152+
}
153+
{
154+
FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), null);
155+
assertThat(
156+
resp.getIndices(),
157+
equalTo(new String[] { "new_green_1", "new_green_2", "new_green_empty", "new_red_1", "new_red_2", "new_red_empty" })
158+
);
159+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
160+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
161+
assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long"));
162+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
163+
assertTrue(resp.getField("yellow_field").get("long").isSearchable());
164+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
165+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
166+
}
167+
}
168+
169+
public void testNewIndicesOnlyWithIndexFilter() throws Exception {
170+
assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
171+
final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
172+
{
173+
FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter);
174+
assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2" }));
175+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
176+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
177+
assertThat(resp.getField("yellow_field").keySet(), contains("integer"));
178+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
179+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
180+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
181+
}
182+
{
183+
FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), indexFilter);
184+
assertThat(resp.getIndices(), equalTo(new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2" }));
185+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
186+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
187+
assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long"));
188+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
189+
assertTrue(resp.getField("yellow_field").get("long").isSearchable());
190+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
191+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
192+
}
193+
}
194+
195+
public void testAllIndices() throws Exception {
196+
assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
197+
FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null);
198+
assertThat(
199+
resp.getIndices(),
200+
equalTo(
201+
new String[] {
202+
"new_green_1",
203+
"new_green_2",
204+
"new_green_empty",
205+
"new_red_1",
206+
"new_red_2",
207+
"new_red_empty",
208+
"old_green_1",
209+
"old_green_2",
210+
"old_green_empty",
211+
"old_red_1",
212+
"old_red_2",
213+
"old_red_empty" }
214+
)
215+
);
216+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
217+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
218+
assertThat(resp.getField("green_field").keySet(), contains("keyword"));
219+
assertTrue(resp.getField("green_field").get("keyword").isSearchable());
220+
assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long"));
221+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
222+
assertTrue(resp.getField("yellow_field").get("long").isSearchable());
223+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
224+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
225+
}
226+
227+
public void testAllIndicesWithIndexFilter() throws Exception {
228+
assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
229+
final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
230+
FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter);
231+
assertThat(
232+
resp.getIndices(),
233+
equalTo(
234+
new String[] {
235+
"new_green_1",
236+
"new_green_2",
237+
"new_red_1",
238+
"new_red_2",
239+
"old_green_1",
240+
"old_green_2",
241+
"old_red_1",
242+
"old_red_2" }
243+
)
244+
);
245+
assertThat(resp.getField("red_field").keySet(), contains("keyword"));
246+
assertTrue(resp.getField("red_field").get("keyword").isSearchable());
247+
assertThat(resp.getField("green_field").keySet(), contains("keyword"));
248+
assertTrue(resp.getField("green_field").get("keyword").isSearchable());
249+
assertThat(resp.getField("yellow_field").keySet(), contains("integer", "long"));
250+
assertTrue(resp.getField("yellow_field").get("integer").isSearchable());
251+
assertTrue(resp.getField("yellow_field").get("long").isSearchable());
252+
assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
253+
assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
254+
}
255+
}

‎test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.Version;
2626
import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
2727
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
28+
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
2829
import org.elasticsearch.client.Request;
2930
import org.elasticsearch.client.RequestOptions;
3031
import org.elasticsearch.client.RequestOptions.Builder;
@@ -48,6 +49,7 @@
4849
import org.elasticsearch.core.TimeValue;
4950
import org.elasticsearch.core.internal.io.IOUtils;
5051
import org.elasticsearch.index.IndexSettings;
52+
import org.elasticsearch.index.query.QueryBuilder;
5153
import org.elasticsearch.index.seqno.ReplicationTracker;
5254
import org.elasticsearch.rest.RestStatus;
5355
import org.elasticsearch.snapshots.SnapshotState;
@@ -1929,4 +1931,22 @@ protected static boolean isNotFoundResponseException(IOException ioe) {
19291931
}
19301932
return false;
19311933
}
1934+
1935+
protected FieldCapabilitiesResponse fieldCaps(List<String> indices, List<String> fields, QueryBuilder indexFilter) throws IOException {
1936+
Request request = new Request("POST", "/_field_caps");
1937+
request.addParameter("index", String.join(",", indices));
1938+
request.addParameter("fields", String.join(",", fields));
1939+
if (indexFilter != null) {
1940+
XContentBuilder body = JsonXContent.contentBuilder();
1941+
body.startObject();
1942+
body.field("index_filter", indexFilter);
1943+
body.endObject();
1944+
request.setJsonEntity(Strings.toString(body));
1945+
}
1946+
Response response = client().performRequest(request);
1947+
assertOK(response);
1948+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, response.getEntity().getContent())) {
1949+
return FieldCapabilitiesResponse.fromXContent(parser);
1950+
}
1951+
}
19321952
}

0 commit comments

Comments
 (0)