Skip to content

Commit 75de8f8

Browse files
DaveCTurnerDaveCTurner
authored
Chunked encoding for RestGetIndicesAction (#92016)
This response scales with the number of indices requested and can reach many MiB in size in a large cluster, let's use chunking here. Relates #89838
1 parent c895331 commit 75de8f8

File tree

6 files changed

+77
-66
lines changed

6 files changed

+77
-66
lines changed

‎modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import static org.elasticsearch.rest.RestStatus.OK;
2626
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
2727
import static org.hamcrest.Matchers.greaterThan;
28+
import static org.hamcrest.Matchers.nullValue;
2829

2930
public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
3031
public void testHeadRoot() throws IOException {
@@ -59,8 +60,8 @@ public void testDocumentExists() throws IOException {
5960

6061
public void testIndexExists() throws IOException {
6162
createTestDoc();
62-
headTestCase("/test", emptyMap(), greaterThan(0));
63-
headTestCase("/test", singletonMap("pretty", "true"), greaterThan(0));
63+
headTestCase("/test", emptyMap(), nullValue(Integer.class));
64+
headTestCase("/test", singletonMap("pretty", "true"), nullValue(Integer.class));
6465
}
6566

6667
public void testAliasExists() throws IOException {
@@ -177,7 +178,8 @@ private void headTestCase(
177178
request.setOptions(expectWarnings(expectedWarnings));
178179
Response response = client().performRequest(request);
179180
assertEquals(expectedStatusCode, response.getStatusLine().getStatusCode());
180-
assertThat(Integer.valueOf(response.getHeader("Content-Length")), matcher);
181+
final var contentLength = response.getHeader("Content-Length");
182+
assertThat(contentLength == null ? null : Integer.valueOf(contentLength), matcher);
181183
assertNull("HEAD requests shouldn't have a response body but " + url + " did", response.getEntity());
182184
}
183185

‎server/src/main/java/org/elasticsearch/action/ActionModule.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -768,7 +768,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
768768
registerHandler.accept(new RestResetFeatureStateAction());
769769
registerHandler.accept(new RestGetFeatureUpgradeStatusAction());
770770
registerHandler.accept(new RestPostFeatureUpgradeAction());
771-
registerHandler.accept(new RestGetIndicesAction(threadPool));
771+
registerHandler.accept(new RestGetIndicesAction());
772772
registerHandler.accept(new RestIndicesStatsAction());
773773
registerHandler.accept(new RestIndicesSegmentsAction());
774774
registerHandler.accept(new RestIndicesShardStoresAction());

‎server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java

Lines changed: 48 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,18 @@
1313
import org.elasticsearch.cluster.metadata.AliasMetadata;
1414
import org.elasticsearch.cluster.metadata.MappingMetadata;
1515
import org.elasticsearch.common.Strings;
16+
import org.elasticsearch.common.collect.Iterators;
1617
import org.elasticsearch.common.io.stream.StreamInput;
1718
import org.elasticsearch.common.io.stream.StreamOutput;
1819
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.common.xcontent.ChunkedToXContent;
1921
import org.elasticsearch.core.RestApiVersion;
2022
import org.elasticsearch.index.mapper.MapperService;
21-
import org.elasticsearch.xcontent.ToXContentObject;
22-
import org.elasticsearch.xcontent.XContentBuilder;
23+
import org.elasticsearch.xcontent.ToXContent;
2324

2425
import java.io.IOException;
2526
import java.util.Arrays;
27+
import java.util.Iterator;
2628
import java.util.List;
2729
import java.util.Map;
2830
import java.util.Objects;
@@ -33,7 +35,7 @@
3335
/**
3436
* A response for a get index action.
3537
*/
36-
public class GetIndexResponse extends ActionResponse implements ToXContentObject {
38+
public class GetIndexResponse extends ActionResponse implements ChunkedToXContent {
3739

3840
private Map<String, MappingMetadata> mappings = Map.of();
3941
private Map<String, List<AliasMetadata>> aliases = Map.of();
@@ -178,59 +180,58 @@ public void writeTo(StreamOutput out) throws IOException {
178180
}
179181

180182
@Override
181-
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
182-
builder.startObject();
183-
{
184-
for (final String index : indices) {
183+
public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
184+
return Iterators.concat(
185+
Iterators.single((builder, params) -> builder.startObject()),
186+
Arrays.stream(indices).<ToXContent>map(index -> (builder, params) -> {
185187
builder.startObject(index);
186-
{
187-
builder.startObject("aliases");
188-
List<AliasMetadata> indexAliases = aliases.get(index);
189-
if (indexAliases != null) {
190-
for (final AliasMetadata alias : indexAliases) {
191-
AliasMetadata.Builder.toXContent(alias, builder, params);
192-
}
193-
}
194-
builder.endObject();
195-
196-
MappingMetadata indexMappings = mappings.get(index);
197-
if (indexMappings == null) {
198-
builder.startObject("mappings").endObject();
199-
} else {
200-
if (builder.getRestApiVersion() == RestApiVersion.V_7
201-
&& params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) {
202-
builder.startObject("mappings");
203-
builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap());
204-
builder.endObject();
205-
} else {
206-
builder.field("mappings", indexMappings.sourceAsMap());
207-
}
208-
}
209188

210-
builder.startObject("settings");
211-
Settings indexSettings = settings.get(index);
212-
if (indexSettings != null) {
213-
indexSettings.toXContent(builder, params);
189+
builder.startObject("aliases");
190+
List<AliasMetadata> indexAliases = aliases.get(index);
191+
if (indexAliases != null) {
192+
for (final AliasMetadata alias : indexAliases) {
193+
AliasMetadata.Builder.toXContent(alias, builder, params);
214194
}
215-
builder.endObject();
195+
}
196+
builder.endObject();
216197

217-
Settings defaultIndexSettings = defaultSettings.get(index);
218-
if (defaultIndexSettings != null && defaultIndexSettings.isEmpty() == false) {
219-
builder.startObject("defaults");
220-
defaultIndexSettings.toXContent(builder, params);
198+
MappingMetadata indexMappings = mappings.get(index);
199+
if (indexMappings == null) {
200+
builder.startObject("mappings").endObject();
201+
} else {
202+
if (builder.getRestApiVersion() == RestApiVersion.V_7
203+
&& params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) {
204+
builder.startObject("mappings");
205+
builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap());
221206
builder.endObject();
207+
} else {
208+
builder.field("mappings", indexMappings.sourceAsMap());
222209
}
210+
}
223211

224-
String dataStream = dataStreams.get(index);
225-
if (dataStream != null) {
226-
builder.field("data_stream", dataStream);
227-
}
212+
builder.startObject("settings");
213+
Settings indexSettings = settings.get(index);
214+
if (indexSettings != null) {
215+
indexSettings.toXContent(builder, params);
228216
}
229217
builder.endObject();
230-
}
231-
}
232-
builder.endObject();
233-
return builder;
218+
219+
Settings defaultIndexSettings = defaultSettings.get(index);
220+
if (defaultIndexSettings != null && defaultIndexSettings.isEmpty() == false) {
221+
builder.startObject("defaults");
222+
defaultIndexSettings.toXContent(builder, params);
223+
builder.endObject();
224+
}
225+
226+
String dataStream = dataStreams.get(index);
227+
if (dataStream != null) {
228+
builder.field("data_stream", dataStream);
229+
}
230+
231+
return builder.endObject();
232+
}).iterator(),
233+
Iterators.single((builder, params) -> builder.endObject())
234+
);
234235
}
235236

236237
@Override

‎server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717
import org.elasticsearch.core.RestApiVersion;
1818
import org.elasticsearch.rest.BaseRestHandler;
1919
import org.elasticsearch.rest.RestRequest;
20-
import org.elasticsearch.rest.action.DispatchingRestToXContentListener;
2120
import org.elasticsearch.rest.action.RestCancellableNodeClient;
22-
import org.elasticsearch.threadpool.ThreadPool;
21+
import org.elasticsearch.rest.action.RestChunkedToXContentListener;
2322

2423
import java.io.IOException;
2524
import java.util.List;
@@ -39,12 +38,6 @@ public class RestGetIndicesAction extends BaseRestHandler {
3938

4039
private static final Set<String> COMPATIBLE_RESPONSE_PARAMS = addToCopy(Settings.FORMAT_PARAMS, INCLUDE_TYPE_NAME_PARAMETER);
4140

42-
private final ThreadPool threadPool;
43-
44-
public RestGetIndicesAction(ThreadPool threadPool) {
45-
this.threadPool = threadPool;
46-
}
47-
4841
@Override
4942
public List<Route> routes() {
5043
return List.of(new Route(GET, "/{index}"), new Route(HEAD, "/{index}"));
@@ -76,10 +69,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
7669
final var httpChannel = request.getHttpChannel();
7770
return channel -> new RestCancellableNodeClient(client, httpChannel).admin()
7871
.indices()
79-
.getIndex(
80-
getIndexRequest,
81-
new DispatchingRestToXContentListener<>(threadPool.executor(ThreadPool.Names.MANAGEMENT), channel, request)
82-
);
72+
.getIndex(getIndexRequest, new RestChunkedToXContentListener<>(channel));
8373
}
8474

8575
/**

‎server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
import org.elasticsearch.common.settings.Settings;
1919
import org.elasticsearch.index.RandomCreateIndexGenerator;
2020
import org.elasticsearch.test.AbstractWireSerializingTestCase;
21+
import org.elasticsearch.xcontent.ToXContent;
2122

23+
import java.io.IOException;
2224
import java.util.ArrayList;
2325
import java.util.Collections;
2426
import java.util.Comparator;
@@ -27,6 +29,9 @@
2729
import java.util.Locale;
2830
import java.util.Map;
2931

32+
import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
33+
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
34+
3035
public class GetIndexResponseTests extends AbstractWireSerializingTestCase<GetIndexResponse> {
3136

3237
@Override
@@ -73,4 +78,18 @@ protected GetIndexResponse createTestInstance() {
7378
}
7479
return new GetIndexResponse(indices, mappings, aliases, settings, defaultSettings, dataStreams);
7580
}
81+
82+
public void testChunking() throws IOException {
83+
final var response = createTestInstance();
84+
85+
try (var builder = jsonBuilder()) {
86+
int chunkCount = 0;
87+
final var iterator = response.toXContentChunked(EMPTY_PARAMS);
88+
while (iterator.hasNext()) {
89+
iterator.next().toXContent(builder, ToXContent.EMPTY_PARAMS);
90+
chunkCount += 1;
91+
}
92+
assertEquals(response.getIndices().length + 2, chunkCount);
93+
} // closing the builder verifies that the XContent is well-formed
94+
}
7695
}

‎server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
package org.elasticsearch.rest.action.admin.indices;
1010

1111
import org.elasticsearch.client.internal.node.NodeClient;
12-
import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
1312
import org.elasticsearch.core.RestApiVersion;
1413
import org.elasticsearch.rest.RestRequest;
1514
import org.elasticsearch.test.ESTestCase;
@@ -37,7 +36,7 @@ public void testIncludeTypeNamesWarning() throws IOException {
3736
Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)
3837
).withMethod(RestRequest.Method.GET).withPath("/some_index").withParams(params).build();
3938

40-
RestGetIndicesAction handler = new RestGetIndicesAction(new DeterministicTaskQueue().getThreadPool());
39+
RestGetIndicesAction handler = new RestGetIndicesAction();
4140
handler.prepareRequest(request, mock(NodeClient.class));
4241
assertCriticalWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE);
4342

@@ -58,7 +57,7 @@ public void testIncludeTypeNamesWarningExists() throws IOException {
5857
Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)
5958
).withMethod(RestRequest.Method.HEAD).withPath("/some_index").withParams(params).build();
6059

61-
RestGetIndicesAction handler = new RestGetIndicesAction(new DeterministicTaskQueue().getThreadPool());
60+
RestGetIndicesAction handler = new RestGetIndicesAction();
6261
handler.prepareRequest(request, mock(NodeClient.class));
6362
}
6463
}

0 commit comments

Comments
 (0)