Skip to content

Commit 81555cc

Browse files
dnhatndnhatn
authored
Allow partial results by default in ES|QL (#125060)
With this change, ES|QL will return partial results instead of failing the entire query when encountering errors. Callers should check the partial_results flag in the response to determine if the result is partial or complete. If returning partial results is not desired, this option can be overridden per request via the allow_partial_results parameter in the query URL or globally via the cluster setting esql.allow_partial_results. Relates #122802
1 parent 0c16ed7 commit 81555cc

File tree

19 files changed

+109
-54
lines changed

19 files changed

+109
-54
lines changed

‎docs/changelog/125060.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
pr: 125060
2+
summary: Allow partial results by default in ES|QL
3+
area: ES|QL
4+
type: breaking
5+
issues: [122802]
6+
7+
breaking:
8+
title: Allow partial results by default in ES|QL
9+
area: ES|QL
10+
details: >-
11+
In earlier versions of {es}, ES|QL would fail the entire query if it encountered any error. ES|QL now returns partial results instead of failing when encountering errors.
12+
13+
impact: >-
14+
Callers should check the `is_partial` flag returned in the response to determine if the result is partial or complete. If returning partial results is not desired, this option can be overridden per request via an `allow_partial_results` parameter in the query URL or globally via the cluster setting `esql.query.allow_partial_results`.
15+
16+
notable: true

‎docs/release-notes/breaking-changes.md

+5
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ To learn how to upgrade, check out <uprade docs>.
1414

1515
% ## Next version [elasticsearch-nextversion-breaking-changes]
1616

17+
## 9.1.0 [elasticsearch-910-breaking-changes]
18+
19+
ES|QL
20+
: * Allow partial results by default in ES|QL [#125060](https://github.com/elastic/elasticsearch/pull/125060)
21+
1722
## 9.0.0 [elasticsearch-900-breaking-changes]
1823

1924
Allocation

‎test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/Clusters.java

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ static ElasticsearchCluster buildCluster() {
2121
.module("test-esql-heap-attack")
2222
.setting("xpack.security.enabled", "false")
2323
.setting("xpack.license.self_generated.type", "trial")
24+
.setting("esql.query.allow_partial_results", "false")
2425
.jvmArg("-Xmx512m");
2526
String javaVersion = JvmInfo.jvmInfo().version();
2627
if (javaVersion.equals("20") || javaVersion.equals("21")) {

‎x-pack/plugin/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
124124
task.skipTest("ml/post_data/Test POST data job api, flush, close and verify DataCounts doc", "Flush API is deprecated")
125125
task.replaceValueInMatch("Size", 49, "Test flamegraph from profiling-events")
126126
task.replaceValueInMatch("Size", 49, "Test flamegraph from test-events")
127+
task.skipTest("esql/63_enrich_int_range/Invalid age as double", "TODO: require disable allow_partial_results")
127128
})
128129

129130
tasks.named('yamlRestCompatTest').configure {

‎x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java

+18-6
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,10 @@ public void testIndexPatternErrorMessageComparison_ESQL_SearchDSL() throws Excep
312312
searchRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", "metadata1_read2"));
313313

314314
// ES|QL query on the same index pattern
315-
var esqlResp = expectThrows(ResponseException.class, () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2"));
315+
var esqlResp = expectThrows(
316+
ResponseException.class,
317+
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2", false)
318+
);
316319
var srchResp = expectThrows(ResponseException.class, () -> client().performRequest(searchRequest));
317320

318321
for (ResponseException r : List.of(esqlResp, srchResp)) {
@@ -331,7 +334,8 @@ public void testLimitedPrivilege() throws Exception {
331334
ResponseException.class,
332335
() -> runESQLCommand(
333336
"metadata1_read2",
334-
"FROM index-user1,index-user2 METADATA _index | STATS sum=sum(value), index=VALUES(_index)"
337+
"FROM index-user1,index-user2 METADATA _index | STATS sum=sum(value), index=VALUES(_index)",
338+
false
335339
)
336340
);
337341
assertThat(
@@ -344,7 +348,7 @@ public void testLimitedPrivilege() throws Exception {
344348

345349
resp = expectThrows(
346350
ResponseException.class,
347-
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 METADATA _index | STATS index=VALUES(_index)")
351+
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 METADATA _index | STATS index=VALUES(_index)", false)
348352
);
349353
assertThat(
350354
EntityUtils.toString(resp.getResponse().getEntity()),
@@ -356,7 +360,7 @@ public void testLimitedPrivilege() throws Exception {
356360

357361
resp = expectThrows(
358362
ResponseException.class,
359-
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)")
363+
() -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)", false)
360364
);
361365
assertThat(
362366
EntityUtils.toString(resp.getResponse().getEntity()),
@@ -368,7 +372,7 @@ public void testLimitedPrivilege() throws Exception {
368372

369373
resp = expectThrows(
370374
ResponseException.class,
371-
() -> runESQLCommand("alias_user1", "FROM first-alias,index-user1 METADATA _index | KEEP _index, org, value | LIMIT 10")
375+
() -> runESQLCommand("alias_user1", "FROM first-alias,index-user1 METADATA _index | KEEP _index, org, value | LIMIT 10", false)
372376
);
373377
assertThat(
374378
EntityUtils.toString(resp.getResponse().getEntity()),
@@ -382,7 +386,8 @@ public void testLimitedPrivilege() throws Exception {
382386
ResponseException.class,
383387
() -> runESQLCommand(
384388
"alias_user2",
385-
"from second-alias,index-user2 METADATA _index | stats sum=sum(value), index=VALUES(_index)"
389+
"from second-alias,index-user2 METADATA _index | stats sum=sum(value), index=VALUES(_index)",
390+
false
386391
)
387392
);
388393
assertThat(
@@ -826,6 +831,10 @@ public void testDataStream() throws IOException {
826831
}
827832

828833
protected Response runESQLCommand(String user, String command) throws IOException {
834+
return runESQLCommand(user, command, null);
835+
}
836+
837+
protected Response runESQLCommand(String user, String command, Boolean allowPartialResults) throws IOException {
829838
if (command.toLowerCase(Locale.ROOT).contains("limit") == false) {
830839
// add a (high) limit to avoid warnings on default limit
831840
command += " | limit 10000000";
@@ -839,6 +848,9 @@ protected Response runESQLCommand(String user, String command) throws IOExceptio
839848
request.setJsonEntity(Strings.toString(json));
840849
request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", user));
841850
request.addParameter("error_trace", "true");
851+
if (allowPartialResults != null) {
852+
request.addParameter("allow_partial_results", Boolean.toString(allowPartialResults));
853+
}
842854
return client().performRequest(request);
843855
}
844856

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/EsqlRestValidationIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,6 @@ private RestClient remoteClusterClient() throws IOException {
8383

8484
@Before
8585
public void skipTestOnOldVersions() {
86-
assumeTrue("skip on old versions", Clusters.localClusterVersion().equals(Version.V_8_16_0));
86+
assumeTrue("skip on old versions", Clusters.localClusterVersion().equals(Version.V_8_19_0));
8787
}
8888
}

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java

+7
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.junit.rules.TestRule;
2929

3030
import java.io.IOException;
31+
import java.util.List;
3132
import java.util.Map;
3233

3334
import static org.elasticsearch.test.MapMatcher.assertMap;
@@ -87,6 +88,12 @@ protected String from(String... indexName) {
8788

8889
@Override
8990
public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException {
91+
if (requestObject.allowPartialResults() != null) {
92+
assumeTrue(
93+
"require allow_partial_results on local cluster",
94+
clusterHasCapability("POST", "/_query", List.of(), List.of("support_partial_results")).orElse(false)
95+
);
96+
}
9097
requestObject.includeCCSMetadata(true);
9198
return super.runEsql(requestObject);
9299
}

‎x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ public void testInvalidPragma() throws IOException {
111111
request.setJsonEntity("{\"f\":" + i + "}");
112112
assertOK(client().performRequest(request));
113113
}
114-
RequestObjectBuilder builder = requestObjectBuilder().query("from test-index | limit 1 | keep f");
114+
RequestObjectBuilder builder = requestObjectBuilder().query("from test-index | limit 1 | keep f").allowPartialResults(false);
115115
builder.pragmas(Settings.builder().put("data_partitioning", "invalid-option").build());
116116
ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(builder));
117117
assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("No enum constant"));

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java

+1
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ private Request createRequest(String indexName) throws IOException {
129129
final var request = new Request("POST", "/_query");
130130
request.addParameter("error_trace", "true");
131131
request.addParameter("pretty", "true");
132+
request.addParameter("allow_partial_results", Boolean.toString(false));
132133
request.setJsonEntity(
133134
Strings.toString(JsonXContent.contentBuilder().startObject().field("query", "from " + indexName).endObject())
134135
);

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java

+13-4
Original file line numberDiff line numberDiff line change
@@ -198,17 +198,26 @@ public void testIndicesDontExist() throws IOException {
198198
int docsTest1 = 0; // we are interested only in the created index, not necessarily that it has data
199199
indexTimestampData(docsTest1, "test1", "2024-11-26", "id1");
200200

201-
ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo"))));
201+
ResponseException e = expectThrows(
202+
ResponseException.class,
203+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo")).allowPartialResults(false))
204+
);
202205
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
203206
assertThat(e.getMessage(), containsString("verification_exception"));
204207
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]")));
205208

206-
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
209+
e = expectThrows(
210+
ResponseException.class,
211+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*")).allowPartialResults(false))
212+
);
207213
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
208214
assertThat(e.getMessage(), containsString("verification_exception"));
209215
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]")));
210216

211-
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1"))));
217+
e = expectThrows(
218+
ResponseException.class,
219+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1")).allowPartialResults(false))
220+
);
212221
assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
213222
assertThat(e.getMessage(), containsString("index_not_found_exception"));
214223
assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]")));
@@ -217,7 +226,7 @@ public void testIndicesDontExist() throws IOException {
217226
var pattern = from("test1");
218227
e = expectThrows(
219228
ResponseException.class,
220-
() -> runEsql(timestampFilter("gte", "2020-01-01").query(pattern + " | LOOKUP JOIN foo ON id1"))
229+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(pattern + " | LOOKUP JOIN foo ON id1").allowPartialResults(false))
221230
);
222231
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
223232
assertThat(

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

+9-5
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ public static class RequestObjectBuilder {
132132
private Boolean includeCCSMetadata = null;
133133

134134
private CheckedConsumer<XContentBuilder, IOException> filter;
135-
private Boolean allPartialResults = null;
135+
private Boolean allowPartialResults = null;
136136

137137
public RequestObjectBuilder() throws IOException {
138138
this(randomFrom(XContentType.values()));
@@ -210,11 +210,15 @@ public RequestObjectBuilder filter(CheckedConsumer<XContentBuilder, IOException>
210210
return this;
211211
}
212212

213-
public RequestObjectBuilder allPartialResults(boolean allPartialResults) {
214-
this.allPartialResults = allPartialResults;
213+
public RequestObjectBuilder allowPartialResults(boolean allowPartialResults) {
214+
this.allowPartialResults = allowPartialResults;
215215
return this;
216216
}
217217

218+
public Boolean allowPartialResults() {
219+
return allowPartialResults;
220+
}
221+
218222
public RequestObjectBuilder build() throws IOException {
219223
if (isBuilt == false) {
220224
if (tables != null) {
@@ -1369,8 +1373,8 @@ protected static Request prepareRequestWithOptions(RequestObjectBuilder requestO
13691373
requestObject.build();
13701374
Request request = prepareRequest(mode);
13711375
String mediaType = attachBody(requestObject, request);
1372-
if (requestObject.allPartialResults != null) {
1373-
request.addParameter("allow_partial_results", String.valueOf(requestObject.allPartialResults));
1376+
if (requestObject.allowPartialResults != null) {
1377+
request.addParameter("allow_partial_results", String.valueOf(requestObject.allowPartialResults));
13741378
}
13751379

13761380
RequestOptions.Builder options = request.getOptions().toBuilder();

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClusterTestCase.java

+6
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.transport.RemoteClusterAware;
2626
import org.elasticsearch.xcontent.XContentBuilder;
2727
import org.elasticsearch.xcontent.json.JsonXContent;
28+
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
2829
import org.junit.After;
2930
import org.junit.Before;
3031

@@ -76,6 +77,11 @@ protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
7677
return plugins;
7778
}
7879

80+
@Override
81+
protected Settings nodeSettings() {
82+
return Settings.builder().put(super.nodeSettings()).put(EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS.getKey(), false).build();
83+
}
84+
7985
public static class InternalExchangePlugin extends Plugin {
8086
@Override
8187
public List<Setting<?>> getSettings() {

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java

+8
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,14 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
139139
return CollectionUtils.appendToCopy(super.nodePlugins(), EsqlPlugin.class);
140140
}
141141

142+
@Override
143+
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
144+
return Settings.builder()
145+
.put(super.nodeSettings(nodeOrdinal, otherSettings))
146+
.put(EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS.getKey(), false)
147+
.build();
148+
}
149+
142150
protected void setRequestCircuitBreakerLimit(ByteSizeValue limit) {
143151
if (limit != null) {
144152
assertAcked(

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterCancellationIT.java

+7-36
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,20 @@
1414
import org.elasticsearch.action.index.IndexRequest;
1515
import org.elasticsearch.action.support.PlainActionFuture;
1616
import org.elasticsearch.action.support.WriteRequest;
17-
import org.elasticsearch.common.settings.Setting;
17+
import org.elasticsearch.common.settings.Settings;
1818
import org.elasticsearch.common.transport.TransportAddress;
1919
import org.elasticsearch.compute.operator.DriverTaskRunner;
2020
import org.elasticsearch.compute.operator.exchange.ExchangeService;
2121
import org.elasticsearch.core.TimeValue;
22-
import org.elasticsearch.plugins.Plugin;
2322
import org.elasticsearch.tasks.TaskCancelledException;
2423
import org.elasticsearch.tasks.TaskInfo;
25-
import org.elasticsearch.test.AbstractMultiClustersTestCase;
2624
import org.elasticsearch.transport.TransportService;
2725
import org.elasticsearch.xcontent.XContentBuilder;
2826
import org.elasticsearch.xcontent.json.JsonXContent;
2927
import org.elasticsearch.xpack.esql.EsqlTestUtils;
3028
import org.elasticsearch.xpack.esql.plugin.ComputeService;
31-
import org.junit.After;
32-
import org.junit.Before;
3329

3430
import java.util.ArrayList;
35-
import java.util.Collection;
3631
import java.util.List;
3732
import java.util.concurrent.TimeUnit;
3833

@@ -44,7 +39,7 @@
4439
import static org.hamcrest.Matchers.hasSize;
4540
import static org.hamcrest.Matchers.instanceOf;
4641

47-
public class CrossClusterCancellationIT extends AbstractMultiClustersTestCase {
42+
public class CrossClusterCancellationIT extends AbstractCrossClusterTestCase {
4843
private static final String REMOTE_CLUSTER = "cluster-a";
4944

5045
@Override
@@ -53,35 +48,11 @@ protected List<String> remoteClusterAlias() {
5348
}
5449

5550
@Override
56-
protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
57-
List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
58-
plugins.add(EsqlPluginWithEnterpriseOrTrialLicense.class);
59-
plugins.add(InternalExchangePlugin.class);
60-
plugins.add(SimplePauseFieldPlugin.class);
61-
return plugins;
62-
}
63-
64-
public static class InternalExchangePlugin extends Plugin {
65-
@Override
66-
public List<Setting<?>> getSettings() {
67-
return List.of(
68-
Setting.timeSetting(
69-
ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING,
70-
TimeValue.timeValueMillis(between(3000, 4000)),
71-
Setting.Property.NodeScope
72-
)
73-
);
74-
}
75-
}
76-
77-
@Before
78-
public void resetPlugin() {
79-
SimplePauseFieldPlugin.resetPlugin();
80-
}
81-
82-
@After
83-
public void releasePlugin() {
84-
SimplePauseFieldPlugin.release();
51+
protected Settings nodeSettings() {
52+
return Settings.builder()
53+
.put(super.nodeSettings())
54+
.put(ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMillis(between(3000, 4000)))
55+
.build();
8556
}
8657

8758
@Override

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/ManyShardsIT.java

+1
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
7171
@Override
7272
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
7373
return Settings.builder()
74+
.put(super.nodeSettings(nodeOrdinal, otherSettings))
7475
.put(ExchangeService.INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMillis(between(3000, 5000)))
7576
.build();
7677
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public class EsqlPlugin extends Plugin implements ActionPlugin {
105105

106106
public static final Setting<Boolean> QUERY_ALLOW_PARTIAL_RESULTS = Setting.boolSetting(
107107
"esql.query.allow_partial_results",
108-
false,
108+
true,
109109
Setting.Property.NodeScope,
110110
Setting.Property.Dynamic
111111
);

‎x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java

+2
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public class CrossClusterEsqlRCS1MissingIndicesIT extends AbstractRemoteClusterS
4747
.apply(commonClusterConfig)
4848
.setting("remote_cluster.port", "0")
4949
.setting("xpack.ml.enabled", "false")
50+
.setting("esql.query.allow_partial_results", "false")
5051
.setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
5152
.setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
5253
.setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
@@ -62,6 +63,7 @@ public class CrossClusterEsqlRCS1MissingIndicesIT extends AbstractRemoteClusterS
6263
.module("x-pack-enrich")
6364
.apply(commonClusterConfig)
6465
.setting("xpack.ml.enabled", "false")
66+
.setting("esql.query.allow_partial_results", "false")
6567
.setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
6668
.build();
6769
}

0 commit comments

Comments
 (0)