Skip to content

Fix missing index exception handling #126738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 37 commits into from
Apr 24, 2025
Merged

Fix missing index exception handling #126738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
436a831
Fix missing index handling for partial-enabled queries
smalyshev Apr 9, 2025
0bacef5
Merge branch 'main' into fix-missing-index-handling
smalyshev Apr 9, 2025
39141cc
Oops shouldn't delete that
smalyshev Apr 9, 2025
5ffdc46
Merge branch 'main' into fix-missing-index-handling
smalyshev Apr 10, 2025
d3421a1
Set ignore_unavaliable to false
smalyshev Apr 11, 2025
374df05
Merge branch 'main' into fix-missing-index-handling
smalyshev Apr 11, 2025
06ee627
Revert
smalyshev Apr 12, 2025
bb890ec
Update docs/changelog/126738.yaml
smalyshev Apr 12, 2025
88602d4
Merge branch 'main' into fix-missing-index-exception
smalyshev Apr 12, 2025
954f753
Merge branch 'main' into fix-missing-index-exception
smalyshev Apr 13, 2025
be41b3e
Merge branch 'main' into fix-missing-index-exception
smalyshev Apr 14, 2025
c2159d2
Merge branch 'main' into fix-missing-index-exception
smalyshev Apr 14, 2025
9f05ded
Test for CCS with filters
smalyshev Apr 14, 2025
6f348b9
[CI] Auto commit changes from spotless
elasticsearchmachine Apr 15, 2025
7313daf
Partial fix for CCS/filters problems
smalyshev Apr 17, 2025
fb7fb6a
fix tests
smalyshev Apr 18, 2025
8610845
Merge branch 'main' into fix-filter-ccs
smalyshev Apr 18, 2025
a51bf2a
More tests
smalyshev Apr 18, 2025
d58d6e6
We can not eliminate some filtered runtime calls since there could be…
smalyshev Apr 18, 2025
9f432eb
Merge branch 'main' into fix-filter-ccs
smalyshev Apr 18, 2025
c730f21
Merge branch 'main' into fix-filter-ccs
smalyshev Apr 21, 2025
d2df0e1
Improve comments
smalyshev Apr 21, 2025
492daae
Add column tests
smalyshev Apr 21, 2025
9cc3fad
More tests for unavailable
smalyshev Apr 21, 2025
14e70bb
Merge branch 'main' into fix-filter-ccs
smalyshev Apr 21, 2025
f9d78f2
Merge branch 'main' into fix-filter-ccs
smalyshev Apr 22, 2025
216bd11
Declutter asserts
smalyshev Apr 22, 2025
6cbfe19
Remove this change, may not be needed
smalyshev Apr 22, 2025
9098695
add async
smalyshev Apr 22, 2025
8ca2529
Merge branch 'main' into fix-missing-index-exception
smalyshev Apr 22, 2025
cd2e7a6
Merge branch 'fix-filter-ccs' into fix-missing-index-exception
smalyshev Apr 22, 2025
c428a43
Update tests
smalyshev Apr 22, 2025
1dad0fb
Remove the test, no longer needed
smalyshev Apr 22, 2025
2a4a1fd
Merge branch 'main' into fix-missing-index-exception
smalyshev Apr 23, 2025
75913eb
Delete docs/changelog/126738.yaml
smalyshev Apr 23, 2025
a8e3efb
Fix merge
smalyshev Apr 23, 2025
44e559a
oops merge dropped this change, restore
smalyshev Apr 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 20 additions & 14 deletions ...lClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.RemoteException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.test.transport.MockTransportService;
Expand Down Expand Up @@ -275,13 +277,13 @@ public void testFilterWithMissingIndex() {
);
assertThat(e.getDetailedMessage(), containsString("Unknown index [missing]"));
// Local index missing + wildcards
// FIXME: planner does not catch this now
// e = expectThrows(VerificationException.class, () -> runQuery("from missing,logs*", randomBoolean(), filter).close());
// assertThat(e.getDetailedMessage(), containsString("Unknown index [missing]"));
// FIXME: planner does not catch this now, it should be VerificationException but for now it's runtime IndexNotFoundException
var ie = expectThrows(IndexNotFoundException.class, () -> runQuery("from missing,logs*", randomBoolean(), filter).close());
assertThat(ie.getDetailedMessage(), containsString("no such index [missing]"));
// Local index missing + existing index
// FIXME: planner does not catch this now
// e = expectThrows(VerificationException.class, () -> runQuery("from missing,logs-1", randomBoolean(), filter).close());
// assertThat(e.getDetailedMessage(), containsString("Unknown index [missing]"));
// FIXME: planner does not catch this now, it should be VerificationException but for now it's runtime IndexNotFoundException
ie = expectThrows(IndexNotFoundException.class, () -> runQuery("from missing,logs-1", randomBoolean(), filter).close());
assertThat(ie.getDetailedMessage(), containsString("no such index [missing]"));
// Local index missing + existing remote
e = expectThrows(VerificationException.class, () -> runQuery("from missing,cluster-a:logs-2", randomBoolean(), filter).close());
assertThat(e.getDetailedMessage(), containsString("Unknown index [missing]"));
Expand Down Expand Up @@ -318,15 +320,19 @@ public void testFilterWithMissingRemoteIndex() {
);
assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]"));
// Local index missing + wildcards
// FIXME: planner does not catch this now
// e = expectThrows(VerificationException.class, () -> runQuery("from cluster-a:missing,cluster-a:logs*", randomBoolean(),
// filter).close());
// assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]"));
// FIXME: planner does not catch this now, it should be VerificationException but for now it's runtime RemoteException
var ie = expectThrows(
RemoteException.class,
() -> runQuery("from cluster-a:missing,cluster-a:logs*", randomBoolean(), filter).close()
);
assertThat(ie.getDetailedMessage(), containsString("no such index [missing]"));
// Local index missing + existing index
// FIXME: planner does not catch this now
// e = expectThrows(VerificationException.class, () -> runQuery("from cluster-a:missing,cluster-a:logs-2", randomBoolean(),
// filter).close());
// assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]"));
// FIXME: planner does not catch this now, it should be VerificationException but for now it's runtime RemoteException
ie = expectThrows(
RemoteException.class,
() -> runQuery("from cluster-a:missing,cluster-a:logs-2", randomBoolean(), filter).close()
);
assertThat(ie.getDetailedMessage(), containsString("no such index [missing]"));
// Local index + missing remote
e = expectThrows(VerificationException.class, () -> runQuery("from logs-1,cluster-a:missing", randomBoolean(), filter).close());
assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]"));
Expand Down
28 changes: 28 additions & 0 deletions ...rTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithPartialResultsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,34 @@ public void testOneRemoteClusterPartial() throws Exception {
}
}

public void testLocalIndexMissing() throws Exception {
populateIndices();
EsqlQueryRequest request = new EsqlQueryRequest();
request.query("FROM ok-local,no_such_index | LIMIT 1");
request.includeCCSMetadata(randomBoolean());
for (boolean allowPartial : Set.of(true, false)) {
request.allowPartialResults(allowPartial);
Exception error = expectThrows(Exception.class, () -> runQuery(request).close());
error = EsqlTestUtils.unwrapIfWrappedInRemoteException(error);
assertThat(error.getMessage(), containsString("no such index"));
assertThat(error.getMessage(), containsString("[no_such_index]"));
}
}

public void testRemoteIndexMissing() throws Exception {
populateIndices();
EsqlQueryRequest request = new EsqlQueryRequest();
request.query("FROM cluster-a:ok-cluster1,cluster-a:no_such_index | LIMIT 1");
request.includeCCSMetadata(randomBoolean());
for (boolean allowPartial : Set.of(true, false)) {
request.allowPartialResults(allowPartial);
Exception error = expectThrows(Exception.class, () -> runQuery(request).close());
error = EsqlTestUtils.unwrapIfWrappedInRemoteException(error);
assertThat(error.getMessage(), containsString("no such index"));
assertThat(error.getMessage(), containsString("[no_such_index]"));
}
}

public void testFailToReceiveClusterResponse() throws Exception {
populateIndices();
Exception simulatedFailure = randomFailure();
Expand Down
26 changes: 0 additions & 26 deletions ...nalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterUsageTelemetryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,32 +138,6 @@ public void testFailed() throws Exception {
assertThat(telemetry.getByRemoteCluster().size(), equalTo(0));
}

// TODO: enable when skip-un patch is merged
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is just old test we never enabled and we don't really need it anymore, so cleaning it up.

// public void testSkipAllRemotes() throws Exception {
// var telemetry = getTelemetryFromQuery("from logs-*,c*:no_such_index | stats sum (v)", "unknown");
//
// assertThat(telemetry.getTotalCount(), equalTo(1L));
// assertThat(telemetry.getSuccessCount(), equalTo(1L));
// assertThat(telemetry.getFailureReasons().size(), equalTo(0));
// assertThat(telemetry.getTook().count(), equalTo(1L));
// assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L));
// assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L));
// assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0));
// assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L));
// assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L));
// assertThat(telemetry.getClientCounts().size(), equalTo(0));
//
// var perCluster = telemetry.getByRemoteCluster();
// assertThat(perCluster.size(), equalTo(3));
// for (String clusterAlias : remoteClusterAlias()) {
// var clusterData = perCluster.get(clusterAlias);
// assertThat(clusterData.getCount(), equalTo(0L));
// assertThat(clusterData.getSkippedCount(), equalTo(1L));
// assertThat(clusterData.getTook().count(), equalTo(0L));
// }
// assertPerClusterCount(perCluster.get(LOCAL_CLUSTER), 1L);
// }

public void testRemoteOnly() throws Exception {
setupClusters();
var telemetry = getTelemetryFromQuery("from c*:logs-* | stats sum (v)", "kibana");
Expand Down
20 changes: 14 additions & 6 deletions .../plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.xpack.esql.plugin;

import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionListenerResponseHandler;
import org.elasticsearch.action.OriginalIndices;
Expand All @@ -16,6 +17,7 @@
import org.elasticsearch.compute.operator.exchange.ExchangeSourceHandler;
import org.elasticsearch.core.Releasable;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.tasks.CancellableTask;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.tasks.TaskCancelledException;
Expand Down Expand Up @@ -88,12 +90,18 @@ void startComputeOnRemoteCluster(
if (receivedResults == false && EsqlCCSUtils.shouldIgnoreRuntimeError(executionInfo, clusterAlias, e)) {
EsqlCCSUtils.markClusterWithFinalStateAndNoShards(executionInfo, clusterAlias, EsqlExecutionInfo.Cluster.Status.SKIPPED, e);
l.onResponse(DriverCompletionInfo.EMPTY);
} else if (configuration.allowPartialResults()) {
EsqlCCSUtils.markClusterWithFinalStateAndNoShards(executionInfo, clusterAlias, EsqlExecutionInfo.Cluster.Status.PARTIAL, e);
l.onResponse(DriverCompletionInfo.EMPTY);
} else {
l.onFailure(e);
}
} else if (configuration.allowPartialResults()
&& (ExceptionsHelper.unwrapCause(e) instanceof IndexNotFoundException) == false) {
EsqlCCSUtils.markClusterWithFinalStateAndNoShards(
executionInfo,
clusterAlias,
EsqlExecutionInfo.Cluster.Status.PARTIAL,
e
);
l.onResponse(DriverCompletionInfo.EMPTY);
} else {
l.onFailure(e);
}
});
ExchangeService.openExchange(
transportService,
Expand Down
5 changes: 4 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.xpack.esql.plugin;

import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.OriginalIndices;
import org.elasticsearch.action.search.SearchRequest;
Expand All @@ -30,6 +31,7 @@
import org.elasticsearch.core.Releasable;
import org.elasticsearch.core.Releasables;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.SearchExecutionContext;
import org.elasticsearch.logging.LogManager;
import org.elasticsearch.logging.Logger;
Expand Down Expand Up @@ -428,7 +430,8 @@ public void executePlan(
);
dataNodesListener.onResponse(r.getCompletionInfo());
}, e -> {
if (configuration.allowPartialResults()) {
if (configuration.allowPartialResults()
&& (ExceptionsHelper.unwrapCause(e) instanceof IndexNotFoundException) == false) {
execInfo.swapCluster(
LOCAL_CLUSTER,
(k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(
Expand Down