Fix race condition when deleting an async search (#53513)

Deleting an async search id can throw a ResourceNotFoundException even if the query was successfully cancelled.
We delete the stored response automatically if the query is cancelled so that creates a race with the delete action that also ensures that the task is removed. This change ensures that we ignore missing async search ids in the async search index if they were successfully cancelled.

Relates #53360
Relates #49931

Author: jimczi

x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java

@@ -201,13 +201,14 @@ void updateExpirationTime(String docId,
      * Deletes the provided <code>searchId</code> from the index if present.
      */
     void deleteResponse(AsyncSearchId searchId,
+                        boolean failIfNotFound,
                         ActionListener<AcknowledgedResponse> listener) {
         DeleteRequest request = new DeleteRequest(INDEX).id(searchId.getDocId());
         createIndexIfNecessary(
             ActionListener.wrap(v -> client.delete(request,
                 ActionListener.wrap(
                     resp -> {
-                        if (resp.status() == RestStatus.NOT_FOUND) {
+                        if (resp.status() == RestStatus.NOT_FOUND && failIfNotFound) {
                             listener.onFailure(new ResourceNotFoundException(searchId.getEncoded()));
                         } else {
                             listener.onResponse(new AcknowledgedResponse(true));

x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java

@@ -61,10 +61,12 @@ protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, Act
     private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener<AcknowledgedResponse> listener) throws IOException {
         AsyncSearchTask task = store.getTask(taskManager, searchId);
         if (task != null) {
-            task.cancelTask(() -> store.deleteResponse(searchId, listener));
+            task.cancelTask(() -> store.deleteResponse(searchId, false, listener));
         } else {
-            // check if the response can be retrieved by the user (handle security) and then delete.
-            store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResponse(searchId, listener), listener::onFailure));
+            // the task is not running anymore so we throw a not found exception if
+            // the search id is also not present in the index (already deleted) or if the user
+            // is not allowed to access it.
+            store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResponse(searchId, true, listener), listener::onFailure));
         }
     }
 }

x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java

@@ -167,7 +167,7 @@ private void onFinalResponse(CancellableTask submitTask,
                                  Runnable nextAction) {
         if (submitTask.isCancelled() || searchTask.isCancelled()) {
             // the user cancelled the submit so we ensure that there is nothing stored in the response index.
-            store.deleteResponse(searchTask.getSearchId(), ActionListener.wrap(() -> {
+            store.deleteResponse(searchTask.getSearchId(), false, ActionListener.wrap(() -> {
                 taskManager.unregister(searchTask);
                 nextAction.run();
             }));