From 750a81d6b37c82c96aaa3206e9535feac7bfa98f Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 4 Dec 2019 12:11:58 +0100 Subject: [PATCH 01/61] Add new x-pack endpoints to asynchronously track the progress of a search --- .../action/search/SearchRequest.java | 2 +- .../action/search/SearchRequestBuilder.java | 2 +- .../elasticsearch/client/node/NodeClient.java | 38 --- .../rest/action/search/RestSearchAction.java | 2 +- .../SearchProgressActionListenerIT.java | 12 +- .../test/rest/yaml/section/DoSection.java | 1 + x-pack/plugin/async-search/build.gradle | 30 +++ x-pack/plugin/async-search/qa/build.gradle | 18 ++ .../plugin/async-search/qa/rest/build.gradle | 24 ++ .../xpack/search/AsyncSearchRestIT.java | 35 +++ .../rest-api-spec/test/search/10_basic.yml | 78 ++++++ .../xpack/search/AsyncSearchRestTestCase.java | 15 ++ .../xpack/search/AsyncSearch.java | 74 ++++++ .../AsyncSearchHistoryTemplateRegistry.java | 95 ++++++++ .../xpack/search/AsyncSearchId.java | 96 ++++++++ .../xpack/search/AsyncSearchStoreService.java | 119 ++++++++++ .../xpack/search/AsyncSearchTask.java | 154 ++++++++++++ .../search/RestDeleteAsyncSearchAction.java | 35 +++ .../search/RestGetAsyncSearchAction.java | 39 +++ .../search/RestSubmitAsyncSearchAction.java | 55 +++++ .../TransportDeleteAsyncSearchAction.java | 108 +++++++++ .../search/TransportGetAsyncSearchAction.java | 136 +++++++++++ .../TransportSubmitAsyncSearchAction.java | 106 +++++++++ .../plugin-metadata/plugin-security.policy | 0 .../xpack/search/AsyncSearchIdTests.java | 45 ++++ .../search/AsyncSearchResponseTests.java | 150 ++++++++++++ .../search/AsyncSearchStoreServiceTests.java | 44 ++++ .../search/DeleteAsyncSearchRequestTests.java | 24 ++ .../search/GetAsyncSearchRequestTests.java | 35 +++ .../search/SubmitAsyncSearchRequestTests.java | 73 ++++++ .../xpack/core/XPackClientPlugin.java | 9 +- .../search/action/AsyncSearchResponse.java | 191 +++++++++++++++ .../action/DeleteAsyncSearchAction.java | 73 ++++++ .../search/action/GetAsyncSearchAction.java | 106 +++++++++ .../search/action/PartialSearchResponse.java | 137 +++++++++++ .../action/SubmitAsyncSearchAction.java | 23 ++ .../action/SubmitAsyncSearchRequest.java | 107 +++++++++ .../async-search-history-ilm-policy.json | 18 ++ .../main/resources/async-search-history.json | 29 +++ .../xpack/security/authz/RBACEngine.java | 3 + .../api/async_search.delete.json | 24 ++ .../rest-api-spec/api/async_search.get.json | 34 +++ .../api/async_search.submit.json | 223 ++++++++++++++++++ 43 files changed, 2579 insertions(+), 43 deletions(-) create mode 100644 x-pack/plugin/async-search/build.gradle create mode 100644 x-pack/plugin/async-search/qa/build.gradle create mode 100644 x-pack/plugin/async-search/qa/rest/build.gradle create mode 100644 x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java create mode 100644 x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml create mode 100644 x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchHistoryTemplateRegistry.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java create mode 100644 x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java create mode 100644 x-pack/plugin/core/src/main/resources/async-search-history-ilm-policy.json create mode 100644 x-pack/plugin/core/src/main/resources/async-search-history.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 8ba9a8c9f0bd7..a719afbdb5ee0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -56,7 +56,7 @@ * @see org.elasticsearch.client.Client#search(SearchRequest) * @see SearchResponse */ -public final class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable { +public class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable { private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false")); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index ceaee96f5c131..b8b791360d30c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -213,7 +213,7 @@ public SearchRequestBuilder setVersion(boolean version) { sourceBuilder().version(version); return this; } - + /** * Should each {@link org.elasticsearch.search.SearchHit} be returned with the * sequence number and primary term of the last modification of the document. diff --git a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java index 40bbf81534b58..cf4ab92baa0c6 100644 --- a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java +++ b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java @@ -23,12 +23,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchTask; -import org.elasticsearch.action.search.SearchProgressActionListener; -import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.Client; import org.elasticsearch.client.support.AbstractClient; @@ -108,38 +102,6 @@ > Task executeLocally(ActionType action, Request request, TaskListener listener::onResponse, listener::onFailure); } - /** - * Execute a {@link SearchRequest} locally and track the progress of the request through - * a {@link SearchProgressActionListener}. - */ - public SearchTask executeSearchLocally(SearchRequest request, SearchProgressActionListener listener) { - // we cannot track the progress if remote cluster requests are splitted. - request.setCcsMinimizeRoundtrips(false); - TransportSearchAction action = (TransportSearchAction) actions.get(SearchAction.INSTANCE); - SearchTask task = (SearchTask) taskManager.register("transport", action.actionName, request); - task.setProgressListener(listener); - action.execute(task, request, new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - try { - taskManager.unregister(task); - } finally { - listener.onResponse(response); - } - } - - @Override - public void onFailure(Exception e) { - try { - taskManager.unregister(task); - } finally { - listener.onFailure(e); - } - } - }); - return task; - } - /** * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by * {@link #executeLocally(ActionType, ActionRequest, TaskListener)}. diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 11dc9f89de532..36a599c6039d3 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -162,7 +162,7 @@ public static void parseSearchRequest(SearchRequest searchRequest, RestRequest r searchRequest.routing(request.param("routing")); searchRequest.preference(request.param("preference")); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); - searchRequest.setCcsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", true)); + searchRequest.setCcsMinimizeRoundtrips(request.paramAsBoolean("ccs_minimize_roundtrips", searchRequest.isCcsMinimizeRoundtrips())); checkRestTotalHits(request, searchRequest); } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 1a6bb6263fb6b..0684c7f49af62 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -29,6 +29,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESSingleNodeTestCase; import java.util.ArrayList; @@ -37,6 +39,7 @@ import java.util.Comparator; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -189,7 +192,14 @@ public void onFailure(Exception e) { throw new AssertionError(); } }; - client.executeSearchLocally(request, listener); + client.executeLocally(SearchAction.INSTANCE, new SearchRequest(request) { + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + SearchTask task = (SearchTask) super.createTask(id, type, action, parentTaskId, headers); + task.setProgressListener(listener); + return task; + } + }, listener); latch.await(); assertThat(shardsListener.get(), equalTo(expectedShards)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index ce94adf73bcd3..63890fad47a0e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -345,6 +345,7 @@ private String formatStatusCodeMessage(ClientYamlTestResponse restTestResponse, private static Map>> catches = new HashMap<>(); static { + catches.put("not_modified", tuple("304", equalTo(304))); catches.put("bad_request", tuple("400", equalTo(400))); catches.put("unauthorized", tuple("401", equalTo(401))); catches.put("forbidden", tuple("403", equalTo(403))); diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle new file mode 100644 index 0000000000000..81210444bb2d5 --- /dev/null +++ b/x-pack/plugin/async-search/build.gradle @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'x-pack-async-search' + description 'A module which allows to track the progress of a search asynchronously.' + classname 'org.elasticsearch.xpack.search.AsyncSearch' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-async-search' + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" + + +dependencies { + compileOnly project(":server") + + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(path: xpackModule('ilm')) +} + +integTest.enabled = false diff --git a/x-pack/plugin/async-search/qa/build.gradle b/x-pack/plugin/async-search/qa/build.gradle new file mode 100644 index 0000000000000..46609933699ca --- /dev/null +++ b/x-pack/plugin/async-search/qa/build.gradle @@ -0,0 +1,18 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.build' +test.enabled = false + +dependencies { + compile project(':test:framework') +} + +subprojects { + project.tasks.withType(RestIntegTestTask) { + final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') + project.copyRestSpec.from(xPackResources) { + include 'rest-api-spec/api/**' + } + } + +} diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle new file mode 100644 index 0000000000000..39bfb37b23877 --- /dev/null +++ b/x-pack/plugin/async-search/qa/rest/build.gradle @@ -0,0 +1,24 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-test' + +dependencies { + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(path: xpackModule('async-search'), configuration: 'runtime') +} + +task restTest(type: RestIntegTestTask) { + mustRunAfter(precommit) +} + +testClusters.restTest { + testDistribution = 'DEFAULT' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'xpack.security.enabled', 'true' + user username: 'async-search-user', password: 'async-search-password' +} + +check.dependsOn restTest +test.enabled = false diff --git a/x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java b/x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java new file mode 100644 index 0000000000000..89bef2430f557 --- /dev/null +++ b/x-pack/plugin/async-search/qa/rest/src/test/java/org/elasticsearch/xpack/search/AsyncSearchRestIT.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +public class AsyncSearchRestIT extends ESClientYamlSuiteTestCase { + + public AsyncSearchRestIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected Settings restClientSettings() { + final String userAuthHeaderValue = basicAuthHeaderValue("async-search-user", + new SecureString("async-search-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", userAuthHeaderValue).build(); + } +} diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml new file mode 100644 index 0000000000000..996ef3c8599fb --- /dev/null +++ b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml @@ -0,0 +1,78 @@ +--- +"Async search": + - do: + indices.create: + index: test-1 + + - do: + indices.create: + index: test-2 + + - do: + indices.create: + index: test-3 + + - do: + index: + index: test-2 + body: { max: 2 } + + - do: + index: + index: test-1 + body: { max: 1 } + + - do: + index: + index: test-3 + body: { max: 3 } + + - do: + indices.refresh: {} + + - do: + async_search.submit: + index: test-* + batched_reduce_size: 2 + body: + query: + match_all: {} + aggs: + 1: + max: + field: max + sort: max + + - set: { id: id } + + - do: + async_search.get: + id: "$id" + wait_for_completion: 10s + + - set: { version: version } + - match: { version: 4 } + - is_false: partial_response + - length: { response.hits.hits: 3 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.1.value: 3.0 } + + - do: + catch: not_modified + async_search.get: + id: "$id" + last_version: "$version" + + - is_false: partial_response + - is_false: response + + - do: + async_search.delete: + id: "$id" + + - match: { acknowledged: true } + + - do: + catch: missing + async_search.delete: + id: "$id" diff --git a/x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java b/x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java new file mode 100644 index 0000000000000..f176efdda4201 --- /dev/null +++ b/x-pack/plugin/async-search/qa/src/main/java/org/elasticsearch/xpack/search/AsyncSearchRestTestCase.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.test.rest.ESRestTestCase; + +public class AsyncSearchRestTestCase extends ESRestTestCase { + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java new file mode 100644 index 0000000000000..221f16fdf3cae --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public final class AsyncSearch extends Plugin implements ActionPlugin { + @Override + public List> getActions() { + return Arrays.asList( + new ActionHandler<>(SubmitAsyncSearchAction.INSTANCE, TransportSubmitAsyncSearchAction.class), + new ActionHandler<>(GetAsyncSearchAction.INSTANCE, TransportGetAsyncSearchAction.class), + new ActionHandler<>(DeleteAsyncSearchAction.INSTANCE, TransportDeleteAsyncSearchAction.class) + ); + } + + @Override + public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + return Arrays.asList( + new RestSubmitAsyncSearchAction(restController), + new RestGetAsyncSearchAction(restController), + new RestDeleteAsyncSearchAction(restController) + ); + } + + @Override + public Collection createComponents(Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry) { + new AsyncSearchHistoryTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry); + return Collections.emptyList(); + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchHistoryTemplateRegistry.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchHistoryTemplateRegistry.java new file mode 100644 index 0000000000000..2f57d219d345d --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchHistoryTemplateRegistry.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; + +/** + * Manage the index template and associated ILM policy for the async search history index. + */ +public class AsyncSearchHistoryTemplateRegistry extends IndexTemplateRegistry { + // history (please add a comment why you increased the version here) + // version 1: initial + public static final String INDEX_TEMPLATE_VERSION = "1"; + public static final String ASYNC_SEARCH_HISTORY_TEMPLATE_VERSION_VARIABLE = "xpack.async-search-history.template.version"; + public static final String ASYNC_SEARCH_HISTORY_TEMPLATE_NAME = ".async-search-history"; + + public static final String ASYNC_SEARCH_POLICY_NAME = "async-search-history-ilm-policy"; + + public static final IndexTemplateConfig TEMPLATE_ASYNC_SEARCH = new IndexTemplateConfig( + ASYNC_SEARCH_HISTORY_TEMPLATE_NAME, + "/async-search-history.json", + INDEX_TEMPLATE_VERSION, + ASYNC_SEARCH_HISTORY_TEMPLATE_VERSION_VARIABLE + ); + + public static final LifecyclePolicyConfig ASYNC_SEARCH_HISTORY_POLICY = new LifecyclePolicyConfig( + ASYNC_SEARCH_POLICY_NAME, + "/async-search-history-ilm-policy.json" + ); + + public AsyncSearchHistoryTemplateRegistry(Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + } + + @Override + protected List getTemplateConfigs() { + return Collections.singletonList(TEMPLATE_ASYNC_SEARCH); + } + + @Override + protected List getPolicyConfigs() { + return Collections.singletonList(ASYNC_SEARCH_HISTORY_POLICY); + } + + @Override + protected String getOrigin() { + return INDEX_LIFECYCLE_ORIGIN; + } + + public boolean validate(ClusterState state) { + boolean allTemplatesPresent = getTemplateConfigs().stream() + .map(IndexTemplateConfig::getTemplateName) + .allMatch(name -> state.metaData().getTemplates().containsKey(name)); + + Optional> maybePolicies = Optional + .ofNullable(state.metaData().custom(IndexLifecycleMetadata.TYPE)) + .map(IndexLifecycleMetadata::getPolicies); + Set policyNames = getPolicyConfigs().stream() + .map(LifecyclePolicyConfig::getPolicyName) + .collect(Collectors.toSet()); + + boolean allPoliciesPresent = maybePolicies + .map(policies -> policies.keySet() + .containsAll(policyNames)) + .orElse(false); + return allTemplatesPresent && allPoliciesPresent; + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java new file mode 100644 index 0000000000000..3a5151e12a952 --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.tasks.TaskId; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Objects; + +/** + * A class that contains all information related to a submitted async search. + */ +class AsyncSearchId { + private final String indexName; + private final String docId; + private final TaskId taskId; + + AsyncSearchId(String indexName, String docId, TaskId taskId) { + this.indexName = indexName; + this.docId = docId; + this.taskId = taskId; + } + + /** + * The index name where to find the response if the task is not running. + */ + String getIndexName() { + return indexName; + } + + /** + * The document id of the response in the index if the task is not running. + */ + String getDocId() { + return docId; + } + + /** + * The {@link TaskId} of the async search in the task manager. + */ + TaskId getTaskId() { + return taskId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AsyncSearchId searchId = (AsyncSearchId) o; + return indexName.equals(searchId.indexName) && + docId.equals(searchId.docId) && + taskId.equals(searchId.taskId); + } + + @Override + public int hashCode() { + return Objects.hash(indexName, docId, taskId); + } + + /** + * Encode the informations needed to retrieve a async search response + * in a base64 encoded string. + */ + static String encode(String indexName, String docId, TaskId taskId) { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeString(indexName); + out.writeString(docId); + out.writeString(taskId.toString()); + return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Decode a base64 encoded string into an {@link AsyncSearchId} that can be used + * to retrieve the response of an async search. + */ + static AsyncSearchId decode(String id) throws IOException { + try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap( Base64.getDecoder().decode(id)))) { + return new AsyncSearchId(in.readString(), in.readString(), new TaskId(in.readString())); + } catch (IOException e) { + throw new IOException("invalid id: " + id); + } + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java new file mode 100644 index 0000000000000..4813dc0ee500d --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Collections; + +import static org.elasticsearch.xpack.search.AsyncSearchHistoryTemplateRegistry.INDEX_TEMPLATE_VERSION; +import static org.elasticsearch.xpack.search.AsyncSearchHistoryTemplateRegistry.ASYNC_SEARCH_HISTORY_TEMPLATE_NAME; + +/** + * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the async + * search history index. + */ +class AsyncSearchStoreService { + static final String ASYNC_SEARCH_HISTORY_ALIAS = ASYNC_SEARCH_HISTORY_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; + static final String RESPONSE_FIELD = "response"; + + private final Client client; + private final NamedWriteableRegistry registry; + + AsyncSearchStoreService(Client client, NamedWriteableRegistry registry) { + this.client = client; + this.registry = registry; + } + + /** + * Store an empty document in the async search history index that is used + * as a place-holder for the future response. + */ + void storeInitialResponse(ActionListener next) { + IndexRequest request = new IndexRequest(ASYNC_SEARCH_HISTORY_ALIAS).source(Collections.emptyMap(), XContentType.JSON); + client.index(request, next); + } + + /** + * Store the final response if the place-holder document is still present (update). + */ + void storeFinalResponse(AsyncSearchResponse response, ActionListener next) throws IOException { + AsyncSearchId searchId = AsyncSearchId.decode(response.id()); + UpdateRequest request = new UpdateRequest().index(searchId.getIndexName()).id(searchId.getDocId()) + .doc(Collections.singletonMap(RESPONSE_FIELD, encodeResponse(response)), XContentType.JSON) + .detectNoop(false); + client.update(request, next); + } + + /** + * Get the final response from the async search history index if present, or delegate a {@link ResourceNotFoundException} + * failure to the provided listener if not. + */ + void getResponse(GetAsyncSearchAction.Request orig, AsyncSearchId searchId, ActionListener next) { + GetRequest request = new GetRequest(searchId.getIndexName()) + .id(searchId.getDocId()) + .storedFields(RESPONSE_FIELD); + client.get(request, ActionListener.wrap( + get -> { + if (get.isExists() == false) { + next.onFailure(new ResourceNotFoundException(request.id() + " not found")); + } else if (get.getFields().containsKey(RESPONSE_FIELD) == false) { + next.onResponse(new AsyncSearchResponse(orig.getId(), new PartialSearchResponse(-1), 0, false)); + } else { + + BytesArray bytesArray = get.getFields().get(RESPONSE_FIELD).getValue(); + next.onResponse(decodeResponse(bytesArray.array(), registry)); + } + }, + exc -> next.onFailure(new ResourceNotFoundException(request.id() + " not found")) + )); + } + + /** + * Encode the provided response in a binary form using base64 encoding. + */ + static String encodeResponse(AsyncSearchResponse response) throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Version.writeVersion(Version.CURRENT, out); + response.writeTo(out); + return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes())); + } + } + + /** + * Decode the provided base-64 bytes into a {@link AsyncSearchResponse}. + */ + static AsyncSearchResponse decodeResponse(byte[] value, NamedWriteableRegistry registry) throws IOException { + try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(value))) { + try (StreamInput in = new NamedWriteableAwareStreamInput(buf, registry)) { + in.setVersion(Version.readVersion(in)); + return new AsyncSearchResponse(in); + } + } + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java new file mode 100644 index 0000000000000..719c372a2d832 --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.search.SearchProgressActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchShard; +import org.elasticsearch.action.search.SearchTask; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.elasticsearch.tasks.TaskId.EMPTY_TASK_ID; + +/** + * Task that tracks the progress of a currently running {@link SearchRequest}. + */ +class AsyncSearchTask extends SearchTask { + private final String searchId; + private final Supplier reduceContextSupplier; + private final Listener progressListener; + + AsyncSearchTask(long id, + String type, + String action, + Map headers, + String searchId, + Supplier reduceContextSupplier) { + super(id, type, action, "async_search", EMPTY_TASK_ID, headers); + this.searchId = searchId; + this.reduceContextSupplier = reduceContextSupplier; + this.progressListener = new Listener(); + setProgressListener(progressListener); + } + + String getSearchId() { + return searchId; + } + + @Override + public SearchProgressActionListener getProgressListener() { + return (Listener) super.getProgressListener(); + } + + /** + * Perform the final reduce on the current {@link AsyncSearchResponse} if requested + * and return the result. + */ + AsyncSearchResponse getAsyncResponse(boolean doFinalReduce) { + return progressListener.response.get(doFinalReduce); + } + + private class Listener extends SearchProgressActionListener { + private int totalShards = -1; + private AtomicInteger version = new AtomicInteger(0); + private AtomicInteger shardFailures = new AtomicInteger(0); + + private volatile Response response; + + Listener() { + final AsyncSearchResponse initial = new AsyncSearchResponse(searchId, + new PartialSearchResponse(totalShards), version.get(), true); + this.response = new Response(initial, false); + } + + @Override + public void onListShards(List shards, boolean fetchPhase) { + this.totalShards = shards.size(); + final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, + new PartialSearchResponse(totalShards), version.incrementAndGet(), true); + response = new Response(newResp, false); + } + + @Override + public void onQueryFailure(int shardIndex, Exception exc) { + shardFailures.incrementAndGet(); + } + + @Override + public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { + final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, + new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs), + version.incrementAndGet(), + true + ); + response = new Response(newResp, aggs != null); + } + + @Override + public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { + final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, + new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs), + version.incrementAndGet(), + true + ); + response = new Response(newResp, false); + } + + @Override + public void onResponse(SearchResponse searchResponse) { + AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, searchResponse, version.incrementAndGet(), false); + response = new Response(newResp, false); + } + + @Override + public void onFailure(Exception exc) { + AsyncSearchResponse current = response.get(true); + response = new Response(new AsyncSearchResponse(searchId, current.getPartialResponse(), + exc != null ? new ElasticsearchException(exc) : null, version.incrementAndGet(), false), false); + } + } + + private class Response { + AsyncSearchResponse internal; + boolean needFinalReduce; + + Response(AsyncSearchResponse response, boolean needFinalReduce) { + this.internal = response; + this.needFinalReduce = needFinalReduce; + } + + /** + * Ensure that we're performing the final reduce only when users explicitly requested + * a response through a {@link GetAsyncSearchAction.Request}. + */ + public synchronized AsyncSearchResponse get(boolean doFinalReduce) { + if (doFinalReduce && needFinalReduce) { + InternalAggregations reducedAggs = internal.getPartialResponse().getAggregations(); + reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(reducedAggs), reduceContextSupplier.get()); + PartialSearchResponse old = internal.getPartialResponse(); + PartialSearchResponse clone = new PartialSearchResponse(old.getTotalShards(), old.getSuccesfullShards(), + old.getShardFailures(), old.getTotalHits(), reducedAggs); + needFinalReduce = false; + return internal = new AsyncSearchResponse(internal.id(), clone, internal.getFailure(), internal.getVersion(), true); + } else { + return internal; + } + } + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java new file mode 100644 index 0000000000000..fd8dc3aefee02 --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; + +import java.io.IOException; + +import static org.elasticsearch.rest.RestRequest.Method.DELETE; + +public class RestDeleteAsyncSearchAction extends BaseRestHandler { + + public RestDeleteAsyncSearchAction(RestController controller) { + controller.registerHandler(DELETE, "/_async_search/{id}", this); + } + + @Override + public String getName() { + return "async_search_delete_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + DeleteAsyncSearchAction.Request delete = new DeleteAsyncSearchAction.Request(request.param("id")); + return channel -> client.execute(DeleteAsyncSearchAction.INSTANCE, delete, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java new file mode 100644 index 0000000000000..1079112e4ca18 --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; + +import java.io.IOException; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestGetAsyncSearchAction extends BaseRestHandler { + + public RestGetAsyncSearchAction(RestController controller) { + controller.registerHandler(GET, "/_async_search/{id}", this); + } + + @Override + public String getName() { + return "async_search_get_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String id = request.param("id"); + int lastVersion = request.paramAsInt("last_version", -1); + TimeValue waitForCompletion = request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1)); + GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(id, waitForCompletion, lastVersion); + return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java new file mode 100644 index 0000000000000..4698ab26439e9 --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; + +import java.io.IOException; +import java.util.function.IntConsumer; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.action.search.RestSearchAction.parseSearchRequest; + +public final class RestSubmitAsyncSearchAction extends BaseRestHandler { + RestSubmitAsyncSearchAction(RestController controller) { + controller.registerHandler(POST, "/_async_search", this); + controller.registerHandler(GET, "/_async_search", this); + controller.registerHandler(POST, "/{index}/_async_search", this); + controller.registerHandler(GET, "/{index}/_async_search", this); + } + + @Override + public String getName() { + return "async_search_submit_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + SubmitAsyncSearchRequest searchRequest = new SubmitAsyncSearchRequest(); + IntConsumer setSize = size -> searchRequest.source().size(size); + request.withContentOrSourceParamParserOrNull(parser -> parseSearchRequest(searchRequest, request, parser, setSize)); + searchRequest.setWaitForCompletion(request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1))); + + ActionRequestValidationException validationException = searchRequest.validate(); + if (validationException != null) { + throw validationException; + } + return channel -> { + RestStatusToXContentListener listener = new RestStatusToXContentListener<>(channel); + client.executeLocally(SubmitAsyncSearchAction.INSTANCE, searchRequest, listener); + }; + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java new file mode 100644 index 0000000000000..cecccbd0e3e50 --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; + +import java.io.IOException; + +public class TransportDeleteAsyncSearchAction extends HandledTransportAction { + private final ClusterService clusterService; + private final TransportService transportService; + private final Client client; + + @Inject + public TransportDeleteAsyncSearchAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + Client client) { + super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new); + this.clusterService = clusterService; + this.transportService = transportService; + this.client = client; + } + + @Override + protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, ActionListener listener) { + try { + AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); + if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { + cancelTaskOnNode(request, searchId, listener); + } else { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); + if (node == null) { + deleteResult(request, searchId, listener, false); + } else { + transportService.sendRequest(node, DeleteAsyncSearchAction.NAME, request, builder.build(), + new ActionListenerResponseHandler<>(listener, AcknowledgedResponse::new, ThreadPool.Names.SAME)); + } + } + } catch (IOException e) { + listener.onFailure(e); + } + } + + private void cancelTaskOnNode(DeleteAsyncSearchAction.Request request, AsyncSearchId searchId, + ActionListener listener) { + Task runningTask = taskManager.getTask(searchId.getTaskId().getId()); + if (runningTask == null) { + deleteResult(request, searchId, listener, false); + return; + } + if (runningTask instanceof AsyncSearchTask) { + AsyncSearchTask searchTask = (AsyncSearchTask) runningTask; + if (searchTask.getSearchId().equals(request.getId()) + && searchTask.isCancelled() == false) { + taskManager.cancel(searchTask, "cancel", () -> {}); + deleteResult(request, searchId, listener, true); + } else { + // Task id has been reused by another task due to a node restart + deleteResult(request, searchId, listener, false); + } + } else { + // Task id has been reused by another task due to a node restart + deleteResult(request, searchId, listener, false); + } + } + + private void deleteResult(DeleteAsyncSearchAction.Request orig, AsyncSearchId searchId, + ActionListener next, boolean foundTask) { + DeleteRequest request = new DeleteRequest(searchId.getIndexName()).id(searchId.getDocId()); + client.delete(request, ActionListener.wrap( + resp -> { + if (resp.status() == RestStatus.NOT_FOUND && foundTask == false) { + next.onFailure(new ResourceNotFoundException("id [{}] not found", orig.getId())); + } else { + next.onResponse(new AcknowledgedResponse(true)); + } + }, + exc -> { + if (foundTask == false) { + next.onFailure(new ResourceNotFoundException("id [{}] not found", orig.getId())); + } else { + next.onResponse(new AcknowledgedResponse(true)); + } + } + )); + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java new file mode 100644 index 0000000000000..4fcbe5693ad1b --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; + +import java.io.IOException; + +public class TransportGetAsyncSearchAction extends HandledTransportAction { + private final ClusterService clusterService; + private final ThreadPool threadPool; + private final TransportService transportService; + private final AsyncSearchStoreService store; + + @Inject + public TransportGetAsyncSearchAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + NamedWriteableRegistry registry, + Client client, + ThreadPool threadPool) { + super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncSearchAction.Request::new); + this.clusterService = clusterService; + this.transportService = transportService; + this.threadPool = threadPool; + this.store = new AsyncSearchStoreService(client, registry); + } + + @Override + protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { + try { + AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); + if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { + getSearchResponseFromTask(task, request, searchId, listener); + } else { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); + if (node == null) { + getSearchResponseFromIndex(task, request, searchId, listener); + } else { + transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), + new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); + } + } + } catch (IOException e) { + listener.onFailure(e); + } + } + + private void getSearchResponseFromTask(Task thisTask, GetAsyncSearchAction.Request request, AsyncSearchId searchId, + ActionListener listener) { + Task runningTask = taskManager.getTask(searchId.getTaskId().getId()); + if (runningTask == null) { + // Task isn't running + getSearchResponseFromIndex(thisTask, request, searchId, listener); + return; + } + if (runningTask instanceof AsyncSearchTask) { + AsyncSearchTask searchTask = (AsyncSearchTask) runningTask; + if (searchTask.getSearchId().equals(request.getId()) == false) { + // Task id has been reused by another task due to a node restart + getSearchResponseFromIndex(thisTask, request, searchId, listener); + return; + } + waitForCompletion(request, searchTask, threadPool.relativeTimeInMillis(), listener); + } else { + // Task id has been reused by another task due to a node restart + getSearchResponseFromIndex(thisTask, request, searchId, listener); + } + } + + private void getSearchResponseFromIndex(Task task, GetAsyncSearchAction.Request request, AsyncSearchId searchId, + ActionListener listener) { + GetRequest get = new GetRequest(searchId.getIndexName(), searchId.getDocId()).storedFields("response"); + get.setParentTask(clusterService.localNode().getId(), task.getId()); + store.getResponse(request, searchId, + ActionListener.wrap( + resp -> { + if (resp.getVersion() <= request.getLastVersion()) { + // return a not-modified response + listener.onResponse(new AsyncSearchResponse(resp.id(), resp.getVersion(), false)); + } else { + listener.onResponse(resp); + } + }, + exc -> listener.onFailure(exc) + ) + ); + } + + void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, long startMs, + ActionListener listener) { + final AsyncSearchResponse response = task.getAsyncResponse(false); + try { + if (response.isFinalResponse()) { + if (response.getVersion() <= request.getLastVersion()) { + // return a not-modified response + listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), false)); + } else { + listener.onResponse(response); + } + } else if (request.getWaitForCompletion().getMillis() < (threadPool.relativeTimeInMillis() - startMs)) { + if (response.getVersion() <= request.getLastVersion()) { + // return a not-modified response + listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), true)); + } else { + listener.onResponse(task.getAsyncResponse(true)); + } + } else { + Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); + threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); + } + } catch (Exception e) { + listener.onFailure(e); + } + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java new file mode 100644 index 0000000000000..bae28e912826d --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchProgressActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; + +import java.util.Map; +import java.util.function.Supplier; + +public class TransportSubmitAsyncSearchAction extends HandledTransportAction { + private final NodeClient nodeClient; + private final Supplier reduceContextSupplier; + private final TransportSearchAction searchAction; + private final AsyncSearchStoreService store; + + @Inject + public TransportSubmitAsyncSearchAction(TransportService transportService, + ActionFilters actionFilters, + NamedWriteableRegistry registry, + Client client, + NodeClient nodeClient, + SearchService searchService, + TransportSearchAction searchAction) { + super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new); + this.nodeClient = nodeClient; + this.reduceContextSupplier = () -> searchService.createReduceContext(true); + this.searchAction = searchAction; + this.store = new AsyncSearchStoreService(client, registry); + } + + @Override + protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener submitListener) { + // add a place holder in the async search history index and fire the async search + store.storeInitialResponse( + ActionListener.wrap( + resp -> executeSearch(request, resp, submitListener), + submitListener::onFailure + ) + ); + } + + private void executeSearch(SubmitAsyncSearchRequest submitRequest, IndexResponse doc, + ActionListener submitListener) { + SearchRequest searchRequest = new SearchRequest(submitRequest) { + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + String searchId = AsyncSearchId.encode(doc.getIndex(), doc.getId(), new TaskId(nodeClient.getLocalNodeId(), id)); + return new AsyncSearchTask(id, type, action, headers, searchId, reduceContextSupplier); + } + }; + + AsyncSearchTask task = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); + SearchProgressActionListener progressListener = task.getProgressListener(); + searchAction.execute(task, searchRequest, + new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + try { + progressListener.onResponse(response); + store.storeFinalResponse(task.getAsyncResponse(true), ActionListener.wrap(() -> taskManager.unregister(task))); + } catch (Exception e) { + taskManager.unregister(task); + } + } + + @Override + public void onFailure(Exception exc) { + try { + progressListener.onFailure(exc); + store.storeFinalResponse(task.getAsyncResponse(true), ActionListener.wrap(() -> taskManager.unregister(task))); + } catch (Exception e) { + taskManager.unregister(task); + } + } + } + ); + + GetAsyncSearchAction.Request getRequest = new GetAsyncSearchAction.Request(task.getSearchId(), + submitRequest.getWaitForCompletion(), -1); + nodeClient.executeLocally(GetAsyncSearchAction.INSTANCE, getRequest, submitListener); + } +} diff --git a/x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/async-search/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java new file mode 100644 index 0000000000000..f57e21311ac7e --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; + +public class AsyncSearchIdTests extends ESTestCase { + public void testEncode() throws Exception { + for (int i = 0; i < 10; i++) { + AsyncSearchId instance = new AsyncSearchId(randomAlphaOfLengthBetween(5, 20), UUIDs.randomBase64UUID(), + new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong())); + String encoded = AsyncSearchId.encode(instance.getIndexName(), instance.getDocId(), instance.getTaskId()); + AsyncSearchId same = AsyncSearchId.decode(encoded); + assertEquals(same, instance); + + AsyncSearchId mutate = mutate(instance); + assertNotEquals(mutate, instance); + assertNotEquals(mutate, same); + } + } + + private AsyncSearchId mutate(AsyncSearchId id) { + int rand = randomIntBetween(0, 2); + switch (rand) { + case 0: + return new AsyncSearchId(randomAlphaOfLength(id.getIndexName().length()+1), id.getDocId(), id.getTaskId()); + + case 1: + return new AsyncSearchId(id.getIndexName(), randomAlphaOfLength(id.getDocId().length()+1), id.getTaskId()); + + case 2: + return new AsyncSearchId(id.getIndexName(), id.getDocId(), + new TaskId(randomAlphaOfLength(id.getTaskId().getNodeId().length()), randomNonNegativeLong())); + + default: + throw new AssertionError(); + } + } +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java new file mode 100644 index 0000000000000..2d75eea425339 --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.metrics.InternalMax; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; +import org.elasticsearch.xpack.core.transform.TransformField; +import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; +import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; + +public class AsyncSearchResponseTests extends ESTestCase { + private SearchResponse searchResponse = randomSearchResponse(); + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), + TimeSyncConfig::new)); + + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); + + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + + protected Writeable.Reader instanceReader() { + return AsyncSearchResponse::new; + } + + protected AsyncSearchResponse createTestInstance() { + return randomAsyncSearchResponse(randomSearchId(), searchResponse); + } + + protected void assertEqualInstances(AsyncSearchResponse expectedInstance, AsyncSearchResponse newInstance) { + assertNotSame(newInstance, expectedInstance); + assertEqualResponses(expectedInstance, newInstance); + } + + public final void testSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + AsyncSearchResponse testInstance = createTestInstance(); + assertSerialization(testInstance); + } + } + + protected final AsyncSearchResponse assertSerialization(AsyncSearchResponse testInstance) throws IOException { + return assertSerialization(testInstance, Version.CURRENT); + } + + protected final AsyncSearchResponse assertSerialization(AsyncSearchResponse testInstance, Version version) throws IOException { + AsyncSearchResponse deserializedInstance = copyInstance(testInstance, version); + assertEqualInstances(testInstance, deserializedInstance); + return deserializedInstance; + } + + protected final AsyncSearchResponse copyInstance(AsyncSearchResponse instance) throws IOException { + return copyInstance(instance, Version.CURRENT); + } + + protected AsyncSearchResponse copyInstance(AsyncSearchResponse instance, Version version) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, instanceReader(), version); + } + + static AsyncSearchResponse randomAsyncSearchResponse(String searchId, SearchResponse searchResponse) { + int rand = randomIntBetween(0, 3); + switch (rand) { + case 0: + return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); + + case 1: + return new AsyncSearchResponse(searchId, searchResponse, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); + + case 2: + return new AsyncSearchResponse(searchId, randomPartialSearchResponse(), + randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); + + case 3: + return new AsyncSearchResponse(searchId, randomPartialSearchResponse(), + new ElasticsearchException(new IOException("boum")), randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); + + default: + throw new AssertionError(); + } + } + + static SearchResponse randomSearchResponse() { + long tookInMillis = randomNonNegativeLong(); + int totalShards = randomIntBetween(1, Integer.MAX_VALUE); + int successfulShards = randomIntBetween(0, totalShards); + int skippedShards = totalShards - successfulShards; + InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty(); + return new SearchResponse(internalSearchResponse, null, totalShards, + successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + } + + private static PartialSearchResponse randomPartialSearchResponse() { + int totalShards = randomIntBetween(0, 10000); + if (randomBoolean()) { + return new PartialSearchResponse(totalShards); + } else { + int successfulShards = randomIntBetween(0, totalShards); + int failedShards = totalShards - successfulShards; + TotalHits totalHits = new TotalHits(randomLongBetween(0, Long.MAX_VALUE), randomFrom(TotalHits.Relation.values())); + InternalMax max = new InternalMax("max", 0f, DocValueFormat.RAW, Collections.emptyList(), Collections.emptyMap()); + InternalAggregations aggs = new InternalAggregations(Collections.singletonList(max)); + return new PartialSearchResponse(totalShards, successfulShards, failedShards, totalHits, aggs); + } + } + + static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchResponse actual) { + assertEquals(expected.id(), actual.id()); + assertEquals(expected.getVersion(), actual.getVersion()); + assertEquals(expected.status(), actual.status()); + assertEquals(expected.getPartialResponse(), actual.getPartialResponse()); + assertEquals(expected.getFailure() == null, actual.getFailure() == null); + // TODO check equals response + } +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java new file mode 100644 index 0000000000000..6277da2a5bd7c --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.junit.Before; + +import java.io.IOException; +import java.util.Base64; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses; +import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse; +import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse; +import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; + +public class AsyncSearchStoreServiceTests extends ESTestCase { + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public void testEncode() throws IOException { + for (int i = 0; i < 10; i++) { + AsyncSearchResponse response = randomAsyncSearchResponse(randomSearchId(), randomSearchResponse()); + String encoded = AsyncSearchStoreService.encodeResponse(response); + AsyncSearchResponse same = AsyncSearchStoreService.decodeResponse(Base64.getDecoder().decode(encoded), namedWriteableRegistry); + assertEqualResponses(response, same); + } + } +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java new file mode 100644 index 0000000000000..f71d859f648a3 --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; + +import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; + +public class DeleteAsyncSearchRequestTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return DeleteAsyncSearchAction.Request::new; + } + + @Override + protected DeleteAsyncSearchAction.Request createTestInstance() { + return new DeleteAsyncSearchAction.Request(randomSearchId()); + } +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java new file mode 100644 index 0000000000000..3e96e738eede1 --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; + +public class GetAsyncSearchRequestTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return GetAsyncSearchAction.Request::new; + } + + @Override + protected GetAsyncSearchAction.Request createTestInstance() { + return new GetAsyncSearchAction.Request(randomSearchId(), TimeValue.timeValueMillis(randomIntBetween(1, 10000)), + randomIntBetween(-1, Integer.MAX_VALUE)); + } + + static String randomSearchId() { + return AsyncSearchId.encode(randomRealisticUnicodeOfLengthBetween(2, 20), UUIDs.randomBase64UUID(), + new TaskId(randomAlphaOfLengthBetween(10, 20), randomLongBetween(0, Long.MAX_VALUE))); + } + + public void testValidateWaitForCompletion() { + + } +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java new file mode 100644 index 0000000000000..bf465b249ae9e --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; +import org.elasticsearch.xpack.core.transform.action.AbstractWireSerializingTransformTestCase; + +public class SubmitAsyncSearchRequestTests extends AbstractWireSerializingTransformTestCase { + @Override + protected Writeable.Reader instanceReader() { + return SubmitAsyncSearchRequest::new; + } + + @Override + protected SubmitAsyncSearchRequest createTestInstance() { + SubmitAsyncSearchRequest searchRequest = new SubmitAsyncSearchRequest(); + searchRequest.allowPartialSearchResults(randomBoolean()); + if (randomBoolean()) { + searchRequest.indices(generateRandomStringArray(10, 10, false, false)); + } + if (randomBoolean()) { + searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + } + if (randomBoolean()) { + searchRequest.preference(randomAlphaOfLengthBetween(3, 10)); + } + if (randomBoolean()) { + searchRequest.requestCache(randomBoolean()); + } + if (randomBoolean()) { + searchRequest.routing(randomAlphaOfLengthBetween(3, 10)); + } + if (randomBoolean()) { + searchRequest.searchType(randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)); + } + if (randomBoolean()) { + searchRequest.source(randomSearchSourceBuilder()); + } + return searchRequest; + } + + protected SearchSourceBuilder randomSearchSourceBuilder() { + SearchSourceBuilder source = new SearchSourceBuilder(); + if (randomBoolean()) { + source.query(QueryBuilders.termQuery("foo", "bar")); + } + if (randomBoolean()) { + source.aggregation(AggregationBuilders.max("max").field("field")); + } + return source; + } + + public void testValidateScroll() { + + } + + public void testValidateSuggestOnly() { + + } + + public void testValidateWaitForCompletion() { + + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index cd8064e4c1350..d5ffa825cd5ab 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -34,6 +34,9 @@ import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; @@ -419,7 +422,11 @@ public List> getClientActions() { DeleteTransformAction.INSTANCE, GetTransformAction.INSTANCE, GetTransformStatsAction.INSTANCE, - PreviewTransformAction.INSTANCE + PreviewTransformAction.INSTANCE, + // Async Search + SubmitAsyncSearchAction.INSTANCE, + GetAsyncSearchAction.INSTANCE, + DeleteAsyncSearchAction.INSTANCE ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java new file mode 100644 index 0000000000000..95de534260e36 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.search.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; + +import static org.elasticsearch.rest.RestStatus.NOT_MODIFIED; +import static org.elasticsearch.rest.RestStatus.PARTIAL_CONTENT; + +/** + * A response of a search progress request that contains a non-null {@link PartialSearchResponse} if the request is running or has failed + * before completion, or a final {@link SearchResponse} if the request succeeded. + */ +public class AsyncSearchResponse extends ActionResponse implements StatusToXContentObject { + private final String id; + private final int version; + private final SearchResponse finalResponse; + private final PartialSearchResponse partialResponse; + private final ElasticsearchException failure; + + private final long timestamp; + private final boolean isRunning; + + public AsyncSearchResponse(String id, int version, boolean isRunning) { + this(id, null, null, null, version, isRunning); + } + + public AsyncSearchResponse(String id, SearchResponse response, int version, boolean isRunning) { + this(id, null, response, null, version, isRunning); + } + + public AsyncSearchResponse(String id, PartialSearchResponse response, int version, boolean isRunning) { + this(id, response, null, null, version, isRunning); + } + + public AsyncSearchResponse(String id, PartialSearchResponse response, ElasticsearchException failure, int version, boolean isRunning) { + this(id, response, null, failure, version, isRunning); + } + + private AsyncSearchResponse(String id, + PartialSearchResponse partialResponse, + SearchResponse finalResponse, + ElasticsearchException failure, + int version, + boolean isRunning) { + this.id = id; + this.version = version; + this.partialResponse = partialResponse; + this.failure = failure; + this.finalResponse = finalResponse; + this.timestamp = System.currentTimeMillis(); + this.isRunning = isRunning; + } + + public AsyncSearchResponse(StreamInput in) throws IOException { + this.id = in.readString(); + this.version = in.readVInt(); + this.partialResponse = in.readOptionalWriteable(PartialSearchResponse::new); + this.failure = in.readOptionalWriteable(ElasticsearchException::new); + this.finalResponse = in.readOptionalWriteable(SearchResponse::new); + this.timestamp = in.readLong(); + this.isRunning = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeVInt(version); + out.writeOptionalWriteable(partialResponse); + out.writeOptionalWriteable(failure); + out.writeOptionalWriteable(finalResponse); + out.writeLong(timestamp); + out.writeBoolean(isRunning); + } + + /** + * Return the id of the search progress request. + */ + public String id() { + return id; + } + + /** + * Return the version of this response. + */ + public int getVersion() { + return version; + } + + /** + * Return true if the request has failed. + */ + public boolean hasFailed() { + return failure != null; + } + + /** + * Return true if a partial response is available through. + */ + public boolean isPartialResponse() { + return partialResponse != null; + } + + /** + * Return true if the final response is available. + */ + public boolean isFinalResponse() { + return finalResponse != null; + } + + /** + * The final {@link SearchResponse} if the request has completed, or null if the + * request is running or failed. + */ + public SearchResponse getSearchResponse() { + return finalResponse; + } + + /** + * The {@link PartialSearchResponse} if the request is running or failed, or null + * if the request has completed. + */ + public PartialSearchResponse getPartialResponse() { + return partialResponse; + } + + /** + * The failure that occurred during the search. + */ + public ElasticsearchException getFailure() { + return failure; + } + + /** + * When this response was created. + */ + public long getTimestamp() { + return timestamp; + } + + /** + * Whether the search is still running in the cluster. + */ + public boolean isRunning() { + return isRunning; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("id", id); + builder.field("version", version); + builder.field("is_running", isRunning); + builder.field("timestamp", timestamp); + if (partialResponse != null) { + builder.field("partial_response", partialResponse); + } else if (finalResponse != null) { + builder.field("response", finalResponse); + } + if (failure != null) { + builder.startObject("failure"); + failure.toXContent(builder, params); + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public RestStatus status() { + if (finalResponse == null && partialResponse == null) { + return NOT_MODIFIED; + } else if (finalResponse == null) { + return failure != null ? failure.status() : PARTIAL_CONTENT; + } else { + return finalResponse.status(); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java new file mode 100644 index 0000000000000..92ed6b1122824 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.search.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.CompositeIndicesRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteAsyncSearchAction extends ActionType { + public static final DeleteAsyncSearchAction INSTANCE = new DeleteAsyncSearchAction(); + public static final String NAME = "indices:data/read/async_search/delete"; + + private DeleteAsyncSearchAction() { + super(NAME, AcknowledgedResponse::new); + } + + @Override + public Writeable.Reader getResponseReader() { + return AcknowledgedResponse::new; + } + + public static class Request extends ActionRequest implements CompositeIndicesRequest { + private final String id; + + public Request(String id) { + this.id = id; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.id = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String getId() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return id.equals(request.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java new file mode 100644 index 0000000000000..f1e89cbf35409 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.search.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.CompositeIndicesRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.Objects; + +public class GetAsyncSearchAction extends ActionType { + public static final GetAsyncSearchAction INSTANCE = new GetAsyncSearchAction(); + public static final String NAME = "indices:data/read/async_search/get"; + + private GetAsyncSearchAction() { + super(NAME, AsyncSearchResponse::new); + } + + @Override + public Writeable.Reader getResponseReader() { + return AsyncSearchResponse::new; + } + + public static class Request extends ActionRequest implements CompositeIndicesRequest { + private final String id; + private final int lastVersion; + private final TimeValue waitForCompletion; + + /** + * Create a new request + * @param id The id of the search progress request. + * @param waitForCompletion The minimum time that the request should wait before returning a partial result. + * @param lastVersion The last version returned by a previous call. + */ + public Request(String id, TimeValue waitForCompletion, int lastVersion) { + this.id = id; + this.waitForCompletion = waitForCompletion; + this.lastVersion = lastVersion; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.id = in.readString(); + this.waitForCompletion = TimeValue.timeValueMillis(in.readLong()); + this.lastVersion = in.readInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + out.writeLong(waitForCompletion.millis()); + out.writeInt(lastVersion); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String getId() { + return id; + } + + /** + * Return the version of the previously retrieved {@link AsyncSearchResponse}. + * Partial hits and aggs are not included in the new response if they match this + * version and the request returns {@link RestStatus#NOT_MODIFIED}. + */ + public int getLastVersion() { + return lastVersion; + } + + /** + * Return the minimum time that the request should wait for completion. + */ + public TimeValue getWaitForCompletion() { + return waitForCompletion; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return lastVersion == request.lastVersion && + id.equals(request.id) && + waitForCompletion.equals(request.waitForCompletion); + } + + @Override + public int hashCode() { + return Objects.hash(id, lastVersion, waitForCompletion); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java new file mode 100644 index 0000000000000..89352572025d8 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.search.action; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.InternalAggregations; + +import java.io.IOException; +import java.util.Objects; + +/** + * A search response that contains partial results. + */ +public class PartialSearchResponse implements ToXContentFragment, Writeable { + private final int totalShards; + private final int successfulShards; + private final int shardFailures; + + private final TotalHits totalHits; + private InternalAggregations aggregations; + + public PartialSearchResponse(int totalShards) { + this(totalShards, 0, 0, null, null); + } + + public PartialSearchResponse(int totalShards, int successfulShards, int shardFailures, + TotalHits totalHits, InternalAggregations aggregations) { + this.totalShards = totalShards; + this.successfulShards = successfulShards; + this.shardFailures = shardFailures; + this.totalHits = totalHits; + this.aggregations = aggregations; + } + + public PartialSearchResponse(StreamInput in) throws IOException { + this.totalShards = in.readVInt(); + this.successfulShards = in.readVInt(); + this.shardFailures = in.readVInt(); + this.totalHits = in.readBoolean() ? Lucene.readTotalHits(in) : null; + this.aggregations = in.readOptionalWriteable(InternalAggregations::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(totalShards); + out.writeVInt(successfulShards); + out.writeVInt(shardFailures); + out.writeBoolean(totalHits != null); + if (totalHits != null) { + Lucene.writeTotalHits(out, totalHits); + } + out.writeOptionalWriteable(aggregations); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("total_shards", totalShards); + builder.field("successful_shards", successfulShards); + builder.field("shard_failures", shardFailures); + if (totalHits != null) { + builder.startObject(SearchHits.Fields.TOTAL); + builder.field("value", totalHits.value); + builder.field("relation", totalHits.relation == TotalHits.Relation.EQUAL_TO ? "eq" : "gte"); + builder.endObject(); + } + if (aggregations != null) { + aggregations.toXContent(builder, params); + } + builder.endObject(); + return builder; + } + + /** + * The total number of shards the search should executed on. + */ + public int getTotalShards() { + return totalShards; + } + + /** + * The successful number of shards the search was executed on. + */ + public int getSuccesfullShards() { + return successfulShards; + } + + /** + * The failed number of shards the search was executed on. + */ + public int getShardFailures() { + return shardFailures; + } + + /** + * Return the partial {@link TotalHits} computed from the shards that + * completed the query phase. + */ + public TotalHits getTotalHits() { + return totalHits; + } + + /** + * Return the partial {@link InternalAggregations} computed from the shards that + * completed the query phase. + */ + public InternalAggregations getAggregations() { + return aggregations; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PartialSearchResponse that = (PartialSearchResponse) o; + return totalShards == that.totalShards && + successfulShards == that.successfulShards && + shardFailures == that.shardFailures && + Objects.equals(totalHits, that.totalHits) && + Objects.equals(aggregations, that.aggregations); + } + + @Override + public int hashCode() { + return Objects.hash(totalShards, successfulShards, shardFailures, totalHits, aggregations); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java new file mode 100644 index 0000000000000..ad3539ceb25c4 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.search.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.io.stream.Writeable; + +public final class SubmitAsyncSearchAction extends ActionType { + public static final SubmitAsyncSearchAction INSTANCE = new SubmitAsyncSearchAction(); + public static final String NAME = "indices:data/read/async_search/submit"; + + private SubmitAsyncSearchAction() { + super(NAME, AsyncSearchResponse::new); + } + + @Override + public Writeable.Reader getResponseReader() { + return AsyncSearchResponse::new; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java new file mode 100644 index 0000000000000..62fcf05a6b00d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.search.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.CompositeIndicesRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * A request to track asynchronously the progress of a search against one or more indices. + * + * @see AsyncSearchResponse + */ +public class SubmitAsyncSearchRequest extends SearchRequest implements CompositeIndicesRequest { + private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1); + + /** + * Create a new request + * @param indices The indices the search will be executed on. + * @param source The source of the search request. + */ + public SubmitAsyncSearchRequest(String[] indices, SearchSourceBuilder source) { + super(indices, source); + setCcsMinimizeRoundtrips(false); + setPreFilterShardSize(1); + setBatchedReduceSize(5); + } + + /** + * Create a new request + * @param indices The indices the search will be executed on. + */ + public SubmitAsyncSearchRequest(String... indices) { + this(indices, new SearchSourceBuilder()); + } + + public SubmitAsyncSearchRequest(StreamInput in) throws IOException { + super(in); + this.waitForCompletion = in.readTimeValue(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeTimeValue(waitForCompletion); + } + + /** + * Set the minimum time that the request should wait before returning a partial result. + */ + public void setWaitForCompletion(TimeValue waitForCompletion) { + this.waitForCompletion = waitForCompletion; + } + + /** + * Return the minimum time that the request should wait before returning a partial result. + */ + public TimeValue getWaitForCompletion() { + return waitForCompletion; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (scroll() != null) { + addValidationError("scroll queries are not supported", validationException); + } + if (isSuggestOnly()) { + validationException = addValidationError("suggest-only queries are not supported", validationException); + } + return validationException; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new Task(id, type, action, getDescription(), parentTaskId, headers); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + SubmitAsyncSearchRequest request = (SubmitAsyncSearchRequest) o; + return waitForCompletion.equals(request.waitForCompletion); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), waitForCompletion); + } +} diff --git a/x-pack/plugin/core/src/main/resources/async-search-history-ilm-policy.json b/x-pack/plugin/core/src/main/resources/async-search-history-ilm-policy.json new file mode 100644 index 0000000000000..fd46244836cb5 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/async-search-history-ilm-policy.json @@ -0,0 +1,18 @@ +{ + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "50GB", + "max_age": "1d" + } + } + }, + "delete": { + "min_age": "5d", + "actions": { + "delete": {} + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/async-search-history.json b/x-pack/plugin/core/src/main/resources/async-search-history.json new file mode 100644 index 0000000000000..9824745cb835d --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/async-search-history.json @@ -0,0 +1,29 @@ +{ + "index_patterns": [ + ".async-search-history-${xpack.async-search-history.template.version}*" + ], + "order": 2147483647, + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.lifecycle.name": "async-search-history-policy", + "index.lifecycle.rollover_alias": ".async-search-history${xpack.async-search-history-template.version}", + "index.format": 1 + }, + "mappings": { + "_doc": { + "dynamic": false, + "_source": { + "excludes": ["*"] + }, + "properties": { + "response": { + "type": "binary", + "store": true, + "doc_values": false + } + } + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index 4b0e99d7290fd..beb53aab5f0b6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -206,6 +206,9 @@ private static boolean shouldAuthorizeIndexActionNameOnly(String action, Transpo case MultiGetAction.NAME: case MultiTermVectorsAction.NAME: case MultiSearchAction.NAME: + case "indices:data/read/async_search/submit": + case "indices:data/read/async_search/get": + case "indices:data/read/async_search/delete": case "indices:data/read/mpercolate": case "indices:data/read/msearch/template": case "indices:data/read/search/template": diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json new file mode 100644 index 0000000000000..4486b90cb9a1d --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.delete.json @@ -0,0 +1,24 @@ +{ + "async_search.delete":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html" + }, + "stability":"experimental", + "url":{ + "paths":[ + { + "path":"/_async_search/{id}", + "methods":[ + "DELETE" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The async search ID" + } + } + } + ] + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json new file mode 100644 index 0000000000000..c69c30ba0cc5e --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json @@ -0,0 +1,34 @@ +{ + "async_search.get":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html" + }, + "stability":"experimental", + "url":{ + "paths":[ + { + "path":"/_async_search/{id}", + "methods":[ + "GET" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The async search ID" + } + } + } + ] + }, + "params":{ + "wait_for_completion":{ + "type":"time", + "description":"Specify the time that the request should block waiting for the final response (default: 1s)" + }, + "last_version":{ + "type":"number", + "description":"Specify the last version returned by a previous call. The request will return 304 (not modified) status if the new version is lower than or equals to the provided one and will remove all details in the response except id and version. (default: -1)" + } + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json new file mode 100644 index 0000000000000..678c22e0f733b --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json @@ -0,0 +1,223 @@ +{ + "async_search.submit":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/async-search.html" + }, + "stability":"experimental", + "url":{ + "paths":[ + { + "path":"/_async_search", + "methods":[ + "GET", + "POST" + ] + }, + { + "path":"/{index}/_async_search", + "methods":[ + "GET", + "POST" + ], + "parts":{ + "index":{ + "type":"list", + "description":"A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" + } + } + } + ] + }, + "params":{ + "wait_for_completion":{ + "type":"time", + "description":"Specify the time that the request should block waiting for the final response (default: 1s)" + }, + "analyzer":{ + "type":"string", + "description":"The analyzer to use for the query string" + }, + "analyze_wildcard":{ + "type":"boolean", + "description":"Specify whether wildcard and prefix queries should be analyzed (default: false)" + }, + "default_operator":{ + "type":"enum", + "options":[ + "AND", + "OR" + ], + "default":"OR", + "description":"The default operator for query string query (AND or OR)" + }, + "df":{ + "type":"string", + "description":"The field to use as default where no field prefix is given in the query string" + }, + "explain":{ + "type":"boolean", + "description":"Specify whether to return detailed information about score computation as part of a hit" + }, + "stored_fields":{ + "type":"list", + "description":"A comma-separated list of stored fields to return as part of a hit" + }, + "docvalue_fields":{ + "type":"list", + "description":"A comma-separated list of fields to return as the docvalue representation of a field for each hit" + }, + "from":{ + "type":"number", + "description":"Starting offset (default: 0)" + }, + "ignore_unavailable":{ + "type":"boolean", + "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)" + }, + "ignore_throttled":{ + "type":"boolean", + "description":"Whether specified concrete, expanded or aliased indices should be ignored when throttled" + }, + "allow_no_indices":{ + "type":"boolean", + "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" + }, + "expand_wildcards":{ + "type":"enum", + "options":[ + "open", + "closed", + "none", + "all" + ], + "default":"open", + "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "lenient":{ + "type":"boolean", + "description":"Specify whether format-based query failures (such as providing text to a numeric field) should be ignored" + }, + "preference":{ + "type":"string", + "description":"Specify the node or shard the operation should be performed on (default: random)" + }, + "q":{ + "type":"string", + "description":"Query in the Lucene query string syntax" + }, + "routing":{ + "type":"list", + "description":"A comma-separated list of specific routing values" + }, + "search_type":{ + "type":"enum", + "options":[ + "query_then_fetch", + "dfs_query_then_fetch" + ], + "description":"Search operation type" + }, + "size":{ + "type":"number", + "description":"Number of hits to return (default: 10)" + }, + "sort":{ + "type":"list", + "description":"A comma-separated list of : pairs" + }, + "_source":{ + "type":"list", + "description":"True or false to return the _source field or not, or a list of fields to return" + }, + "_source_excludes":{ + "type":"list", + "description":"A list of fields to exclude from the returned _source field" + }, + "_source_includes":{ + "type":"list", + "description":"A list of fields to extract and return from the _source field" + }, + "terminate_after":{ + "type":"number", + "description":"The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early." + }, + "stats":{ + "type":"list", + "description":"Specific 'tag' of the request for logging and statistical purposes" + }, + "suggest_field":{ + "type":"string", + "description":"Specify which field to use for suggestions" + }, + "suggest_mode":{ + "type":"enum", + "options":[ + "missing", + "popular", + "always" + ], + "default":"missing", + "description":"Specify suggest mode" + }, + "suggest_size":{ + "type":"number", + "description":"How many suggestions to return in response" + }, + "suggest_text":{ + "type":"string", + "description":"The source text for which the suggestions should be returned" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + }, + "track_scores":{ + "type":"boolean", + "description":"Whether to calculate and return scores even if they are not used for sorting" + }, + "track_total_hits":{ + "type":"boolean", + "description":"Indicate if the number of documents that match the query should be tracked" + }, + "allow_partial_search_results":{ + "type":"boolean", + "default":true, + "description":"Indicate if an error should be returned if there is a partial search failure or timeout" + }, + "typed_keys":{ + "type":"boolean", + "description":"Specify whether aggregation and suggester names should be prefixed by their respective types in the response" + }, + "version":{ + "type":"boolean", + "description":"Specify whether to return document version as part of a hit" + }, + "seq_no_primary_term":{ + "type":"boolean", + "description":"Specify whether to return sequence number and primary term of the last modification of each hit" + }, + "request_cache":{ + "type":"boolean", + "description":"Specify if request cache should be used for this request or not, defaults to index level setting" + }, + "batched_reduce_size":{ + "type":"number", + "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.", + "default":5 + }, + "max_concurrent_shard_requests":{ + "type":"number", + "description":"The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests", + "default":5 + }, + "pre_filter_shard_size":{ + "type":"number", + "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.", + "default":1 + } + }, + "body":{ + "description":"The search definition using the Query DSL" + } + } +} From c28766c52ebb05d85ac18c1be8e1a56c84b6856f Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 6 Dec 2019 21:26:05 +0100 Subject: [PATCH 02/61] notify partial reduce even on top-docs query --- .../org/elasticsearch/action/search/SearchPhaseController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 27b5c9cf3b2a8..d1db2954aaea0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -638,7 +638,7 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) { } numReducePhases++; index = 1; - if (hasAggs) { + if (hasAggs || hasTopDocs) { progressListener.notifyPartialReduce(progressListener.searchShards(processedShards), topDocsStats.getTotalHits(), aggsBuffer[0], numReducePhases); } From 4bfafb4ee215dd098808ede1d918461591108a15 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 6 Dec 2019 22:14:14 +0100 Subject: [PATCH 03/61] fix npe --- .../org/elasticsearch/action/search/SearchPhaseController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index d1db2954aaea0..6e15f492d46b5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -640,7 +640,7 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) { index = 1; if (hasAggs || hasTopDocs) { progressListener.notifyPartialReduce(progressListener.searchShards(processedShards), - topDocsStats.getTotalHits(), aggsBuffer[0], numReducePhases); + topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0] : null, numReducePhases); } } final int i = index++; From 3ace355280130bb9be3a4075cdf8d286f76c7d6a Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 11 Dec 2019 13:55:45 +0100 Subject: [PATCH 04/61] use a single object named response in the xcontent serialization of the async search response --- .../search/AsyncSearchResponseTests.java | 2 +- .../search/action/AsyncSearchResponse.java | 34 +++++++++++++------ .../search/action/PartialSearchResponse.java | 1 + 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index 2d75eea425339..e7bfa61cbf1f9 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -145,6 +145,6 @@ static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchRespon assertEquals(expected.status(), actual.status()); assertEquals(expected.getPartialResponse(), actual.getPartialResponse()); assertEquals(expected.getFailure() == null, actual.getFailure() == null); - // TODO check equals response + // TODO check equal SearchResponse } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 95de534260e36..6dad4ef688a05 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -59,7 +59,7 @@ private AsyncSearchResponse(String id, this.version = version; this.partialResponse = partialResponse; this.failure = failure; - this.finalResponse = finalResponse; + this.finalResponse = finalResponse != null ? wrapFinalResponse(finalResponse) : null; this.timestamp = System.currentTimeMillis(); this.isRunning = isRunning; } @@ -157,6 +157,17 @@ public boolean isRunning() { return isRunning; } + @Override + public RestStatus status() { + if (finalResponse == null && partialResponse == null) { + return NOT_MODIFIED; + } else if (finalResponse == null) { + return failure != null ? failure.status() : PARTIAL_CONTENT; + } else { + return finalResponse.status(); + } + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -164,8 +175,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("version", version); builder.field("is_running", isRunning); builder.field("timestamp", timestamp); + if (partialResponse != null) { - builder.field("partial_response", partialResponse); + builder.field("response", partialResponse); } else if (finalResponse != null) { builder.field("response", finalResponse); } @@ -178,14 +190,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - @Override - public RestStatus status() { - if (finalResponse == null && partialResponse == null) { - return NOT_MODIFIED; - } else if (finalResponse == null) { - return failure != null ? failure.status() : PARTIAL_CONTENT; - } else { - return finalResponse.status(); - } + private static SearchResponse wrapFinalResponse(SearchResponse response) { + // Adds a partial flag set to false in the xcontent serialization + return new SearchResponse(response) { + @Override + public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("is_partial", false); + return super.innerToXContent(builder, params); + } + }; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java index 89352572025d8..8e6eb4fa44df9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java @@ -65,6 +65,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field("is_partial", true); builder.field("total_shards", totalShards); builder.field("successful_shards", successfulShards); builder.field("shard_failures", shardFailures); From 5d000cadd37d248939a2651365fbe292c9f1ed0c Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 11 Dec 2019 15:31:38 +0100 Subject: [PATCH 05/61] delete frozen search response automatically --- .../action/search/SearchResponse.java | 5 ++ .../rest-api-spec/test/search/10_basic.yml | 26 ++-------- .../xpack/search/AsyncSearchStoreService.java | 12 ++--- .../search/TransportGetAsyncSearchAction.java | 48 ++++++++++++++----- 4 files changed, 49 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index cb36dbd0cd8f2..e19f31fb89d8b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -108,6 +108,11 @@ public SearchResponse(SearchResponseSections internalResponse, String scrollId, assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards; } + public SearchResponse(SearchResponse clone) { + this(clone.internalResponse, clone.scrollId, clone.totalShards, clone.successfulShards, clone.skippedShards, + clone.tookInMillis, clone.shardFailures, clone.clusters); + } + @Override public RestStatus status() { return RestStatus.status(successfulShards, totalShards, shardFailures); diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml index 996ef3c8599fb..ebc7528674862 100644 --- a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml +++ b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml @@ -34,6 +34,7 @@ async_search.submit: index: test-* batched_reduce_size: 2 + wait_for_completion: 10s body: query: match_all: {} @@ -44,35 +45,14 @@ sort: max - set: { id: id } - - - do: - async_search.get: - id: "$id" - wait_for_completion: 10s - - set: { version: version } - match: { version: 4 } - - is_false: partial_response + - match: { response.is_partial: false } - length: { response.hits.hits: 3 } - match: { response.hits.hits.0._source.max: 1 } - match: { response.aggregations.1.value: 3.0 } - - do: - catch: not_modified - async_search.get: - id: "$id" - last_version: "$version" - - - is_false: partial_response - - is_false: response - - - do: - async_search.delete: - id: "$id" - - - match: { acknowledged: true } - - do: catch: missing - async_search.delete: + async_search.get: id: "$id" diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 4813dc0ee500d..a1cf4439dcfc4 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -74,23 +74,23 @@ void storeFinalResponse(AsyncSearchResponse response, ActionListener next) { - GetRequest request = new GetRequest(searchId.getIndexName()) + void getResponse(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener next) { + GetRequest internalGet = new GetRequest(searchId.getIndexName()) .id(searchId.getDocId()) .storedFields(RESPONSE_FIELD); - client.get(request, ActionListener.wrap( + client.get(internalGet, ActionListener.wrap( get -> { if (get.isExists() == false) { - next.onFailure(new ResourceNotFoundException(request.id() + " not found")); + next.onFailure(new ResourceNotFoundException(request.getId() + " not found")); } else if (get.getFields().containsKey(RESPONSE_FIELD) == false) { - next.onResponse(new AsyncSearchResponse(orig.getId(), new PartialSearchResponse(-1), 0, false)); + next.onResponse(new AsyncSearchResponse(request.getId(), new PartialSearchResponse(-1), 0, false)); } else { BytesArray bytesArray = get.getFields().get(RESPONSE_FIELD).getValue(); next.onResponse(decodeResponse(bytesArray.array(), registry)); } }, - exc -> next.onFailure(new ResourceNotFoundException(request.id() + " not found")) + exc -> next.onFailure(new ResourceNotFoundException(request.getId() + " not found")) )); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 4fcbe5693ad1b..9bf0fb7f9271b 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -25,11 +26,14 @@ import java.io.IOException; +import static org.elasticsearch.action.ActionListener.wrap; + public class TransportGetAsyncSearchAction extends HandledTransportAction { private final ClusterService clusterService; private final ThreadPool threadPool; private final TransportService transportService; private final AsyncSearchStoreService store; + private final Client client; @Inject public TransportGetAsyncSearchAction(TransportService transportService, @@ -43,12 +47,14 @@ public TransportGetAsyncSearchAction(TransportService transportService, this.transportService = transportService; this.threadPool = threadPool; this.store = new AsyncSearchStoreService(client, registry); + this.client = client; } @Override protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); + listener = wrapCleanupListener(searchId, listener); if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { getSearchResponseFromTask(task, request, searchId, listener); } else { @@ -81,7 +87,7 @@ private void getSearchResponseFromTask(Task thisTask, GetAsyncSearchAction.Reque getSearchResponseFromIndex(thisTask, request, searchId, listener); return; } - waitForCompletion(request, searchTask, threadPool.relativeTimeInMillis(), listener); + waitForCompletion(request, searchTask, searchId, threadPool.relativeTimeInMillis(), listener); } else { // Task id has been reused by another task due to a node restart getSearchResponseFromIndex(thisTask, request, searchId, listener); @@ -93,7 +99,7 @@ private void getSearchResponseFromIndex(Task task, GetAsyncSearchAction.Request GetRequest get = new GetRequest(searchId.getIndexName(), searchId.getDocId()).storedFields("response"); get.setParentTask(clusterService.localNode().getId(), task.getId()); store.getResponse(request, searchId, - ActionListener.wrap( + wrap( resp -> { if (resp.getVersion() <= request.getLastVersion()) { // return a not-modified response @@ -107,30 +113,46 @@ private void getSearchResponseFromIndex(Task task, GetAsyncSearchAction.Request ); } - void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, long startMs, - ActionListener listener) { + void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, + AsyncSearchId searchId, + long startMs, ActionListener listener) { final AsyncSearchResponse response = task.getAsyncResponse(false); try { - if (response.isFinalResponse()) { - if (response.getVersion() <= request.getLastVersion()) { - // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), false)); - } else { - listener.onResponse(response); - } + if (response.isRunning() == false) { + listener.onResponse(response); } else if (request.getWaitForCompletion().getMillis() < (threadPool.relativeTimeInMillis() - startMs)) { if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), true)); } else { - listener.onResponse(task.getAsyncResponse(true)); + final AsyncSearchResponse ret = task.getAsyncResponse(true); + listener.onResponse(ret); } } else { - Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); + Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, searchId, startMs, listener)); threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); } } catch (Exception e) { listener.onFailure(e); } } + + /** + * Returns a new listener that delegates the response to another listener and + * then deletes the async search document from the system index if the response is + * frozen (because the task has completed, failed or the coordinating node crashed). + */ + private ActionListener wrapCleanupListener(AsyncSearchId id, + ActionListener listener) { + return ActionListener.wrap( + resp -> { + listener.onResponse(resp); + if (resp.isRunning() == false) { + DeleteRequest delete = new DeleteRequest(id.getIndexName()).id(id.getDocId()); + client.delete(delete, wrap(() -> {})); + } + }, + listener::onFailure + ); + } } From 1cd2a9acd06ade129c8e4f5ad62a7f106fdf59db Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 12 Dec 2019 21:24:06 +0100 Subject: [PATCH 06/61] add integration tests --- x-pack/plugin/async-search/build.gradle | 39 +- x-pack/plugin/async-search/qa/build.gradle | 1 - .../xpack/search/AsyncSearchTask.java | 25 +- .../search/TransportGetAsyncSearchAction.java | 9 +- .../xpack/search/AsyncSearchIT.java | 213 ++++++++++ .../search/AsyncSearchIntegTestCase.java | 381 ++++++++++++++++++ .../search/action/AsyncSearchResponse.java | 4 +- .../search/action/PartialSearchResponse.java | 2 +- .../action/SubmitAsyncSearchRequest.java | 17 +- 9 files changed, 667 insertions(+), 24 deletions(-) create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle index 81210444bb2d5..81e3bcfd9793f 100644 --- a/x-pack/plugin/async-search/build.gradle +++ b/x-pack/plugin/async-search/build.gradle @@ -18,13 +18,42 @@ archivesBaseName = 'x-pack-async-search' compileJava.options.compilerArgs << "-Xlint:-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" +integTest.enabled = false + +// Instead we create a separate task to run the +// tests based on ESIntegTestCase +task internalClusterTest(type: Test) { + description = 'Java fantasy integration tests' + mustRunAfter test + + include '**/*IT.class' +} + +check.dependsOn internalClusterTest + +// add all sub-projects of the qa sub-project +gradle.projectsEvaluated { + project.subprojects + .find { it.path == project.path + ":qa" } + .subprojects + .findAll { it.path.startsWith(project.path + ":qa") } + .each { check.dependsOn it.check } +} dependencies { - compileOnly project(":server") + compileOnly project(":server") - compileOnly project(path: xpackModule('core'), configuration: 'default') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') - testCompile project(path: xpackModule('ilm')) + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(path: xpackModule('ilm')) } -integTest.enabled = false +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +testingConventions.naming { + IT { + baseClass "org.elasticsearch.xpack.search.AsyncSearchIntegTestCase" + } +} diff --git a/x-pack/plugin/async-search/qa/build.gradle b/x-pack/plugin/async-search/qa/build.gradle index 46609933699ca..d3e95d997c3fb 100644 --- a/x-pack/plugin/async-search/qa/build.gradle +++ b/x-pack/plugin/async-search/qa/build.gradle @@ -14,5 +14,4 @@ subprojects { include 'rest-api-spec/api/**' } } - } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 719c372a2d832..38895f5c24cb9 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -69,6 +69,9 @@ private class Listener extends SearchProgressActionListener { private AtomicInteger version = new AtomicInteger(0); private AtomicInteger shardFailures = new AtomicInteger(0); + private int lastSuccess = 0; + private int lastFailures = 0; + private volatile Response response; Listener() { @@ -92,8 +95,10 @@ public void onQueryFailure(int shardIndex, Exception exc) { @Override public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { + lastSuccess = shards.size(); + lastFailures = shardFailures.get(); final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, - new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs), + new PartialSearchResponse(totalShards, lastSuccess, lastFailures, totalHits, aggs), version.incrementAndGet(), true ); @@ -102,9 +107,11 @@ public void onPartialReduce(List shards, TotalHits totalHits, Inter @Override public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { + int failures = shardFailures.get(); + int ver = (lastSuccess == shards.size() && lastFailures == failures) ? version.get() : version.incrementAndGet(); final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, - new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs), - version.incrementAndGet(), + new PartialSearchResponse(totalShards, shards.size(), failures, totalHits, aggs), + ver, true ); response = new Response(newResp, false); @@ -118,10 +125,16 @@ public void onResponse(SearchResponse searchResponse) { @Override public void onFailure(Exception exc) { - AsyncSearchResponse current = response.get(true); - response = new Response(new AsyncSearchResponse(searchId, current.getPartialResponse(), + AsyncSearchResponse previous = response.get(true); + response = new Response(new AsyncSearchResponse(searchId, newPartialResponse(previous, shardFailures.get()), exc != null ? new ElasticsearchException(exc) : null, version.incrementAndGet(), false), false); } + + private PartialSearchResponse newPartialResponse(AsyncSearchResponse response, int numFailures) { + PartialSearchResponse old = response.getPartialResponse(); + return response.hasPartialResponse() ? new PartialSearchResponse(totalShards, old.getSuccessfulShards(), shardFailures.get(), + old.getTotalHits(), old.getAggregations()) : null; + } } private class Response { @@ -142,7 +155,7 @@ public synchronized AsyncSearchResponse get(boolean doFinalReduce) { InternalAggregations reducedAggs = internal.getPartialResponse().getAggregations(); reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(reducedAggs), reduceContextSupplier.get()); PartialSearchResponse old = internal.getPartialResponse(); - PartialSearchResponse clone = new PartialSearchResponse(old.getTotalShards(), old.getSuccesfullShards(), + PartialSearchResponse clone = new PartialSearchResponse(old.getTotalShards(), old.getSuccessfulShards(), old.getShardFailures(), old.getTotalHits(), reducedAggs); needFinalReduce = false; return internal = new AsyncSearchResponse(internal.id(), clone, internal.getFailure(), internal.getVersion(), true); diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 9bf0fb7f9271b..0f13dc71f6593 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -17,12 +17,14 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.RestResponse; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.rest.RestChannel; import java.io.IOException; @@ -96,7 +98,9 @@ private void getSearchResponseFromTask(Task thisTask, GetAsyncSearchAction.Reque private void getSearchResponseFromIndex(Task task, GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener listener) { - GetRequest get = new GetRequest(searchId.getIndexName(), searchId.getDocId()).storedFields("response"); + GetRequest get = new GetRequest(searchId.getIndexName(), searchId.getDocId()) + .routing(searchId.getDocId()) + .storedFields("response"); get.setParentTask(clusterService.localNode().getId(), task.getId()); store.getResponse(request, searchId, wrap( @@ -141,6 +145,9 @@ void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask tas * Returns a new listener that delegates the response to another listener and * then deletes the async search document from the system index if the response is * frozen (because the task has completed, failed or the coordinating node crashed). + * + * TODO: We should ensure that the response was successfully sent to the user before deleting + * (see {@link RestChannel#sendResponse(RestResponse)}. */ private ActionListener wrapCleanupListener(AsyncSearchId id, ActionListener listener) { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java new file mode 100644 index 0000000000000..0dad22901dc80 --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms; +import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; +import org.elasticsearch.search.aggregations.metrics.InternalMax; +import org.elasticsearch.search.aggregations.metrics.InternalMin; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class AsyncSearchIT extends AsyncSearchIntegTestCase { + private String indexName; + private int numShards; + private int numDocs; + + private int numKeywords; + private Map keywordFreqs; + private float maxMetric = Float.NEGATIVE_INFINITY; + private float minMetric = Float.POSITIVE_INFINITY; + + @Before + public void indexDocuments() throws InterruptedException { + indexName = "test-async"; + numShards = randomIntBetween(internalCluster().numDataNodes(), internalCluster().numDataNodes()*10); + numDocs = randomIntBetween(numShards, numShards*10); + createIndex(indexName, Settings.builder().put("index.number_of_shards", numShards).build()); + numKeywords = randomIntBetween(1, 100); + keywordFreqs = new HashMap<>(); + String[] keywords = new String[numKeywords]; + for (int i = 0; i < numKeywords; i++) { + keywords[i] = randomAlphaOfLengthBetween(10, 20); + } + for (int i = 0; i < numDocs; i++) { + List reqs = new ArrayList<>(); + float metric = randomFloat(); + maxMetric = Math.max(metric, maxMetric); + minMetric = Math.min(metric, minMetric); + String keyword = keywords[randomIntBetween(0, numKeywords-1)]; + keywordFreqs.compute(keyword, + (k, v) -> { + if (v == null) { + return new AtomicInteger(1); + } + v.incrementAndGet(); + return v; + }); + reqs.add(client().prepareIndex(indexName).setSource("terms", keyword, "metric", metric)); + indexRandom(true, true, reqs); + } + ensureGreen("test-async"); + } + + public void testMaxMinAggregation() throws Exception { + int step = numShards > 2 ? randomIntBetween(2, numShards) : 2; + int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0; + SearchSourceBuilder source = new SearchSourceBuilder() + .aggregation(AggregationBuilders.min("min").field("metric")) + .aggregation(AggregationBuilders.max("max").field("metric")); + try (SearchResponseIterator it = + assertBlockingIterator(indexName, source, numFailures, step)) { + AsyncSearchResponse response = it.next(); + while (it.hasNext()) { + response = it.next(); + if (response.hasPartialResponse() && response.getPartialResponse().getSuccessfulShards() > 0) { + assertNotNull(response.getPartialResponse().getAggregations()); + assertNotNull(response.getPartialResponse().getAggregations().get("max")); + assertNotNull(response.getPartialResponse().getAggregations().get("min")); + InternalMax max = response.getPartialResponse().getAggregations().get("max"); + InternalMin min = response.getPartialResponse().getAggregations().get("min"); + assertThat((float) min.getValue(), greaterThanOrEqualTo(minMetric)); + assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric)); + } + } + if (numFailures == numShards) { + assertTrue(response.hasFailed()); + } else { + assertTrue(response.isFinalResponse()); + assertNotNull(response.getSearchResponse().getAggregations()); + assertNotNull(response.getSearchResponse().getAggregations().get("max")); + assertNotNull(response.getSearchResponse().getAggregations().get("min")); + InternalMax max = response.getSearchResponse().getAggregations().get("max"); + InternalMin min = response.getSearchResponse().getAggregations().get("min"); + if (numFailures == 0) { + assertThat((float) min.getValue(), equalTo(minMetric)); + assertThat((float) max.getValue(), equalTo(maxMetric)); + } else { + assertThat((float) min.getValue(), greaterThanOrEqualTo(minMetric)); + assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric)); + } + } + waitTaskRemoval(response.id()); + } + } + + public void testTermsAggregation() throws Exception { + int step = numShards > 2 ? randomIntBetween(2, numShards) : 2; + int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0; + int termsSize = randomIntBetween(1, numKeywords); + SearchSourceBuilder source = new SearchSourceBuilder() + .aggregation(AggregationBuilders.terms("terms").field("terms.keyword").size(termsSize).shardSize(termsSize*2)); + try (SearchResponseIterator it = + assertBlockingIterator(indexName, source, numFailures, step)) { + AsyncSearchResponse response = it.next(); + while (it.hasNext()) { + response = it.next(); + if (response.hasPartialResponse() && response.getPartialResponse().getSuccessfulShards() > 0) { + assertNotNull(response.getPartialResponse().getAggregations()); + assertNotNull(response.getPartialResponse().getAggregations().get("terms")); + StringTerms terms = response.getPartialResponse().getAggregations().get("terms"); + assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0)); + assertThat(terms.getBuckets().size(), lessThanOrEqualTo(termsSize)); + for (InternalTerms.Bucket bucket : terms.getBuckets()) { + long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get(); + assertThat(bucket.getDocCount(), lessThanOrEqualTo(count)); + } + } + } + if (numFailures == numShards) { + assertTrue(response.hasFailed()); + } else { + assertTrue(response.isFinalResponse()); + assertNotNull(response.getSearchResponse().getAggregations()); + assertNotNull(response.getSearchResponse().getAggregations().get("terms")); + StringTerms terms = response.getSearchResponse().getAggregations().get("terms"); + assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0)); + assertThat(terms.getBuckets().size(), lessThanOrEqualTo(termsSize)); + for (InternalTerms.Bucket bucket : terms.getBuckets()) { + long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get(); + if (numFailures > 0) { + assertThat(bucket.getDocCount(), lessThanOrEqualTo(count)); + } else { + assertThat(bucket.getDocCount(), equalTo(count)); + } + } + } + waitTaskRemoval(response.id()); + } + } + + public void testRestartAfterCompletion() throws Exception { + final AsyncSearchResponse initial; + try (SearchResponseIterator it = + assertBlockingIterator(indexName, new SearchSourceBuilder(), 0, 2)) { + initial = it.next(); + } + waitTaskCompletion(initial.id()); + restartTaskNode(initial.id()); + AsyncSearchResponse response = getAsyncSearch(initial.id()); + assertTrue(response.isFinalResponse()); + assertFalse(response.isRunning()); + assertFalse(response.hasPartialResponse()); + waitTaskRemoval(response.id()); + } + + public void testDeleteCancelRunningTask() throws Exception { + final AsyncSearchResponse initial; + SearchResponseIterator it = + assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); + initial = it.next(); + deleteAsyncSearch(initial.id()); + it.close(); + waitTaskCompletion(initial.id()); + waitTaskRemoval(initial.id()); + } + + public void testDeleteCleanupIndex() throws Exception { + SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { indexName }); + request.setWaitForCompletion(TimeValue.timeValueMillis(1)); + SearchResponseIterator it = + assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); + AsyncSearchResponse response = it.next(); + deleteAsyncSearch(response.id()); + it.close(); + waitTaskCompletion(response.id()); + waitTaskRemoval(response.id()); + } + + public void testCleanupOnFailure() throws Exception { + final AsyncSearchResponse initial; + try (SearchResponseIterator it = + assertBlockingIterator(indexName, new SearchSourceBuilder(), numShards, 2)) { + initial = it.next(); + } + waitTaskCompletion(initial.id()); + AsyncSearchResponse response = getAsyncSearch(initial.id()); + assertTrue(response.hasFailed()); + assertTrue(response.hasPartialResponse()); + assertThat(response.getPartialResponse().getTotalShards(), equalTo(numShards)); + assertThat(response.getPartialResponse().getShardFailures(), equalTo(numShards)); + waitTaskRemoval(initial.id()); + } +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java new file mode 100644 index 0000000000000..7f07bd5715763 --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -0,0 +1,381 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.apache.lucene.search.Query; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; +import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup; +import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; +import org.elasticsearch.xpack.ilm.IndexLifecycle; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public abstract class AsyncSearchIntegTestCase extends ESIntegTestCase { + interface SearchResponseIterator extends Iterator, Closeable {} + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class, QueryBlockPlugin.class); + } + + /** + * Restart the node that runs the {@link TaskId} decoded from the provided {@link AsyncSearchId}. + */ + protected void restartTaskNode(String id) throws Exception { + AsyncSearchId searchId = AsyncSearchId.decode(id); + final ClusterStateResponse clusterState = client().admin().cluster() + .prepareState().clear().setNodes(true).get(); + DiscoveryNode node = clusterState.getState().nodes().get(searchId.getTaskId().getNodeId()); + internalCluster().restartNode(node.getName(), new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) throws Exception { + return super.onNodeStopped(nodeName); + } + }); + ensureGreen(searchId.getIndexName()); + } + + protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request) throws ExecutionException, InterruptedException { + return client().execute(SubmitAsyncSearchAction.INSTANCE, request).get(); + } + + protected AsyncSearchResponse getAsyncSearch(String id) throws ExecutionException, InterruptedException { + return client().execute(GetAsyncSearchAction.INSTANCE, + new GetAsyncSearchAction.Request(id, TimeValue.timeValueMillis(1), -1)).get(); + } + + protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionException, InterruptedException { + return client().execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(id)).get(); + } + + /** + * Wait the removal of the document decoded from the provided {@link AsyncSearchId}. + */ + protected void waitTaskRemoval(String id) throws InterruptedException, IOException { + AsyncSearchId searchId = AsyncSearchId.decode(id); + assertTrue(waitUntil(() -> client().prepareGet().setRouting(searchId.getDocId()) + .setIndex(searchId.getIndexName()).setId(searchId.getDocId()).get().isExists() == false) + ); + } + + /** + * Wait the completion of the {@link TaskId} decoded from the provided {@link AsyncSearchId}. + */ + protected void waitTaskCompletion(String id) throws InterruptedException { + assertTrue(waitUntil( + () -> { + try { + TaskId taskId = AsyncSearchId.decode(id).getTaskId(); + GetTaskResponse resp = client().admin().cluster() + .prepareGetTask(taskId).get(); + return resp.getTask() == null; + } catch (Exception e) { + return true; + } + })); + } + + protected SearchResponseIterator assertBlockingIterator(String indexName, + SearchSourceBuilder source, + int numFailures, + int progressStep) throws Exception { + SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { indexName }, source); + request.setBatchedReduceSize(progressStep); + request.setWaitForCompletion(TimeValue.timeValueMillis(1)); + ClusterSearchShardsResponse response = dataNodeClient().admin().cluster().prepareSearchShards(request.indices()).get(); + AtomicInteger failures = new AtomicInteger(numFailures); + Map shardLatchMap = Arrays.stream(response.getGroups()) + .map(ClusterSearchShardsGroup::getShardId) + .collect( + Collectors.toMap( + Function.identity(), + id -> new ShardIdLatch(id, new CountDownLatch(1), failures.decrementAndGet() >= 0 ? true : false) + ) + ); + ShardIdLatch[] shardLatchArray = shardLatchMap.values().stream() + .sorted(Comparator.comparing(ShardIdLatch::shard)) + .toArray(ShardIdLatch[]::new); + resetPluginsLatch(shardLatchMap); + request.source().query(new BlockQueryBuilder(shardLatchMap)); + + final AsyncSearchResponse initial; + { + AsyncSearchResponse resp = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get(); + while (resp.getPartialResponse().getSuccessfulShards() == -1) { + resp = client().execute(GetAsyncSearchAction.INSTANCE, + new GetAsyncSearchAction.Request(resp.id(), TimeValue.timeValueSeconds(1), resp.getVersion())).get(); + } + initial = resp; + } + + assertTrue(initial.hasPartialResponse()); + assertThat(initial.status(), equalTo(RestStatus.PARTIAL_CONTENT)); + assertThat(initial.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); + assertThat(initial.getPartialResponse().getSuccessfulShards(), equalTo(0)); + assertThat(initial.getPartialResponse().getShardFailures(), equalTo(0)); + + return new SearchResponseIterator() { + private AsyncSearchResponse response = initial; + private int lastVersion = initial.getVersion(); + private int shardIndex = 0; + private boolean isFirst = true; + private int shardFailures = 0; + + @Override + public boolean hasNext() { + return response.isRunning(); + } + + @Override + public AsyncSearchResponse next() { + if (isFirst) { + isFirst = false; + return response; + } + AtomicReference atomic = new AtomicReference<>(); + AtomicReference exc = new AtomicReference<>(); + int step = shardIndex == 0 ? progressStep+1 : progressStep-1; + int index = 0; + while (index < step && shardIndex < shardLatchArray.length) { + if (shardLatchArray[shardIndex].shouldFail == false) { + ++ index; + } else { + ++ shardFailures; + } + shardLatchArray[shardIndex++].countDown(); + } + try { + assertTrue(waitUntil(() -> { + try { + AsyncSearchResponse newResp = client().execute(GetAsyncSearchAction.INSTANCE, + new GetAsyncSearchAction.Request(response.id(), TimeValue.timeValueMillis(10), lastVersion) + ).get(); + atomic.set(newResp); + return newResp.status() != RestStatus.NOT_MODIFIED; + } catch (Exception e) { + exc.set(e); + return true; + } + })); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (exc.get() != null) { + throw new ElasticsearchException(exc.get()); + } + AsyncSearchResponse newResponse = atomic.get(); + lastVersion = newResponse.getVersion(); + + if (newResponse.isRunning()) { + assertThat(newResponse.status(), equalTo(RestStatus.PARTIAL_CONTENT)); + assertTrue(newResponse.hasPartialResponse()); + assertFalse(newResponse.hasFailed()); + assertFalse(newResponse.isFinalResponse()); + assertThat(newResponse.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); + assertThat(newResponse.getPartialResponse().getShardFailures(), lessThanOrEqualTo(numFailures)); + } else if (numFailures == shardLatchArray.length) { + assertThat(newResponse.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); + assertTrue(newResponse.hasFailed()); + assertTrue(newResponse.hasPartialResponse()); + assertFalse(newResponse.isFinalResponse()); + assertThat(newResponse.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); + assertThat(newResponse.getPartialResponse().getSuccessfulShards(), equalTo(0)); + assertThat(newResponse.getPartialResponse().getShardFailures(), equalTo(numFailures)); + assertNull(newResponse.getPartialResponse().getAggregations()); + assertNull(newResponse.getPartialResponse().getTotalHits()); + } else { + assertThat(newResponse.status(), equalTo(RestStatus.OK)); + assertTrue(newResponse.isFinalResponse()); + assertFalse(newResponse.hasPartialResponse()); + assertThat(newResponse.status(), equalTo(RestStatus.OK)); + assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length)); + assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures)); + assertThat(newResponse.getSearchResponse().getSuccessfulShards(), + equalTo(shardLatchArray.length-newResponse.getSearchResponse().getShardFailures().length)); + } + return response = newResponse; + } + + @Override + public void close() { + Arrays.stream(shardLatchArray).forEach(shard -> { + if (shard.latch.getCount() == 1) { + shard.latch.countDown(); + } + }); + } + }; + } + + private void resetPluginsLatch(Map newLatch) { + for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) { + pluginsService.filterPlugins(QueryBlockPlugin.class).forEach(p -> p.reset(newLatch)); + } + } + + public static class QueryBlockPlugin extends Plugin implements SearchPlugin { + private Map shardsLatch; + + public QueryBlockPlugin() { + this.shardsLatch = null; + } + + public void reset(Map newLatch) { + shardsLatch = newLatch; + } + + @Override + public List> getQueries() { + return Collections.singletonList( + new QuerySpec<>("block_match_all", + in -> new BlockQueryBuilder(in, shardsLatch), + p -> BlockQueryBuilder.fromXContent(p, shardsLatch)) + ); + } + } + + private static class BlockQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "block_match_all"; + private final Map shardsLatch; + + private BlockQueryBuilder(Map shardsLatch) { + super(); + this.shardsLatch = shardsLatch; + } + + BlockQueryBuilder(StreamInput in, Map shardsLatch) throws IOException { + super(in); + this.shardsLatch = shardsLatch; + } + + private BlockQueryBuilder() { + this.shardsLatch = null; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException {} + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.endObject(); + } + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, BlockQueryBuilder::new); + + public static BlockQueryBuilder fromXContent(XContentParser parser, Map shardsLatch) { + try { + PARSER.apply(parser, null); + return new BlockQueryBuilder(shardsLatch); + } catch (IllegalArgumentException e) { + throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e); + } + } + + @Override + protected Query doToQuery(QueryShardContext context) { + if (shardsLatch != null) { + try { + final ShardIdLatch latch = shardsLatch.get(new ShardId(context.index(), context.getShardId())); + latch.await(); + if (latch.shouldFail) { + throw new IllegalStateException("boum"); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return Queries.newMatchAllQuery(); + } + + @Override + protected boolean doEquals(BlockQueryBuilder other) { + return true; + } + + @Override + protected int doHashCode() { + return 0; + } + + @Override + public String getWriteableName() { + return NAME; + } + } + + private static class ShardIdLatch { + private final ShardId shard; + private final CountDownLatch latch; + private final boolean shouldFail; + + private ShardIdLatch(ShardId shard, CountDownLatch latch, boolean shouldFail) { + this.shard = shard; + this.latch = latch; + this.shouldFail = shouldFail; + } + + ShardId shard() { + return shard; + } + + void countDown() { + latch.countDown(); + } + + void await() throws InterruptedException { + latch.await(); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 6dad4ef688a05..7dffd21847a89 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -107,9 +107,9 @@ public boolean hasFailed() { } /** - * Return true if a partial response is available through. + * Return true if a partial response is available. */ - public boolean isPartialResponse() { + public boolean hasPartialResponse() { return partialResponse != null; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java index 8e6eb4fa44df9..3537d8acf26b3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java @@ -92,7 +92,7 @@ public int getTotalShards() { /** * The successful number of shards the search was executed on. */ - public int getSuccesfullShards() { + public int getSuccessfulShards() { return successfulShards; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 62fcf05a6b00d..04d59ce3863de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -29,6 +29,14 @@ public class SubmitAsyncSearchRequest extends SearchRequest implements CompositeIndicesRequest { private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1); + /** + * Create a new request + * @param indices The indices the search will be executed on. + */ + public SubmitAsyncSearchRequest(String... indices) { + this(indices, new SearchSourceBuilder()); + } + /** * Create a new request * @param indices The indices the search will be executed on. @@ -39,14 +47,7 @@ public SubmitAsyncSearchRequest(String[] indices, SearchSourceBuilder source) { setCcsMinimizeRoundtrips(false); setPreFilterShardSize(1); setBatchedReduceSize(5); - } - - /** - * Create a new request - * @param indices The indices the search will be executed on. - */ - public SubmitAsyncSearchRequest(String... indices) { - this(indices, new SearchSourceBuilder()); + requestCache(true); } public SubmitAsyncSearchRequest(StreamInput in) throws IOException { From 52085bbe616926f1b3f99edaccedc0a6cb192e51 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 12 Dec 2019 21:38:12 +0100 Subject: [PATCH 07/61] simplify schema for the .async-search index --- .../xpack/search/AsyncSearch.java | 2 +- .../xpack/search/AsyncSearchStoreService.java | 35 ++++++++----------- ....java => AsyncSearchTemplateRegistry.java} | 32 ++++++++--------- .../search/AsyncSearchStoreServiceTests.java | 4 +-- .../action/SubmitAsyncSearchRequest.java | 2 +- .../main/resources/async-search-history.json | 29 --------------- ...licy.json => async-search-ilm-policy.json} | 0 .../core/src/main/resources/async-search.json | 25 +++++++++++++ 8 files changed, 59 insertions(+), 70 deletions(-) rename x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/{AsyncSearchHistoryTemplateRegistry.java => AsyncSearchTemplateRegistry.java} (71%) delete mode 100644 x-pack/plugin/core/src/main/resources/async-search-history.json rename x-pack/plugin/core/src/main/resources/{async-search-history-ilm-policy.json => async-search-ilm-policy.json} (100%) create mode 100644 x-pack/plugin/core/src/main/resources/async-search.json diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index 221f16fdf3cae..edb67b0fb0614 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -68,7 +68,7 @@ public Collection createComponents(Client client, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - new AsyncSearchHistoryTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry); + new AsyncSearchTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry); return Collections.emptyList(); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index a1cf4439dcfc4..6737fc384963f 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -31,16 +30,15 @@ import java.util.Base64; import java.util.Collections; -import static org.elasticsearch.xpack.search.AsyncSearchHistoryTemplateRegistry.INDEX_TEMPLATE_VERSION; -import static org.elasticsearch.xpack.search.AsyncSearchHistoryTemplateRegistry.ASYNC_SEARCH_HISTORY_TEMPLATE_NAME; +import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.INDEX_TEMPLATE_VERSION; +import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_TEMPLATE_NAME; /** - * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the async - * search history index. + * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the .async-search index. */ class AsyncSearchStoreService { - static final String ASYNC_SEARCH_HISTORY_ALIAS = ASYNC_SEARCH_HISTORY_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; - static final String RESPONSE_FIELD = "response"; + static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; + static final String RESULT_FIELD = "result"; private final Client client; private final NamedWriteableRegistry registry; @@ -51,11 +49,11 @@ class AsyncSearchStoreService { } /** - * Store an empty document in the async search history index that is used + * Store an empty document in the .async-search index that is used * as a place-holder for the future response. */ void storeInitialResponse(ActionListener next) { - IndexRequest request = new IndexRequest(ASYNC_SEARCH_HISTORY_ALIAS).source(Collections.emptyMap(), XContentType.JSON); + IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS).source(Collections.emptyMap(), XContentType.JSON); client.index(request, next); } @@ -65,29 +63,26 @@ void storeInitialResponse(ActionListener next) { void storeFinalResponse(AsyncSearchResponse response, ActionListener next) throws IOException { AsyncSearchId searchId = AsyncSearchId.decode(response.id()); UpdateRequest request = new UpdateRequest().index(searchId.getIndexName()).id(searchId.getDocId()) - .doc(Collections.singletonMap(RESPONSE_FIELD, encodeResponse(response)), XContentType.JSON) + .doc(Collections.singletonMap(RESULT_FIELD, encodeResponse(response)), XContentType.JSON) .detectNoop(false); client.update(request, next); } /** - * Get the final response from the async search history index if present, or delegate a {@link ResourceNotFoundException} + * Get the final response from the .async-search index if present, or delegate a {@link ResourceNotFoundException} * failure to the provided listener if not. */ void getResponse(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener next) { - GetRequest internalGet = new GetRequest(searchId.getIndexName()) - .id(searchId.getDocId()) - .storedFields(RESPONSE_FIELD); + GetRequest internalGet = new GetRequest(searchId.getIndexName()).id(searchId.getDocId()); client.get(internalGet, ActionListener.wrap( get -> { if (get.isExists() == false) { next.onFailure(new ResourceNotFoundException(request.getId() + " not found")); - } else if (get.getFields().containsKey(RESPONSE_FIELD) == false) { + } else if (get.getSource().containsKey(RESULT_FIELD) == false) { next.onResponse(new AsyncSearchResponse(request.getId(), new PartialSearchResponse(-1), 0, false)); } else { - - BytesArray bytesArray = get.getFields().get(RESPONSE_FIELD).getValue(); - next.onResponse(decodeResponse(bytesArray.array(), registry)); + String encoded = (String) get.getSource().get(RESULT_FIELD); + next.onResponse(decodeResponse(encoded, registry)); } }, exc -> next.onFailure(new ResourceNotFoundException(request.getId() + " not found")) @@ -108,8 +103,8 @@ static String encodeResponse(AsyncSearchResponse response) throws IOException { /** * Decode the provided base-64 bytes into a {@link AsyncSearchResponse}. */ - static AsyncSearchResponse decodeResponse(byte[] value, NamedWriteableRegistry registry) throws IOException { - try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(value))) { + static AsyncSearchResponse decodeResponse(String value, NamedWriteableRegistry registry) throws IOException { + try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getDecoder().decode(value)))) { try (StreamInput in = new NamedWriteableAwareStreamInput(buf, registry)) { in.setVersion(Version.readVersion(in)); return new AsyncSearchResponse(in); diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchHistoryTemplateRegistry.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java similarity index 71% rename from x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchHistoryTemplateRegistry.java rename to x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java index 2f57d219d345d..56355fc3d8a49 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchHistoryTemplateRegistry.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java @@ -28,34 +28,34 @@ import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; /** - * Manage the index template and associated ILM policy for the async search history index. + * Manage the index template and associated ILM policy for the .async-search index. */ -public class AsyncSearchHistoryTemplateRegistry extends IndexTemplateRegistry { +public class AsyncSearchTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial public static final String INDEX_TEMPLATE_VERSION = "1"; - public static final String ASYNC_SEARCH_HISTORY_TEMPLATE_VERSION_VARIABLE = "xpack.async-search-history.template.version"; - public static final String ASYNC_SEARCH_HISTORY_TEMPLATE_NAME = ".async-search-history"; + public static final String ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE = "xpack.async-search.template.version"; + public static final String ASYNC_SEARCH_TEMPLATE_NAME = ".async-search"; - public static final String ASYNC_SEARCH_POLICY_NAME = "async-search-history-ilm-policy"; + public static final String ASYNC_SEARCH_POLICY_NAME = "async-search-ilm-policy"; public static final IndexTemplateConfig TEMPLATE_ASYNC_SEARCH = new IndexTemplateConfig( - ASYNC_SEARCH_HISTORY_TEMPLATE_NAME, - "/async-search-history.json", + ASYNC_SEARCH_TEMPLATE_NAME, + "/async-search.json", INDEX_TEMPLATE_VERSION, - ASYNC_SEARCH_HISTORY_TEMPLATE_VERSION_VARIABLE + ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE ); - public static final LifecyclePolicyConfig ASYNC_SEARCH_HISTORY_POLICY = new LifecyclePolicyConfig( + public static final LifecyclePolicyConfig ASYNC_SEARCH_POLICY = new LifecyclePolicyConfig( ASYNC_SEARCH_POLICY_NAME, - "/async-search-history-ilm-policy.json" + "/async-search-ilm-policy.json" ); - public AsyncSearchHistoryTemplateRegistry(Settings nodeSettings, - ClusterService clusterService, - ThreadPool threadPool, - Client client, - NamedXContentRegistry xContentRegistry) { + public AsyncSearchTemplateRegistry(Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry); } @@ -66,7 +66,7 @@ protected List getTemplateConfigs() { @Override protected List getPolicyConfigs() { - return Collections.singletonList(ASYNC_SEARCH_HISTORY_POLICY); + return Collections.singletonList(ASYNC_SEARCH_POLICY); } @Override diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java index 6277da2a5bd7c..6dad76dbbdbcd 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java @@ -13,7 +13,6 @@ import org.junit.Before; import java.io.IOException; -import java.util.Base64; import java.util.List; import static java.util.Collections.emptyList; @@ -28,7 +27,6 @@ public class AsyncSearchStoreServiceTests extends ESTestCase { @Before public void registerNamedObjects() { SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - List namedWriteables = searchModule.getNamedWriteables(); namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); } @@ -37,7 +35,7 @@ public void testEncode() throws IOException { for (int i = 0; i < 10; i++) { AsyncSearchResponse response = randomAsyncSearchResponse(randomSearchId(), randomSearchResponse()); String encoded = AsyncSearchStoreService.encodeResponse(response); - AsyncSearchResponse same = AsyncSearchStoreService.decodeResponse(Base64.getDecoder().decode(encoded), namedWriteableRegistry); + AsyncSearchResponse same = AsyncSearchStoreService.decodeResponse(encoded, namedWriteableRegistry); assertEqualResponses(response, same); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 04d59ce3863de..504ab94e3b87c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -36,7 +36,7 @@ public class SubmitAsyncSearchRequest extends SearchRequest implements Composite public SubmitAsyncSearchRequest(String... indices) { this(indices, new SearchSourceBuilder()); } - + /** * Create a new request * @param indices The indices the search will be executed on. diff --git a/x-pack/plugin/core/src/main/resources/async-search-history.json b/x-pack/plugin/core/src/main/resources/async-search-history.json deleted file mode 100644 index 9824745cb835d..0000000000000 --- a/x-pack/plugin/core/src/main/resources/async-search-history.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "index_patterns": [ - ".async-search-history-${xpack.async-search-history.template.version}*" - ], - "order": 2147483647, - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", - "index.lifecycle.name": "async-search-history-policy", - "index.lifecycle.rollover_alias": ".async-search-history${xpack.async-search-history-template.version}", - "index.format": 1 - }, - "mappings": { - "_doc": { - "dynamic": false, - "_source": { - "excludes": ["*"] - }, - "properties": { - "response": { - "type": "binary", - "store": true, - "doc_values": false - } - } - } - } -} diff --git a/x-pack/plugin/core/src/main/resources/async-search-history-ilm-policy.json b/x-pack/plugin/core/src/main/resources/async-search-ilm-policy.json similarity index 100% rename from x-pack/plugin/core/src/main/resources/async-search-history-ilm-policy.json rename to x-pack/plugin/core/src/main/resources/async-search-ilm-policy.json diff --git a/x-pack/plugin/core/src/main/resources/async-search.json b/x-pack/plugin/core/src/main/resources/async-search.json new file mode 100644 index 0000000000000..9f5fae1594b67 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/async-search.json @@ -0,0 +1,25 @@ +{ + "index_patterns": [ + ".async-search-${xpack.async-search.template.version}*" + ], + "order": 2147483647, + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.lifecycle.name": "async-search", + "index.lifecycle.rollover_alias": ".async-search-${xpack.async-search-template.version}", + "index.format": 1 + }, + "mappings": { + "_doc": { + "dynamic": false, + "properties": { + "result": { + "type": "object", + "enabled": false + } + } + } + } +} From da7329d7c39af7f8b76a0b0449fc6996c305b581 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 12 Dec 2019 21:52:45 +0100 Subject: [PATCH 08/61] switch to true random uuuids --- .../xpack/search/AsyncSearchStoreService.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 6737fc384963f..63d8080e0e69b 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -29,6 +30,7 @@ import java.nio.ByteBuffer; import java.util.Base64; import java.util.Collections; +import java.util.Random; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.INDEX_TEMPLATE_VERSION; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_TEMPLATE_NAME; @@ -42,10 +44,12 @@ class AsyncSearchStoreService { private final Client client; private final NamedWriteableRegistry registry; + private final Random random; AsyncSearchStoreService(Client client, NamedWriteableRegistry registry) { this.client = client; this.registry = registry; + this.random = new Random(System.nanoTime()); } /** @@ -53,7 +57,9 @@ class AsyncSearchStoreService { * as a place-holder for the future response. */ void storeInitialResponse(ActionListener next) { - IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS).source(Collections.emptyMap(), XContentType.JSON); + IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS) + .id(UUIDs.randomBase64UUID()) + .source(Collections.emptyMap(), XContentType.JSON); client.index(request, next); } From 20c24a098f63e7323d4b302145258464e456e578 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 13 Dec 2019 11:49:27 +0100 Subject: [PATCH 09/61] iter --- .../xpack/search/AsyncSearchStoreService.java | 6 ++++-- .../search/TransportGetAsyncSearchAction.java | 19 +++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 63d8080e0e69b..083fde47e94a8 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -58,7 +58,7 @@ class AsyncSearchStoreService { */ void storeInitialResponse(ActionListener next) { IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS) - .id(UUIDs.randomBase64UUID()) + .id(UUIDs.randomBase64UUID(random)) .source(Collections.emptyMap(), XContentType.JSON); client.index(request, next); } @@ -79,7 +79,9 @@ void storeFinalResponse(AsyncSearchResponse response, ActionListener next) { - GetRequest internalGet = new GetRequest(searchId.getIndexName()).id(searchId.getDocId()); + GetRequest internalGet = new GetRequest(searchId.getIndexName()) + .id(searchId.getDocId()) + .routing(searchId.getDocId()); client.get(internalGet, ActionListener.wrap( get -> { if (get.isExists() == false) { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 0f13dc71f6593..55008ae32b137 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; @@ -58,12 +57,12 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); listener = wrapCleanupListener(searchId, listener); if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { - getSearchResponseFromTask(task, request, searchId, listener); + getSearchResponseFromTask(request, searchId, listener); } else { TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); if (node == null) { - getSearchResponseFromIndex(task, request, searchId, listener); + getSearchResponseFromIndex(request, searchId, listener); } else { transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); @@ -74,34 +73,30 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action } } - private void getSearchResponseFromTask(Task thisTask, GetAsyncSearchAction.Request request, AsyncSearchId searchId, + private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener listener) { Task runningTask = taskManager.getTask(searchId.getTaskId().getId()); if (runningTask == null) { // Task isn't running - getSearchResponseFromIndex(thisTask, request, searchId, listener); + getSearchResponseFromIndex(request, searchId, listener); return; } if (runningTask instanceof AsyncSearchTask) { AsyncSearchTask searchTask = (AsyncSearchTask) runningTask; if (searchTask.getSearchId().equals(request.getId()) == false) { // Task id has been reused by another task due to a node restart - getSearchResponseFromIndex(thisTask, request, searchId, listener); + getSearchResponseFromIndex(request, searchId, listener); return; } waitForCompletion(request, searchTask, searchId, threadPool.relativeTimeInMillis(), listener); } else { // Task id has been reused by another task due to a node restart - getSearchResponseFromIndex(thisTask, request, searchId, listener); + getSearchResponseFromIndex(request, searchId, listener); } } - private void getSearchResponseFromIndex(Task task, GetAsyncSearchAction.Request request, AsyncSearchId searchId, + private void getSearchResponseFromIndex(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener listener) { - GetRequest get = new GetRequest(searchId.getIndexName(), searchId.getDocId()) - .routing(searchId.getDocId()) - .storedFields("response"); - get.setParentTask(clusterService.localNode().getId(), task.getId()); store.getResponse(request, searchId, wrap( resp -> { From 438a7b064156f5367699c25a4d1bbfb24b70f461 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 13 Dec 2019 13:57:13 +0100 Subject: [PATCH 10/61] replace waitUntil with assertBusy --- .../search/AsyncSearchIntegTestCase.java | 77 ++++++++++--------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index 7f07bd5715763..fcee4464abbef 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -7,11 +7,12 @@ package org.elasticsearch.xpack.search; import org.apache.lucene.search.Query; -import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.ParsingException; @@ -102,28 +103,34 @@ protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionExce /** * Wait the removal of the document decoded from the provided {@link AsyncSearchId}. */ - protected void waitTaskRemoval(String id) throws InterruptedException, IOException { + protected void waitTaskRemoval(String id) throws Exception { AsyncSearchId searchId = AsyncSearchId.decode(id); - assertTrue(waitUntil(() -> client().prepareGet().setRouting(searchId.getDocId()) - .setIndex(searchId.getIndexName()).setId(searchId.getDocId()).get().isExists() == false) - ); + assertBusy(() -> { + GetResponse resp = client().prepareGet() + .setRouting(searchId.getDocId()) + .setIndex(searchId.getIndexName()) + .setId(searchId.getDocId()) + .get(); + assertFalse(resp.isExists()); + }); } /** * Wait the completion of the {@link TaskId} decoded from the provided {@link AsyncSearchId}. */ - protected void waitTaskCompletion(String id) throws InterruptedException { - assertTrue(waitUntil( - () -> { - try { - TaskId taskId = AsyncSearchId.decode(id).getTaskId(); - GetTaskResponse resp = client().admin().cluster() - .prepareGetTask(taskId).get(); - return resp.getTask() == null; - } catch (Exception e) { - return true; + protected void waitTaskCompletion(String id) throws Exception { + assertBusy(() -> { + TaskId taskId = AsyncSearchId.decode(id).getTaskId(); + try { + GetTaskResponse resp = client().admin().cluster() + .prepareGetTask(taskId).get(); + assertNull(resp.getTask()); + } catch (Exception exc) { + if (exc.getCause() instanceof ResourceNotFoundException == false) { + throw exc; } - })); + } + }); } protected SearchResponseIterator assertBlockingIterator(String indexName, @@ -179,6 +186,14 @@ public boolean hasNext() { @Override public AsyncSearchResponse next() { + try { + return doNext(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private AsyncSearchResponse doNext() throws Exception { if (isFirst) { isFirst = false; return response; @@ -189,31 +204,19 @@ public AsyncSearchResponse next() { int index = 0; while (index < step && shardIndex < shardLatchArray.length) { if (shardLatchArray[shardIndex].shouldFail == false) { - ++ index; + ++index; } else { - ++ shardFailures; + ++shardFailures; } shardLatchArray[shardIndex++].countDown(); - } - try { - assertTrue(waitUntil(() -> { - try { - AsyncSearchResponse newResp = client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(response.id(), TimeValue.timeValueMillis(10), lastVersion) - ).get(); - atomic.set(newResp); - return newResp.status() != RestStatus.NOT_MODIFIED; - } catch (Exception e) { - exc.set(e); - return true; - } - })); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (exc.get() != null) { - throw new ElasticsearchException(exc.get()); } + assertBusy(() -> { + AsyncSearchResponse newResp = client().execute(GetAsyncSearchAction.INSTANCE, + new GetAsyncSearchAction.Request(response.id(), TimeValue.timeValueMillis(10), lastVersion) + ).get(); + atomic.set(newResp); + assertNotEquals(RestStatus.NOT_MODIFIED, newResp.status()); + }); AsyncSearchResponse newResponse = atomic.get(); lastVersion = newResponse.getVersion(); From 5a907b9942f1b797d6d4d575c8b77644a6fe3581 Mon Sep 17 00:00:00 2001 From: jimczi Date: Mon, 16 Dec 2019 10:29:44 +0100 Subject: [PATCH 11/61] add a security layer that restricts the usage of the async search APIs to the user that submitted the initial rquest --- .../support/tasks/TransportTasksAction.java | 7 +- .../async-search/qa/security/build.gradle | 19 +++ .../plugin/async-search/qa/security/roles.yml | 33 ++++ .../xpack/search/AsyncSearchSecurityIT.java | 150 ++++++++++++++++++ .../xpack/search/AsyncSearchId.java | 20 +++ .../xpack/search/AsyncSearchStoreService.java | 53 +++++-- .../xpack/search/AsyncSearchTask.java | 32 ++-- .../search/RestGetAsyncSearchAction.java | 3 +- .../search/RestSubmitAsyncSearchAction.java | 14 +- .../TransportDeleteAsyncSearchAction.java | 85 +++------- .../search/TransportGetAsyncSearchAction.java | 101 +++++++++--- .../TransportSubmitAsyncSearchAction.java | 34 ++-- .../search/AsyncSearchIntegTestCase.java | 11 +- .../search/GetAsyncSearchRequestTests.java | 2 +- .../search/SubmitAsyncSearchRequestTests.java | 33 ++-- .../search/action/AsyncSearchResponse.java | 1 + .../search/action/GetAsyncSearchAction.java | 13 +- .../action/SubmitAsyncSearchAction.java | 6 - .../action/SubmitAsyncSearchRequest.java | 65 +++++--- .../core/src/main/resources/async-search.json | 4 + 20 files changed, 505 insertions(+), 181 deletions(-) create mode 100644 x-pack/plugin/async-search/qa/security/build.gradle create mode 100644 x-pack/plugin/async-search/qa/security/roles.yml create mode 100644 x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index a20e72d853af5..7cdc9d331ab3a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -99,7 +99,12 @@ protected void doExecute(Task task, TasksRequest request, ActionListener listener) { TasksRequest request = nodeTaskRequest.tasksRequest; List tasks = new ArrayList<>(); - processTasks(request, tasks::add); + try { + processTasks(request, tasks::add); + } catch (Exception exc) { + listener.onFailure(exc); + return; + } if (tasks.isEmpty()) { listener.onResponse(new NodeTasksResponse(clusterService.localNode().getId(), emptyList(), emptyList())); return; diff --git a/x-pack/plugin/async-search/qa/security/build.gradle b/x-pack/plugin/async-search/qa/security/build.gradle new file mode 100644 index 0000000000000..0337749a9eebd --- /dev/null +++ b/x-pack/plugin/async-search/qa/security/build.gradle @@ -0,0 +1,19 @@ +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(path: xpackModule('async-search'), configuration: 'runtime') + testCompile project(':x-pack:plugin:async-search:qa') +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' + setting 'xpack.security.enabled', 'true' + extraConfigFile 'roles.yml', file('roles.yml') + user username: "test-admin", password: 'x-pack-test-password', role: "test-admin" + user username: "user1", password: 'x-pack-test-password', role: "user1" + user username: "user2", password: 'x-pack-test-password', role: "user2" +} diff --git a/x-pack/plugin/async-search/qa/security/roles.yml b/x-pack/plugin/async-search/qa/security/roles.yml new file mode 100644 index 0000000000000..4ab3be5ff0571 --- /dev/null +++ b/x-pack/plugin/async-search/qa/security/roles.yml @@ -0,0 +1,33 @@ +# All cluster rights +# All operations on all indices +# Run as all users +test-admin: + cluster: + - all + indices: + - names: '*' + privileges: [ all ] + run_as: + - '*' + +user1: + cluster: + - cluster:monitor/main + indices: + - names: ['index-user1', 'index' ] + privileges: + - read + - write + - create_index + - indices:admin/refresh + +user2: + cluster: + - cluster:monitor/main + indices: + - names: [ 'index-user2', 'index' ] + privileges: + - read + - write + - create_index + - indices:admin/refresh diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java new file mode 100644 index 0000000000000..0497a2ffc9592 --- /dev/null +++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER; +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class AsyncSearchSecurityIT extends AsyncSearchRestTestCase { + private static final String tokenUser1 = basicAuthHeaderValue("user1", new SecureString("x-pack-test-password".toCharArray())); + private static final String tokenUser2 = basicAuthHeaderValue("user2", new SecureString("x-pack-test-password".toCharArray())); + + /** + * All tests run as a superuser but use es-security-runas-user to become a less privileged user. + */ + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("test-admin", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } + + @Before + private void indexDocuments() throws IOException { + createIndex("index", Settings.EMPTY); + index("index", "0", "foo", "bar"); + refresh("index"); + + createIndex("index-user1", Settings.EMPTY); + index("index-user1", "0", "foo", "bar"); + refresh("index-user1"); + + createIndex("index-user2", Settings.EMPTY); + index("index-user2", "0", "foo", "bar"); + refresh("index-user2"); + } + + public void testWithUsers() throws Exception { + testCase("user1", "user2"); + testCase("user2", "user1"); + } + + private void testCase(String user, String other) throws Exception { + for (String indexName : new String[] {"index", "index-" + user}) { + Response submitResp = submitAsyncSearch(indexName, "foo:bar", user); + assertOK(submitResp); + String id = extractResponseId(submitResp); + Response getResp = getAsyncSearch(id, user); + assertOK(getResp); + + // other cannot access the result + ResponseException exc = expectThrows(ResponseException.class, () -> getAsyncSearch(id, other)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + + // other cannot delete the result + exc = expectThrows(ResponseException.class, () -> deleteAsyncSearch(id, other)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + + Response delResp = deleteAsyncSearch(id, user); + assertOK(delResp); + } + ResponseException exc = expectThrows(ResponseException.class, + () -> submitAsyncSearch("index-" + other, "*", user)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(500)); + assertThat(exc.getMessage(), containsString("unauthorized")); + } + + static String extractResponseId(Response response) throws IOException { + Map map = toMap(response); + return (String) map.get("id"); + } + + static void index(String index, String id, Object... fields) throws IOException { + XContentBuilder document = jsonBuilder().startObject(); + for (int i = 0; i < fields.length; i += 2) { + document.field((String) fields[i], fields[i + 1]); + } + document.endObject(); + final Request request = new Request("POST", "/" + index + "/_doc/" + id); + request.setJsonEntity(Strings.toString(document)); + assertOK(client().performRequest(request)); + } + + static void refresh(String index) throws IOException { + assertOK(adminClient().performRequest(new Request("POST", "/" + index + "/_refresh"))); + } + + static Response submitAsyncSearch(String indexName, String query, String user) throws IOException { + final Request request = new Request("GET", indexName + "/_async_search"); + setRunAsHeader(request, user); + request.addParameter("q", query); + request.addParameter("wait_for_completion", "0ms"); + // we do the cleanup explicitly + request.addParameter("clean_on_completion", "false"); + return client().performRequest(request); + } + + static Response getAsyncSearch(String id, String user) throws IOException { + final Request request = new Request("GET", "/_async_search/" + id); + setRunAsHeader(request, user); + request.addParameter("wait_for_completion", "0ms"); + // we do the cleanup explicitly + request.addParameter("clean_on_completion", "false"); + return client().performRequest(request); + } + + static Response deleteAsyncSearch(String id, String user) throws IOException { + final Request request = new Request("DELETE", "/_async_search/" + id); + setRunAsHeader(request, user); + return client().performRequest(request); + } + + static Map toMap(Response response) throws IOException { + return toMap(EntityUtils.toString(response.getEntity())); + } + + static Map toMap(String response) { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, response, false); + } + + static void setRunAsHeader(Request request, String user) { + final RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.addHeader(RUN_AS_USER_HEADER, user); + request.setOptions(builder); + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java index 3a5151e12a952..9dd9849cea6bc 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java @@ -24,11 +24,24 @@ class AsyncSearchId { private final String indexName; private final String docId; private final TaskId taskId; + private final String encoded; AsyncSearchId(String indexName, String docId, TaskId taskId) { this.indexName = indexName; this.docId = docId; this.taskId = taskId; + this.encoded = encode(indexName, docId, taskId); + } + + AsyncSearchId(String id) throws IOException { + try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap( Base64.getDecoder().decode(id)))) { + this.indexName = in.readString(); + this.docId = in.readString(); + this.taskId = new TaskId(in.readString()); + this.encoded = id; + } catch (IOException e) { + throw new IOException("invalid id: " + id); + } } /** @@ -52,6 +65,13 @@ TaskId getTaskId() { return taskId; } + /** + * Get the encoded string that represents this search. + */ + String getEncoded() { + return encoded; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 083fde47e94a8..91f4e57c595d6 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; @@ -23,17 +24,21 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; +import org.elasticsearch.xpack.core.security.authc.Authentication; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Base64; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Random; +import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.INDEX_TEMPLATE_VERSION; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_TEMPLATE_NAME; +import static org.elasticsearch.xpack.search.TransportGetAsyncSearchAction.ensureAuthenticatedUserIsSame; /** * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the .async-search index. @@ -41,59 +46,81 @@ class AsyncSearchStoreService { static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; static final String RESULT_FIELD = "result"; + static final String HEADERS_FIELD = "headers"; private final Client client; private final NamedWriteableRegistry registry; private final Random random; AsyncSearchStoreService(Client client, NamedWriteableRegistry registry) { - this.client = client; + this.client = new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN); this.registry = registry; this.random = new Random(System.nanoTime()); } + Client getClient() { + return client; + } + /** * Store an empty document in the .async-search index that is used * as a place-holder for the future response. */ - void storeInitialResponse(ActionListener next) { + void storeInitialResponse(Map headers, ActionListener next) { IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS) .id(UUIDs.randomBase64UUID(random)) - .source(Collections.emptyMap(), XContentType.JSON); + .source(Collections.singletonMap(HEADERS_FIELD, headers), XContentType.JSON); client.index(request, next); } /** * Store the final response if the place-holder document is still present (update). */ - void storeFinalResponse(AsyncSearchResponse response, ActionListener next) throws IOException { + void storeFinalResponse(Map headers, AsyncSearchResponse response, + ActionListener next) throws IOException { AsyncSearchId searchId = AsyncSearchId.decode(response.id()); + Map source = new HashMap<>(); + source.put(RESULT_FIELD, encodeResponse(response)); + source.put(HEADERS_FIELD, headers); UpdateRequest request = new UpdateRequest().index(searchId.getIndexName()).id(searchId.getDocId()) - .doc(Collections.singletonMap(RESULT_FIELD, encodeResponse(response)), XContentType.JSON) + .doc(source, XContentType.JSON) .detectNoop(false); client.update(request, next); } /** - * Get the final response from the .async-search index if present, or delegate a {@link ResourceNotFoundException} + * Get the response from the .async-search index if present, or delegate a {@link ResourceNotFoundException} * failure to the provided listener if not. */ - void getResponse(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener next) { + void getResponse(AsyncSearchId searchId, ActionListener next) { + final Authentication current = Authentication.getAuthentication(client.threadPool().getThreadContext()); GetRequest internalGet = new GetRequest(searchId.getIndexName()) .id(searchId.getDocId()) .routing(searchId.getDocId()); client.get(internalGet, ActionListener.wrap( get -> { if (get.isExists() == false) { - next.onFailure(new ResourceNotFoundException(request.getId() + " not found")); - } else if (get.getSource().containsKey(RESULT_FIELD) == false) { - next.onResponse(new AsyncSearchResponse(request.getId(), new PartialSearchResponse(-1), 0, false)); + next.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); + return; + } + + @SuppressWarnings("unchecked") + Map headers = (Map) get.getSource().get(HEADERS_FIELD); + if (ensureAuthenticatedUserIsSame(headers, current) == false) { + next.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); + return; + } + + @SuppressWarnings("unchecked") + String encoded = (String) get.getSource().get(RESULT_FIELD); + if (encoded == null) { + next.onResponse(new AsyncSearchResponse(searchId.getEncoded(), + new PartialSearchResponse(-1), 0, false)); } else { - String encoded = (String) get.getSource().get(RESULT_FIELD); next.onResponse(decodeResponse(encoded, registry)); } }, - exc -> next.onFailure(new ResourceNotFoundException(request.getId() + " not found")) + next::onFailure )); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 38895f5c24cb9..ef71562c42442 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -30,24 +30,32 @@ * Task that tracks the progress of a currently running {@link SearchRequest}. */ class AsyncSearchTask extends SearchTask { - private final String searchId; + private final AsyncSearchId searchId; private final Supplier reduceContextSupplier; private final Listener progressListener; + private final Map originHeaders; + AsyncSearchTask(long id, String type, String action, - Map headers, - String searchId, + Map originHeaders, + Map taskHeaders, + AsyncSearchId searchId, Supplier reduceContextSupplier) { - super(id, type, action, "async_search", EMPTY_TASK_ID, headers); + super(id, type, action, "async_search", EMPTY_TASK_ID, taskHeaders); + this.originHeaders = originHeaders; this.searchId = searchId; this.reduceContextSupplier = reduceContextSupplier; this.progressListener = new Listener(); setProgressListener(progressListener); } - String getSearchId() { + Map getOriginHeaders() { + return originHeaders; + } + + AsyncSearchId getSearchId() { return searchId; } @@ -75,7 +83,7 @@ private class Listener extends SearchProgressActionListener { private volatile Response response; Listener() { - final AsyncSearchResponse initial = new AsyncSearchResponse(searchId, + final AsyncSearchResponse initial = new AsyncSearchResponse(searchId.getEncoded(), new PartialSearchResponse(totalShards), version.get(), true); this.response = new Response(initial, false); } @@ -83,7 +91,7 @@ private class Listener extends SearchProgressActionListener { @Override public void onListShards(List shards, boolean fetchPhase) { this.totalShards = shards.size(); - final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, + final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), new PartialSearchResponse(totalShards), version.incrementAndGet(), true); response = new Response(newResp, false); } @@ -97,7 +105,7 @@ public void onQueryFailure(int shardIndex, Exception exc) { public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { lastSuccess = shards.size(); lastFailures = shardFailures.get(); - final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, + final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), new PartialSearchResponse(totalShards, lastSuccess, lastFailures, totalHits, aggs), version.incrementAndGet(), true @@ -109,7 +117,7 @@ public void onPartialReduce(List shards, TotalHits totalHits, Inter public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { int failures = shardFailures.get(); int ver = (lastSuccess == shards.size() && lastFailures == failures) ? version.get() : version.incrementAndGet(); - final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, + final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), new PartialSearchResponse(totalShards, shards.size(), failures, totalHits, aggs), ver, true @@ -119,14 +127,16 @@ public void onReduce(List shards, TotalHits totalHits, InternalAggr @Override public void onResponse(SearchResponse searchResponse) { - AsyncSearchResponse newResp = new AsyncSearchResponse(searchId, searchResponse, version.incrementAndGet(), false); + AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), + searchResponse, version.incrementAndGet(), false); response = new Response(newResp, false); } @Override public void onFailure(Exception exc) { AsyncSearchResponse previous = response.get(true); - response = new Response(new AsyncSearchResponse(searchId, newPartialResponse(previous, shardFailures.get()), + response = new Response(new AsyncSearchResponse(searchId.getEncoded(), + newPartialResponse(previous, shardFailures.get()), exc != null ? new ElasticsearchException(exc) : null, version.incrementAndGet(), false), false); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index 1079112e4ca18..191edbbfe0635 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -33,7 +33,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli String id = request.param("id"); int lastVersion = request.paramAsInt("last_version", -1); TimeValue waitForCompletion = request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1)); - GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(id, waitForCompletion, lastVersion); + boolean cleanOnCompletion = request.paramAsBoolean("clean_on_completion", true); + GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(id, waitForCompletion, lastVersion, cleanOnCompletion); return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java index 4698ab26439e9..7ef20bee03cc4 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -38,18 +38,20 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - SubmitAsyncSearchRequest searchRequest = new SubmitAsyncSearchRequest(); - IntConsumer setSize = size -> searchRequest.source().size(size); - request.withContentOrSourceParamParserOrNull(parser -> parseSearchRequest(searchRequest, request, parser, setSize)); - searchRequest.setWaitForCompletion(request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1))); + SubmitAsyncSearchRequest submitRequest = new SubmitAsyncSearchRequest(); + IntConsumer setSize = size -> submitRequest.getSearchRequest().source().size(size); + request.withContentOrSourceParamParserOrNull(parser -> + parseSearchRequest(submitRequest.getSearchRequest(), request, parser, setSize)); + submitRequest.setWaitForCompletion(request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1))); + submitRequest.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", true)); - ActionRequestValidationException validationException = searchRequest.validate(); + ActionRequestValidationException validationException = submitRequest.validate(); if (validationException != null) { throw validationException; } return channel -> { RestStatusToXContentListener listener = new RestStatusToXContentListener<>(channel); - client.executeLocally(SubmitAsyncSearchAction.INSTANCE, searchRequest, listener); + client.execute(SubmitAsyncSearchAction.INSTANCE, submitRequest, listener); }; } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java index cecccbd0e3e50..28de8e0aca4cd 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -7,102 +7,69 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; import java.io.IOException; public class TransportDeleteAsyncSearchAction extends HandledTransportAction { - private final ClusterService clusterService; - private final TransportService transportService; - private final Client client; + private final NodeClient nodeClient; + private final AsyncSearchStoreService store; @Inject public TransportDeleteAsyncSearchAction(TransportService transportService, ActionFilters actionFilters, - ClusterService clusterService, + NamedWriteableRegistry registry, + NodeClient nodeClient, Client client) { super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new); - this.clusterService = clusterService; - this.transportService = transportService; - this.client = client; + this.nodeClient = nodeClient; + this.store = new AsyncSearchStoreService(client, registry); } @Override protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); - if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { - cancelTaskOnNode(request, searchId, listener); - } else { - TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); - DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); - if (node == null) { - deleteResult(request, searchId, listener, false); - } else { - transportService.sendRequest(node, DeleteAsyncSearchAction.NAME, request, builder.build(), - new ActionListenerResponseHandler<>(listener, AcknowledgedResponse::new, ThreadPool.Names.SAME)); - } - } - } catch (IOException e) { - listener.onFailure(e); + // check if the response can be retrieved (handle security) + store.getResponse(searchId, ActionListener.wrap(res -> cancelTask(searchId, listener), listener::onFailure)); + } catch (IOException exc) { + listener.onFailure(exc); } } - private void cancelTaskOnNode(DeleteAsyncSearchAction.Request request, AsyncSearchId searchId, - ActionListener listener) { - Task runningTask = taskManager.getTask(searchId.getTaskId().getId()); - if (runningTask == null) { - deleteResult(request, searchId, listener, false); - return; - } - if (runningTask instanceof AsyncSearchTask) { - AsyncSearchTask searchTask = (AsyncSearchTask) runningTask; - if (searchTask.getSearchId().equals(request.getId()) - && searchTask.isCancelled() == false) { - taskManager.cancel(searchTask, "cancel", () -> {}); - deleteResult(request, searchId, listener, true); - } else { - // Task id has been reused by another task due to a node restart - deleteResult(request, searchId, listener, false); - } - } else { - // Task id has been reused by another task due to a node restart - deleteResult(request, searchId, listener, false); + private void cancelTask(AsyncSearchId searchId, ActionListener listener) { + try { + nodeClient.execute(CancelTasksAction.INSTANCE, new CancelTasksRequest().setTaskId(searchId.getTaskId()), + ActionListener.wrap(() -> deleteResult(searchId, listener))); + } catch (Exception e) { + deleteResult(searchId, listener); } } - private void deleteResult(DeleteAsyncSearchAction.Request orig, AsyncSearchId searchId, - ActionListener next, boolean foundTask) { + private void deleteResult(AsyncSearchId searchId, ActionListener next) { DeleteRequest request = new DeleteRequest(searchId.getIndexName()).id(searchId.getDocId()); - client.delete(request, ActionListener.wrap( + store.getClient().delete(request, ActionListener.wrap( resp -> { - if (resp.status() == RestStatus.NOT_FOUND && foundTask == false) { - next.onFailure(new ResourceNotFoundException("id [{}] not found", orig.getId())); + if (resp.status() == RestStatus.NOT_FOUND) { + next.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded())); } else { next.onResponse(new AcknowledgedResponse(true)); } }, - exc -> { - if (foundTask == false) { - next.onFailure(new ResourceNotFoundException("id [{}] not found", orig.getId())); - } else { - next.onResponse(new AcknowledgedResponse(true)); - } - } - )); + exc -> next.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded()))) + ); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 55008ae32b137..bcadee6752e9b 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -5,9 +5,11 @@ */ package org.elasticsearch.xpack.search; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; -import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; @@ -16,20 +18,23 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.rest.RestResponse; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; -import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.xpack.core.security.authc.Authentication; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; public class TransportGetAsyncSearchAction extends HandledTransportAction { + private Logger logger = LogManager.getLogger(getClass()); private final ClusterService clusterService; private final ThreadPool threadPool; private final TransportService transportService; @@ -53,9 +58,9 @@ public TransportGetAsyncSearchAction(TransportService transportService, @Override protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { + listener = wrapCleanupListener(request, listener); try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); - listener = wrapCleanupListener(searchId, listener); if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { getSearchResponseFromTask(request, searchId, listener); } else { @@ -74,7 +79,7 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action } private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, AsyncSearchId searchId, - ActionListener listener) { + ActionListener listener) throws IOException { Task runningTask = taskManager.getTask(searchId.getTaskId().getId()); if (runningTask == null) { // Task isn't running @@ -83,12 +88,19 @@ private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, Asy } if (runningTask instanceof AsyncSearchTask) { AsyncSearchTask searchTask = (AsyncSearchTask) runningTask; - if (searchTask.getSearchId().equals(request.getId()) == false) { + if (searchTask.getSearchId().getEncoded().equals(request.getId()) == false) { // Task id has been reused by another task due to a node restart getSearchResponseFromIndex(request, searchId, listener); return; } - waitForCompletion(request, searchTask, searchId, threadPool.relativeTimeInMillis(), listener); + + // Check authentication for the user + final Authentication auth = Authentication.getAuthentication(threadPool.getThreadContext()); + if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { + listener.onFailure(new ResourceNotFoundException(request.getId())); + return; + } + waitForCompletion(request, searchTask, threadPool.relativeTimeInMillis(), listener); } else { // Task id has been reused by another task due to a node restart getSearchResponseFromIndex(request, searchId, listener); @@ -97,7 +109,7 @@ private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, Asy private void getSearchResponseFromIndex(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener listener) { - store.getResponse(request, searchId, + store.getResponse(searchId, wrap( resp -> { if (resp.getVersion() <= request.getLastVersion()) { @@ -107,13 +119,12 @@ private void getSearchResponseFromIndex(GetAsyncSearchAction.Request request, As listener.onResponse(resp); } }, - exc -> listener.onFailure(exc) + listener::onFailure ) ); } void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, - AsyncSearchId searchId, long startMs, ActionListener listener) { final AsyncSearchResponse response = task.getAsyncResponse(false); try { @@ -128,7 +139,7 @@ void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask tas listener.onResponse(ret); } } else { - Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, searchId, startMs, listener)); + Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); } } catch (Exception e) { @@ -136,25 +147,75 @@ void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask tas } } + static boolean ensureAuthenticatedUserIsSame(Map originHeaders, Authentication current) throws IOException { + if (originHeaders == null || originHeaders.containsKey(AUTHENTICATION_KEY) == false) { + return true; + } + if (current == null) { + return false; + } + Authentication origin = Authentication.decode(originHeaders.get(AUTHENTICATION_KEY)); + return ensureAuthenticatedUserIsSame(origin, current); + } + + /** + * Compares the {@link Authentication} that was used to create the {@link AsyncSearchId} with the + * current authentication. + */ + static boolean ensureAuthenticatedUserIsSame(Authentication original, Authentication current) { + final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal()); + final boolean sameRealmType; + if (original.getUser().isRunAs()) { + if (current.getUser().isRunAs()) { + sameRealmType = original.getLookedUpBy().getType().equals(current.getLookedUpBy().getType()); + } else { + sameRealmType = original.getLookedUpBy().getType().equals(current.getAuthenticatedBy().getType()); + } + } else if (current.getUser().isRunAs()) { + sameRealmType = original.getAuthenticatedBy().getType().equals(current.getLookedUpBy().getType()); + } else { + sameRealmType = original.getAuthenticatedBy().getType().equals(current.getAuthenticatedBy().getType()); + } + return samePrincipal && sameRealmType; + } + /** * Returns a new listener that delegates the response to another listener and * then deletes the async search document from the system index if the response is * frozen (because the task has completed, failed or the coordinating node crashed). - * - * TODO: We should ensure that the response was successfully sent to the user before deleting - * (see {@link RestChannel#sendResponse(RestResponse)}. */ - private ActionListener wrapCleanupListener(AsyncSearchId id, + private ActionListener wrapCleanupListener(GetAsyncSearchAction.Request request, ActionListener listener) { return ActionListener.wrap( resp -> { - listener.onResponse(resp); - if (resp.isRunning() == false) { - DeleteRequest delete = new DeleteRequest(id.getIndexName()).id(id.getDocId()); - client.delete(delete, wrap(() -> {})); + if (request.isCleanOnCompletion() && resp.isRunning() == false) { + // TODO: We could ensure that the response was successfully sent to the user + // before deleting, see {@link RestChannel#sendResponse(RestResponse)}. + cleanupAsyncSearch(resp, null, listener); + } else { + listener.onResponse(resp); } }, - listener::onFailure + exc -> { + if (request.isCleanOnCompletion() && exc instanceof ResourceNotFoundException == false) { + // remove the task on real failures + cleanupAsyncSearch(null, exc, listener); + } else { + listener.onFailure(exc); + } + } ); } + + private void cleanupAsyncSearch(AsyncSearchResponse resp, Exception exc, ActionListener listener) { + client.execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(resp.id()), + ActionListener.wrap( + () -> { + if (exc == null) { + listener.onResponse(resp); + } else { + listener.onFailure(exc); + } + })); + } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index bae28e912826d..369f8a8dbfb8e 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; +import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; @@ -54,22 +55,20 @@ public TransportSubmitAsyncSearchAction(TransportService transportService, @Override protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener submitListener) { - // add a place holder in the async search history index and fire the async search - store.storeInitialResponse( - ActionListener.wrap( - resp -> executeSearch(request, resp, submitListener), - submitListener::onFailure - ) - ); + Map headers = new HashMap<>(nodeClient.threadPool().getThreadContext().getHeaders()); + + // add a place holder in the search index and fire the async search + store.storeInitialResponse(headers, + ActionListener.wrap(resp -> executeSearch(request, resp, submitListener, headers), submitListener::onFailure)); } private void executeSearch(SubmitAsyncSearchRequest submitRequest, IndexResponse doc, - ActionListener submitListener) { - SearchRequest searchRequest = new SearchRequest(submitRequest) { + ActionListener submitListener, Map originHeaders) { + final SearchRequest searchRequest = new SearchRequest(submitRequest.getSearchRequest()) { @Override - public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - String searchId = AsyncSearchId.encode(doc.getIndex(), doc.getId(), new TaskId(nodeClient.getLocalNodeId(), id)); - return new AsyncSearchTask(id, type, action, headers, searchId, reduceContextSupplier); + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map taskHeaders) { + AsyncSearchId searchId = new AsyncSearchId(doc.getIndex(), doc.getId(), new TaskId(nodeClient.getLocalNodeId(), id)); + return new AsyncSearchTask(id, type, action, originHeaders, taskHeaders, searchId, reduceContextSupplier); } }; @@ -81,7 +80,8 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, public void onResponse(SearchResponse response) { try { progressListener.onResponse(response); - store.storeFinalResponse(task.getAsyncResponse(true), ActionListener.wrap(() -> taskManager.unregister(task))); + store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), + ActionListener.wrap(() -> taskManager.unregister(task))); } catch (Exception e) { taskManager.unregister(task); } @@ -91,16 +91,16 @@ public void onResponse(SearchResponse response) { public void onFailure(Exception exc) { try { progressListener.onFailure(exc); - store.storeFinalResponse(task.getAsyncResponse(true), ActionListener.wrap(() -> taskManager.unregister(task))); + store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), + ActionListener.wrap(() -> taskManager.unregister(task))); } catch (Exception e) { taskManager.unregister(task); } } } ); - - GetAsyncSearchAction.Request getRequest = new GetAsyncSearchAction.Request(task.getSearchId(), - submitRequest.getWaitForCompletion(), -1); + GetAsyncSearchAction.Request getRequest = new GetAsyncSearchAction.Request(task.getSearchId().getEncoded(), + submitRequest.getWaitForCompletion(), -1, submitRequest.isCleanOnCompletion()); nodeClient.executeLocally(GetAsyncSearchAction.INSTANCE, getRequest, submitListener); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index fcee4464abbef..baaefbe2ce431 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -93,7 +93,7 @@ protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request protected AsyncSearchResponse getAsyncSearch(String id) throws ExecutionException, InterruptedException { return client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(id, TimeValue.timeValueMillis(1), -1)).get(); + new GetAsyncSearchAction.Request(id, TimeValue.MINUS_ONE, -1, true)).get(); } protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionException, InterruptedException { @@ -137,10 +137,11 @@ protected SearchResponseIterator assertBlockingIterator(String indexName, SearchSourceBuilder source, int numFailures, int progressStep) throws Exception { - SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { indexName }, source); + SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(source, indexName); request.setBatchedReduceSize(progressStep); request.setWaitForCompletion(TimeValue.timeValueMillis(1)); - ClusterSearchShardsResponse response = dataNodeClient().admin().cluster().prepareSearchShards(request.indices()).get(); + ClusterSearchShardsResponse response = dataNodeClient().admin().cluster() + .prepareSearchShards(request.getSearchRequest().indices()).get(); AtomicInteger failures = new AtomicInteger(numFailures); Map shardLatchMap = Arrays.stream(response.getGroups()) .map(ClusterSearchShardsGroup::getShardId) @@ -161,7 +162,7 @@ protected SearchResponseIterator assertBlockingIterator(String indexName, AsyncSearchResponse resp = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get(); while (resp.getPartialResponse().getSuccessfulShards() == -1) { resp = client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(resp.id(), TimeValue.timeValueSeconds(1), resp.getVersion())).get(); + new GetAsyncSearchAction.Request(resp.id(), TimeValue.timeValueSeconds(1), resp.getVersion(), true)).get(); } initial = resp; } @@ -212,7 +213,7 @@ private AsyncSearchResponse doNext() throws Exception { } assertBusy(() -> { AsyncSearchResponse newResp = client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(response.id(), TimeValue.timeValueMillis(10), lastVersion) + new GetAsyncSearchAction.Request(response.id(), TimeValue.timeValueMillis(10), lastVersion, true) ).get(); atomic.set(newResp); assertNotEquals(RestStatus.NOT_MODIFIED, newResp.status()); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java index 3e96e738eede1..1645163b4fdb1 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java @@ -21,7 +21,7 @@ protected Writeable.Reader instanceReader() { @Override protected GetAsyncSearchAction.Request createTestInstance() { return new GetAsyncSearchAction.Request(randomSearchId(), TimeValue.timeValueMillis(randomIntBetween(1, 10000)), - randomIntBetween(-1, Integer.MAX_VALUE)); + randomIntBetween(-1, Integer.MAX_VALUE), randomBoolean()); } static String randomSearchId() { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java index bf465b249ae9e..271c4529e47b1 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java @@ -22,28 +22,31 @@ protected Writeable.Reader instanceReader() { @Override protected SubmitAsyncSearchRequest createTestInstance() { - SubmitAsyncSearchRequest searchRequest = new SubmitAsyncSearchRequest(); - searchRequest.allowPartialSearchResults(randomBoolean()); + final SubmitAsyncSearchRequest searchRequest; if (randomBoolean()) { - searchRequest.indices(generateRandomStringArray(10, 10, false, false)); + searchRequest = new SubmitAsyncSearchRequest(generateRandomStringArray(10, 10, false, false)); + } else { + searchRequest = new SubmitAsyncSearchRequest(); } if (randomBoolean()) { - searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + searchRequest.getSearchRequest() + .indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); } if (randomBoolean()) { - searchRequest.preference(randomAlphaOfLengthBetween(3, 10)); + searchRequest.getSearchRequest() + .preference(randomAlphaOfLengthBetween(3, 10)); } if (randomBoolean()) { - searchRequest.requestCache(randomBoolean()); + searchRequest.getSearchRequest().requestCache(randomBoolean()); } if (randomBoolean()) { - searchRequest.routing(randomAlphaOfLengthBetween(3, 10)); + searchRequest.getSearchRequest().routing(randomAlphaOfLengthBetween(3, 10)); } if (randomBoolean()) { - searchRequest.searchType(randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)); + searchRequest.getSearchRequest().searchType(randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)); } if (randomBoolean()) { - searchRequest.source(randomSearchSourceBuilder()); + searchRequest.getSearchRequest().source(randomSearchSourceBuilder()); } return searchRequest; } @@ -58,16 +61,4 @@ protected SearchSourceBuilder randomSearchSourceBuilder() { } return source; } - - public void testValidateScroll() { - - } - - public void testValidateSuggestOnly() { - - } - - public void testValidateWaitForCompletion() { - - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 7dffd21847a89..2943a2f12fbe4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -15,6 +15,7 @@ import org.elasticsearch.rest.RestStatus; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.rest.RestStatus.NOT_MODIFIED; import static org.elasticsearch.rest.RestStatus.PARTIAL_CONTENT; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java index f1e89cbf35409..e82bed950680e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -35,6 +35,7 @@ public static class Request extends ActionRequest implements CompositeIndicesReq private final String id; private final int lastVersion; private final TimeValue waitForCompletion; + private final boolean cleanOnCompletion; /** * Create a new request @@ -42,10 +43,11 @@ public static class Request extends ActionRequest implements CompositeIndicesReq * @param waitForCompletion The minimum time that the request should wait before returning a partial result. * @param lastVersion The last version returned by a previous call. */ - public Request(String id, TimeValue waitForCompletion, int lastVersion) { + public Request(String id, TimeValue waitForCompletion, int lastVersion, boolean cleanOnCompletion) { this.id = id; this.waitForCompletion = waitForCompletion; this.lastVersion = lastVersion; + this.cleanOnCompletion = cleanOnCompletion; } public Request(StreamInput in) throws IOException { @@ -53,6 +55,7 @@ public Request(StreamInput in) throws IOException { this.id = in.readString(); this.waitForCompletion = TimeValue.timeValueMillis(in.readLong()); this.lastVersion = in.readInt(); + this.cleanOnCompletion = in.readBoolean(); } @Override @@ -61,6 +64,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeLong(waitForCompletion.millis()); out.writeInt(lastVersion); + out.writeBoolean(cleanOnCompletion); } @Override @@ -88,6 +92,13 @@ public TimeValue getWaitForCompletion() { return waitForCompletion; } + /** + * Cleanup the resource on completion or failure. + */ + public boolean isCleanOnCompletion() { + return cleanOnCompletion; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java index ad3539ceb25c4..a91c861c64e16 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchAction.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.core.search.action; import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.io.stream.Writeable; public final class SubmitAsyncSearchAction extends ActionType { public static final SubmitAsyncSearchAction INSTANCE = new SubmitAsyncSearchAction(); @@ -15,9 +14,4 @@ public final class SubmitAsyncSearchAction extends ActionType getResponseReader() { - return AsyncSearchResponse::new; - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 504ab94e3b87c..5f773f1e64ea9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.search.action; +import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.search.SearchRequest; @@ -26,39 +27,45 @@ * * @see AsyncSearchResponse */ -public class SubmitAsyncSearchRequest extends SearchRequest implements CompositeIndicesRequest { +public class SubmitAsyncSearchRequest extends ActionRequest implements CompositeIndicesRequest { private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1); + private boolean cleanOnCompletion = true; + + private final SearchRequest request; /** * Create a new request - * @param indices The indices the search will be executed on. */ public SubmitAsyncSearchRequest(String... indices) { - this(indices, new SearchSourceBuilder()); + this(new SearchSourceBuilder(), indices); } /** * Create a new request - * @param indices The indices the search will be executed on. - * @param source The source of the search request. */ - public SubmitAsyncSearchRequest(String[] indices, SearchSourceBuilder source) { - super(indices, source); - setCcsMinimizeRoundtrips(false); - setPreFilterShardSize(1); - setBatchedReduceSize(5); - requestCache(true); + public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) { + this.request = new SearchRequest(indices, source); + request.setCcsMinimizeRoundtrips(false); + request.setPreFilterShardSize(1); + request.setBatchedReduceSize(5); + request.requestCache(true); } public SubmitAsyncSearchRequest(StreamInput in) throws IOException { - super(in); + this.request = new SearchRequest(in); this.waitForCompletion = in.readTimeValue(); + this.cleanOnCompletion = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); + request.writeTo(out); out.writeTimeValue(waitForCompletion); + out.writeBoolean(cleanOnCompletion); + } + + public SearchRequest getSearchRequest() { + return request; } /** @@ -75,13 +82,32 @@ public TimeValue getWaitForCompletion() { return waitForCompletion; } + public void setBatchedReduceSize(int size) { + request.setBatchedReduceSize(size); + } + + public SearchSourceBuilder source() { + return request.source(); + } + + /** + * Should the resource be removed on completion or failure. + */ + public boolean isCleanOnCompletion() { + return cleanOnCompletion; + } + + public void setCleanOnCompletion(boolean value) { + this.cleanOnCompletion = value; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (scroll() != null) { + if (request.scroll() != null) { addValidationError("scroll queries are not supported", validationException); } - if (isSuggestOnly()) { + if (request.isSuggestOnly()) { validationException = addValidationError("suggest-only queries are not supported", validationException); } return validationException; @@ -96,13 +122,14 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - SubmitAsyncSearchRequest request = (SubmitAsyncSearchRequest) o; - return waitForCompletion.equals(request.waitForCompletion); + SubmitAsyncSearchRequest request1 = (SubmitAsyncSearchRequest) o; + return cleanOnCompletion == request1.cleanOnCompletion && + Objects.equals(waitForCompletion, request1.waitForCompletion) && + request.equals(request1.request); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), waitForCompletion); + return Objects.hash(waitForCompletion, cleanOnCompletion, request); } } diff --git a/x-pack/plugin/core/src/main/resources/async-search.json b/x-pack/plugin/core/src/main/resources/async-search.json index 9f5fae1594b67..e5c801eddbb04 100644 --- a/x-pack/plugin/core/src/main/resources/async-search.json +++ b/x-pack/plugin/core/src/main/resources/async-search.json @@ -15,6 +15,10 @@ "_doc": { "dynamic": false, "properties": { + "headers": { + "type": "object", + "enabled": false + }, "result": { "type": "object", "enabled": false From 2180ddeddc02c840263b5f61c492d8c7db8f7035 Mon Sep 17 00:00:00 2001 From: jimczi Date: Mon, 16 Dec 2019 11:22:52 +0100 Subject: [PATCH 12/61] fix checkstyle --- .../xpack/core/search/action/AsyncSearchResponse.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 2943a2f12fbe4..7dffd21847a89 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -15,7 +15,6 @@ import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.util.Map; import static org.elasticsearch.rest.RestStatus.NOT_MODIFIED; import static org.elasticsearch.rest.RestStatus.PARTIAL_CONTENT; From be0ba055c0a6cef9b3ed53cb5fc1449a19698708 Mon Sep 17 00:00:00 2001 From: jimczi Date: Mon, 16 Dec 2019 15:07:32 +0100 Subject: [PATCH 13/61] fix double clean --- .../xpack/search/TransportGetAsyncSearchAction.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index bcadee6752e9b..6defab7a5bb6e 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -58,16 +58,15 @@ public TransportGetAsyncSearchAction(TransportService transportService, @Override protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { - listener = wrapCleanupListener(request, listener); try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { - getSearchResponseFromTask(request, searchId, listener); + getSearchResponseFromTask(request, searchId, wrapCleanupListener(request, listener)); } else { TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); if (node == null) { - getSearchResponseFromIndex(request, searchId, listener); + getSearchResponseFromIndex(request, searchId, wrapCleanupListener(request, listener)); } else { transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); From 3c1ffdd8eec3386772695df1afe1fdb30ee88fde Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 17 Dec 2019 22:21:41 +0100 Subject: [PATCH 14/61] iter --- .../xpack/search/AsyncSearchSecurityIT.java | 25 ++-- .../xpack/search/AsyncSearchStoreService.java | 52 +++++--- .../xpack/search/AsyncSearchTask.java | 22 ++-- .../TransportDeleteAsyncSearchAction.java | 27 +--- .../search/TransportGetAsyncSearchAction.java | 119 ++++++++++-------- .../TransportSubmitAsyncSearchAction.java | 36 ++++-- .../search/action/AsyncSearchResponse.java | 2 +- .../action/SubmitAsyncSearchRequest.java | 3 +- 8 files changed, 164 insertions(+), 122 deletions(-) diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java index 0497a2ffc9592..3ee3216e7daf0 100644 --- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java +++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -30,9 +31,6 @@ import static org.hamcrest.Matchers.equalTo; public class AsyncSearchSecurityIT extends AsyncSearchRestTestCase { - private static final String tokenUser1 = basicAuthHeaderValue("user1", new SecureString("x-pack-test-password".toCharArray())); - private static final String tokenUser2 = basicAuthHeaderValue("user2", new SecureString("x-pack-test-password".toCharArray())); - /** * All tests run as a superuser but use es-security-runas-user to become a less privileged user. */ @@ -65,8 +63,8 @@ public void testWithUsers() throws Exception { } private void testCase(String user, String other) throws Exception { - for (String indexName : new String[] {"index", "index-" + user}) { - Response submitResp = submitAsyncSearch(indexName, "foo:bar", user); + for (String indexName : new String[] {"index", "index-" + user}) { + Response submitResp = submitAsyncSearch(indexName, "foo:bar", TimeValue.timeValueSeconds(10), user); assertOK(submitResp); String id = extractResponseId(submitResp); Response getResp = getAsyncSearch(id, user); @@ -84,8 +82,8 @@ private void testCase(String user, String other) throws Exception { assertOK(delResp); } ResponseException exc = expectThrows(ResponseException.class, - () -> submitAsyncSearch("index-" + other, "*", user)); - assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(500)); + () -> submitAsyncSearch("index-" + other, "*", TimeValue.timeValueSeconds(10), user)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403)); assertThat(exc.getMessage(), containsString("unauthorized")); } @@ -110,15 +108,26 @@ static void refresh(String index) throws IOException { } static Response submitAsyncSearch(String indexName, String query, String user) throws IOException { + return submitAsyncSearch(indexName, query, TimeValue.MINUS_ONE, user); + } + + static Response submitAsyncSearch(String indexName, String query, TimeValue waitForCompletion, String user) throws IOException { final Request request = new Request("GET", indexName + "/_async_search"); setRunAsHeader(request, user); request.addParameter("q", query); - request.addParameter("wait_for_completion", "0ms"); + request.addParameter("wait_for_completion", waitForCompletion.toString()); // we do the cleanup explicitly request.addParameter("clean_on_completion", "false"); return client().performRequest(request); } + static Response search(String indexName, String query, String user) throws IOException { + final Request request = new Request("GET", indexName + "/_search"); + setRunAsHeader(request, user); + request.addParameter("q", query); + return client().performRequest(request); + } + static Response getAsyncSearch(String id, String user) throws IOException { final Request request = new Request("GET", "/_async_search/" + id); setRunAsHeader(request, user); diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 91f4e57c595d6..acfd3df43a54e 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -5,12 +5,17 @@ */ package org.elasticsearch.xpack.search; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; @@ -23,8 +28,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; import java.io.IOException; @@ -44,6 +49,8 @@ * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the .async-search index. */ class AsyncSearchStoreService { + private static final Logger logger = LogManager.getLogger(AsyncSearchStoreService.class); + static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; static final String RESULT_FIELD = "result"; static final String HEADERS_FIELD = "headers"; @@ -58,26 +65,22 @@ class AsyncSearchStoreService { this.random = new Random(System.nanoTime()); } - Client getClient() { - return client; - } - /** * Store an empty document in the .async-search index that is used * as a place-holder for the future response. */ - void storeInitialResponse(Map headers, ActionListener next) { + void storeInitialResponse(Map headers, ActionListener listener) { IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS) .id(UUIDs.randomBase64UUID(random)) .source(Collections.singletonMap(HEADERS_FIELD, headers), XContentType.JSON); - client.index(request, next); + client.index(request, listener); } /** * Store the final response if the place-holder document is still present (update). */ void storeFinalResponse(Map headers, AsyncSearchResponse response, - ActionListener next) throws IOException { + ActionListener listener) throws IOException { AsyncSearchId searchId = AsyncSearchId.decode(response.id()); Map source = new HashMap<>(); source.put(RESULT_FIELD, encodeResponse(response)); @@ -85,14 +88,14 @@ void storeFinalResponse(Map headers, AsyncSearchResponse respons UpdateRequest request = new UpdateRequest().index(searchId.getIndexName()).id(searchId.getDocId()) .doc(source, XContentType.JSON) .detectNoop(false); - client.update(request, next); + client.update(request, listener); } /** * Get the response from the .async-search index if present, or delegate a {@link ResourceNotFoundException} * failure to the provided listener if not. */ - void getResponse(AsyncSearchId searchId, ActionListener next) { + void getResponse(AsyncSearchId searchId, ActionListener listener) { final Authentication current = Authentication.getAuthentication(client.threadPool().getThreadContext()); GetRequest internalGet = new GetRequest(searchId.getIndexName()) .id(searchId.getDocId()) @@ -100,28 +103,41 @@ void getResponse(AsyncSearchId searchId, ActionListener nex client.get(internalGet, ActionListener.wrap( get -> { if (get.isExists() == false) { - next.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); return; } + // check the authentication of the current user against the user that initiated the async search @SuppressWarnings("unchecked") Map headers = (Map) get.getSource().get(HEADERS_FIELD); if (ensureAuthenticatedUserIsSame(headers, current) == false) { - next.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); return; } @SuppressWarnings("unchecked") String encoded = (String) get.getSource().get(RESULT_FIELD); - if (encoded == null) { - next.onResponse(new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(-1), 0, false)); + listener.onResponse(encoded != null ? decodeResponse(encoded, registry) : null); + }, + listener::onFailure + )); + } + + void deleteResult(AsyncSearchId searchId, ActionListener listener) { + DeleteRequest request = new DeleteRequest(searchId.getIndexName()).id(searchId.getDocId()); + client.delete(request, ActionListener.wrap( + resp -> { + if (resp.status() == RestStatus.NOT_FOUND) { + listener.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded())); } else { - next.onResponse(decodeResponse(encoded, registry)); + listener.onResponse(new AcknowledgedResponse(true)); } }, - next::onFailure - )); + exc -> { + logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc); + listener.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded())); + }) + ); } /** diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index ef71562c42442..95d50f5e38d8c 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -65,11 +65,13 @@ public SearchProgressActionListener getProgressListener() { } /** - * Perform the final reduce on the current {@link AsyncSearchResponse} if requested - * and return the result. + * Perform the final reduce on the current {@link AsyncSearchResponse} if doFinalReduce + * is set to true and return the result. + * Note that this function returns null until {@link Listener#onListShards} + * or {@link Listener#onFailure} is called on the search task. */ AsyncSearchResponse getAsyncResponse(boolean doFinalReduce) { - return progressListener.response.get(doFinalReduce); + return progressListener.response != null ? progressListener.response.get(doFinalReduce) : null; } private class Listener extends SearchProgressActionListener { @@ -82,12 +84,6 @@ private class Listener extends SearchProgressActionListener { private volatile Response response; - Listener() { - final AsyncSearchResponse initial = new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(totalShards), version.get(), true); - this.response = new Response(initial, false); - } - @Override public void onListShards(List shards, boolean fetchPhase) { this.totalShards = shards.size(); @@ -134,15 +130,15 @@ public void onResponse(SearchResponse searchResponse) { @Override public void onFailure(Exception exc) { - AsyncSearchResponse previous = response.get(true); + AsyncSearchResponse previous = response != null ? response.get(true) : null; response = new Response(new AsyncSearchResponse(searchId.getEncoded(), - newPartialResponse(previous, shardFailures.get()), - exc != null ? new ElasticsearchException(exc) : null, version.incrementAndGet(), false), false); + previous != null ? newPartialResponse(previous, shardFailures.get()) : null, + exc != null ? ElasticsearchException.guessRootCauses(exc)[0] : null, version.incrementAndGet(), false), false); } private PartialSearchResponse newPartialResponse(AsyncSearchResponse response, int numFailures) { PartialSearchResponse old = response.getPartialResponse(); - return response.hasPartialResponse() ? new PartialSearchResponse(totalShards, old.getSuccessfulShards(), shardFailures.get(), + return response.hasPartialResponse() ? new PartialSearchResponse(totalShards, old.getSuccessfulShards(), numFailures, old.getTotalHits(), old.getAggregations()) : null; } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java index 28de8e0aca4cd..027d2169edc79 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -5,11 +5,9 @@ */ package org.elasticsearch.xpack.search; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -17,7 +15,6 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; @@ -43,33 +40,19 @@ public TransportDeleteAsyncSearchAction(TransportService transportService, protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); - // check if the response can be retrieved (handle security) - store.getResponse(searchId, ActionListener.wrap(res -> cancelTask(searchId, listener), listener::onFailure)); + // check if the response can be retrieved by the user (handle security) and then cancel/delete. + store.getResponse(searchId, ActionListener.wrap(res -> cancelTaskAndDeleteResult(searchId, listener), listener::onFailure)); } catch (IOException exc) { listener.onFailure(exc); } } - private void cancelTask(AsyncSearchId searchId, ActionListener listener) { + private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener listener) { try { nodeClient.execute(CancelTasksAction.INSTANCE, new CancelTasksRequest().setTaskId(searchId.getTaskId()), - ActionListener.wrap(() -> deleteResult(searchId, listener))); + ActionListener.wrap(() -> store.deleteResult(searchId, listener))); } catch (Exception e) { - deleteResult(searchId, listener); + store.deleteResult(searchId, listener); } } - - private void deleteResult(AsyncSearchId searchId, ActionListener next) { - DeleteRequest request = new DeleteRequest(searchId.getIndexName()).id(searchId.getDocId()); - store.getClient().delete(request, ActionListener.wrap( - resp -> { - if (resp.status() == RestStatus.NOT_FOUND) { - next.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded())); - } else { - next.onResponse(new AcknowledgedResponse(true)); - } - }, - exc -> next.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded()))) - ); - } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 6defab7a5bb6e..5f409a2af9a7a 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -7,11 +7,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; @@ -23,23 +25,21 @@ import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.security.authc.Authentication; import java.io.IOException; import java.util.Map; -import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; public class TransportGetAsyncSearchAction extends HandledTransportAction { - private Logger logger = LogManager.getLogger(getClass()); + private static final Logger logger = LogManager.getLogger(TransportGetAsyncSearchAction.class); + private final ClusterService clusterService; private final ThreadPool threadPool; private final TransportService transportService; private final AsyncSearchStoreService store; - private final Client client; @Inject public TransportGetAsyncSearchAction(TransportService transportService, @@ -53,7 +53,6 @@ public TransportGetAsyncSearchAction(TransportService transportService, this.transportService = transportService; this.threadPool = threadPool; this.store = new AsyncSearchStoreService(client, registry); - this.client = client; } @Override @@ -61,12 +60,12 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { - getSearchResponseFromTask(request, searchId, wrapCleanupListener(request, listener)); + getSearchResponseFromTask(request, searchId, wrapCleanupListener(request, searchId, listener)); } else { TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); if (node == null) { - getSearchResponseFromIndex(request, searchId, wrapCleanupListener(request, listener)); + getSearchResponseFromIndex(request, searchId, wrapCleanupListener(request, searchId, listener)); } else { transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); @@ -108,41 +107,64 @@ private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, Asy private void getSearchResponseFromIndex(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener listener) { - store.getResponse(searchId, - wrap( - resp -> { - if (resp.getVersion() <= request.getLastVersion()) { + store.getResponse(searchId, new ActionListener<>() { + @Override + public void onResponse(AsyncSearchResponse response) { + if (response == null) { + // the task failed to store a response but we still have the placeholder in the index so we + // force the deletion and throw a resource not found exception. + store.deleteResult(searchId, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + listener.onFailure(new ResourceNotFoundException(request.getId() + " not found")); + } + + @Override + public void onFailure(Exception exc) { + logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", request.getId()), exc); + listener.onFailure(new ResourceNotFoundException(request.getId() + " not found")); + } + }); + } else if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response - listener.onResponse(new AsyncSearchResponse(resp.id(), resp.getVersion(), false)); + listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), false)); } else { - listener.onResponse(resp); + listener.onResponse(response); } - }, - listener::onFailure - ) - ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); } void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, long startMs, ActionListener listener) { final AsyncSearchResponse response = task.getAsyncResponse(false); - try { - if (response.isRunning() == false) { - listener.onResponse(response); - } else if (request.getWaitForCompletion().getMillis() < (threadPool.relativeTimeInMillis() - startMs)) { - if (response.getVersion() <= request.getLastVersion()) { - // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), true)); + if (response == null) { + // the search task is not fully initialized + Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); + threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); + } else { + try { + if (response.isRunning() == false) { + listener.onResponse(response); + } else if (request.getWaitForCompletion().getMillis() < (threadPool.relativeTimeInMillis() - startMs)) { + if (response.getVersion() <= request.getLastVersion()) { + // return a not-modified response + listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), true)); + } else { + listener.onResponse(task.getAsyncResponse(true)); + } } else { - final AsyncSearchResponse ret = task.getAsyncResponse(true); - listener.onResponse(ret); + Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); + threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); } - } else { - Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); - threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); + } catch (Exception e) { + listener.onFailure(e); } - } catch (Exception e) { - listener.onFailure(e); } } @@ -184,37 +206,30 @@ static boolean ensureAuthenticatedUserIsSame(Authentication original, Authentica * frozen (because the task has completed, failed or the coordinating node crashed). */ private ActionListener wrapCleanupListener(GetAsyncSearchAction.Request request, + AsyncSearchId searchId, ActionListener listener) { return ActionListener.wrap( resp -> { if (request.isCleanOnCompletion() && resp.isRunning() == false) { // TODO: We could ensure that the response was successfully sent to the user // before deleting, see {@link RestChannel#sendResponse(RestResponse)}. - cleanupAsyncSearch(resp, null, listener); + store.deleteResult(searchId, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + listener.onResponse(resp); + } + + @Override + public void onFailure(Exception exc) { + logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", request.getId()), exc); + listener.onResponse(resp); + } + }); } else { listener.onResponse(resp); } }, - exc -> { - if (request.isCleanOnCompletion() && exc instanceof ResourceNotFoundException == false) { - // remove the task on real failures - cleanupAsyncSearch(null, exc, listener); - } else { - listener.onFailure(exc); - } - } + listener::onFailure ); } - - private void cleanupAsyncSearch(AsyncSearchResponse resp, Exception exc, ActionListener listener) { - client.execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(resp.id()), - ActionListener.wrap( - () -> { - if (exc == null) { - listener.onResponse(resp); - } else { - listener.onFailure(exc); - } - })); - } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 369f8a8dbfb8e..511586e3744e7 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -5,7 +5,11 @@ */ package org.elasticsearch.xpack.search; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchProgressActionListener; @@ -14,6 +18,7 @@ import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.inject.Inject; @@ -33,6 +38,8 @@ import java.util.function.Supplier; public class TransportSubmitAsyncSearchAction extends HandledTransportAction { + private static final Logger logger = LogManager.getLogger(TransportSubmitAsyncSearchAction.class); + private final NodeClient nodeClient; private final Supplier reduceContextSupplier; private final TransportSearchAction searchAction; @@ -55,8 +62,12 @@ public TransportSubmitAsyncSearchAction(TransportService transportService, @Override protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener submitListener) { - Map headers = new HashMap<>(nodeClient.threadPool().getThreadContext().getHeaders()); + ActionRequestValidationException exc = request.validate(); + if (exc != null) { + submitListener.onFailure(exc); + } + Map headers = new HashMap<>(nodeClient.threadPool().getThreadContext().getHeaders()); // add a place holder in the search index and fire the async search store.storeInitialResponse(headers, ActionListener.wrap(resp -> executeSearch(request, resp, submitListener, headers), submitListener::onFailure)); @@ -74,16 +85,28 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, AsyncSearchTask task = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); SearchProgressActionListener progressListener = task.getProgressListener(); + + final ActionListener finishHim = new ActionListener<>() { + @Override + public void onResponse(UpdateResponse updateResponse) { + taskManager.unregister(task); + } + + @Override + public void onFailure(Exception exc) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); + taskManager.unregister(task); + } + }; searchAction.execute(task, searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse response) { try { progressListener.onResponse(response); - store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), - ActionListener.wrap(() -> taskManager.unregister(task))); + store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), finishHim); } catch (Exception e) { - taskManager.unregister(task); + finishHim.onFailure(e); } } @@ -91,10 +114,9 @@ public void onResponse(SearchResponse response) { public void onFailure(Exception exc) { try { progressListener.onFailure(exc); - store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), - ActionListener.wrap(() -> taskManager.unregister(task))); + store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), finishHim); } catch (Exception e) { - taskManager.unregister(task); + finishHim.onFailure(e); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 7dffd21847a89..558dd70f0e909 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -160,7 +160,7 @@ public boolean isRunning() { @Override public RestStatus status() { if (finalResponse == null && partialResponse == null) { - return NOT_MODIFIED; + return failure != null ? failure.status() : NOT_MODIFIED; } else if (finalResponse == null) { return failure != null ? failure.status() : PARTIAL_CONTENT; } else { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 5f773f1e64ea9..14d82dee43735 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -103,13 +103,14 @@ public void setCleanOnCompletion(boolean value) { @Override public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; + ActionRequestValidationException validationException = request.validate(); if (request.scroll() != null) { addValidationError("scroll queries are not supported", validationException); } if (request.isSuggestOnly()) { validationException = addValidationError("suggest-only queries are not supported", validationException); } + return validationException; } From b1d24146a4e670aef992aeb0c9d416cc43a36d84 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 18 Dec 2019 10:06:21 +0100 Subject: [PATCH 15/61] add more logging --- .../xpack/search/TransportSubmitAsyncSearchAction.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 511586e3744e7..226fed2a5e85a 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -104,6 +104,7 @@ public void onFailure(Exception exc) { public void onResponse(SearchResponse response) { try { progressListener.onResponse(response); + logger.info(() -> new ParameterizedMessage("store async-search [{}]", task.getSearchId().getEncoded())); store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), finishHim); } catch (Exception e) { finishHim.onFailure(e); @@ -114,6 +115,7 @@ public void onResponse(SearchResponse response) { public void onFailure(Exception exc) { try { progressListener.onFailure(exc); + logger.info(() -> new ParameterizedMessage("store failed async-search [{}]", task.getSearchId().getEncoded()), exc); store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), finishHim); } catch (Exception e) { finishHim.onFailure(e); From 542fff26472e646d51e662bea00d58827b8fd4a3 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 18 Dec 2019 12:53:26 +0100 Subject: [PATCH 16/61] replace routing with preference in get requests --- .../elasticsearch/xpack/search/AsyncSearchStoreService.java | 4 ++-- .../xpack/search/TransportSubmitAsyncSearchAction.java | 3 +-- .../elasticsearch/xpack/search/AsyncSearchIntegTestCase.java | 1 - .../xpack/search/SubmitAsyncSearchRequestTests.java | 3 --- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index acfd3df43a54e..47337f8f2664b 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -98,8 +98,8 @@ void storeFinalResponse(Map headers, AsyncSearchResponse respons void getResponse(AsyncSearchId searchId, ActionListener listener) { final Authentication current = Authentication.getAuthentication(client.threadPool().getThreadContext()); GetRequest internalGet = new GetRequest(searchId.getIndexName()) - .id(searchId.getDocId()) - .routing(searchId.getDocId()); + .preference(searchId.getEncoded()) + .id(searchId.getDocId()); client.get(internalGet, ActionListener.wrap( get -> { if (get.isExists() == false) { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 226fed2a5e85a..354557fc20967 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -33,7 +33,6 @@ import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; -import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; @@ -67,7 +66,7 @@ protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionList submitListener.onFailure(exc); } - Map headers = new HashMap<>(nodeClient.threadPool().getThreadContext().getHeaders()); + final Map headers = nodeClient.threadPool().getThreadContext().getHeaders(); // add a place holder in the search index and fire the async search store.storeInitialResponse(headers, ActionListener.wrap(resp -> executeSearch(request, resp, submitListener, headers), submitListener::onFailure)); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index baaefbe2ce431..309bdbb96da35 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -107,7 +107,6 @@ protected void waitTaskRemoval(String id) throws Exception { AsyncSearchId searchId = AsyncSearchId.decode(id); assertBusy(() -> { GetResponse resp = client().prepareGet() - .setRouting(searchId.getDocId()) .setIndex(searchId.getIndexName()) .setId(searchId.getDocId()) .get(); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java index 271c4529e47b1..57434edf818ca 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java @@ -39,9 +39,6 @@ protected SubmitAsyncSearchRequest createTestInstance() { if (randomBoolean()) { searchRequest.getSearchRequest().requestCache(randomBoolean()); } - if (randomBoolean()) { - searchRequest.getSearchRequest().routing(randomAlphaOfLengthBetween(3, 10)); - } if (randomBoolean()) { searchRequest.getSearchRequest().searchType(randomFrom(SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)); } From 177e3039f78258ae8bec81e76a1ef7699f81fa4a Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 18 Dec 2019 21:53:26 +0100 Subject: [PATCH 17/61] iter This commit ensures that the rollover index is created before launching an async search. It also changes the design to not store any document if the submit action returns a final response. --- .../xpack/search/AsyncSearchStoreService.java | 70 ++++- .../xpack/search/AsyncSearchTask.java | 19 +- .../search/AsyncSearchTemplateRegistry.java | 28 +- .../search/TransportGetAsyncSearchAction.java | 14 +- .../TransportSubmitAsyncSearchAction.java | 140 +++++++--- .../xpack/search/AsyncSearchIT.java | 20 +- .../search/AsyncSearchIntegTestCase.java | 14 +- .../search/AsyncSearchStoreServiceTests.java | 250 +++++++++++++++++- 8 files changed, 455 insertions(+), 100 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 47337f8f2664b..1d49c046737e1 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -8,9 +8,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; @@ -20,7 +23,8 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; -import org.elasticsearch.common.UUIDs; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -38,7 +42,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Random; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.INDEX_TEMPLATE_VERSION; @@ -52,26 +55,79 @@ class AsyncSearchStoreService { private static final Logger logger = LogManager.getLogger(AsyncSearchStoreService.class); static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; + static final String ASYNC_SEARCH_INDEX_PREFIX = ASYNC_SEARCH_ALIAS + "-"; static final String RESULT_FIELD = "result"; static final String HEADERS_FIELD = "headers"; private final Client client; private final NamedWriteableRegistry registry; - private final Random random; AsyncSearchStoreService(Client client, NamedWriteableRegistry registry) { this.client = new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN); this.registry = registry; - this.random = new Random(System.nanoTime()); + } + + /** + * Checks if the async-search index exists, and if not, creates it. + * The provided {@link ActionListener} is called with the index name that should + * be used to store the response. + */ + void ensureAsyncSearchIndex(ClusterState state, ActionListener andThen) { + final String initialIndexName = ASYNC_SEARCH_INDEX_PREFIX + "000001"; + final AliasOrIndex current = state.metaData().getAliasAndIndexLookup().get(ASYNC_SEARCH_ALIAS); + final AliasOrIndex initialIndex = state.metaData().getAliasAndIndexLookup().get(initialIndexName); + if (current == null && initialIndex == null) { + // No alias or index exists with the expected names, so create the index with appropriate alias + client.admin().indices().prepareCreate(initialIndexName) + .setWaitForActiveShards(1) + .addAlias(new Alias(ASYNC_SEARCH_ALIAS).writeIndex(true)) + .execute(new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse response) { + andThen.onResponse(initialIndexName); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + // The index didn't exist before we made the call, there was probably a race - just ignore this + andThen.onResponse(initialIndexName); + } else { + andThen.onFailure(e); + } + } + }); + } else if (current == null) { + // alias does not exist but initial index does, something is broken + andThen.onFailure(new IllegalStateException("async-search index [" + initialIndexName + + "] already exists but does not have alias [" + ASYNC_SEARCH_ALIAS + "]")); + } else if (current.isAlias() && current instanceof AliasOrIndex.Alias) { + AliasOrIndex.Alias alias = (AliasOrIndex.Alias) current; + if (alias.getWriteIndex() != null) { + // The alias exists and has a write index, so we're good + andThen.onResponse(alias.getWriteIndex().getIndex().getName()); + } else { + // The alias does not have a write index, so we can't index into it + andThen.onFailure(new IllegalStateException("async-search alias [" + ASYNC_SEARCH_ALIAS + "] does not have a write index")); + } + } else if (current.isAlias() == false) { + // This is not an alias, error out + andThen.onFailure(new IllegalStateException("async-search alias [" + ASYNC_SEARCH_ALIAS + + "] already exists as concrete index")); + } else { + logger.error("unexpected IndexOrAlias for [{}]: [{}]", ASYNC_SEARCH_ALIAS, current); + andThen.onFailure(new IllegalStateException("unexpected IndexOrAlias for async-search index")); + assert false : ASYNC_SEARCH_ALIAS + " cannot be both an alias and not an alias simultaneously"; + } } /** * Store an empty document in the .async-search index that is used * as a place-holder for the future response. */ - void storeInitialResponse(Map headers, ActionListener listener) { - IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS) - .id(UUIDs.randomBase64UUID(random)) + void storeInitialResponse(Map headers, String index, String docID, ActionListener listener) { + IndexRequest request = new IndexRequest(index) + .id(docID) .source(Collections.singletonMap(HEADERS_FIELD, headers), XContentType.JSON); client.index(request, listener); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 95d50f5e38d8c..b41d7d5c084f8 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -7,6 +7,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.search.SearchProgressActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -34,6 +35,9 @@ class AsyncSearchTask extends SearchTask { private final Supplier reduceContextSupplier; private final Listener progressListener; + // indicate if the user retrieved the final response + private boolean isFinalResponseRetrieved; + private final Map originHeaders; AsyncSearchTask(long id, @@ -70,8 +74,19 @@ public SearchProgressActionListener getProgressListener() { * Note that this function returns null until {@link Listener#onListShards} * or {@link Listener#onFailure} is called on the search task. */ - AsyncSearchResponse getAsyncResponse(boolean doFinalReduce) { - return progressListener.response != null ? progressListener.response.get(doFinalReduce) : null; + synchronized AsyncSearchResponse getAsyncResponse(boolean doFinalReduce, boolean cleanOnCompletion) { + AsyncSearchResponse resp = progressListener.response != null ? progressListener.response.get(doFinalReduce) : null; + if (resp != null + && doFinalReduce + && cleanOnCompletion + && resp.isRunning() == false) { + if (isFinalResponseRetrieved) { + // the response was already retrieved in a previous call + throw new ResourceNotFoundException(resp.id() + " not found"); + } + isFinalResponseRetrieved = true; + } + return resp; } private class Listener extends SearchProgressActionListener { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java index 56355fc3d8a49..bd0a91c06b7dc 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java @@ -7,23 +7,16 @@ package org.elasticsearch.xpack.search; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; -import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; @@ -35,7 +28,7 @@ public class AsyncSearchTemplateRegistry extends IndexTemplateRegistry { // version 1: initial public static final String INDEX_TEMPLATE_VERSION = "1"; public static final String ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE = "xpack.async-search.template.version"; - public static final String ASYNC_SEARCH_TEMPLATE_NAME = ".async-search"; + public static final String ASYNC_SEARCH_TEMPLATE_NAME = "async-search"; public static final String ASYNC_SEARCH_POLICY_NAME = "async-search-ilm-policy"; @@ -73,23 +66,4 @@ protected List getPolicyConfigs() { protected String getOrigin() { return INDEX_LIFECYCLE_ORIGIN; } - - public boolean validate(ClusterState state) { - boolean allTemplatesPresent = getTemplateConfigs().stream() - .map(IndexTemplateConfig::getTemplateName) - .allMatch(name -> state.metaData().getTemplates().containsKey(name)); - - Optional> maybePolicies = Optional - .ofNullable(state.metaData().custom(IndexLifecycleMetadata.TYPE)) - .map(IndexLifecycleMetadata::getPolicies); - Set policyNames = getPolicyConfigs().stream() - .map(LifecyclePolicyConfig::getPolicyName) - .collect(Collectors.toSet()); - - boolean allPoliciesPresent = maybePolicies - .map(policies -> policies.keySet() - .containsAll(policyNames)) - .orElse(false); - return allTemplatesPresent && allPoliciesPresent; - } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 5f409a2af9a7a..96d0b656f3725 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -59,6 +59,10 @@ public TransportGetAsyncSearchAction(TransportService transportService, protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); + if (searchId.getIndexName().startsWith(AsyncSearchStoreService.ASYNC_SEARCH_ALIAS) == false) { + listener.onFailure(new IllegalArgumentException("invalid id [" + request.getId() + "] that references the wrong index [" + + searchId.getIndexName() + "]")); + } if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { getSearchResponseFromTask(request, searchId, wrapCleanupListener(request, searchId, listener)); } else { @@ -142,7 +146,7 @@ public void onFailure(Exception e) { void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, long startMs, ActionListener listener) { - final AsyncSearchResponse response = task.getAsyncResponse(false); + final AsyncSearchResponse response = task.getAsyncResponse(false, false); if (response == null) { // the search task is not fully initialized Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); @@ -150,20 +154,20 @@ void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask tas } else { try { if (response.isRunning() == false) { - listener.onResponse(response); + listener.onResponse(task.getAsyncResponse(true, request.isCleanOnCompletion())); } else if (request.getWaitForCompletion().getMillis() < (threadPool.relativeTimeInMillis() - startMs)) { if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), true)); } else { - listener.onResponse(task.getAsyncResponse(true)); + listener.onResponse(task.getAsyncResponse(true, request.isCleanOnCompletion())); } } else { Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); } - } catch (Exception e) { - listener.onFailure(e); + } catch (Exception exc) { + listener.onFailure(exc); } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 354557fc20967..16aed124d091f 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchProgressActionListener; import org.elasticsearch.action.search.SearchRequest; @@ -21,31 +20,42 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; +import java.io.IOException; import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; public class TransportSubmitAsyncSearchAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportSubmitAsyncSearchAction.class); private final NodeClient nodeClient; + private final ClusterService clusterService; + private final ThreadPool threadPool; private final Supplier reduceContextSupplier; private final TransportSearchAction searchAction; private final AsyncSearchStoreService store; + private final Random random; @Inject - public TransportSubmitAsyncSearchAction(TransportService transportService, + public TransportSubmitAsyncSearchAction(ClusterService clusterService, + TransportService transportService, ActionFilters actionFilters, NamedWriteableRegistry registry, Client client, @@ -53,10 +63,13 @@ public TransportSubmitAsyncSearchAction(TransportService transportService, SearchService searchService, TransportSearchAction searchAction) { super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new); + this.clusterService = clusterService; + this.threadPool = transportService.getThreadPool(); this.nodeClient = nodeClient; this.reduceContextSupplier = () -> searchService.createReduceContext(true); this.searchAction = searchAction; this.store = new AsyncSearchStoreService(client, registry); + this.random = new Random(System.nanoTime()); } @Override @@ -64,66 +77,119 @@ protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionList ActionRequestValidationException exc = request.validate(); if (exc != null) { submitListener.onFailure(exc); + return; } - final Map headers = nodeClient.threadPool().getThreadContext().getHeaders(); - // add a place holder in the search index and fire the async search - store.storeInitialResponse(headers, - ActionListener.wrap(resp -> executeSearch(request, resp, submitListener, headers), submitListener::onFailure)); + store.ensureAsyncSearchIndex(clusterService.state(), new ActionListener<>() { + @Override + public void onResponse(String indexName) { + executeSearch(request, indexName, UUIDs.randomBase64UUID(random), submitListener); + } + + @Override + public void onFailure(Exception exc) { + submitListener.onFailure(exc); + } + }); } - private void executeSearch(SubmitAsyncSearchRequest submitRequest, IndexResponse doc, - ActionListener submitListener, Map originHeaders) { + private void executeSearch(SubmitAsyncSearchRequest submitRequest, String indexName, String docID, + ActionListener submitListener) { + final Map originHeaders = nodeClient.threadPool().getThreadContext().getHeaders(); final SearchRequest searchRequest = new SearchRequest(submitRequest.getSearchRequest()) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map taskHeaders) { - AsyncSearchId searchId = new AsyncSearchId(doc.getIndex(), doc.getId(), new TaskId(nodeClient.getLocalNodeId(), id)); + AsyncSearchId searchId = new AsyncSearchId(indexName, docID, new TaskId(nodeClient.getLocalNodeId(), id)); return new AsyncSearchTask(id, type, action, originHeaders, taskHeaders, searchId, reduceContextSupplier); } }; + // trigger the async search + final AtomicReference shouldStoreResult = new AtomicReference<>(); AsyncSearchTask task = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); SearchProgressActionListener progressListener = task.getProgressListener(); - - final ActionListener finishHim = new ActionListener<>() { - @Override - public void onResponse(UpdateResponse updateResponse) { - taskManager.unregister(task); - } - - @Override - public void onFailure(Exception exc) { - logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); - taskManager.unregister(task); - } - }; searchAction.execute(task, searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse response) { - try { - progressListener.onResponse(response); - logger.info(() -> new ParameterizedMessage("store async-search [{}]", task.getSearchId().getEncoded())); - store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), finishHim); - } catch (Exception e) { - finishHim.onFailure(e); - } + progressListener.onResponse(response); + logger.debug(() -> new ParameterizedMessage("store async-search [{}]", task.getSearchId().getEncoded())); + onTaskCompletion(threadPool, task, shouldStoreResult); } @Override public void onFailure(Exception exc) { - try { - progressListener.onFailure(exc); - logger.info(() -> new ParameterizedMessage("store failed async-search [{}]", task.getSearchId().getEncoded()), exc); - store.storeFinalResponse(originHeaders, task.getAsyncResponse(true), finishHim); - } catch (Exception e) { - finishHim.onFailure(e); - } + progressListener.onFailure(exc); + logger.error(() -> new ParameterizedMessage("store failed async-search [{}]", task.getSearchId().getEncoded()), exc); + onTaskCompletion(threadPool, task, shouldStoreResult); } } ); + + // and get the response asynchronously GetAsyncSearchAction.Request getRequest = new GetAsyncSearchAction.Request(task.getSearchId().getEncoded(), submitRequest.getWaitForCompletion(), -1, submitRequest.isCleanOnCompletion()); - nodeClient.executeLocally(GetAsyncSearchAction.INSTANCE, getRequest, submitListener); + nodeClient.executeLocally(GetAsyncSearchAction.INSTANCE, getRequest, + new ActionListener<>() { + @Override + public void onResponse(AsyncSearchResponse response) { + if (response.isRunning() || submitRequest.isCleanOnCompletion() == false) { + // the task is still running and the user cannot wait more so we create + // an empty document for further retrieval + store.storeInitialResponse(originHeaders, indexName, docID, + ActionListener.wrap(() -> { + shouldStoreResult.set(true); + submitListener.onResponse(response); + })); + } else { + // the user will get a final response directly so no need to store the result on completion + shouldStoreResult.set(false); + submitListener.onResponse(response); + } + } + + @Override + public void onFailure(Exception e) { + // we don't need to store the result if the submit failed + shouldStoreResult.set(false); + submitListener.onFailure(e); + } + }); + } + + private void onTaskCompletion(ThreadPool threadPool, AsyncSearchTask task, + AtomicReference reference) { + final Boolean shouldStoreResult = reference.get(); + try { + if (shouldStoreResult == null) { + // the user is still waiting for a response so we schedule a retry in 100ms + threadPool.schedule(() -> onTaskCompletion(threadPool, task, reference), + TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); + } else if (shouldStoreResult) { + // the user retrieved an initial partial response so we need to store the final one + // for further retrieval + store.storeFinalResponse(task.getOriginHeaders(), task.getAsyncResponse(true, false), + new ActionListener<>() { + @Override + public void onResponse(UpdateResponse updateResponse) { + logger.debug(() -> new ParameterizedMessage("store async-search [{}]", task.getSearchId().getEncoded())); + taskManager.unregister(task); + } + + @Override + public void onFailure(Exception exc) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", + task.getSearchId().getEncoded()), exc); + taskManager.unregister(task); + } + }); + } else { + // the user retrieved the final response already so we don't need to store it + taskManager.unregister(task); + } + } catch (IOException exc) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); + taskManager.unregister(task); + } } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java index 0dad22901dc80..329088ff52bbc 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java @@ -109,7 +109,7 @@ public void testMaxMinAggregation() throws Exception { assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric)); } } - waitTaskRemoval(response.id()); + ensureTaskRemoval(response.id()); } } @@ -154,7 +154,7 @@ public void testTermsAggregation() throws Exception { } } } - waitTaskRemoval(response.id()); + ensureTaskRemoval(response.id()); } } @@ -164,13 +164,13 @@ public void testRestartAfterCompletion() throws Exception { assertBlockingIterator(indexName, new SearchSourceBuilder(), 0, 2)) { initial = it.next(); } - waitTaskCompletion(initial.id()); + ensureTaskCompletion(initial.id()); restartTaskNode(initial.id()); AsyncSearchResponse response = getAsyncSearch(initial.id()); assertTrue(response.isFinalResponse()); assertFalse(response.isRunning()); assertFalse(response.hasPartialResponse()); - waitTaskRemoval(response.id()); + ensureTaskRemoval(response.id()); } public void testDeleteCancelRunningTask() throws Exception { @@ -180,8 +180,8 @@ public void testDeleteCancelRunningTask() throws Exception { initial = it.next(); deleteAsyncSearch(initial.id()); it.close(); - waitTaskCompletion(initial.id()); - waitTaskRemoval(initial.id()); + ensureTaskCompletion(initial.id()); + ensureTaskRemoval(initial.id()); } public void testDeleteCleanupIndex() throws Exception { @@ -192,8 +192,8 @@ public void testDeleteCleanupIndex() throws Exception { AsyncSearchResponse response = it.next(); deleteAsyncSearch(response.id()); it.close(); - waitTaskCompletion(response.id()); - waitTaskRemoval(response.id()); + ensureTaskCompletion(response.id()); + ensureTaskRemoval(response.id()); } public void testCleanupOnFailure() throws Exception { @@ -202,12 +202,12 @@ public void testCleanupOnFailure() throws Exception { assertBlockingIterator(indexName, new SearchSourceBuilder(), numShards, 2)) { initial = it.next(); } - waitTaskCompletion(initial.id()); + ensureTaskCompletion(initial.id()); AsyncSearchResponse response = getAsyncSearch(initial.id()); assertTrue(response.hasFailed()); assertTrue(response.hasPartialResponse()); assertThat(response.getPartialResponse().getTotalShards(), equalTo(numShards)); assertThat(response.getPartialResponse().getShardFailures(), equalTo(numShards)); - waitTaskRemoval(initial.id()); + ensureTaskRemoval(initial.id()); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index 309bdbb96da35..ec55b085a01b8 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -103,7 +103,7 @@ protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionExce /** * Wait the removal of the document decoded from the provided {@link AsyncSearchId}. */ - protected void waitTaskRemoval(String id) throws Exception { + protected void ensureTaskRemoval(String id) throws Exception { AsyncSearchId searchId = AsyncSearchId.decode(id); assertBusy(() -> { GetResponse resp = client().prepareGet() @@ -117,7 +117,7 @@ protected void waitTaskRemoval(String id) throws Exception { /** * Wait the completion of the {@link TaskId} decoded from the provided {@link AsyncSearchId}. */ - protected void waitTaskCompletion(String id) throws Exception { + protected void ensureTaskCompletion(String id) throws Exception { assertBusy(() -> { TaskId taskId = AsyncSearchId.decode(id).getTaskId(); try { @@ -156,15 +156,7 @@ protected SearchResponseIterator assertBlockingIterator(String indexName, resetPluginsLatch(shardLatchMap); request.source().query(new BlockQueryBuilder(shardLatchMap)); - final AsyncSearchResponse initial; - { - AsyncSearchResponse resp = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get(); - while (resp.getPartialResponse().getSuccessfulShards() == -1) { - resp = client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(resp.id(), TimeValue.timeValueSeconds(1), resp.getVersion(), true)).get(); - } - initial = resp; - } + final AsyncSearchResponse initial = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get(); assertTrue(initial.hasPartialResponse()); assertThat(initial.status(), equalTo(RestStatus.PARTIAL_CONTENT)); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java index 6dad76dbbdbcd..85a91f9cce29a 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java @@ -5,30 +5,72 @@ */ package org.elasticsearch.xpack.search; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.junit.After; import org.junit.Before; import java.io.IOException; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static java.util.Collections.emptyList; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch; import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses; import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse; import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_ALIAS; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.core.IsEqual.equalTo; public class AsyncSearchStoreServiceTests extends ESTestCase { private NamedWriteableRegistry namedWriteableRegistry; + private ThreadPool threadPool; + private VerifyingClient client; + private AsyncSearchStoreService store; @Before - public void registerNamedObjects() { + public void setup() { SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); List namedWriteables = searchModule.getNamedWriteables(); namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + store = new AsyncSearchStoreService(client, namedWriteableRegistry); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); } public void testEncode() throws IOException { @@ -39,4 +81,210 @@ public void testEncode() throws IOException { assertEqualResponses(response, same); } } + + public void testIndexNeedsCreation() throws InterruptedException { + ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) + .metaData(MetaData.builder()) + .build(); + + client.setVerifier((a, r, l) -> { + assertThat(a, instanceOf(CreateIndexAction.class)); + assertThat(r, instanceOf(CreateIndexRequest.class)); + CreateIndexRequest request = (CreateIndexRequest) r; + assertThat(request.aliases(), hasSize(1)); + request.aliases().forEach(alias -> { + assertThat(alias.name(), equalTo(ASYNC_SEARCH_ALIAS)); + assertTrue(alias.writeIndex()); + }); + return new CreateIndexResponse(true, true, request.index()); + }); + + CountDownLatch latch = new CountDownLatch(1); + store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( + name -> assertThat(name, equalTo(ASYNC_SEARCH_INDEX_PREFIX + "000001")), + ex -> { + logger.error(ex); + fail("should have called onResponse, not onFailure"); + }), latch)); + + awaitLatch(latch, 10, TimeUnit.SECONDS); + } + + public void testIndexProperlyExistsAlready() throws InterruptedException { + ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) + .metaData(MetaData.builder() + .put(IndexMetaData.builder(ASYNC_SEARCH_INDEX_PREFIX + "000001") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(randomIntBetween(1,10)) + .numberOfReplicas(randomIntBetween(1,10)) + .putAlias(AliasMetaData.builder(ASYNC_SEARCH_ALIAS) + .writeIndex(true) + .build()))) + .build(); + + client.setVerifier((a, r, l) -> { + fail("no client calls should have been made"); + return null; + }); + + CountDownLatch latch = new CountDownLatch(1); + store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( + name -> assertThat(name, equalTo(ASYNC_SEARCH_INDEX_PREFIX + "000001")), + ex -> { + logger.error(ex); + fail("should have called onResponse, not onFailure"); + }), latch)); + + awaitLatch(latch, 10, TimeUnit.SECONDS); + } + + public void testIndexHasNoWriteIndex() throws InterruptedException { + ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) + .metaData(MetaData.builder() + .put(IndexMetaData.builder(ASYNC_SEARCH_INDEX_PREFIX + "000001") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(randomIntBetween(1,10)) + .numberOfReplicas(randomIntBetween(1,10)) + .putAlias(AliasMetaData.builder(ASYNC_SEARCH_ALIAS) + .build())) + .put(IndexMetaData.builder(randomAlphaOfLength(5)) + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(randomIntBetween(1,10)) + .numberOfReplicas(randomIntBetween(1,10)) + .putAlias(AliasMetaData.builder(ASYNC_SEARCH_ALIAS) + .build()))) + .build(); + + client.setVerifier((a, r, l) -> { + fail("no client calls should have been made"); + return null; + }); + + CountDownLatch latch = new CountDownLatch(1); + store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( + name -> fail("should have called onFailure, not onResponse"), + ex -> { + assertThat(ex, instanceOf(IllegalStateException.class)); + assertThat(ex.getMessage(), containsString("async-search alias [" + ASYNC_SEARCH_ALIAS + + "] does not have a write index")); + }), latch)); + + awaitLatch(latch, 10, TimeUnit.SECONDS); + } + + public void testIndexNotAlias() throws InterruptedException { + ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) + .metaData(MetaData.builder() + .put(IndexMetaData.builder(ASYNC_SEARCH_ALIAS) + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(randomIntBetween(1,10)) + .numberOfReplicas(randomIntBetween(1,10)))) + .build(); + + client.setVerifier((a, r, l) -> { + fail("no client calls should have been made"); + return null; + }); + + CountDownLatch latch = new CountDownLatch(1); + store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( + name -> fail("should have called onFailure, not onResponse"), + ex -> { + assertThat(ex, instanceOf(IllegalStateException.class)); + assertThat(ex.getMessage(), containsString("async-search alias [" + ASYNC_SEARCH_ALIAS + + "] already exists as concrete index")); + }), latch)); + + awaitLatch(latch, 10, TimeUnit.SECONDS); + } + + public void testIndexCreatedConcurrently() throws InterruptedException { + ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) + .metaData(MetaData.builder()) + .build(); + + client.setVerifier((a, r, l) -> { + assertThat(a, instanceOf(CreateIndexAction.class)); + assertThat(r, instanceOf(CreateIndexRequest.class)); + CreateIndexRequest request = (CreateIndexRequest) r; + assertThat(request.aliases(), hasSize(1)); + request.aliases().forEach(alias -> { + assertThat(alias.name(), equalTo(ASYNC_SEARCH_ALIAS)); + assertTrue(alias.writeIndex()); + }); + throw new ResourceAlreadyExistsException("that index already exists"); + }); + + CountDownLatch latch = new CountDownLatch(1); + store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( + name -> assertThat(name, equalTo(ASYNC_SEARCH_INDEX_PREFIX + "000001")), + ex -> { + logger.error(ex); + fail("should have called onResponse, not onFailure"); + }), latch)); + + awaitLatch(latch, 10, TimeUnit.SECONDS); + } + + public void testAliasDoesntExistButIndexDoes() throws InterruptedException { + final String initialIndex = ASYNC_SEARCH_INDEX_PREFIX + "000001"; + ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) + .metaData(MetaData.builder() + .put(IndexMetaData.builder(initialIndex) + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(randomIntBetween(1,10)) + .numberOfReplicas(randomIntBetween(1,10)))) + .build(); + + client.setVerifier((a, r, l) -> { + fail("no client calls should have been made"); + return null; + }); + + CountDownLatch latch = new CountDownLatch(1); + store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( + name -> { + logger.error(name); + fail("should have called onFailure, not onResponse"); + }, + ex -> { + assertThat(ex, instanceOf(IllegalStateException.class)); + assertThat(ex.getMessage(), containsString("async-search index [" + initialIndex + + "] already exists but does not have alias [" + ASYNC_SEARCH_ALIAS + "]")); + }), latch)); + + awaitLatch(latch, 10, TimeUnit.SECONDS); + } + + /** + * A client that delegates to a verifying function for action/request/listener + */ + public static class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute(ActionType action, + Request request, + ActionListener listener) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } + } } From baa498d84d71d87c4afad294845c3c4f68853672 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 18 Dec 2019 22:05:42 +0100 Subject: [PATCH 18/61] validate the index name when receiving a search id --- .../search/TransportGetAsyncSearchAction.java | 3 ++- .../xpack/search/AsyncSearchIT.java | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 96d0b656f3725..dd0cab2934936 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -32,6 +32,7 @@ import java.util.Map; import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; public class TransportGetAsyncSearchAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportGetAsyncSearchAction.class); @@ -59,7 +60,7 @@ public TransportGetAsyncSearchAction(TransportService transportService, protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); - if (searchId.getIndexName().startsWith(AsyncSearchStoreService.ASYNC_SEARCH_ALIAS) == false) { + if (searchId.getIndexName().startsWith(ASYNC_SEARCH_INDEX_PREFIX) == false) { listener.onFailure(new IllegalArgumentException("invalid id [" + request.getId() + "] that references the wrong index [" + searchId.getIndexName() + "]")); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java index 329088ff52bbc..ccdc33208bbef 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java @@ -23,10 +23,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; public class AsyncSearchIT extends AsyncSearchIntegTestCase { @@ -210,4 +213,21 @@ public void testCleanupOnFailure() throws Exception { assertThat(response.getPartialResponse().getShardFailures(), equalTo(numShards)); ensureTaskRemoval(initial.id()); } + + public void testInvalidId() throws Exception { + SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { indexName }); + request.setWaitForCompletion(TimeValue.timeValueMillis(1)); + SearchResponseIterator it = + assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); + AsyncSearchResponse response = it.next(); + AsyncSearchId original = AsyncSearchId.decode(response.id()); + String invalid = AsyncSearchId.encode("another_index", original.getDocId(), original.getTaskId()); + ExecutionException exc = expectThrows(ExecutionException.class, () -> getAsyncSearch(invalid)); + assertThat(exc.getCause(), instanceOf(IllegalArgumentException.class)); + assertThat(exc.getMessage(), containsString("invalid id")); + while (it.hasNext()) { + response = it.next(); + } + assertTrue(response.isFinalResponse()); + } } From fe5c150622fc856ed579f7259e0dbc92a543d102 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 19 Dec 2019 02:34:24 +0100 Subject: [PATCH 19/61] add authentication test --- .../xpack/search/AsyncSearchStoreService.java | 68 ++++++++++++++++++- .../TransportDeleteAsyncSearchAction.java | 4 +- .../search/TransportGetAsyncSearchAction.java | 63 ++--------------- .../TransportSubmitAsyncSearchAction.java | 2 +- .../xpack/search/AsyncSearchIT.java | 2 +- .../search/AsyncSearchStoreServiceTests.java | 67 +++++++++++++++++- 6 files changed, 142 insertions(+), 64 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 1d49c046737e1..4e54ecac496b9 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -33,6 +33,9 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; @@ -44,9 +47,9 @@ import java.util.Map; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.INDEX_TEMPLATE_VERSION; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_TEMPLATE_NAME; -import static org.elasticsearch.xpack.search.TransportGetAsyncSearchAction.ensureAuthenticatedUserIsSame; /** * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the .async-search index. @@ -59,10 +62,14 @@ class AsyncSearchStoreService { static final String RESULT_FIELD = "result"; static final String HEADERS_FIELD = "headers"; + private final TaskManager taskManager; + private final ThreadPool threadPool; private final Client client; private final NamedWriteableRegistry registry; - AsyncSearchStoreService(Client client, NamedWriteableRegistry registry) { + AsyncSearchStoreService(TaskManager taskManager, ThreadPool threadPool, Client client, NamedWriteableRegistry registry) { + this.taskManager = taskManager; + this.threadPool = threadPool; this.client = new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN); this.registry = registry; } @@ -147,6 +154,24 @@ void storeFinalResponse(Map headers, AsyncSearchResponse respons client.update(request, listener); } + AsyncSearchTask getTask(AsyncSearchId searchId) throws IOException { + Task task = taskManager.getTask(searchId.getTaskId().getId()); + if (task == null || task instanceof AsyncSearchTask == false) { + return null; + } + AsyncSearchTask searchTask = (AsyncSearchTask) task; + if (searchTask.getSearchId().equals(searchId) == false) { + return null; + } + + // Check authentication for the user + final Authentication auth = Authentication.getAuthentication(threadPool.getThreadContext()); + if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { + throw new ResourceNotFoundException(searchId.getEncoded() + " not found"); + } + return searchTask; + } + /** * Get the response from the .async-search index if present, or delegate a {@link ResourceNotFoundException} * failure to the provided listener if not. @@ -196,6 +221,45 @@ void deleteResult(AsyncSearchId searchId, ActionListener l ); } + /** + * Extracts the authentication from the original headers and checks that it matches + * the current user. This function returns always true if the provided + * headers do not contain any authentication. + */ + static boolean ensureAuthenticatedUserIsSame(Map originHeaders, Authentication current) throws IOException { + if (originHeaders == null || originHeaders.containsKey(AUTHENTICATION_KEY) == false) { + // no authorization attached to the original request + return true; + } + if (current == null) { + // origin is an authenticated user but current is not + return false; + } + Authentication origin = Authentication.decode(originHeaders.get(AUTHENTICATION_KEY)); + return ensureAuthenticatedUserIsSame(origin, current); + } + + /** + * Compares the {@link Authentication} that was used to create the {@link AsyncSearchId} with the + * current authentication. + */ + static boolean ensureAuthenticatedUserIsSame(Authentication original, Authentication current) { + final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal()); + final boolean sameRealmType; + if (original.getUser().isRunAs()) { + if (current.getUser().isRunAs()) { + sameRealmType = original.getLookedUpBy().getType().equals(current.getLookedUpBy().getType()); + } else { + sameRealmType = original.getLookedUpBy().getType().equals(current.getAuthenticatedBy().getType()); + } + } else if (current.getUser().isRunAs()) { + sameRealmType = original.getAuthenticatedBy().getType().equals(current.getLookedUpBy().getType()); + } else { + sameRealmType = original.getAuthenticatedBy().getType().equals(current.getAuthenticatedBy().getType()); + } + return samePrincipal && sameRealmType; + } + /** * Encode the provided response in a binary form using base64 encoding. */ diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java index 027d2169edc79..a49ac174f923c 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; @@ -28,12 +29,13 @@ public class TransportDeleteAsyncSearchAction extends HandledTransportAction { @@ -53,7 +50,7 @@ public TransportGetAsyncSearchAction(TransportService transportService, this.clusterService = clusterService; this.transportService = transportService; this.threadPool = threadPool; - this.store = new AsyncSearchStoreService(client, registry); + this.store = new AsyncSearchStoreService(taskManager, threadPool, client, registry); } @Override @@ -83,29 +80,11 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, AsyncSearchId searchId, ActionListener listener) throws IOException { - Task runningTask = taskManager.getTask(searchId.getTaskId().getId()); - if (runningTask == null) { - // Task isn't running - getSearchResponseFromIndex(request, searchId, listener); - return; - } - if (runningTask instanceof AsyncSearchTask) { - AsyncSearchTask searchTask = (AsyncSearchTask) runningTask; - if (searchTask.getSearchId().getEncoded().equals(request.getId()) == false) { - // Task id has been reused by another task due to a node restart - getSearchResponseFromIndex(request, searchId, listener); - return; - } - - // Check authentication for the user - final Authentication auth = Authentication.getAuthentication(threadPool.getThreadContext()); - if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { - listener.onFailure(new ResourceNotFoundException(request.getId())); - return; - } - waitForCompletion(request, searchTask, threadPool.relativeTimeInMillis(), listener); + final AsyncSearchTask task = store.getTask(searchId); + if (task != null) { + waitForCompletion(request, task, threadPool.relativeTimeInMillis(), listener); } else { - // Task id has been reused by another task due to a node restart + // Task isn't running getSearchResponseFromIndex(request, searchId, listener); } } @@ -173,38 +152,6 @@ void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask tas } } - static boolean ensureAuthenticatedUserIsSame(Map originHeaders, Authentication current) throws IOException { - if (originHeaders == null || originHeaders.containsKey(AUTHENTICATION_KEY) == false) { - return true; - } - if (current == null) { - return false; - } - Authentication origin = Authentication.decode(originHeaders.get(AUTHENTICATION_KEY)); - return ensureAuthenticatedUserIsSame(origin, current); - } - - /** - * Compares the {@link Authentication} that was used to create the {@link AsyncSearchId} with the - * current authentication. - */ - static boolean ensureAuthenticatedUserIsSame(Authentication original, Authentication current) { - final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal()); - final boolean sameRealmType; - if (original.getUser().isRunAs()) { - if (current.getUser().isRunAs()) { - sameRealmType = original.getLookedUpBy().getType().equals(current.getLookedUpBy().getType()); - } else { - sameRealmType = original.getLookedUpBy().getType().equals(current.getAuthenticatedBy().getType()); - } - } else if (current.getUser().isRunAs()) { - sameRealmType = original.getAuthenticatedBy().getType().equals(current.getLookedUpBy().getType()); - } else { - sameRealmType = original.getAuthenticatedBy().getType().equals(current.getAuthenticatedBy().getType()); - } - return samePrincipal && sameRealmType; - } - /** * Returns a new listener that delegates the response to another listener and * then deletes the async search document from the system index if the response is diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 16aed124d091f..668a44e6fd21c 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -68,7 +68,7 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService, this.nodeClient = nodeClient; this.reduceContextSupplier = () -> searchService.createReduceContext(true); this.searchAction = searchAction; - this.store = new AsyncSearchStoreService(client, registry); + this.store = new AsyncSearchStoreService(taskManager, threadPool, client, registry); this.random = new Random(System.nanoTime()); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java index ccdc33208bbef..ae5c8dcae46ea 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java @@ -76,7 +76,7 @@ public void indexDocuments() throws InterruptedException { public void testMaxMinAggregation() throws Exception { int step = numShards > 2 ? randomIntBetween(2, numShards) : 2; - int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0; + int numFailures = numShards;//randomBoolean() ? randomIntBetween(0, numShards) : 0; SearchSourceBuilder source = new SearchSourceBuilder() .aggregation(AggregationBuilders.min("min").field("metric")) .aggregation(AggregationBuilders.max("max").field("metric")); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java index 85a91f9cce29a..1093a1b107af8 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java @@ -23,16 +23,21 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.user.User; import org.junit.After; import org.junit.Before; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -44,11 +49,13 @@ import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse; import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_ALIAS; import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ensureAuthenticatedUserIsSame; import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.Mockito.mock; public class AsyncSearchStoreServiceTests extends ESTestCase { private NamedWriteableRegistry namedWriteableRegistry; @@ -63,7 +70,8 @@ public void setup() { namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); threadPool = new TestThreadPool(this.getClass().getName()); client = new VerifyingClient(threadPool); - store = new AsyncSearchStoreService(client, namedWriteableRegistry); + TaskManager taskManager = mock(TaskManager.class); + store = new AsyncSearchStoreService(taskManager, threadPool, client, namedWriteableRegistry); } @After @@ -256,6 +264,63 @@ public void testAliasDoesntExistButIndexDoes() throws InterruptedException { awaitLatch(latch, 10, TimeUnit.SECONDS); } + public void testEnsuredAuthenticatedUserIsSame() throws IOException { + Authentication original = + new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null); + Authentication current = randomBoolean() ? original : + new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null); + assertTrue(ensureAuthenticatedUserIsSame(original, current)); + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + original.writeToContext(threadContext); + assertTrue(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); + + // original is not set + assertTrue(ensureAuthenticatedUserIsSame(Collections.emptyMap(), current)); + // current is not set + assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), null)); + + // original user being run as + User user = new User(new User("test", "role"), new User("authenticated", "runas")); + current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), + new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node")); + assertTrue(ensureAuthenticatedUserIsSame(original, current)); + assertTrue(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); + + // both user are run as + current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), + new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node")); + Authentication runAs = current; + assertTrue(ensureAuthenticatedUserIsSame(runAs, current)); + threadContext = new ThreadContext(Settings.EMPTY); + original.writeToContext(threadContext); + assertTrue(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); + + // different authenticated by type + Authentication differentRealmType = + new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", randomAlphaOfLength(5), "node"), null); + threadContext = new ThreadContext(Settings.EMPTY); + original.writeToContext(threadContext); + assertFalse(ensureAuthenticatedUserIsSame(original, differentRealmType)); + assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), differentRealmType)); + + // wrong user + Authentication differentUser = + new Authentication(new User("test2", "role"), new Authentication.RealmRef("realm", "realm", "node"), null); + assertFalse(ensureAuthenticatedUserIsSame(original, differentUser)); + + // run as different user + Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")), + new Authentication.RealmRef("realm", "file", "node1"), new Authentication.RealmRef("realm", "file", "node1")); + assertFalse(ensureAuthenticatedUserIsSame(original, diffRunAs)); + assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), diffRunAs)); + + // run as different looked up by type + Authentication runAsDiffType = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), + new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node")); + assertFalse(ensureAuthenticatedUserIsSame(original, runAsDiffType)); + assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), runAsDiffType)); + } + /** * A client that delegates to a verifying function for action/request/listener */ From b4fa0de75a67eb37ef42b22dd90733cf9bf72501 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 19 Dec 2019 10:02:06 +0100 Subject: [PATCH 20/61] fix block plugin to throw exceptions in createWeight rather than toQuery --- .../xpack/search/AsyncSearchIT.java | 2 +- .../search/AsyncSearchIntegTestCase.java | 44 ++++++++++++++----- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java index ae5c8dcae46ea..ccdc33208bbef 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java @@ -76,7 +76,7 @@ public void indexDocuments() throws InterruptedException { public void testMaxMinAggregation() throws Exception { int step = numShards > 2 ? randomIntBetween(2, numShards) : 2; - int numFailures = numShards;//randomBoolean() ? randomIntBetween(0, numShards) : 0; + int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0; SearchSourceBuilder source = new SearchSourceBuilder() .aggregation(AggregationBuilders.min("min").field("metric")) .aggregation(AggregationBuilders.max("max").field("metric")); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index ec55b085a01b8..b2df8644b7f4c 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -6,7 +6,10 @@ package org.elasticsearch.xpack.search; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Weight; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup; @@ -320,18 +323,39 @@ public static BlockQueryBuilder fromXContent(XContentParser parser, Map Date: Fri, 20 Dec 2019 12:22:24 +0100 Subject: [PATCH 21/61] remove clean_on_completion option after the initial submit --- x-pack/plugin/async-search/build.gradle | 17 ---- .../rest-api-spec/test/search/10_basic.yml | 45 +++++++++++ .../xpack/search/AsyncSearchSecurityIT.java | 2 - .../xpack/search/AsyncSearchId.java | 11 --- .../xpack/search/AsyncSearchStoreService.java | 13 +-- .../xpack/search/AsyncSearchTask.java | 23 ++---- .../search/RestGetAsyncSearchAction.java | 3 +- .../search/TransportGetAsyncSearchAction.java | 71 ++-------------- .../TransportSubmitAsyncSearchAction.java | 51 ++++++------ ...rchIT.java => AsyncSearchActionTests.java} | 46 ++++++----- .../search/AsyncSearchIntegTestCase.java | 10 +-- .../search/AsyncSearchResponseTests.java | 2 +- .../search/GetAsyncSearchRequestTests.java | 2 +- .../search/action/AsyncSearchResponse.java | 80 ++++++++++++------- .../search/action/GetAsyncSearchAction.java | 13 +-- .../action/SubmitAsyncSearchRequest.java | 2 +- .../api/async_search.submit.json | 4 + 17 files changed, 185 insertions(+), 210 deletions(-) rename x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/{AsyncSearchIT.java => AsyncSearchActionTests.java} (89%) diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle index 81e3bcfd9793f..8aac2b9f885ac 100644 --- a/x-pack/plugin/async-search/build.gradle +++ b/x-pack/plugin/async-search/build.gradle @@ -20,17 +20,6 @@ compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" integTest.enabled = false -// Instead we create a separate task to run the -// tests based on ESIntegTestCase -task internalClusterTest(type: Test) { - description = 'Java fantasy integration tests' - mustRunAfter test - - include '**/*IT.class' -} - -check.dependsOn internalClusterTest - // add all sub-projects of the qa sub-project gradle.projectsEvaluated { project.subprojects @@ -51,9 +40,3 @@ dependencies { dependencyLicenses { ignoreSha 'x-pack-core' } - -testingConventions.naming { - IT { - baseClass "org.elasticsearch.xpack.search.AsyncSearchIntegTestCase" - } -} diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml index ebc7528674862..2fd5a1a2e62af 100644 --- a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml +++ b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml @@ -44,6 +44,28 @@ field: max sort: max + - is_false: id + - match: { version: 4 } + - match: { response.is_partial: false } + - length: { response.hits.hits: 3 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.1.value: 3.0 } + + - do: + async_search.submit: + index: test-* + batched_reduce_size: 2 + wait_for_completion: 10s + clean_on_completion: false + body: + query: + match_all: {} + aggs: + 1: + max: + field: max + sort: max + - set: { id: id } - set: { version: version } - match: { version: 4 } @@ -52,7 +74,30 @@ - match: { response.hits.hits.0._source.max: 1 } - match: { response.aggregations.1.value: 3.0 } + - do: + async_search.get: + id: "$id" + + - match: { version: 4 } + - match: { response.is_partial: false } + - length: { response.hits.hits: 3 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.1.value: 3.0 } + + - do: + async_search.delete: + id: "$id" + + - match: { acknowledged: true } + - do: catch: missing async_search.get: id: "$id" + + - do: + catch: missing + async_search.delete: + id: "$id" + + diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java index 3ee3216e7daf0..d22570b822323 100644 --- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java +++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java @@ -132,8 +132,6 @@ static Response getAsyncSearch(String id, String user) throws IOException { final Request request = new Request("GET", "/_async_search/" + id); setRunAsHeader(request, user); request.addParameter("wait_for_completion", "0ms"); - // we do the cleanup explicitly - request.addParameter("clean_on_completion", "false"); return client().performRequest(request); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java index 9dd9849cea6bc..f7ea683bdb895 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java @@ -33,17 +33,6 @@ class AsyncSearchId { this.encoded = encode(indexName, docId, taskId); } - AsyncSearchId(String id) throws IOException { - try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap( Base64.getDecoder().decode(id)))) { - this.indexName = in.readString(); - this.docId = in.readString(); - this.taskId = new TaskId(in.readString()); - this.encoded = id; - } catch (IOException e) { - throw new IOException("invalid id: " + id); - } - } - /** * The index name where to find the response if the task is not running. */ diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 4e54ecac496b9..4e8b152173152 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -42,7 +42,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Base64; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -132,10 +131,14 @@ public void onFailure(Exception e) { * Store an empty document in the .async-search index that is used * as a place-holder for the future response. */ - void storeInitialResponse(Map headers, String index, String docID, ActionListener listener) { + void storeInitialResponse(Map headers, String index, String docID, AsyncSearchResponse response, + ActionListener listener) throws IOException { + Map source = new HashMap<>(); + source.put(RESULT_FIELD, encodeResponse(response)); + source.put(HEADERS_FIELD, headers); IndexRequest request = new IndexRequest(index) .id(docID) - .source(Collections.singletonMap(HEADERS_FIELD, headers), XContentType.JSON); + .source(source, XContentType.JSON); client.index(request, listener); } @@ -144,7 +147,7 @@ void storeInitialResponse(Map headers, String index, String docI */ void storeFinalResponse(Map headers, AsyncSearchResponse response, ActionListener listener) throws IOException { - AsyncSearchId searchId = AsyncSearchId.decode(response.id()); + AsyncSearchId searchId = AsyncSearchId.decode(response.getId()); Map source = new HashMap<>(); source.put(RESULT_FIELD, encodeResponse(response)); source.put(HEADERS_FIELD, headers); @@ -216,7 +219,7 @@ void deleteResult(AsyncSearchId searchId, ActionListener l }, exc -> { logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc); - listener.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded())); + listener.onFailure(exc); }) ); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index b41d7d5c084f8..c11595f137a10 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -7,7 +7,6 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.search.SearchProgressActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -35,9 +34,6 @@ class AsyncSearchTask extends SearchTask { private final Supplier reduceContextSupplier; private final Listener progressListener; - // indicate if the user retrieved the final response - private boolean isFinalResponseRetrieved; - private final Map originHeaders; AsyncSearchTask(long id, @@ -74,19 +70,12 @@ public SearchProgressActionListener getProgressListener() { * Note that this function returns null until {@link Listener#onListShards} * or {@link Listener#onFailure} is called on the search task. */ - synchronized AsyncSearchResponse getAsyncResponse(boolean doFinalReduce, boolean cleanOnCompletion) { - AsyncSearchResponse resp = progressListener.response != null ? progressListener.response.get(doFinalReduce) : null; - if (resp != null - && doFinalReduce - && cleanOnCompletion - && resp.isRunning() == false) { - if (isFinalResponseRetrieved) { - // the response was already retrieved in a previous call - throw new ResourceNotFoundException(resp.id() + " not found"); - } - isFinalResponseRetrieved = true; + AsyncSearchResponse getAsyncResponse(boolean doFinalReduce) { + AsyncSearchResponse response = progressListener.response != null ? progressListener.response.get(doFinalReduce) : null; + if (response != null) { + response.addTaskInfo(taskInfo(searchId.getTaskId().getNodeId(), false)); } - return resp; + return response; } private class Listener extends SearchProgressActionListener { @@ -179,7 +168,7 @@ public synchronized AsyncSearchResponse get(boolean doFinalReduce) { PartialSearchResponse clone = new PartialSearchResponse(old.getTotalShards(), old.getSuccessfulShards(), old.getShardFailures(), old.getTotalHits(), reducedAggs); needFinalReduce = false; - return internal = new AsyncSearchResponse(internal.id(), clone, internal.getFailure(), internal.getVersion(), true); + return internal = new AsyncSearchResponse(internal.getId(), clone, internal.getFailure(), internal.getVersion(), true); } else { return internal; } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index 191edbbfe0635..1079112e4ca18 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -33,8 +33,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli String id = request.param("id"); int lastVersion = request.paramAsInt("last_version", -1); TimeValue waitForCompletion = request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1)); - boolean cleanOnCompletion = request.paramAsBoolean("clean_on_completion", true); - GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(id, waitForCompletion, lastVersion, cleanOnCompletion); + GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(id, waitForCompletion, lastVersion); return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 97436f53d13fe..d3722fdd7ff8d 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -5,15 +5,10 @@ */ package org.elasticsearch.xpack.search; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; @@ -32,8 +27,6 @@ import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; public class TransportGetAsyncSearchAction extends HandledTransportAction { - private static final Logger logger = LogManager.getLogger(TransportGetAsyncSearchAction.class); - private final ClusterService clusterService; private final ThreadPool threadPool; private final TransportService transportService; @@ -62,12 +55,12 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action + searchId.getIndexName() + "]")); } if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { - getSearchResponseFromTask(request, searchId, wrapCleanupListener(request, searchId, listener)); + getSearchResponseFromTask(request, searchId, listener); } else { TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); if (node == null) { - getSearchResponseFromIndex(request, searchId, wrapCleanupListener(request, searchId, listener)); + getSearchResponseFromIndex(request, searchId, listener); } else { transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); @@ -94,24 +87,9 @@ private void getSearchResponseFromIndex(GetAsyncSearchAction.Request request, As store.getResponse(searchId, new ActionListener<>() { @Override public void onResponse(AsyncSearchResponse response) { - if (response == null) { - // the task failed to store a response but we still have the placeholder in the index so we - // force the deletion and throw a resource not found exception. - store.deleteResult(searchId, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - listener.onFailure(new ResourceNotFoundException(request.getId() + " not found")); - } - - @Override - public void onFailure(Exception exc) { - logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", request.getId()), exc); - listener.onFailure(new ResourceNotFoundException(request.getId() + " not found")); - } - }); - } else if (response.getVersion() <= request.getLastVersion()) { + if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), false)); + listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), false)); } else { listener.onResponse(response); } @@ -126,7 +104,7 @@ public void onFailure(Exception e) { void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, long startMs, ActionListener listener) { - final AsyncSearchResponse response = task.getAsyncResponse(false, false); + final AsyncSearchResponse response = task.getAsyncResponse(false); if (response == null) { // the search task is not fully initialized Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); @@ -134,13 +112,13 @@ void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask tas } else { try { if (response.isRunning() == false) { - listener.onResponse(task.getAsyncResponse(true, request.isCleanOnCompletion())); + listener.onResponse(task.getAsyncResponse(true)); } else if (request.getWaitForCompletion().getMillis() < (threadPool.relativeTimeInMillis() - startMs)) { if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.id(), response.getVersion(), true)); + listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), true)); } else { - listener.onResponse(task.getAsyncResponse(true, request.isCleanOnCompletion())); + listener.onResponse(task.getAsyncResponse(true)); } } else { Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); @@ -151,37 +129,4 @@ void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask tas } } } - - /** - * Returns a new listener that delegates the response to another listener and - * then deletes the async search document from the system index if the response is - * frozen (because the task has completed, failed or the coordinating node crashed). - */ - private ActionListener wrapCleanupListener(GetAsyncSearchAction.Request request, - AsyncSearchId searchId, - ActionListener listener) { - return ActionListener.wrap( - resp -> { - if (request.isCleanOnCompletion() && resp.isRunning() == false) { - // TODO: We could ensure that the response was successfully sent to the user - // before deleting, see {@link RestChannel#sendResponse(RestResponse)}. - store.deleteResult(searchId, new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse acknowledgedResponse) { - listener.onResponse(resp); - } - - @Override - public void onFailure(Exception exc) { - logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", request.getId()), exc); - listener.onResponse(resp); - } - }); - } else { - listener.onResponse(resp); - } - }, - listener::onFailure - ); - } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 668a44e6fd21c..77f10ff4ce1fe 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -79,11 +79,11 @@ protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionList submitListener.onFailure(exc); return; } - + final String docID = UUIDs.randomBase64UUID(random); store.ensureAsyncSearchIndex(clusterService.state(), new ActionListener<>() { @Override public void onResponse(String indexName) { - executeSearch(request, indexName, UUIDs.randomBase64UUID(random), submitListener); + executeSearch(request, indexName, docID, submitListener); } @Override @@ -113,14 +113,12 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public void onResponse(SearchResponse response) { progressListener.onResponse(response); - logger.debug(() -> new ParameterizedMessage("store async-search [{}]", task.getSearchId().getEncoded())); onTaskCompletion(threadPool, task, shouldStoreResult); } @Override public void onFailure(Exception exc) { progressListener.onFailure(exc); - logger.error(() -> new ParameterizedMessage("store failed async-search [{}]", task.getSearchId().getEncoded()), exc); onTaskCompletion(threadPool, task, shouldStoreResult); } } @@ -128,7 +126,7 @@ public void onFailure(Exception exc) { // and get the response asynchronously GetAsyncSearchAction.Request getRequest = new GetAsyncSearchAction.Request(task.getSearchId().getEncoded(), - submitRequest.getWaitForCompletion(), -1, submitRequest.isCleanOnCompletion()); + submitRequest.getWaitForCompletion(), -1); nodeClient.executeLocally(GetAsyncSearchAction.INSTANCE, getRequest, new ActionListener<>() { @Override @@ -136,15 +134,19 @@ public void onResponse(AsyncSearchResponse response) { if (response.isRunning() || submitRequest.isCleanOnCompletion() == false) { // the task is still running and the user cannot wait more so we create // an empty document for further retrieval - store.storeInitialResponse(originHeaders, indexName, docID, - ActionListener.wrap(() -> { - shouldStoreResult.set(true); - submitListener.onResponse(response); - })); + try { + store.storeInitialResponse(originHeaders, indexName, docID, response, + ActionListener.wrap(() -> { + shouldStoreResult.set(true); + submitListener.onResponse(response); + })); + } catch (IOException exc) { + onFailure(exc); + } } else { // the user will get a final response directly so no need to store the result on completion shouldStoreResult.set(false); - submitListener.onResponse(response); + submitListener.onResponse(new AsyncSearchResponse(null, response)); } } @@ -160,19 +162,18 @@ public void onFailure(Exception e) { private void onTaskCompletion(ThreadPool threadPool, AsyncSearchTask task, AtomicReference reference) { final Boolean shouldStoreResult = reference.get(); - try { - if (shouldStoreResult == null) { - // the user is still waiting for a response so we schedule a retry in 100ms - threadPool.schedule(() -> onTaskCompletion(threadPool, task, reference), - TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); - } else if (shouldStoreResult) { - // the user retrieved an initial partial response so we need to store the final one - // for further retrieval - store.storeFinalResponse(task.getOriginHeaders(), task.getAsyncResponse(true, false), + if (shouldStoreResult == null) { + // the user is still waiting for a response so we schedule a retry in 100ms + threadPool.schedule(() -> onTaskCompletion(threadPool, task, reference), + TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); + } else if (shouldStoreResult) { + // the user retrieved an initial partial response so we need to store the final one + // for further retrieval + try { + store.storeFinalResponse(task.getOriginHeaders(), task.getAsyncResponse(true), new ActionListener<>() { @Override public void onResponse(UpdateResponse updateResponse) { - logger.debug(() -> new ParameterizedMessage("store async-search [{}]", task.getSearchId().getEncoded())); taskManager.unregister(task); } @@ -183,12 +184,12 @@ public void onFailure(Exception exc) { taskManager.unregister(task); } }); - } else { - // the user retrieved the final response already so we don't need to store it + } catch (IOException exc) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); taskManager.unregister(task); } - } catch (IOException exc) { - logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); + } else { + // the user retrieved the final response already so we don't need to store it taskManager.unregister(task); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java similarity index 89% rename from x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java rename to x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index ccdc33208bbef..153aa926981f3 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class AsyncSearchIT extends AsyncSearchIntegTestCase { +public class AsyncSearchActionTests extends AsyncSearchIntegTestCase { private String indexName; private int numShards; private int numDocs; @@ -98,7 +98,7 @@ public void testMaxMinAggregation() throws Exception { if (numFailures == numShards) { assertTrue(response.hasFailed()); } else { - assertTrue(response.isFinalResponse()); + assertTrue(response.hasResponse()); assertNotNull(response.getSearchResponse().getAggregations()); assertNotNull(response.getSearchResponse().getAggregations().get("max")); assertNotNull(response.getSearchResponse().getAggregations().get("min")); @@ -112,7 +112,8 @@ public void testMaxMinAggregation() throws Exception { assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric)); } } - ensureTaskRemoval(response.id()); + deleteAsyncSearch(response.getId()); + ensureTaskRemoval(response.getId()); } } @@ -142,7 +143,7 @@ public void testTermsAggregation() throws Exception { if (numFailures == numShards) { assertTrue(response.hasFailed()); } else { - assertTrue(response.isFinalResponse()); + assertTrue(response.hasResponse()); assertNotNull(response.getSearchResponse().getAggregations()); assertNotNull(response.getSearchResponse().getAggregations().get("terms")); StringTerms terms = response.getSearchResponse().getAggregations().get("terms"); @@ -157,7 +158,8 @@ public void testTermsAggregation() throws Exception { } } } - ensureTaskRemoval(response.id()); + deleteAsyncSearch(response.getId()); + ensureTaskRemoval(response.getId()); } } @@ -167,13 +169,14 @@ public void testRestartAfterCompletion() throws Exception { assertBlockingIterator(indexName, new SearchSourceBuilder(), 0, 2)) { initial = it.next(); } - ensureTaskCompletion(initial.id()); - restartTaskNode(initial.id()); - AsyncSearchResponse response = getAsyncSearch(initial.id()); - assertTrue(response.isFinalResponse()); + ensureTaskCompletion(initial.getId()); + restartTaskNode(initial.getId()); + AsyncSearchResponse response = getAsyncSearch(initial.getId()); + assertTrue(response.hasResponse()); assertFalse(response.isRunning()); assertFalse(response.hasPartialResponse()); - ensureTaskRemoval(response.id()); + deleteAsyncSearch(response.getId()); + ensureTaskRemoval(response.getId()); } public void testDeleteCancelRunningTask() throws Exception { @@ -181,10 +184,10 @@ public void testDeleteCancelRunningTask() throws Exception { SearchResponseIterator it = assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); initial = it.next(); - deleteAsyncSearch(initial.id()); + deleteAsyncSearch(initial.getId()); it.close(); - ensureTaskCompletion(initial.id()); - ensureTaskRemoval(initial.id()); + ensureTaskCompletion(initial.getId()); + ensureTaskRemoval(initial.getId()); } public void testDeleteCleanupIndex() throws Exception { @@ -193,10 +196,10 @@ public void testDeleteCleanupIndex() throws Exception { SearchResponseIterator it = assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); AsyncSearchResponse response = it.next(); - deleteAsyncSearch(response.id()); + deleteAsyncSearch(response.getId()); it.close(); - ensureTaskCompletion(response.id()); - ensureTaskRemoval(response.id()); + ensureTaskCompletion(response.getId()); + ensureTaskRemoval(response.getId()); } public void testCleanupOnFailure() throws Exception { @@ -205,13 +208,14 @@ public void testCleanupOnFailure() throws Exception { assertBlockingIterator(indexName, new SearchSourceBuilder(), numShards, 2)) { initial = it.next(); } - ensureTaskCompletion(initial.id()); - AsyncSearchResponse response = getAsyncSearch(initial.id()); + ensureTaskCompletion(initial.getId()); + AsyncSearchResponse response = getAsyncSearch(initial.getId()); assertTrue(response.hasFailed()); assertTrue(response.hasPartialResponse()); assertThat(response.getPartialResponse().getTotalShards(), equalTo(numShards)); assertThat(response.getPartialResponse().getShardFailures(), equalTo(numShards)); - ensureTaskRemoval(initial.id()); + deleteAsyncSearch(initial.getId()); + ensureTaskRemoval(initial.getId()); } public void testInvalidId() throws Exception { @@ -220,7 +224,7 @@ public void testInvalidId() throws Exception { SearchResponseIterator it = assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); AsyncSearchResponse response = it.next(); - AsyncSearchId original = AsyncSearchId.decode(response.id()); + AsyncSearchId original = AsyncSearchId.decode(response.getId()); String invalid = AsyncSearchId.encode("another_index", original.getDocId(), original.getTaskId()); ExecutionException exc = expectThrows(ExecutionException.class, () -> getAsyncSearch(invalid)); assertThat(exc.getCause(), instanceOf(IllegalArgumentException.class)); @@ -228,6 +232,6 @@ public void testInvalidId() throws Exception { while (it.hasNext()) { response = it.next(); } - assertTrue(response.isFinalResponse()); + assertFalse(response.isRunning()); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index b2df8644b7f4c..bc0e8edcce426 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -96,7 +96,7 @@ protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request protected AsyncSearchResponse getAsyncSearch(String id) throws ExecutionException, InterruptedException { return client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(id, TimeValue.MINUS_ONE, -1, true)).get(); + new GetAsyncSearchAction.Request(id, TimeValue.MINUS_ONE, -1)).get(); } protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionException, InterruptedException { @@ -207,7 +207,7 @@ private AsyncSearchResponse doNext() throws Exception { } assertBusy(() -> { AsyncSearchResponse newResp = client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(response.id(), TimeValue.timeValueMillis(10), lastVersion, true) + new GetAsyncSearchAction.Request(response.getId(), TimeValue.timeValueMillis(10), lastVersion) ).get(); atomic.set(newResp); assertNotEquals(RestStatus.NOT_MODIFIED, newResp.status()); @@ -219,14 +219,14 @@ private AsyncSearchResponse doNext() throws Exception { assertThat(newResponse.status(), equalTo(RestStatus.PARTIAL_CONTENT)); assertTrue(newResponse.hasPartialResponse()); assertFalse(newResponse.hasFailed()); - assertFalse(newResponse.isFinalResponse()); + assertFalse(newResponse.hasResponse()); assertThat(newResponse.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); assertThat(newResponse.getPartialResponse().getShardFailures(), lessThanOrEqualTo(numFailures)); } else if (numFailures == shardLatchArray.length) { assertThat(newResponse.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); assertTrue(newResponse.hasFailed()); assertTrue(newResponse.hasPartialResponse()); - assertFalse(newResponse.isFinalResponse()); + assertFalse(newResponse.hasResponse()); assertThat(newResponse.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); assertThat(newResponse.getPartialResponse().getSuccessfulShards(), equalTo(0)); assertThat(newResponse.getPartialResponse().getShardFailures(), equalTo(numFailures)); @@ -234,7 +234,7 @@ private AsyncSearchResponse doNext() throws Exception { assertNull(newResponse.getPartialResponse().getTotalHits()); } else { assertThat(newResponse.status(), equalTo(RestStatus.OK)); - assertTrue(newResponse.isFinalResponse()); + assertTrue(newResponse.hasResponse()); assertFalse(newResponse.hasPartialResponse()); assertThat(newResponse.status(), equalTo(RestStatus.OK)); assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length)); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index e7bfa61cbf1f9..63ce2602d36e0 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -140,7 +140,7 @@ private static PartialSearchResponse randomPartialSearchResponse() { } static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchResponse actual) { - assertEquals(expected.id(), actual.id()); + assertEquals(expected.getId(), actual.getId()); assertEquals(expected.getVersion(), actual.getVersion()); assertEquals(expected.status(), actual.status()); assertEquals(expected.getPartialResponse(), actual.getPartialResponse()); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java index 1645163b4fdb1..3e96e738eede1 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java @@ -21,7 +21,7 @@ protected Writeable.Reader instanceReader() { @Override protected GetAsyncSearchAction.Request createTestInstance() { return new GetAsyncSearchAction.Request(randomSearchId(), TimeValue.timeValueMillis(randomIntBetween(1, 10000)), - randomIntBetween(-1, Integer.MAX_VALUE), randomBoolean()); + randomIntBetween(-1, Integer.MAX_VALUE)); } static String randomSearchId() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 558dd70f0e909..67b6adc3c52ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -8,11 +8,13 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; @@ -24,15 +26,17 @@ * before completion, or a final {@link SearchResponse} if the request succeeded. */ public class AsyncSearchResponse extends ActionResponse implements StatusToXContentObject { + @Nullable private final String id; private final int version; - private final SearchResponse finalResponse; + private final SearchResponse response; private final PartialSearchResponse partialResponse; private final ElasticsearchException failure; - - private final long timestamp; private final boolean isRunning; + private long startDateMillis; + private long runningTimeNanos; + public AsyncSearchResponse(String id, int version, boolean isRunning) { this(id, null, null, null, version, isRunning); } @@ -49,46 +53,61 @@ public AsyncSearchResponse(String id, PartialSearchResponse response, Elasticsea this(id, response, null, failure, version, isRunning); } + public AsyncSearchResponse(String id, AsyncSearchResponse clone) { + this(id, clone.partialResponse, clone.response, clone.failure, clone.version, clone.isRunning); + this.startDateMillis = clone.startDateMillis; + this.runningTimeNanos = clone.runningTimeNanos; + } + private AsyncSearchResponse(String id, PartialSearchResponse partialResponse, - SearchResponse finalResponse, + SearchResponse response, ElasticsearchException failure, int version, boolean isRunning) { + assert id != null || isRunning == false; this.id = id; this.version = version; this.partialResponse = partialResponse; this.failure = failure; - this.finalResponse = finalResponse != null ? wrapFinalResponse(finalResponse) : null; - this.timestamp = System.currentTimeMillis(); + this.response = response != null ? wrapFinalResponse(response) : null; this.isRunning = isRunning; } public AsyncSearchResponse(StreamInput in) throws IOException { - this.id = in.readString(); + this.id = in.readOptionalString(); this.version = in.readVInt(); this.partialResponse = in.readOptionalWriteable(PartialSearchResponse::new); this.failure = in.readOptionalWriteable(ElasticsearchException::new); - this.finalResponse = in.readOptionalWriteable(SearchResponse::new); - this.timestamp = in.readLong(); + this.response = in.readOptionalWriteable(SearchResponse::new); this.isRunning = in.readBoolean(); + this.startDateMillis = in.readLong(); + this.runningTimeNanos = in.readLong(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(id); + out.writeOptionalString(id); out.writeVInt(version); out.writeOptionalWriteable(partialResponse); out.writeOptionalWriteable(failure); - out.writeOptionalWriteable(finalResponse); - out.writeLong(timestamp); + out.writeOptionalWriteable(response); out.writeBoolean(isRunning); + out.writeLong(startDateMillis); + out.writeLong(runningTimeNanos); + } + + public void addTaskInfo(TaskInfo taskInfo) { + this.startDateMillis = taskInfo.getStartTime(); + this.runningTimeNanos = taskInfo.getRunningTimeNanos(); } /** - * Return the id of the search progress request. + * Return the id of the async search request or null if the response + * was cleaned on completion. */ - public String id() { + @Nullable + public String getId() { return id; } @@ -116,8 +135,8 @@ public boolean hasPartialResponse() { /** * Return true if the final response is available. */ - public boolean isFinalResponse() { - return finalResponse != null; + public boolean hasResponse() { + return response != null; } /** @@ -125,7 +144,7 @@ public boolean isFinalResponse() { * request is running or failed. */ public SearchResponse getSearchResponse() { - return finalResponse; + return response; } /** @@ -144,10 +163,14 @@ public ElasticsearchException getFailure() { } /** - * When this response was created. + * When this response was created as a timestamp in milliseconds since epoch. */ - public long getTimestamp() { - return timestamp; + public long getStartDate() { + return startDateMillis; + } + + public long getRunningTimeNanos() { + return runningTimeNanos; } /** @@ -159,27 +182,30 @@ public boolean isRunning() { @Override public RestStatus status() { - if (finalResponse == null && partialResponse == null) { + if (response == null && partialResponse == null) { return failure != null ? failure.status() : NOT_MODIFIED; - } else if (finalResponse == null) { + } else if (response == null) { return failure != null ? failure.status() : PARTIAL_CONTENT; } else { - return finalResponse.status(); + return response.status(); } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field("id", id); + if (id != null) { + builder.field("id", id); + } builder.field("version", version); builder.field("is_running", isRunning); - builder.field("timestamp", timestamp); + builder.field("start_date_in_millis", startDateMillis); + builder.field("running_time_in_nanos", runningTimeNanos); if (partialResponse != null) { builder.field("response", partialResponse); - } else if (finalResponse != null) { - builder.field("response", finalResponse); + } else if (response != null) { + builder.field("response", response); } if (failure != null) { builder.startObject("failure"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java index e82bed950680e..f1e89cbf35409 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -35,7 +35,6 @@ public static class Request extends ActionRequest implements CompositeIndicesReq private final String id; private final int lastVersion; private final TimeValue waitForCompletion; - private final boolean cleanOnCompletion; /** * Create a new request @@ -43,11 +42,10 @@ public static class Request extends ActionRequest implements CompositeIndicesReq * @param waitForCompletion The minimum time that the request should wait before returning a partial result. * @param lastVersion The last version returned by a previous call. */ - public Request(String id, TimeValue waitForCompletion, int lastVersion, boolean cleanOnCompletion) { + public Request(String id, TimeValue waitForCompletion, int lastVersion) { this.id = id; this.waitForCompletion = waitForCompletion; this.lastVersion = lastVersion; - this.cleanOnCompletion = cleanOnCompletion; } public Request(StreamInput in) throws IOException { @@ -55,7 +53,6 @@ public Request(StreamInput in) throws IOException { this.id = in.readString(); this.waitForCompletion = TimeValue.timeValueMillis(in.readLong()); this.lastVersion = in.readInt(); - this.cleanOnCompletion = in.readBoolean(); } @Override @@ -64,7 +61,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeLong(waitForCompletion.millis()); out.writeInt(lastVersion); - out.writeBoolean(cleanOnCompletion); } @Override @@ -92,13 +88,6 @@ public TimeValue getWaitForCompletion() { return waitForCompletion; } - /** - * Cleanup the resource on completion or failure. - */ - public boolean isCleanOnCompletion() { - return cleanOnCompletion; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 14d82dee43735..87f0e083cb9fd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -125,7 +125,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; SubmitAsyncSearchRequest request1 = (SubmitAsyncSearchRequest) o; return cleanOnCompletion == request1.cleanOnCompletion && - Objects.equals(waitForCompletion, request1.waitForCompletion) && + waitForCompletion.equals(request1.waitForCompletion) && request.equals(request1.request); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json index 678c22e0f733b..bba7c39723fea 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json @@ -33,6 +33,10 @@ "type":"time", "description":"Specify the time that the request should block waiting for the final response (default: 1s)" }, + "clean_on_completion":{ + "type":"boolean", + "description":"Specify whether the response should not be stored in the cluster if it completed within the provided wait_for_completion time (default: true)" + }, "analyzer":{ "type":"string", "description":"The analyzer to use for the query string" From 93189d38054961070647eaf9734c922af275c720 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 20 Dec 2019 17:27:19 +0100 Subject: [PATCH 22/61] move shard stats in a _shards section for partial response --- .../core/search/action/AsyncSearchResponse.java | 17 +++++++++-------- .../search/action/PartialSearchResponse.java | 6 +++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 67b6adc3c52ff..c5a82ada58006 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -17,6 +17,7 @@ import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.rest.RestStatus.NOT_MODIFIED; import static org.elasticsearch.rest.RestStatus.PARTIAL_CONTENT; @@ -35,7 +36,7 @@ public class AsyncSearchResponse extends ActionResponse implements StatusToXCont private final boolean isRunning; private long startDateMillis; - private long runningTimeNanos; + private long runningTimeMillis; public AsyncSearchResponse(String id, int version, boolean isRunning) { this(id, null, null, null, version, isRunning); @@ -56,7 +57,7 @@ public AsyncSearchResponse(String id, PartialSearchResponse response, Elasticsea public AsyncSearchResponse(String id, AsyncSearchResponse clone) { this(id, clone.partialResponse, clone.response, clone.failure, clone.version, clone.isRunning); this.startDateMillis = clone.startDateMillis; - this.runningTimeNanos = clone.runningTimeNanos; + this.runningTimeMillis = clone.runningTimeMillis; } private AsyncSearchResponse(String id, @@ -82,7 +83,7 @@ public AsyncSearchResponse(StreamInput in) throws IOException { this.response = in.readOptionalWriteable(SearchResponse::new); this.isRunning = in.readBoolean(); this.startDateMillis = in.readLong(); - this.runningTimeNanos = in.readLong(); + this.runningTimeMillis = in.readLong(); } @Override @@ -94,12 +95,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(response); out.writeBoolean(isRunning); out.writeLong(startDateMillis); - out.writeLong(runningTimeNanos); + out.writeLong(runningTimeMillis); } public void addTaskInfo(TaskInfo taskInfo) { this.startDateMillis = taskInfo.getStartTime(); - this.runningTimeNanos = taskInfo.getRunningTimeNanos(); + this.runningTimeMillis = TimeUnit.NANOSECONDS.toMillis(taskInfo.getRunningTimeNanos()); } /** @@ -169,8 +170,8 @@ public long getStartDate() { return startDateMillis; } - public long getRunningTimeNanos() { - return runningTimeNanos; + public long getRunningTime() { + return runningTimeMillis; } /** @@ -200,7 +201,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("version", version); builder.field("is_running", isRunning); builder.field("start_date_in_millis", startDateMillis); - builder.field("running_time_in_nanos", runningTimeNanos); + builder.field("running_time_in_millis", runningTimeMillis); if (partialResponse != null) { builder.field("response", partialResponse); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java index 3537d8acf26b3..74b083a4c239f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -66,9 +67,8 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("is_partial", true); - builder.field("total_shards", totalShards); - builder.field("successful_shards", successfulShards); - builder.field("shard_failures", shardFailures); + RestActions.buildBroadcastShardsHeader(builder, params, totalShards, successfulShards, 0, + shardFailures, null); if (totalHits != null) { builder.startObject(SearchHits.Fields.TOTAL); builder.field("value", totalHits.value); From d03b9402192b03a592e08907e07b9ac10f5f5515 Mon Sep 17 00:00:00 2001 From: jimczi Date: Sun, 22 Dec 2019 10:54:18 +0100 Subject: [PATCH 23/61] address review --- .../xpack/search/AsyncSearchId.java | 25 +++++++++++++------ .../xpack/search/AsyncSearchStoreService.java | 7 ++++++ .../TransportDeleteAsyncSearchAction.java | 18 +++---------- .../search/TransportGetAsyncSearchAction.java | 6 ----- .../TransportSubmitAsyncSearchAction.java | 15 +++-------- .../xpack/search/AsyncSearchIdTests.java | 8 +++--- 6 files changed, 37 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java index f7ea683bdb895..b80dd85013d9b 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java @@ -12,11 +12,12 @@ import org.elasticsearch.tasks.TaskId; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.util.Base64; import java.util.Objects; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; + /** * A class that contains all information related to a submitted async search. */ @@ -85,9 +86,9 @@ static String encode(String indexName, String docId, TaskId taskId) { out.writeString(indexName); out.writeString(docId); out.writeString(taskId.toString()); - return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes())); + return Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(out.bytes())); } catch (IOException e) { - throw new UncheckedIOException(e); + throw new IllegalArgumentException(e); } } @@ -95,11 +96,21 @@ static String encode(String indexName, String docId, TaskId taskId) { * Decode a base64 encoded string into an {@link AsyncSearchId} that can be used * to retrieve the response of an async search. */ - static AsyncSearchId decode(String id) throws IOException { - try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap( Base64.getDecoder().decode(id)))) { - return new AsyncSearchId(in.readString(), in.readString(), new TaskId(in.readString())); + static AsyncSearchId decode(String id) { + final AsyncSearchId searchId; + try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap( Base64.getUrlDecoder().decode(id)))) { + searchId = new AsyncSearchId(in.readString(), in.readString(), new TaskId(in.readString())); } catch (IOException e) { - throw new IOException("invalid id: " + id); + throw new IllegalArgumentException("invalid id: " + id); + } + validateAsyncSearchId(searchId); + return searchId; + } + + static void validateAsyncSearchId(AsyncSearchId searchId) { + if (searchId.getIndexName().startsWith(ASYNC_SEARCH_INDEX_PREFIX) == false) { + throw new IllegalArgumentException("invalid id [" + searchId.getEncoded() + "] that references the wrong index [" + + searchId.getIndexName() + "]"); } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 4e8b152173152..0beb3129cfebc 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -73,6 +73,13 @@ class AsyncSearchStoreService { this.registry = registry; } + /** + * Return the internal client with origin. + */ + Client getClient() { + return client; + } + /** * Checks if the async-search index exists, and if not, creates it. * The provided {@link ActionListener} is called with the index name that should diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java index a49ac174f923c..887d3b1196824 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -6,13 +6,10 @@ package org.elasticsearch.xpack.search; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.tasks.Task; @@ -20,10 +17,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; -import java.io.IOException; - public class TransportDeleteAsyncSearchAction extends HandledTransportAction { - private final NodeClient nodeClient; private final AsyncSearchStoreService store; @Inject @@ -31,10 +25,8 @@ public TransportDeleteAsyncSearchAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool, NamedWriteableRegistry registry, - NodeClient nodeClient, Client client) { super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new); - this.nodeClient = nodeClient; this.store = new AsyncSearchStoreService(taskManager, threadPool, client, registry); } @@ -44,17 +36,13 @@ protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, Act AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); // check if the response can be retrieved by the user (handle security) and then cancel/delete. store.getResponse(searchId, ActionListener.wrap(res -> cancelTaskAndDeleteResult(searchId, listener), listener::onFailure)); - } catch (IOException exc) { + } catch (Exception exc) { listener.onFailure(exc); } } private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener listener) { - try { - nodeClient.execute(CancelTasksAction.INSTANCE, new CancelTasksRequest().setTaskId(searchId.getTaskId()), - ActionListener.wrap(() -> store.deleteResult(searchId, listener))); - } catch (Exception e) { - store.deleteResult(searchId, listener); - } + store.getClient().admin().cluster().prepareCancelTasks().setTaskId(searchId.getTaskId()) + .execute(ActionListener.wrap(() -> store.deleteResult(searchId, listener))); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index d3722fdd7ff8d..e028ee4613484 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -24,8 +24,6 @@ import java.io.IOException; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; - public class TransportGetAsyncSearchAction extends HandledTransportAction { private final ClusterService clusterService; private final ThreadPool threadPool; @@ -50,10 +48,6 @@ public TransportGetAsyncSearchAction(TransportService transportService, protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); - if (searchId.getIndexName().startsWith(ASYNC_SEARCH_INDEX_PREFIX) == false) { - listener.onFailure(new IllegalArgumentException("invalid id [" + request.getId() + "] that references the wrong index [" - + searchId.getIndexName() + "]")); - } if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { getSearchResponseFromTask(request, searchId, listener); } else { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 77f10ff4ce1fe..934ec5405299f 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -80,17 +80,10 @@ protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionList return; } final String docID = UUIDs.randomBase64UUID(random); - store.ensureAsyncSearchIndex(clusterService.state(), new ActionListener<>() { - @Override - public void onResponse(String indexName) { - executeSearch(request, indexName, docID, submitListener); - } - - @Override - public void onFailure(Exception exc) { - submitListener.onFailure(exc); - } - }); + store.ensureAsyncSearchIndex(clusterService.state(), ActionListener.wrap( + indexName -> executeSearch(request, indexName, docID, submitListener), + submitListener::onFailure + )); } private void executeSearch(SubmitAsyncSearchRequest submitRequest, String indexName, String docID, diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java index f57e21311ac7e..3e22664431de5 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java @@ -10,11 +10,13 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; + public class AsyncSearchIdTests extends ESTestCase { - public void testEncode() throws Exception { + public void testEncode() { for (int i = 0; i < 10; i++) { - AsyncSearchId instance = new AsyncSearchId(randomAlphaOfLengthBetween(5, 20), UUIDs.randomBase64UUID(), - new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong())); + AsyncSearchId instance = new AsyncSearchId(ASYNC_SEARCH_INDEX_PREFIX + randomAlphaOfLengthBetween(5, 20), + UUIDs.randomBase64UUID(), new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong())); String encoded = AsyncSearchId.encode(instance.getIndexName(), instance.getDocId(), instance.getTaskId()); AsyncSearchId same = AsyncSearchId.decode(encoded); assertEquals(same, instance); From 7ffb1dde347870e4541ad05532b1acdeaccf2b14 Mon Sep 17 00:00:00 2001 From: jimczi Date: Mon, 6 Jan 2020 16:40:32 +0100 Subject: [PATCH 24/61] address review --- .../rest-api-spec/test/search/10_basic.yml | 15 ++++-- .../xpack/search/AsyncSearchStoreService.java | 4 +- .../search/AsyncSearchTemplateRegistry.java | 4 +- .../action/DeleteAsyncSearchAction.java | 3 +- .../search/action/GetAsyncSearchAction.java | 3 +- .../action/SubmitAsyncSearchRequest.java | 3 +- .../xpack/security/authz/RBACEngine.java | 49 +++++++++++++------ 7 files changed, 53 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml index 2fd5a1a2e62af..cb3b0ee37b796 100644 --- a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml +++ b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml @@ -3,14 +3,23 @@ - do: indices.create: index: test-1 + body: + settings: + number_of_shards: "2" - do: indices.create: index: test-2 + body: + settings: + number_of_shards: "1" - do: indices.create: index: test-3 + body: + settings: + number_of_shards: "3" - do: index: @@ -45,7 +54,7 @@ sort: max - is_false: id - - match: { version: 4 } + - match: { version: 7 } - match: { response.is_partial: false } - length: { response.hits.hits: 3 } - match: { response.hits.hits.0._source.max: 1 } @@ -68,7 +77,7 @@ - set: { id: id } - set: { version: version } - - match: { version: 4 } + - match: { version: 7 } - match: { response.is_partial: false } - length: { response.hits.hits: 3 } - match: { response.hits.hits.0._source.max: 1 } @@ -78,7 +87,7 @@ async_search.get: id: "$id" - - match: { version: 4 } + - match: { version: 7 } - match: { response.is_partial: false } - length: { response.hits.hits: 3 } - match: { response.hits.hits.0._source.max: 1 } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 0beb3129cfebc..c11e4c9cb240f 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -45,7 +45,7 @@ import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; +import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.INDEX_TEMPLATE_VERSION; import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_TEMPLATE_NAME; @@ -69,7 +69,7 @@ class AsyncSearchStoreService { AsyncSearchStoreService(TaskManager taskManager, ThreadPool threadPool, Client client, NamedWriteableRegistry registry) { this.taskManager = taskManager; this.threadPool = threadPool; - this.client = new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN); + this.client = new OriginSettingClient(client, TASKS_ORIGIN); this.registry = registry; } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java index bd0a91c06b7dc..501d33607fd60 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java @@ -18,7 +18,7 @@ import java.util.Collections; import java.util.List; -import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; +import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; /** * Manage the index template and associated ILM policy for the .async-search index. @@ -64,6 +64,6 @@ protected List getPolicyConfigs() { @Override protected String getOrigin() { - return INDEX_LIFECYCLE_ORIGIN; + return TASKS_ORIGIN; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java index 92ed6b1122824..d69de80d2293e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/DeleteAsyncSearchAction.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -30,7 +29,7 @@ public Writeable.Reader getResponseReader() { return AcknowledgedResponse::new; } - public static class Request extends ActionRequest implements CompositeIndicesRequest { + public static class Request extends ActionRequest { private final String id; public Request(String id) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java index f1e89cbf35409..cc96d8d913300 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -31,7 +30,7 @@ public Writeable.Reader getResponseReader() { return AsyncSearchResponse::new; } - public static class Request extends ActionRequest implements CompositeIndicesRequest { + public static class Request extends ActionRequest { private final String id; private final int lastVersion; private final TimeValue waitForCompletion; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 87f0e083cb9fd..58cbc1bc03d40 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -7,7 +7,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -27,7 +26,7 @@ * * @see AsyncSearchResponse */ -public class SubmitAsyncSearchRequest extends ActionRequest implements CompositeIndicesRequest { +public class SubmitAsyncSearchRequest extends ActionRequest { private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1); private boolean cleanOnCompletion = true; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index beb53aab5f0b6..b1898bdcb801a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -32,6 +32,10 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; +import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest; import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction; @@ -206,9 +210,6 @@ private static boolean shouldAuthorizeIndexActionNameOnly(String action, Transpo case MultiGetAction.NAME: case MultiTermVectorsAction.NAME: case MultiSearchAction.NAME: - case "indices:data/read/async_search/submit": - case "indices:data/read/async_search/get": - case "indices:data/read/async_search/delete": case "indices:data/read/mpercolate": case "indices:data/read/msearch/template": case "indices:data/read/search/template": @@ -242,17 +243,18 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth // need to validate that the action is allowed and then move on authorizeIndexActionName(action, authorizationInfo, null, listener); } else if (request instanceof IndicesRequest == false && request instanceof IndicesAliasesRequest == false) { - // scroll is special - // some APIs are indices requests that are not actually associated with indices. For example, - // search scroll request, is categorized under the indices context, but doesn't hold indices names - // (in this case, the security check on the indices was done on the search request that initialized - // the scroll. Given that scroll is implemented using a context on the node holding the shard, we - // piggyback on it and enhance the context with the original authentication. This serves as our method - // to validate the scroll id only stays with the same user! - // note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any - // indices permission as it's categorized under cluster. This is why the scroll check is performed - // even before checking if the user has any indices permission. if (isScrollRelatedAction(action)) { + // scroll is special + // some APIs are indices requests that are not actually associated with indices. For example, + // search scroll request, is categorized under the indices context, but doesn't hold indices names + // (in this case, the security check on the indices was done on the search request that initialized + // the scroll. Given that scroll is implemented using a context on the node holding the shard, we + // piggyback on it and enhance the context with the original authentication. This serves as our method + // to validate the scroll id only stays with the same user! + // note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any + // indices permission as it's categorized under cluster. This is why the scroll check is performed + // even before checking if the user has any indices permission. + // if the action is a search scroll action, we first authorize that the user can execute the action for some // index and if they cannot, we can fail the request early before we allow the execution of the action and in // turn the shard actions @@ -264,10 +266,21 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth // information such as the index and the incoming address of the request listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES)); } + } else if (isAsyncSearchRelatedAction(action)) { + if (SubmitAsyncSearchAction.NAME.equals(action)) { + // we check if the user has any indices permission when submitting an async-search request in order to be + // able to fail the request early. Fine grained index-level permissions are handled by the search action + // that is triggered internally by the submit API. + authorizeIndexActionName(action, authorizationInfo, null, listener); + } else { + // async-search actions other than submit have a custom security layer that checks if the current user is + // the same as the user that submitted the original request so we can skip security here. + listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES)); + } } else { assert false : - "only scroll related requests are known indices api that don't support retrieving the indices they relate to"; - listener.onFailure(new IllegalStateException("only scroll related requests are known indices api that don't support " + + "only scroll and async-search related requests are known indices api that don't support retrieving the indices they relate to"; + listener.onFailure(new IllegalStateException("only scroll and async-search related requests are known indices api that don't support " + "retrieving the indices they relate to")); } } else if (request instanceof IndicesRequest && @@ -570,4 +583,10 @@ private static boolean isScrollRelatedAction(String action) { action.equals("indices:data/read/sql/close_cursor") || action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME); } + + private static boolean isAsyncSearchRelatedAction(String action) { + return action.equals(SubmitAsyncSearchAction.NAME) || + action.equals(GetAsyncSearchAction.NAME) || + action.equals(DeleteAsyncSearchAction.NAME); + } } From 9930ccd29f32b40dbc0360935aeec68ffc2c870a Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 8 Jan 2020 10:27:06 +0100 Subject: [PATCH 25/61] unused import --- .../java/org/elasticsearch/xpack/security/authz/RBACEngine.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index b1898bdcb801a..aa9b115792fb0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -35,7 +35,6 @@ import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; -import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest; import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction; From 9d9870626332a139d17871947d15129ad0c88ac0 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 8 Jan 2020 15:22:00 +0100 Subject: [PATCH 26/61] line len --- .../elasticsearch/xpack/security/authz/RBACEngine.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index aa9b115792fb0..801bcb400d5ce 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -277,10 +277,10 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES)); } } else { - assert false : - "only scroll and async-search related requests are known indices api that don't support retrieving the indices they relate to"; - listener.onFailure(new IllegalStateException("only scroll and async-search related requests are known indices api that don't support " + - "retrieving the indices they relate to")); + assert false : "only scroll and async-search related requests are known indices api that don't " + + "support retrieving the indices they relate to"; + listener.onFailure(new IllegalStateException("only scroll and async-search related requests are known indices " + + "api that don't support retrieving the indices they relate to")); } } else if (request instanceof IndicesRequest && IndicesAndAliasesResolver.allowsRemoteIndices((IndicesRequest) request)) { From 9056b873039b5f087a7f98dee1fd31d3850dde11 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 15 Jan 2020 13:15:59 +0100 Subject: [PATCH 27/61] address review --- .../action/search/SearchProgressListener.java | 2 +- .../rest-api-spec/test/search/10_basic.yml | 30 +-- .../xpack/search/AsyncSearchId.java | 10 +- .../xpack/search/AsyncSearchStoreService.java | 12 +- .../xpack/search/AsyncSearchTask.java | 171 ++++++++++-------- .../TransportDeleteAsyncSearchAction.java | 2 +- .../search/TransportGetAsyncSearchAction.java | 48 ++--- .../TransportSubmitAsyncSearchAction.java | 127 ++++++++----- .../search/AsyncSearchIntegTestCase.java | 1 - .../search/AsyncSearchResponseTests.java | 2 +- .../search/AsyncSearchStoreServiceTests.java | 2 +- .../search/action/AsyncSearchResponse.java | 14 +- .../search/action/PartialSearchResponse.java | 24 ++- .../action/SubmitAsyncSearchRequest.java | 8 +- .../core/src/main/resources/async-search.json | 2 +- 15 files changed, 254 insertions(+), 201 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java index 87146719a0f52..572d4f6e2a88c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java @@ -37,7 +37,7 @@ /** * A listener that allows to track progress of the {@link SearchAction}. */ -abstract class SearchProgressListener { +public abstract class SearchProgressListener { private static final Logger logger = LogManager.getLogger(SearchProgressListener.class); public static final SearchProgressListener NOOP = new SearchProgressListener() {}; diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml index cb3b0ee37b796..c66a2f152d877 100644 --- a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml +++ b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml @@ -54,11 +54,11 @@ sort: max - is_false: id - - match: { version: 7 } - - match: { response.is_partial: false } + - match: { version: 7 } + - match: { response.is_partial: false } - length: { response.hits.hits: 3 } - - match: { response.hits.hits.0._source.max: 1 } - - match: { response.aggregations.1.value: 3.0 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.1.value: 3.0 } - do: async_search.submit: @@ -75,23 +75,23 @@ field: max sort: max - - set: { id: id } - - set: { version: version } - - match: { version: 7 } - - match: { response.is_partial: false } + - set: { id: id } + - set: { version: version } + - match: { version: 7 } + - match: { response.is_partial: false } - length: { response.hits.hits: 3 } - - match: { response.hits.hits.0._source.max: 1 } - - match: { response.aggregations.1.value: 3.0 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.1.value: 3.0 } - do: async_search.get: id: "$id" - - match: { version: 7 } - - match: { response.is_partial: false } - - length: { response.hits.hits: 3 } - - match: { response.hits.hits.0._source.max: 1 } - - match: { response.aggregations.1.value: 3.0 } + - match: { version: 7 } + - match: { response.is_partial: false } + - length: { response.hits.hits: 3 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.1.value: 3.0 } - do: async_search.delete: diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java index b80dd85013d9b..7c5976be57c77 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java @@ -98,10 +98,13 @@ static String encode(String indexName, String docId, TaskId taskId) { */ static AsyncSearchId decode(String id) { final AsyncSearchId searchId; - try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap( Base64.getUrlDecoder().decode(id)))) { + try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getUrlDecoder().decode(id)))) { searchId = new AsyncSearchId(in.readString(), in.readString(), new TaskId(in.readString())); + if (in.available() > 0) { + throw new IllegalArgumentException("invalid id:[" + id + "]"); + } } catch (IOException e) { - throw new IllegalArgumentException("invalid id: " + id); + throw new IllegalArgumentException("invalid id:[" + id + "]"); } validateAsyncSearchId(searchId); return searchId; @@ -109,8 +112,7 @@ static AsyncSearchId decode(String id) { static void validateAsyncSearchId(AsyncSearchId searchId) { if (searchId.getIndexName().startsWith(ASYNC_SEARCH_INDEX_PREFIX) == false) { - throw new IllegalArgumentException("invalid id [" + searchId.getEncoded() + "] that references the wrong index [" - + searchId.getIndexName() + "]"); + throw new IllegalArgumentException("invalid id:[" + searchId.getEncoded() + "]"); } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index c11e4c9cb240f..6c2fca9e1e328 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -31,11 +31,11 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; @@ -62,13 +62,13 @@ class AsyncSearchStoreService { static final String HEADERS_FIELD = "headers"; private final TaskManager taskManager; - private final ThreadPool threadPool; + private final ThreadContext threadContext; private final Client client; private final NamedWriteableRegistry registry; - AsyncSearchStoreService(TaskManager taskManager, ThreadPool threadPool, Client client, NamedWriteableRegistry registry) { + AsyncSearchStoreService(TaskManager taskManager, ThreadContext threadContext, Client client, NamedWriteableRegistry registry) { this.taskManager = taskManager; - this.threadPool = threadPool; + this.threadContext = threadContext; this.client = new OriginSettingClient(client, TASKS_ORIGIN); this.registry = registry; } @@ -114,7 +114,7 @@ public void onFailure(Exception e) { // alias does not exist but initial index does, something is broken andThen.onFailure(new IllegalStateException("async-search index [" + initialIndexName + "] already exists but does not have alias [" + ASYNC_SEARCH_ALIAS + "]")); - } else if (current.isAlias() && current instanceof AliasOrIndex.Alias) { + } else if (current.isAlias()) { AliasOrIndex.Alias alias = (AliasOrIndex.Alias) current; if (alias.getWriteIndex() != null) { // The alias exists and has a write index, so we're good @@ -175,7 +175,7 @@ AsyncSearchTask getTask(AsyncSearchId searchId) throws IOException { } // Check authentication for the user - final Authentication auth = Authentication.getAuthentication(threadPool.getThreadContext()); + final Authentication auth = Authentication.getAuthentication(threadContext); if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { throw new ResourceNotFoundException(searchId.getEncoded() + " not found"); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index c11595f137a10..5955330a98041 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -15,12 +15,13 @@ import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; @@ -36,6 +37,31 @@ class AsyncSearchTask extends SearchTask { private final Map originHeaders; + // a latch that notifies when the first response is available + private final CountDownLatch initLatch = new CountDownLatch(1); + // a latch that notifies the completion (or failure) of the request + private final CountDownLatch completionLatch = new CountDownLatch(1); + + // the current response, updated last in mutually exclusive methods below (onListShards, onReduce, + // onPartialReduce, onResponse and onFailure) + private volatile AsyncSearchResponse response; + + // set once in onListShards (when the search starts) + private volatile int totalShards = -1; + private final AtomicInteger version = new AtomicInteger(0); + private final AtomicInteger shardFailures = new AtomicInteger(0); + + /** + * Creates an instance of {@link AsyncSearchTask}. + * + * @param id The id of the task. + * @param type The type of the task. + * @param action The action name. + * @param originHeaders All the request context headers. + * @param taskHeaders The filtered request headers for the task. + * @param searchId The {@link AsyncSearchId} of the task. + * @param reduceContextSupplier A supplier to create final reduce contexts. + */ AsyncSearchTask(long id, String type, String action, @@ -51,10 +77,16 @@ class AsyncSearchTask extends SearchTask { setProgressListener(progressListener); } + /** + * Returns all of the request contexts headers + */ Map getOriginHeaders() { return originHeaders; } + /** + * Returns the {@link AsyncSearchId} of the task + */ AsyncSearchId getSearchId() { return searchId; } @@ -65,35 +97,57 @@ public SearchProgressActionListener getProgressListener() { } /** - * Perform the final reduce on the current {@link AsyncSearchResponse} if doFinalReduce - * is set to true and return the result. - * Note that this function returns null until {@link Listener#onListShards} - * or {@link Listener#onFailure} is called on the search task. + * Waits up to the provided waitForCompletionMillis for the task completion and then returns a not-modified + * response if the provided version is less than or equals to the current version, and the full response otherwise. + * + * Consumers should fork in a different thread to avoid blocking a network thread. */ - AsyncSearchResponse getAsyncResponse(boolean doFinalReduce) { - AsyncSearchResponse response = progressListener.response != null ? progressListener.response.get(doFinalReduce) : null; - if (response != null) { - response.addTaskInfo(taskInfo(searchId.getTaskId().getNodeId(), false)); + AsyncSearchResponse getAsyncResponse(long waitForCompletionMillis, int minimumVersion) throws InterruptedException { + if (waitForCompletionMillis > 0) { + completionLatch.await(waitForCompletionMillis, TimeUnit.MILLISECONDS); } - return response; + initLatch.await(); + assert response != null; + + AsyncSearchResponse resp = response; + // return a not-modified response + AsyncSearchResponse newResp = createFinalResponse(resp); + //resp.getVersion() > minimumVersion ? createFinalResponse(resp) : + // new AsyncSearchResponse(resp.getId(), resp.getVersion(), resp.isRunning()); + // not-modified response + newResp.setTaskInfo(taskInfo(searchId.getTaskId().getNodeId(), false)); + return createFinalResponse(resp); } - private class Listener extends SearchProgressActionListener { - private int totalShards = -1; - private AtomicInteger version = new AtomicInteger(0); - private AtomicInteger shardFailures = new AtomicInteger(0); - - private int lastSuccess = 0; - private int lastFailures = 0; - - private volatile Response response; + private AsyncSearchResponse createFinalResponse(AsyncSearchResponse resp) { + PartialSearchResponse partialResp = resp.getPartialResponse(); + if (partialResp != null) { + InternalAggregations newAggs = resp.getPartialResponse().getAggregations(); + if (partialResp.isFinalReduce() == false && newAggs != null) { + newAggs = InternalAggregations.topLevelReduce(Collections.singletonList(newAggs), reduceContextSupplier.get()); + } + partialResp = new PartialSearchResponse(partialResp.getTotalShards(), partialResp.getSuccessfulShards(), + shardFailures.get(), // update shard failures + partialResp.getTotalHits(), + newAggs, true // update aggs + ); + } + AsyncSearchResponse newResp = new AsyncSearchResponse(resp.getId(), + partialResp, // update partial response, + resp.getSearchResponse(), resp.getFailure(), resp.getVersion(), resp.isRunning()); + return newResp; + } + private class Listener extends SearchProgressActionListener { @Override public void onListShards(List shards, boolean fetchPhase) { - this.totalShards = shards.size(); - final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(totalShards), version.incrementAndGet(), true); - response = new Response(newResp, false); + try { + totalShards = shards.size(); + response = new AsyncSearchResponse(searchId.getEncoded(), + new PartialSearchResponse(totalShards), version.incrementAndGet(), true); + } finally { + initLatch.countDown(); + } } @Override @@ -103,74 +157,43 @@ public void onQueryFailure(int shardIndex, Exception exc) { @Override public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { - lastSuccess = shards.size(); - lastFailures = shardFailures.get(); - final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(totalShards, lastSuccess, lastFailures, totalHits, aggs), + response = new AsyncSearchResponse(searchId.getEncoded(), + new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs, false), version.incrementAndGet(), true ); - response = new Response(newResp, aggs != null); } @Override public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { - int failures = shardFailures.get(); - int ver = (lastSuccess == shards.size() && lastFailures == failures) ? version.get() : version.incrementAndGet(); - final AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(totalShards, shards.size(), failures, totalHits, aggs), - ver, + response = new AsyncSearchResponse(searchId.getEncoded(), + new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs, true), + version.incrementAndGet(), true ); - response = new Response(newResp, false); } @Override public void onResponse(SearchResponse searchResponse) { - AsyncSearchResponse newResp = new AsyncSearchResponse(searchId.getEncoded(), - searchResponse, version.incrementAndGet(), false); - response = new Response(newResp, false); + try { + response = new AsyncSearchResponse(searchId.getEncoded(), + searchResponse, version.incrementAndGet(), false); + } finally { + completionLatch.countDown(); + } } @Override public void onFailure(Exception exc) { - AsyncSearchResponse previous = response != null ? response.get(true) : null; - response = new Response(new AsyncSearchResponse(searchId.getEncoded(), - previous != null ? newPartialResponse(previous, shardFailures.get()) : null, - exc != null ? ElasticsearchException.guessRootCauses(exc)[0] : null, version.incrementAndGet(), false), false); - } - - private PartialSearchResponse newPartialResponse(AsyncSearchResponse response, int numFailures) { - PartialSearchResponse old = response.getPartialResponse(); - return response.hasPartialResponse() ? new PartialSearchResponse(totalShards, old.getSuccessfulShards(), numFailures, - old.getTotalHits(), old.getAggregations()) : null; - } - } - - private class Response { - AsyncSearchResponse internal; - boolean needFinalReduce; - - Response(AsyncSearchResponse response, boolean needFinalReduce) { - this.internal = response; - this.needFinalReduce = needFinalReduce; - } - - /** - * Ensure that we're performing the final reduce only when users explicitly requested - * a response through a {@link GetAsyncSearchAction.Request}. - */ - public synchronized AsyncSearchResponse get(boolean doFinalReduce) { - if (doFinalReduce && needFinalReduce) { - InternalAggregations reducedAggs = internal.getPartialResponse().getAggregations(); - reducedAggs = InternalAggregations.topLevelReduce(Collections.singletonList(reducedAggs), reduceContextSupplier.get()); - PartialSearchResponse old = internal.getPartialResponse(); - PartialSearchResponse clone = new PartialSearchResponse(old.getTotalShards(), old.getSuccessfulShards(), - old.getShardFailures(), old.getTotalHits(), reducedAggs); - needFinalReduce = false; - return internal = new AsyncSearchResponse(internal.getId(), clone, internal.getFailure(), internal.getVersion(), true); - } else { - return internal; + try { + response = new AsyncSearchResponse(searchId.getEncoded(), response != null ? response.getPartialResponse() : null, + exc != null ? ElasticsearchException.guessRootCauses(exc)[0] : null, version.incrementAndGet(), false); + } finally { + if (initLatch.getCount() == 1) { + // the failure happened before the initialization of the query phase + initLatch.countDown(); + } + completionLatch.countDown(); } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java index 887d3b1196824..30b95ac16e89c 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -27,7 +27,7 @@ public TransportDeleteAsyncSearchAction(TransportService transportService, NamedWriteableRegistry registry, Client client) { super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new); - this.store = new AsyncSearchStoreService(taskManager, threadPool, client, registry); + this.store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, registry); } @Override diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index e028ee4613484..9f9b89f992745 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -14,7 +14,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequestOptions; @@ -23,10 +22,11 @@ import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import java.io.IOException; +import java.util.concurrent.Executor; public class TransportGetAsyncSearchAction extends HandledTransportAction { private final ClusterService clusterService; - private final ThreadPool threadPool; + private final Executor generic; private final TransportService transportService; private final AsyncSearchStoreService store; @@ -40,8 +40,8 @@ public TransportGetAsyncSearchAction(TransportService transportService, super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncSearchAction.Request::new); this.clusterService = clusterService; this.transportService = transportService; - this.threadPool = threadPool; - this.store = new AsyncSearchStoreService(taskManager, threadPool, client, registry); + this.generic = threadPool.generic(); + this.store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, registry); } @Override @@ -69,7 +69,17 @@ private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, Asy ActionListener listener) throws IOException { final AsyncSearchTask task = store.getTask(searchId); if (task != null) { - waitForCompletion(request, task, threadPool.relativeTimeInMillis(), listener); + // don't block on a network thread + generic.execute(() -> { + final AsyncSearchResponse response; + try { + response = task.getAsyncResponse(request.getWaitForCompletion().millis(), request.getLastVersion()); + } catch (Exception exc) { + listener.onFailure(exc); + return; + } + listener.onResponse(response); + }); } else { // Task isn't running getSearchResponseFromIndex(request, searchId, listener); @@ -95,32 +105,4 @@ public void onFailure(Exception e) { } }); } - - void waitForCompletion(GetAsyncSearchAction.Request request, AsyncSearchTask task, - long startMs, ActionListener listener) { - final AsyncSearchResponse response = task.getAsyncResponse(false); - if (response == null) { - // the search task is not fully initialized - Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); - threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); - } else { - try { - if (response.isRunning() == false) { - listener.onResponse(task.getAsyncResponse(true)); - } else if (request.getWaitForCompletion().getMillis() < (threadPool.relativeTimeInMillis() - startMs)) { - if (response.getVersion() <= request.getLastVersion()) { - // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), true)); - } else { - listener.onResponse(task.getAsyncResponse(true)); - } - } else { - Runnable runnable = threadPool.preserveContext(() -> waitForCompletion(request, task, startMs, listener)); - threadPool.schedule(runnable, TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); - } - } catch (Exception exc) { - listener.onFailure(exc); - } - } - } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 934ec5405299f..1084329708172 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.search.SearchAction; @@ -24,22 +25,20 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; -import java.io.IOException; import java.util.Map; -import java.util.Random; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; import java.util.function.Supplier; public class TransportSubmitAsyncSearchAction extends HandledTransportAction { @@ -47,11 +46,11 @@ public class TransportSubmitAsyncSearchAction extends HandledTransportAction reduceContextSupplier; private final TransportSearchAction searchAction; private final AsyncSearchStoreService store; - private final Random random; @Inject public TransportSubmitAsyncSearchAction(ClusterService clusterService, @@ -64,12 +63,12 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService, TransportSearchAction searchAction) { super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new); this.clusterService = clusterService; - this.threadPool = transportService.getThreadPool(); + this.threadContext= transportService.getThreadPool().getThreadContext(); + this.generic = transportService.getThreadPool().generic(); this.nodeClient = nodeClient; this.reduceContextSupplier = () -> searchService.createReduceContext(true); this.searchAction = searchAction; - this.store = new AsyncSearchStoreService(taskManager, threadPool, client, registry); - this.random = new Random(System.nanoTime()); + this.store = new AsyncSearchStoreService(taskManager, threadContext, client, registry); } @Override @@ -79,7 +78,7 @@ protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionList submitListener.onFailure(exc); return; } - final String docID = UUIDs.randomBase64UUID(random); + final String docID = UUIDs.randomBase64UUID(); store.ensureAsyncSearchIndex(clusterService.state(), ActionListener.wrap( indexName -> executeSearch(request, indexName, docID, submitListener), submitListener::onFailure @@ -98,7 +97,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, }; // trigger the async search - final AtomicReference shouldStoreResult = new AtomicReference<>(); + final FutureBoolean shouldStoreResult = new FutureBoolean(); AsyncSearchTask task = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); SearchProgressActionListener progressListener = task.getProgressListener(); searchAction.execute(task, searchRequest, @@ -106,13 +105,28 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public void onResponse(SearchResponse response) { progressListener.onResponse(response); - onTaskCompletion(threadPool, task, shouldStoreResult); + onFinish(); } @Override public void onFailure(Exception exc) { progressListener.onFailure(exc); - onTaskCompletion(threadPool, task, shouldStoreResult); + onFinish(); + } + + private void onFinish() { + try { + // don't block on a network thread + generic.execute(() -> { + if (shouldStoreResult.getValue()) { + storeTask(task); + } else { + taskManager.unregister(task); + } + }); + } catch (Exception e){ + taskManager.unregister(task); + } } } ); @@ -130,15 +144,15 @@ public void onResponse(AsyncSearchResponse response) { try { store.storeInitialResponse(originHeaders, indexName, docID, response, ActionListener.wrap(() -> { - shouldStoreResult.set(true); + shouldStoreResult.setValue(true); submitListener.onResponse(response); })); - } catch (IOException exc) { + } catch (Exception exc) { onFailure(exc); } } else { // the user will get a final response directly so no need to store the result on completion - shouldStoreResult.set(false); + shouldStoreResult.setValue(false); submitListener.onResponse(new AsyncSearchResponse(null, response)); } } @@ -146,44 +160,63 @@ public void onResponse(AsyncSearchResponse response) { @Override public void onFailure(Exception e) { // we don't need to store the result if the submit failed - shouldStoreResult.set(false); + shouldStoreResult.setValue(false); submitListener.onFailure(e); } }); } - private void onTaskCompletion(ThreadPool threadPool, AsyncSearchTask task, - AtomicReference reference) { - final Boolean shouldStoreResult = reference.get(); - if (shouldStoreResult == null) { - // the user is still waiting for a response so we schedule a retry in 100ms - threadPool.schedule(() -> onTaskCompletion(threadPool, task, reference), - TimeValue.timeValueMillis(100), ThreadPool.Names.GENERIC); - } else if (shouldStoreResult) { - // the user retrieved an initial partial response so we need to store the final one - // for further retrieval + private void storeTask(AsyncSearchTask task) { + try { + store.storeFinalResponse(task.getOriginHeaders(), task.getAsyncResponse(0, -1), + new ActionListener<>() { + @Override + public void onResponse(UpdateResponse updateResponse) { + taskManager.unregister(task); + } + + @Override + public void onFailure(Exception exc) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", + task.getSearchId().getEncoded()), exc); + taskManager.unregister(task); + } + }); + } catch (Exception exc) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); + taskManager.unregister(task); + } + } + + /** + * A condition variable for booleans that notifies consumers when the value is set. + */ + private static class FutureBoolean { + final SetOnce value = new SetOnce<>(); + final CountDownLatch latch = new CountDownLatch(1); + + /** + * Sets the value and notifies the consumers. + */ + void setValue(boolean v) { try { - store.storeFinalResponse(task.getOriginHeaders(), task.getAsyncResponse(true), - new ActionListener<>() { - @Override - public void onResponse(UpdateResponse updateResponse) { - taskManager.unregister(task); - } + value.set(v); + } finally { + latch.countDown(); + } + } - @Override - public void onFailure(Exception exc) { - logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", - task.getSearchId().getEncoded()), exc); - taskManager.unregister(task); - } - }); - } catch (IOException exc) { - logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); - taskManager.unregister(task); + /** + * Waits for the value to be set and returns it. + * Consumers should fork in a different thread to avoid blocking a network thread. + */ + boolean getValue() { + try { + latch.await(); + } catch (InterruptedException e) { + return false; } - } else { - // the user retrieved the final response already so we don't need to store it - taskManager.unregister(task); + return value.get(); } } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index bc0e8edcce426..901d6a5177dc9 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -194,7 +194,6 @@ private AsyncSearchResponse doNext() throws Exception { return response; } AtomicReference atomic = new AtomicReference<>(); - AtomicReference exc = new AtomicReference<>(); int step = shardIndex == 0 ? progressStep+1 : progressStep-1; int index = 0; while (index < step && shardIndex < shardLatchArray.length) { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index 63ce2602d36e0..6ac5598c5d7cf 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -135,7 +135,7 @@ private static PartialSearchResponse randomPartialSearchResponse() { TotalHits totalHits = new TotalHits(randomLongBetween(0, Long.MAX_VALUE), randomFrom(TotalHits.Relation.values())); InternalMax max = new InternalMax("max", 0f, DocValueFormat.RAW, Collections.emptyList(), Collections.emptyMap()); InternalAggregations aggs = new InternalAggregations(Collections.singletonList(max)); - return new PartialSearchResponse(totalShards, successfulShards, failedShards, totalHits, aggs); + return new PartialSearchResponse(totalShards, successfulShards, failedShards, totalHits, aggs, randomBoolean()); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java index 1093a1b107af8..47b21ca403d81 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java @@ -71,7 +71,7 @@ public void setup() { threadPool = new TestThreadPool(this.getClass().getName()); client = new VerifyingClient(threadPool); TaskManager taskManager = mock(TaskManager.class); - store = new AsyncSearchStoreService(taskManager, threadPool, client, namedWriteableRegistry); + store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, namedWriteableRegistry); } @After diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index c5a82ada58006..620ce1b199037 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -60,12 +60,12 @@ public AsyncSearchResponse(String id, AsyncSearchResponse clone) { this.runningTimeMillis = clone.runningTimeMillis; } - private AsyncSearchResponse(String id, - PartialSearchResponse partialResponse, - SearchResponse response, - ElasticsearchException failure, - int version, - boolean isRunning) { + public AsyncSearchResponse(String id, + PartialSearchResponse partialResponse, + SearchResponse response, + ElasticsearchException failure, + int version, + boolean isRunning) { assert id != null || isRunning == false; this.id = id; this.version = version; @@ -98,7 +98,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(runningTimeMillis); } - public void addTaskInfo(TaskInfo taskInfo) { + public void setTaskInfo(TaskInfo taskInfo) { this.startDateMillis = taskInfo.getStartTime(); this.runningTimeMillis = TimeUnit.NANOSECONDS.toMillis(taskInfo.getRunningTimeNanos()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java index 74b083a4c239f..276fae61a021a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java @@ -14,10 +14,15 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.joda.time.Partial; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.function.Supplier; /** * A search response that contains partial results. @@ -28,19 +33,21 @@ public class PartialSearchResponse implements ToXContentFragment, Writeable { private final int shardFailures; private final TotalHits totalHits; - private InternalAggregations aggregations; + private final InternalAggregations aggregations; + private final boolean isFinalReduce; public PartialSearchResponse(int totalShards) { - this(totalShards, 0, 0, null, null); + this(totalShards, 0, 0, null, null, false); } public PartialSearchResponse(int totalShards, int successfulShards, int shardFailures, - TotalHits totalHits, InternalAggregations aggregations) { + TotalHits totalHits, InternalAggregations aggregations, boolean isFinalReduce) { this.totalShards = totalShards; this.successfulShards = successfulShards; this.shardFailures = shardFailures; this.totalHits = totalHits; this.aggregations = aggregations; + this.isFinalReduce = isFinalReduce; } public PartialSearchResponse(StreamInput in) throws IOException { @@ -49,6 +56,7 @@ public PartialSearchResponse(StreamInput in) throws IOException { this.shardFailures = in.readVInt(); this.totalHits = in.readBoolean() ? Lucene.readTotalHits(in) : null; this.aggregations = in.readOptionalWriteable(InternalAggregations::new); + this.isFinalReduce = in.readBoolean(); } @Override @@ -61,6 +69,7 @@ public void writeTo(StreamOutput out) throws IOException { Lucene.writeTotalHits(out, totalHits); } out.writeOptionalWriteable(aggregations); + out.writeBoolean(isFinalReduce); } @Override @@ -119,6 +128,10 @@ public InternalAggregations getAggregations() { return aggregations; } + public boolean isFinalReduce() { + return isFinalReduce; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -128,11 +141,12 @@ public boolean equals(Object o) { successfulShards == that.successfulShards && shardFailures == that.shardFailures && Objects.equals(totalHits, that.totalHits) && - Objects.equals(aggregations, that.aggregations); + Objects.equals(aggregations, that.aggregations) && + isFinalReduce == that.isFinalReduce; } @Override public int hashCode() { - return Objects.hash(totalShards, successfulShards, shardFailures, totalHits, aggregations); + return Objects.hash(totalShards, successfulShards, shardFailures, totalHits, aggregations, isFinalReduce); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 58cbc1bc03d40..8942333ac1da6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -33,14 +33,14 @@ public class SubmitAsyncSearchRequest extends ActionRequest { private final SearchRequest request; /** - * Create a new request + * Creates a new request */ public SubmitAsyncSearchRequest(String... indices) { this(new SearchSourceBuilder(), indices); } /** - * Create a new request + * Creates a new request */ public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) { this.request = new SearchRequest(indices, source); @@ -68,14 +68,14 @@ public SearchRequest getSearchRequest() { } /** - * Set the minimum time that the request should wait before returning a partial result. + * Sets the minimum time that the request should wait before returning a partial result. */ public void setWaitForCompletion(TimeValue waitForCompletion) { this.waitForCompletion = waitForCompletion; } /** - * Return the minimum time that the request should wait before returning a partial result. + * Returns the minimum time that the request should wait before returning a partial result. */ public TimeValue getWaitForCompletion() { return waitForCompletion; diff --git a/x-pack/plugin/core/src/main/resources/async-search.json b/x-pack/plugin/core/src/main/resources/async-search.json index e5c801eddbb04..9f5c6ea5ce79b 100644 --- a/x-pack/plugin/core/src/main/resources/async-search.json +++ b/x-pack/plugin/core/src/main/resources/async-search.json @@ -13,7 +13,7 @@ }, "mappings": { "_doc": { - "dynamic": false, + "dynamic": "strict", "properties": { "headers": { "type": "object", From 8a69a2163a35b32055b6509f00200cc91ae5d189 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 15 Jan 2020 13:59:13 +0100 Subject: [PATCH 28/61] add more tests --- .../xpack/search/AsyncSearchSecurityIT.java | 14 +++++++++++ .../xpack/search/AsyncSearchActionTests.java | 23 +++++++++++++++---- .../search/action/PartialSearchResponse.java | 5 ---- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java index d22570b822323..9f977f560108a 100644 --- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java +++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java @@ -78,6 +78,14 @@ private void testCase(String user, String other) throws Exception { exc = expectThrows(ResponseException.class, () -> deleteAsyncSearch(id, other)); assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + // other and user cannot access the result from direct get calls + AsyncSearchId searchId = AsyncSearchId.decode(id); + for (String runAs : new String[] {user, other}) { + exc = expectThrows(ResponseException.class, () -> get(searchId.getIndexName(), searchId.getDocId(), runAs)); + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403)); + assertThat(exc.getMessage(), containsString("unauthorized")); + } + Response delResp = deleteAsyncSearch(id, user); assertOK(delResp); } @@ -107,6 +115,12 @@ static void refresh(String index) throws IOException { assertOK(adminClient().performRequest(new Request("POST", "/" + index + "/_refresh"))); } + static Response get(String index, String id, String user) throws IOException { + final Request request = new Request("GET", "/" + index + "/_doc/" + id); + setRunAsHeader(request, user); + return client().performRequest(request); + } + static Response submitAsyncSearch(String indexName, String query, String user) throws IOException { return submitAsyncSearch(indexName, query, TimeValue.MINUS_ONE, user); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index 153aa926981f3..7b39a0746086f 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.search; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -191,8 +192,6 @@ public void testDeleteCancelRunningTask() throws Exception { } public void testDeleteCleanupIndex() throws Exception { - SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { indexName }); - request.setWaitForCompletion(TimeValue.timeValueMillis(1)); SearchResponseIterator it = assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); AsyncSearchResponse response = it.next(); @@ -219,8 +218,6 @@ public void testCleanupOnFailure() throws Exception { } public void testInvalidId() throws Exception { - SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { indexName }); - request.setWaitForCompletion(TimeValue.timeValueMillis(1)); SearchResponseIterator it = assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); AsyncSearchResponse response = it.next(); @@ -234,4 +231,22 @@ public void testInvalidId() throws Exception { } assertFalse(response.isRunning()); } + + public void testNoIndex() throws Exception { + SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { "invalid-*" }); + request.setWaitForCompletion(TimeValue.timeValueMillis(1)); + AsyncSearchResponse response = submitAsyncSearch(request); + assertTrue(response.hasResponse()); + assertFalse(response.isRunning()); + assertThat(response.getSearchResponse().getTotalShards(), equalTo(0)); + + request = new SubmitAsyncSearchRequest(new String[] { "invalid" }); + request.setWaitForCompletion(TimeValue.timeValueMillis(1)); + response = submitAsyncSearch(request); + assertFalse(response.hasResponse()); + assertTrue(response.hasFailed()); + assertFalse(response.isRunning()); + ElasticsearchException exc = response.getFailure(); + assertThat(exc.getMessage(), containsString("no such index")); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java index 276fae61a021a..9cb44310bd872 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java @@ -14,15 +14,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.InternalAggregations; -import org.joda.time.Partial; import java.io.IOException; -import java.util.Collections; -import java.util.List; import java.util.Objects; -import java.util.function.Supplier; /** * A search response that contains partial results. From 78c2ca45e33d0194c6b2b281a48b6023399daaec Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 15 Jan 2020 17:01:51 +0100 Subject: [PATCH 29/61] remove unrelated change --- .../org/elasticsearch/action/search/SearchProgressListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java index 572d4f6e2a88c..87146719a0f52 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java @@ -37,7 +37,7 @@ /** * A listener that allows to track progress of the {@link SearchAction}. */ -public abstract class SearchProgressListener { +abstract class SearchProgressListener { private static final Logger logger = LogManager.getLogger(SearchProgressListener.class); public static final SearchProgressListener NOOP = new SearchProgressListener() {}; From 5eb71485923e0e9208324788444895efb978b27d Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 15 Jan 2020 18:39:40 +0100 Subject: [PATCH 30/61] fix nocommit --- .../org/elasticsearch/xpack/search/AsyncSearchTask.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 5955330a98041..20ba85c87c837 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -111,12 +111,10 @@ AsyncSearchResponse getAsyncResponse(long waitForCompletionMillis, int minimumVe AsyncSearchResponse resp = response; // return a not-modified response - AsyncSearchResponse newResp = createFinalResponse(resp); - //resp.getVersion() > minimumVersion ? createFinalResponse(resp) : - // new AsyncSearchResponse(resp.getId(), resp.getVersion(), resp.isRunning()); - // not-modified response + AsyncSearchResponse newResp = resp.getVersion() > minimumVersion ? createFinalResponse(resp) : + new AsyncSearchResponse(resp.getId(), resp.getVersion(), resp.isRunning()); // not-modified response newResp.setTaskInfo(taskInfo(searchId.getTaskId().getNodeId(), false)); - return createFinalResponse(resp); + return newResp; } private AsyncSearchResponse createFinalResponse(AsyncSearchResponse resp) { From fb3364f832c7ddacd64ba01ef75d9ccd9fc1f666 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 22 Jan 2020 11:35:12 +0100 Subject: [PATCH 31/61] add more statistics to the search progress listener (number of skipped shards, remote clusters statistics) --- .../search/AbstractSearchAsyncAction.java | 13 ++++--- .../action/search/DfsQueryPhase.java | 5 +-- .../SearchDfsQueryThenFetchAsyncAction.java | 5 +++ .../action/search/SearchPhaseController.java | 5 ++- .../action/search/SearchProgressListener.java | 39 +++++++++++-------- .../SearchQueryThenFetchAsyncAction.java | 7 ++-- .../action/search/SearchResponse.java | 4 ++ .../java/org/elasticsearch/tasks/Task.java | 7 ++++ .../search/SearchPhaseControllerTests.java | 2 +- .../SearchProgressActionListenerIT.java | 6 ++- 10 files changed, 58 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index ca68bb4008146..f99d26b5adbc3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -87,7 +87,7 @@ abstract class AbstractSearchAsyncAction exten private final SearchTimeProvider timeProvider; private final SearchResponse.Clusters clusters; - private final GroupShardsIterator toSkipShardsIts; + protected final GroupShardsIterator toSkipShardsIts; protected final GroupShardsIterator shardsIts; private final int expectedTotalOps; private final AtomicInteger totalOps = new AtomicInteger(); @@ -382,7 +382,7 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, logger.trace(new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e); } } - onShardGroupFailure(shardIndex, e); + onShardGroupFailure(shardIndex, shardTarget, e); onPhaseDone(); } else { final ShardRouting nextShard = shardIt.nextOrNull(); @@ -402,7 +402,7 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e); } } - onShardGroupFailure(shardIndex, e); + onShardGroupFailure(shardIndex, shardTarget, e); } } } @@ -410,10 +410,11 @@ private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, /** * Executed once for every {@link ShardId} that failed on all available shard routing. * - * @param shardIndex the shard target that failed - * @param exc the final failure reason + * @param shardIndex the shard index that failed + * @param shardTarget the last shard target for this failure + * @param exc the last failure reason */ - protected void onShardGroupFailure(int shardIndex, Exception exc) {} + protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {} /** * Executed once for every failed shard level request. This method is invoked before the next replica is tried for the given diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index b4d52fa418e10..1170893150d20 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.dfs.AggregatedDfs; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.query.QuerySearchRequest; @@ -72,8 +71,6 @@ public void run() throws IOException { final CountedCollector counter = new CountedCollector<>(queryResult::consumeResult, resultList.size(), () -> context.executeNextPhase(this, nextPhaseFactory.apply(queryResult)), context); - final SearchSourceBuilder sourceBuilder = context.getRequest().source(); - progressListener.notifyListShards(progressListener.searchShards(resultList), sourceBuilder == null || sourceBuilder.size() != 0); for (final DfsSearchResult dfsResult : resultList) { final SearchShardTarget searchShardTarget = dfsResult.getSearchShardTarget(); Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); @@ -97,7 +94,7 @@ public void onFailure(Exception exception) { try { context.getLogger().debug(() -> new ParameterizedMessage("[{}] Failed to execute query phase", querySearchRequest.id()), exception); - progressListener.notifyQueryFailure(shardIndex, exception); + progressListener.notifyQueryFailure(shardIndex, searchShardTarget, exception); counter.onFailure(shardIndex, searchShardTarget, exception); } finally { // the query might not have been executed at all (for example because thread pool rejected diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java index 0782fbb310b65..5778a45e21685 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.transport.Transport; @@ -48,6 +49,10 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; + SearchProgressListener progressListener = task.getProgressListener(); + SearchSourceBuilder sourceBuilder = request.source(); + progressListener.notifyListShards(progressListener.searchShards(this.shardsIts), + progressListener.searchShards(toSkipShardsIts), clusters, sourceBuilder == null || sourceBuilder.size() != 0); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 6e15f492d46b5..5aee59c7d44b6 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -669,7 +669,7 @@ public ReducedQueryPhase reduce() { ReducedQueryPhase reducePhase = controller.reducedQueryPhase(results.asList(), getRemainingAggs(), getRemainingTopDocs(), topDocsStats, numReducePhases, false, performFinalReduce); progressListener.notifyReduce(progressListener.searchShards(results.asList()), - reducePhase.totalHits, reducePhase.aggregations); + reducePhase.totalHits, reducePhase.aggregations, reducePhase.numReducePhases); return reducePhase; } @@ -723,7 +723,8 @@ ReducedQueryPhase reduce() { List resultList = results.asList(); final ReducedQueryPhase reducePhase = reducedQueryPhase(resultList, isScrollRequest, trackTotalHitsUpTo, request.isFinalReduce()); - listener.notifyReduce(listener.searchShards(resultList), reducePhase.totalHits, reducePhase.aggregations); + listener.notifyReduce(listener.searchShards(resultList), reducePhase.totalHits, + reducePhase.aggregations, reducePhase.numReducePhases); return reducePhase; } }; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java index 87146719a0f52..80eda195ad7e9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchResponse.Clusters; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; @@ -48,24 +49,27 @@ abstract class SearchProgressListener { * Executed when shards are ready to be queried. * * @param shards The list of shards to query. + * @param skippedShards The list of skipped shards. + * @param clusters The statistics for remote clusters included in the search. * @param fetchPhase true if the search needs a fetch phase, false otherwise. **/ - public void onListShards(List shards, boolean fetchPhase) {} + public void onListShards(List shards, List skippedShards, Clusters clusters, boolean fetchPhase) {} /** * Executed when a shard returns a query result. * - * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)} )}. + * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards} )}. */ public void onQueryResult(int shardIndex) {} /** * Executed when a shard reports a query failure. * - * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}. + * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}. + * @param shardTarget The last shard target that thrown an exception. * @param exc The cause of the failure. */ - public void onQueryFailure(int shardIndex, Exception exc) {} + public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) {} /** * Executed when a partial reduce is created. The number of partial reduce can be controlled via @@ -74,9 +78,9 @@ public void onQueryFailure(int shardIndex, Exception exc) {} * @param shards The list of shards that are part of this reduce. * @param totalHits The total number of hits in this reduce. * @param aggs The partial result for aggregations. - * @param version The version number for this reduce. + * @param reducePhase The version number for this reduce. */ - public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int version) {} + public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {} /** * Executed once when the final reduce is created. @@ -84,28 +88,29 @@ public void onPartialReduce(List shards, TotalHits totalHits, Inter * @param shards The list of shards that are part of this reduce. * @param totalHits The total number of hits in this reduce. * @param aggs The final result for aggregations. + * @param reducePhase The version number for this reduce. */ - public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) {} + public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) {} /** * Executed when a shard returns a fetch result. * - * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}. + * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}. */ public void onFetchResult(int shardIndex) {} /** * Executed when a shard reports a fetch failure. * - * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards(List, boolean)})}. + * @param shardIndex The index of the shard in the list provided by {@link SearchProgressListener#onListShards})}. * @param exc The cause of the failure. */ public void onFetchFailure(int shardIndex, Exception exc) {} - final void notifyListShards(List shards, boolean fetchPhase) { + final void notifyListShards(List shards, List skippedShards, Clusters clusters, boolean fetchPhase) { this.shards = shards; try { - onListShards(shards, fetchPhase); + onListShards(shards, skippedShards, clusters, fetchPhase); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on list shards"), e); } @@ -120,26 +125,26 @@ final void notifyQueryResult(int shardIndex) { } } - final void notifyQueryFailure(int shardIndex, Exception exc) { + final void notifyQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) { try { - onQueryFailure(shardIndex, exc); + onQueryFailure(shardIndex, shardTarget, exc); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] Failed to execute progress listener on query failure", shards.get(shardIndex)), e); } } - final void notifyPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int version) { + final void notifyPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { try { - onPartialReduce(shards, totalHits, aggs, version); + onPartialReduce(shards, totalHits, aggs, reducePhase); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on partial reduce"), e); } } - final void notifyReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { + final void notifyReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { try { - onReduce(shards, totalHits, aggs); + onReduce(shards, totalHits, aggs, reducePhase); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("Failed to execute progress listener on reduce"), e); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java index d5060b728347d..57162edbbb2f4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.search.SearchPhaseResult; +import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.transport.Transport; @@ -54,7 +55,7 @@ final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction shards, boolean fetchPhase) { + public void onListShards(List shards, List skippedShards, + SearchResponse.Clusters clusters, boolean fetchPhase) { shardsListener.set(shards); assertEquals(fetchPhase, hasFetchPhase); } @@ -154,7 +156,7 @@ public void onQueryResult(int shardIndex) { } @Override - public void onQueryFailure(int shardIndex, Exception exc) { + public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) { assertThat(shardIndex, lessThan(shardsListener.get().size())); numQueryFailures.incrementAndGet(); } From 1e102fb45b24ad078a5caba733c06296bc8b9ffa Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 22 Jan 2020 15:14:19 +0100 Subject: [PATCH 32/61] Replace generic thread pool execution with asynchronous listener that waits for completion in the task --- .../action/search/SearchShard.java | 2 +- .../rest-api-spec/test/search/10_basic.yml | 19 +- .../xpack/search/AsyncSearchTask.java | 253 +++++++++++------- .../xpack/search/MutableSearchResponse.java | 164 ++++++++++++ .../search/TransportGetAsyncSearchAction.java | 31 +-- .../TransportSubmitAsyncSearchAction.java | 140 +++------- .../xpack/search/AsyncSearchActionTests.java | 50 ++-- .../search/AsyncSearchIntegTestCase.java | 44 +-- .../search/AsyncSearchResponseTests.java | 41 +-- .../xpack/search/AsyncSearchTaskTests.java | 162 +++++++++++ .../search/action/AsyncSearchResponse.java | 172 +++++------- .../search/action/PartialSearchResponse.java | 147 ---------- 12 files changed, 671 insertions(+), 554 deletions(-) create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java index 16459d81885ce..448c9ffee5818 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchShard.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchShard.java @@ -34,7 +34,7 @@ public class SearchShard implements Comparable { private final String clusterAlias; private final ShardId shardId; - SearchShard(@Nullable String clusterAlias, ShardId shardId) { + public SearchShard(@Nullable String clusterAlias, ShardId shardId) { this.clusterAlias = clusterAlias; this.shardId = shardId; } diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml index c66a2f152d877..e151a204b8a8c 100644 --- a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml +++ b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml @@ -54,9 +54,9 @@ sort: max - is_false: id - - match: { version: 7 } - - match: { response.is_partial: false } - - length: { response.hits.hits: 3 } + - match: { version: 6 } + - match: { is_partial: false } + - length: { response.hits.hits: 3 } - match: { response.hits.hits.0._source.max: 1 } - match: { response.aggregations.1.value: 3.0 } @@ -75,11 +75,10 @@ field: max sort: max - - set: { id: id } - - set: { version: version } - - match: { version: 7 } - - match: { response.is_partial: false } - - length: { response.hits.hits: 3 } + - set: { id: id } + - match: { version: 6 } + - match: { is_partial: false } + - length: { response.hits.hits: 3 } - match: { response.hits.hits.0._source.max: 1 } - match: { response.aggregations.1.value: 3.0 } @@ -87,8 +86,8 @@ async_search.get: id: "$id" - - match: { version: 7 } - - match: { response.is_partial: false } + - match: { version: 6 } + - match: { is_partial: false } - length: { response.hits.hits: 3 } - match: { response.hits.hits.0._source.max: 1 } - match: { response.aggregations.1.value: 3.0 } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 20ba85c87c837..5697adc5b9336 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -6,23 +6,29 @@ package org.elasticsearch.xpack.search; import org.apache.lucene.search.TotalHits; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.search.SearchProgressActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponse.Clusters; import org.elasticsearch.action.search.SearchShard; import org.elasticsearch.action.search.SearchTask; -import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.threadpool.Scheduler.Cancellable; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; -import java.util.Collections; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.function.Supplier; import static org.elasticsearch.tasks.TaskId.EMPTY_TASK_ID; @@ -32,24 +38,19 @@ */ class AsyncSearchTask extends SearchTask { private final AsyncSearchId searchId; - private final Supplier reduceContextSupplier; + private final ThreadPool threadPool; + private final Supplier reduceContextSupplier; private final Listener progressListener; private final Map originHeaders; - // a latch that notifies when the first response is available - private final CountDownLatch initLatch = new CountDownLatch(1); - // a latch that notifies the completion (or failure) of the request - private final CountDownLatch completionLatch = new CountDownLatch(1); + private boolean hasInitialized; + private boolean hasCompleted; + private long completionId; + private final List initListeners = new ArrayList<>(); + private final Map> completionListeners = new HashMap<>(); - // the current response, updated last in mutually exclusive methods below (onListShards, onReduce, - // onPartialReduce, onResponse and onFailure) - private volatile AsyncSearchResponse response; - - // set once in onListShards (when the search starts) - private volatile int totalShards = -1; - private final AtomicInteger version = new AtomicInteger(0); - private final AtomicInteger shardFailures = new AtomicInteger(0); + private MutableSearchResponse searchResponse; /** * Creates an instance of {@link AsyncSearchTask}. @@ -60,6 +61,7 @@ class AsyncSearchTask extends SearchTask { * @param originHeaders All the request context headers. * @param taskHeaders The filtered request headers for the task. * @param searchId The {@link AsyncSearchId} of the task. + * @param threadPool The threadPool to schedule runnable. * @param reduceContextSupplier A supplier to create final reduce contexts. */ AsyncSearchTask(long id, @@ -68,10 +70,12 @@ class AsyncSearchTask extends SearchTask { Map originHeaders, Map taskHeaders, AsyncSearchId searchId, - Supplier reduceContextSupplier) { - super(id, type, action, "async_search", EMPTY_TASK_ID, taskHeaders); + ThreadPool threadPool, + Supplier reduceContextSupplier) { + super(id, type, action, null, EMPTY_TASK_ID, taskHeaders); this.originHeaders = originHeaders; this.searchId = searchId; + this.threadPool = threadPool; this.reduceContextSupplier = reduceContextSupplier; this.progressListener = new Listener(); setProgressListener(progressListener); @@ -93,106 +97,169 @@ AsyncSearchId getSearchId() { @Override public SearchProgressActionListener getProgressListener() { - return (Listener) super.getProgressListener(); + return progressListener; } /** - * Waits up to the provided waitForCompletionMillis for the task completion and then returns a not-modified - * response if the provided version is less than or equals to the current version, and the full response otherwise. - * - * Consumers should fork in a different thread to avoid blocking a network thread. + * Creates a listener that listens for an {@link AsyncSearchResponse} and executes the + * consumer when the task is finished or when the provided waitForCompletion + * timeout occurs. In such case the consumed {@link AsyncSearchResponse} will contain partial results. + */ + public void addCompletionListener(Consumer listener, TimeValue waitForCompletion) { + boolean executeImmediatly = false; + synchronized (this) { + if (hasCompleted) { + executeImmediatly = true; + } else { + addInitListener(() -> internalAddCompletionListener(listener, waitForCompletion)); + } + } + if (executeImmediatly) { + listener.accept(getResponse()); + } + } + + /** + * Creates a listener that listens for an {@link AsyncSearchResponse} and executes the + * consumer when the task is finished. */ - AsyncSearchResponse getAsyncResponse(long waitForCompletionMillis, int minimumVersion) throws InterruptedException { - if (waitForCompletionMillis > 0) { - completionLatch.await(waitForCompletionMillis, TimeUnit.MILLISECONDS); - } - initLatch.await(); - assert response != null; - - AsyncSearchResponse resp = response; - // return a not-modified response - AsyncSearchResponse newResp = resp.getVersion() > minimumVersion ? createFinalResponse(resp) : - new AsyncSearchResponse(resp.getId(), resp.getVersion(), resp.isRunning()); // not-modified response - newResp.setTaskInfo(taskInfo(searchId.getTaskId().getNodeId(), false)); - return newResp; + public void addCompletionListener(Consumer listener) { + boolean executeImmediatly = false; + synchronized (this) { + if (hasCompleted == false) { + completionListeners.put(completionId++, resp -> listener.accept(getResponse())); + } else { + executeImmediatly = true; + } + } + if (executeImmediatly) { + listener.accept(getResponse()); + } + } + + private void internalAddCompletionListener(Consumer listener, TimeValue waitForCompletion) { + boolean executeImmediatly = false; + synchronized (this) { + if (hasCompleted == false) { + // ensure that we consumes the listener only once + AtomicBoolean hasRun = new AtomicBoolean(false); + long id = completionId++; + Cancellable cancellable = + threadPool.schedule(() -> { + if (hasRun.compareAndSet(false, true)) { + // timeout occurred before completion + removeCompletionListener(id); + listener.accept(getResponse()); + } + }, waitForCompletion, "generic"); + completionListeners.put(id, resp -> { + if (hasRun.compareAndSet(false, true)) { + // completion occurred before timeout + cancellable.cancel(); + listener.accept(resp); + } + }); + } else { + executeImmediatly = true; + } + } + if (executeImmediatly) { + listener.accept(getResponse()); + } + } + + private void removeCompletionListener(long id) { + synchronized (this) { + if (hasCompleted == false) { + completionListeners.remove(id); + } + } } - private AsyncSearchResponse createFinalResponse(AsyncSearchResponse resp) { - PartialSearchResponse partialResp = resp.getPartialResponse(); - if (partialResp != null) { - InternalAggregations newAggs = resp.getPartialResponse().getAggregations(); - if (partialResp.isFinalReduce() == false && newAggs != null) { - newAggs = InternalAggregations.topLevelReduce(Collections.singletonList(newAggs), reduceContextSupplier.get()); + private void addInitListener(Runnable listener) { + boolean executeImmediatly = false; + synchronized (this) { + if (hasInitialized) { + executeImmediatly = true; + } else { + initListeners.add(listener); } - partialResp = new PartialSearchResponse(partialResp.getTotalShards(), partialResp.getSuccessfulShards(), - shardFailures.get(), // update shard failures - partialResp.getTotalHits(), - newAggs, true // update aggs - ); - } - AsyncSearchResponse newResp = new AsyncSearchResponse(resp.getId(), - partialResp, // update partial response, - resp.getSearchResponse(), resp.getFailure(), resp.getVersion(), resp.isRunning()); - return newResp; + } + if (executeImmediatly) { + listener.run(); + } + } + + private void executeInitListeners() { + synchronized (this) { + if (hasInitialized) { + return; + } + hasInitialized = true; + } + for (Runnable listener : initListeners) { + listener.run(); + } + initListeners.clear(); + } + + private void executeCompletionListeners() { + synchronized (this) { + hasCompleted = true; + } + AsyncSearchResponse finalResponse = getResponse(); + for (Consumer listener : completionListeners.values()) { + listener.accept(finalResponse); + } + completionListeners.clear(); + } + + private AsyncSearchResponse getResponse() { + assert searchResponse != null; + return searchResponse.toAsyncSearchResponse(this); } private class Listener extends SearchProgressActionListener { @Override - public void onListShards(List shards, boolean fetchPhase) { - try { - totalShards = shards.size(); - response = new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(totalShards), version.incrementAndGet(), true); - } finally { - initLatch.countDown(); - } + public void onListShards(List shards, List skipped, Clusters clusters, boolean fetchPhase) { + searchResponse = new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier); + executeInitListeners(); } @Override - public void onQueryFailure(int shardIndex, Exception exc) { - shardFailures.incrementAndGet(); + public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) { + searchResponse.addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget)); } @Override public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { - response = new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs, false), - version.incrementAndGet(), - true - ); + searchResponse.updatePartialResponse(shards.size(), + new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs, + null, null, false, null, reducePhase), aggs == null); } @Override - public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { - response = new AsyncSearchResponse(searchId.getEncoded(), - new PartialSearchResponse(totalShards, shards.size(), shardFailures.get(), totalHits, aggs, true), - version.incrementAndGet(), - true - ); + public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { + searchResponse.updatePartialResponse(shards.size(), + new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs, + null, null, false, null, reducePhase), true); } @Override - public void onResponse(SearchResponse searchResponse) { - try { - response = new AsyncSearchResponse(searchId.getEncoded(), - searchResponse, version.incrementAndGet(), false); - } finally { - completionLatch.countDown(); - } + public void onResponse(SearchResponse response) { + searchResponse.updateFinalResponse(response.getSuccessfulShards(), response.getInternalResponse()); + executeCompletionListeners(); } @Override public void onFailure(Exception exc) { - try { - response = new AsyncSearchResponse(searchId.getEncoded(), response != null ? response.getPartialResponse() : null, - exc != null ? ElasticsearchException.guessRootCauses(exc)[0] : null, version.incrementAndGet(), false); - } finally { - if (initLatch.getCount() == 1) { - // the failure happened before the initialization of the query phase - initLatch.countDown(); - } - completionLatch.countDown(); + if (searchResponse == null) { + // if the failure occurred before calling onListShards + searchResponse = new MutableSearchResponse(-1, -1, null, reduceContextSupplier); } + searchResponse.updateWithFailure(exc); + executeInitListeners(); + executeCompletionListeners(); } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java new file mode 100644 index 0000000000000..1d4ef11a43279 --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponse.Clusters; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static java.util.Collections.singletonList; +import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO; +import static org.elasticsearch.search.aggregations.InternalAggregations.topLevelReduce; + +/** + * A mutable search response that allows to update and create partial response synchronously. + * Synchronized methods ensure that updates of the content are blocked if another thread is + * creating an async response concurrently. This limits the number of final reduction that can + * run concurrently to 1 and ensures that we pause the search progress when an {@link AsyncSearchResponse} is built. + */ +class MutableSearchResponse { + private final int totalShards; + private final int skippedShards; + private final Clusters clusters; + private final AtomicArray shardFailures; + private final Supplier reduceContextSupplier; + + private int version; + private boolean isPartial; + private boolean isFinalReduce; + private int successfulShards; + private SearchResponseSections sections; + private ElasticsearchException failure; + + private boolean frozen; + + MutableSearchResponse(int totalShards, int skippedShards, Clusters clusters, Supplier reduceContextSupplier) { + this.totalShards = totalShards; + this.skippedShards = skippedShards; + this.clusters = clusters; + this.reduceContextSupplier = reduceContextSupplier; + this.version = 0; + this.shardFailures = totalShards == -1 ? null : new AtomicArray<>(totalShards-skippedShards); + this.isPartial = true; + this.sections = totalShards == -1 ? null : new InternalSearchResponse( + new SearchHits(SearchHits.EMPTY, new TotalHits(0, GREATER_THAN_OR_EQUAL_TO), Float.NaN), + null, null, null, false, null, 0); + } + + /** + * Updates the response with the partial {@link SearchResponseSections} merged from #successfulShards + * shards. + */ + synchronized void updatePartialResponse(int successfulShards, SearchResponseSections newSections, boolean isFinalReduce) { + failIfFrozen(); + if (newSections.getNumReducePhases() < sections.getNumReducePhases()) { + // should never happen since partial response are called under a lock + throw new IllegalStateException("received partial response out of order: " + + newSections.getNumReducePhases() + " < " + sections.getNumReducePhases()); + } + failIfFrozen(); + ++ version; + this.successfulShards = successfulShards; + this.sections = newSections; + this.isPartial = true; + this.isFinalReduce = isFinalReduce; + } + + /** + * Updates the response with the final {@link SearchResponseSections} merged from #successfulShards + * shards. + */ + synchronized void updateFinalResponse(int successfulShards, SearchResponseSections newSections) { + assert newSections.getNumReducePhases() != sections.getNumReducePhases(); + failIfFrozen(); + ++ version; + this.successfulShards = successfulShards; + this.sections = newSections; + this.isPartial = false; + this.isFinalReduce = true; + this.frozen = true; + } + + /** + * Updates the response with a fatal failure. This method preserves the partial response + * received from previous updates + */ + synchronized void updateWithFailure(Exception exc) { + failIfFrozen(); + ++ version; + this.isPartial = true; + this.failure = ElasticsearchException.guessRootCauses(exc)[0]; + this.frozen = true; + } + + /** + * Adds a shard failure concurrently (non-blocking). + */ + void addShardFailure(int shardIndex, ShardSearchFailure failure) { + failIfFrozen(); + shardFailures.set(shardIndex, failure); + } + + /** + * Creates an {@link AsyncSearchResponse} based on the current state + * of the mutable response. The final reduction of aggregations is + * executed if needed in the synchronized block to ensure that only one + * can run concurrently. + */ + synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task) { + final SearchResponse resp; + if (totalShards != -1) { + if (sections.aggregations() != null && isFinalReduce == false) { + InternalAggregations oldAggs = (InternalAggregations) sections.aggregations(); + InternalAggregations newAggs = topLevelReduce(singletonList(oldAggs), reduceContextSupplier.get()); + sections = new InternalSearchResponse(sections.hits(), newAggs, sections.suggest(), + null, sections.timedOut(), sections.terminatedEarly(), sections.getNumReducePhases()); + isFinalReduce = true; + } + long tookInMillis = TimeValue.timeValueNanos(System.nanoTime() - task.getStartTimeNanos()).getMillis(); + resp = new SearchResponse(sections, null, totalShards, successfulShards, + skippedShards, tookInMillis, buildShardFailures(), clusters); + } else { + resp = null; + } + return new AsyncSearchResponse(task.getSearchId().getEncoded(), version, resp, failure, isPartial, + frozen == false, task.getStartTime()); + } + + private void failIfFrozen() { + if (frozen) { + throw new IllegalStateException("invalid update received after the completion of the request"); + } + } + + private ShardSearchFailure[] buildShardFailures() { + if (shardFailures == null) { + return new ShardSearchFailure[0]; + } + List failures = new ArrayList<>(); + for (int i = 0; i < shardFailures.length(); i++) { + ShardSearchFailure failure = shardFailures.get(i); + if (failure != null) { + failures.add(failure); + } + } + return failures.toArray(ShardSearchFailure[]::new); + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 9f9b89f992745..7b2549041b576 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -22,11 +22,9 @@ import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import java.io.IOException; -import java.util.concurrent.Executor; public class TransportGetAsyncSearchAction extends HandledTransportAction { private final ClusterService clusterService; - private final Executor generic; private final TransportService transportService; private final AsyncSearchStoreService store; @@ -40,7 +38,6 @@ public TransportGetAsyncSearchAction(TransportService transportService, super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncSearchAction.Request::new); this.clusterService = clusterService; this.transportService = transportService; - this.generic = threadPool.generic(); this.store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, registry); } @@ -69,17 +66,20 @@ private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, Asy ActionListener listener) throws IOException { final AsyncSearchTask task = store.getTask(searchId); if (task != null) { - // don't block on a network thread - generic.execute(() -> { - final AsyncSearchResponse response; - try { - response = task.getAsyncResponse(request.getWaitForCompletion().millis(), request.getLastVersion()); - } catch (Exception exc) { - listener.onFailure(exc); - return; - } - listener.onResponse(response); - }); + try { + task.addCompletionListener( + response -> { + if (response.getVersion() <= request.getLastVersion()) { + // return a not-modified response + listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), + response.isPartial(), response.isRunning(), response.getStartDate())); + } else { + listener.onResponse(response); + } + }, request.getWaitForCompletion()); + } catch (Exception exc) { + listener.onFailure(exc); + } } else { // Task isn't running getSearchResponseFromIndex(request, searchId, listener); @@ -93,7 +93,8 @@ private void getSearchResponseFromIndex(GetAsyncSearchAction.Request request, As public void onResponse(AsyncSearchResponse response) { if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), false)); + listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), + response.isPartial(), false, response.getStartDate())); } else { listener.onResponse(response); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 1084329708172..74cf95ffc341d 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -8,13 +8,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchProgressActionListener; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -27,18 +25,15 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.search.SearchService; -import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; import java.util.function.Supplier; public class TransportSubmitAsyncSearchAction extends HandledTransportAction { @@ -47,8 +42,7 @@ public class TransportSubmitAsyncSearchAction extends HandledTransportAction reduceContextSupplier; + private final Supplier reduceContextSupplier; private final TransportSearchAction searchAction; private final AsyncSearchStoreService store; @@ -64,7 +58,6 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService, super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new); this.clusterService = clusterService; this.threadContext= transportService.getThreadPool().getThreadContext(); - this.generic = transportService.getThreadPool().generic(); this.nodeClient = nodeClient; this.reduceContextSupplier = () -> searchService.createReduceContext(true); this.searchAction = searchAction; @@ -92,83 +85,52 @@ private void executeSearch(SubmitAsyncSearchRequest submitRequest, String indexN @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map taskHeaders) { AsyncSearchId searchId = new AsyncSearchId(indexName, docID, new TaskId(nodeClient.getLocalNodeId(), id)); - return new AsyncSearchTask(id, type, action, originHeaders, taskHeaders, searchId, reduceContextSupplier); + return new AsyncSearchTask(id, type, action, originHeaders, taskHeaders, searchId, + nodeClient.threadPool(), reduceContextSupplier); } }; // trigger the async search - final FutureBoolean shouldStoreResult = new FutureBoolean(); AsyncSearchTask task = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); - SearchProgressActionListener progressListener = task.getProgressListener(); - searchAction.execute(task, searchRequest, - new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - progressListener.onResponse(response); - onFinish(); - } - - @Override - public void onFailure(Exception exc) { - progressListener.onFailure(exc); - onFinish(); - } + searchAction.execute(task, searchRequest, task.getProgressListener()); + task.addCompletionListener(searchResponse -> { + if (searchResponse.isRunning() || submitRequest.isCleanOnCompletion() == false) { + // the task is still running and the user cannot wait more so we create + // a document for further retrieval + try { + store.storeInitialResponse(originHeaders, indexName, docID, searchResponse, + new ActionListener<>() { + @Override + public void onResponse(IndexResponse r) { + // store the final response + task.addCompletionListener(finalResponse -> storeFinalResponse(task, finalResponse)); + submitListener.onResponse(searchResponse); + } - private void onFinish() { - try { - // don't block on a network thread - generic.execute(() -> { - if (shouldStoreResult.getValue()) { - storeTask(task); - } else { + @Override + public void onFailure(Exception exc) { + // TODO: cancel search taskManager.unregister(task); + submitListener.onFailure(exc); } }); - } catch (Exception e){ - taskManager.unregister(task); - } + } catch (Exception exc) { + // TODO: cancel search + taskManager.unregister(task); + submitListener.onFailure(exc); } + } else { + // the task completed within the timeout so the response is sent back to the user + // with a null id since nothing was stored on the cluster. + taskManager.unregister(task); + submitListener.onResponse(searchResponse.clone(null)); } - ); - - // and get the response asynchronously - GetAsyncSearchAction.Request getRequest = new GetAsyncSearchAction.Request(task.getSearchId().getEncoded(), - submitRequest.getWaitForCompletion(), -1); - nodeClient.executeLocally(GetAsyncSearchAction.INSTANCE, getRequest, - new ActionListener<>() { - @Override - public void onResponse(AsyncSearchResponse response) { - if (response.isRunning() || submitRequest.isCleanOnCompletion() == false) { - // the task is still running and the user cannot wait more so we create - // an empty document for further retrieval - try { - store.storeInitialResponse(originHeaders, indexName, docID, response, - ActionListener.wrap(() -> { - shouldStoreResult.setValue(true); - submitListener.onResponse(response); - })); - } catch (Exception exc) { - onFailure(exc); - } - } else { - // the user will get a final response directly so no need to store the result on completion - shouldStoreResult.setValue(false); - submitListener.onResponse(new AsyncSearchResponse(null, response)); - } - } - - @Override - public void onFailure(Exception e) { - // we don't need to store the result if the submit failed - shouldStoreResult.setValue(false); - submitListener.onFailure(e); - } - }); + }, submitRequest.getWaitForCompletion()); } - private void storeTask(AsyncSearchTask task) { + private void storeFinalResponse(AsyncSearchTask task, AsyncSearchResponse response) { try { - store.storeFinalResponse(task.getOriginHeaders(), task.getAsyncResponse(0, -1), + store.storeFinalResponse(task.getOriginHeaders(), response, new ActionListener<>() { @Override public void onResponse(UpdateResponse updateResponse) { @@ -187,36 +149,4 @@ public void onFailure(Exception exc) { taskManager.unregister(task); } } - - /** - * A condition variable for booleans that notifies consumers when the value is set. - */ - private static class FutureBoolean { - final SetOnce value = new SetOnce<>(); - final CountDownLatch latch = new CountDownLatch(1); - - /** - * Sets the value and notifies the consumers. - */ - void setValue(boolean v) { - try { - value.set(v); - } finally { - latch.countDown(); - } - } - - /** - * Waits for the value to be set and returns it. - * Consumers should fork in a different thread to avoid blocking a network thread. - */ - boolean getValue() { - try { - latch.await(); - } catch (InterruptedException e) { - return false; - } - return value.get(); - } - } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index 7b39a0746086f..ad1359dfd6a74 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -86,20 +86,21 @@ public void testMaxMinAggregation() throws Exception { AsyncSearchResponse response = it.next(); while (it.hasNext()) { response = it.next(); - if (response.hasPartialResponse() && response.getPartialResponse().getSuccessfulShards() > 0) { - assertNotNull(response.getPartialResponse().getAggregations()); - assertNotNull(response.getPartialResponse().getAggregations().get("max")); - assertNotNull(response.getPartialResponse().getAggregations().get("min")); - InternalMax max = response.getPartialResponse().getAggregations().get("max"); - InternalMin min = response.getPartialResponse().getAggregations().get("min"); + assertNotNull(response.getSearchResponse()); + if (response.getSearchResponse().getSuccessfulShards() > 0) { + assertNotNull(response.getSearchResponse().getAggregations()); + assertNotNull(response.getSearchResponse().getAggregations().get("max")); + assertNotNull(response.getSearchResponse().getAggregations().get("min")); + InternalMax max = response.getSearchResponse().getAggregations().get("max"); + InternalMin min = response.getSearchResponse().getAggregations().get("min"); assertThat((float) min.getValue(), greaterThanOrEqualTo(minMetric)); assertThat((float) max.getValue(), lessThanOrEqualTo(maxMetric)); } } if (numFailures == numShards) { - assertTrue(response.hasFailed()); + assertNotNull(response.getFailure()); } else { - assertTrue(response.hasResponse()); + assertNotNull(response.getSearchResponse()); assertNotNull(response.getSearchResponse().getAggregations()); assertNotNull(response.getSearchResponse().getAggregations().get("max")); assertNotNull(response.getSearchResponse().getAggregations().get("min")); @@ -120,7 +121,7 @@ public void testMaxMinAggregation() throws Exception { public void testTermsAggregation() throws Exception { int step = numShards > 2 ? randomIntBetween(2, numShards) : 2; - int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0; + int numFailures = 0;//randomBoolean() ? randomIntBetween(0, numShards) : 0; int termsSize = randomIntBetween(1, numKeywords); SearchSourceBuilder source = new SearchSourceBuilder() .aggregation(AggregationBuilders.terms("terms").field("terms.keyword").size(termsSize).shardSize(termsSize*2)); @@ -129,10 +130,11 @@ public void testTermsAggregation() throws Exception { AsyncSearchResponse response = it.next(); while (it.hasNext()) { response = it.next(); - if (response.hasPartialResponse() && response.getPartialResponse().getSuccessfulShards() > 0) { - assertNotNull(response.getPartialResponse().getAggregations()); - assertNotNull(response.getPartialResponse().getAggregations().get("terms")); - StringTerms terms = response.getPartialResponse().getAggregations().get("terms"); + assertNotNull(response.getSearchResponse()); + if (response.getSearchResponse().getSuccessfulShards() > 0) { + assertNotNull(response.getSearchResponse().getAggregations()); + assertNotNull(response.getSearchResponse().getAggregations().get("terms")); + StringTerms terms = response.getSearchResponse().getAggregations().get("terms"); assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0)); assertThat(terms.getBuckets().size(), lessThanOrEqualTo(termsSize)); for (InternalTerms.Bucket bucket : terms.getBuckets()) { @@ -142,9 +144,9 @@ public void testTermsAggregation() throws Exception { } } if (numFailures == numShards) { - assertTrue(response.hasFailed()); + assertNotNull(response.getFailure()); } else { - assertTrue(response.hasResponse()); + assertNotNull(response.getSearchResponse()); assertNotNull(response.getSearchResponse().getAggregations()); assertNotNull(response.getSearchResponse().getAggregations().get("terms")); StringTerms terms = response.getSearchResponse().getAggregations().get("terms"); @@ -173,9 +175,9 @@ public void testRestartAfterCompletion() throws Exception { ensureTaskCompletion(initial.getId()); restartTaskNode(initial.getId()); AsyncSearchResponse response = getAsyncSearch(initial.getId()); - assertTrue(response.hasResponse()); + assertNotNull(response.getSearchResponse()); assertFalse(response.isRunning()); - assertFalse(response.hasPartialResponse()); + assertFalse(response.isPartial()); deleteAsyncSearch(response.getId()); ensureTaskRemoval(response.getId()); } @@ -209,10 +211,10 @@ public void testCleanupOnFailure() throws Exception { } ensureTaskCompletion(initial.getId()); AsyncSearchResponse response = getAsyncSearch(initial.getId()); - assertTrue(response.hasFailed()); - assertTrue(response.hasPartialResponse()); - assertThat(response.getPartialResponse().getTotalShards(), equalTo(numShards)); - assertThat(response.getPartialResponse().getShardFailures(), equalTo(numShards)); + assertNotNull(response.getFailure()); + assertTrue(response.isPartial()); + assertThat(response.getSearchResponse().getTotalShards(), equalTo(numShards)); + assertThat(response.getSearchResponse().getShardFailures().length, equalTo(numShards)); deleteAsyncSearch(initial.getId()); ensureTaskRemoval(initial.getId()); } @@ -236,15 +238,15 @@ public void testNoIndex() throws Exception { SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new String[] { "invalid-*" }); request.setWaitForCompletion(TimeValue.timeValueMillis(1)); AsyncSearchResponse response = submitAsyncSearch(request); - assertTrue(response.hasResponse()); + assertNotNull(response.getSearchResponse()); assertFalse(response.isRunning()); assertThat(response.getSearchResponse().getTotalShards(), equalTo(0)); request = new SubmitAsyncSearchRequest(new String[] { "invalid" }); request.setWaitForCompletion(TimeValue.timeValueMillis(1)); response = submitAsyncSearch(request); - assertFalse(response.hasResponse()); - assertTrue(response.hasFailed()); + assertNull(response.getSearchResponse()); + assertNotNull(response.getFailure()); assertFalse(response.isRunning()); ElasticsearchException exc = response.getFailure(); assertThat(exc.getMessage(), containsString("no such index")); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index 901d6a5177dc9..36ba9cc2a447d 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -161,11 +161,11 @@ protected SearchResponseIterator assertBlockingIterator(String indexName, final AsyncSearchResponse initial = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get(); - assertTrue(initial.hasPartialResponse()); - assertThat(initial.status(), equalTo(RestStatus.PARTIAL_CONTENT)); - assertThat(initial.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); - assertThat(initial.getPartialResponse().getSuccessfulShards(), equalTo(0)); - assertThat(initial.getPartialResponse().getShardFailures(), equalTo(0)); + assertTrue(initial.isPartial()); + assertThat(initial.status(), equalTo(RestStatus.OK)); + assertThat(initial.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length)); + assertThat(initial.getSearchResponse().getSuccessfulShards(), equalTo(0)); + assertThat(initial.getSearchResponse().getShardFailures().length, equalTo(0)); return new SearchResponseIterator() { private AsyncSearchResponse response = initial; @@ -209,32 +209,32 @@ private AsyncSearchResponse doNext() throws Exception { new GetAsyncSearchAction.Request(response.getId(), TimeValue.timeValueMillis(10), lastVersion) ).get(); atomic.set(newResp); - assertNotEquals(RestStatus.NOT_MODIFIED, newResp.status()); + assertNotEquals(lastVersion, newResp.getVersion()); }); AsyncSearchResponse newResponse = atomic.get(); lastVersion = newResponse.getVersion(); if (newResponse.isRunning()) { - assertThat(newResponse.status(), equalTo(RestStatus.PARTIAL_CONTENT)); - assertTrue(newResponse.hasPartialResponse()); - assertFalse(newResponse.hasFailed()); - assertFalse(newResponse.hasResponse()); - assertThat(newResponse.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); - assertThat(newResponse.getPartialResponse().getShardFailures(), lessThanOrEqualTo(numFailures)); + assertThat(newResponse.status(), equalTo(RestStatus.OK)); + assertTrue(newResponse.isPartial()); + assertFalse(newResponse.getFailure() != null); + assertNotNull(newResponse.getSearchResponse()); + assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length)); + assertThat(newResponse.getSearchResponse().getShardFailures().length, lessThanOrEqualTo(numFailures)); } else if (numFailures == shardLatchArray.length) { assertThat(newResponse.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR)); - assertTrue(newResponse.hasFailed()); - assertTrue(newResponse.hasPartialResponse()); - assertFalse(newResponse.hasResponse()); - assertThat(newResponse.getPartialResponse().getTotalShards(), equalTo(shardLatchArray.length)); - assertThat(newResponse.getPartialResponse().getSuccessfulShards(), equalTo(0)); - assertThat(newResponse.getPartialResponse().getShardFailures(), equalTo(numFailures)); - assertNull(newResponse.getPartialResponse().getAggregations()); - assertNull(newResponse.getPartialResponse().getTotalHits()); + assertTrue(newResponse.getFailure() != null); + assertTrue(newResponse.isPartial()); + assertNotNull(newResponse.getSearchResponse()); + assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length)); + assertThat(newResponse.getSearchResponse().getSuccessfulShards(), equalTo(0)); + assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures)); + assertNull(newResponse.getSearchResponse().getAggregations()); + assertNull(newResponse.getSearchResponse().getHits().getTotalHits()); } else { assertThat(newResponse.status(), equalTo(RestStatus.OK)); - assertTrue(newResponse.hasResponse()); - assertFalse(newResponse.hasPartialResponse()); + assertNotNull(newResponse.getSearchResponse()); + assertFalse(newResponse.isPartial()); assertThat(newResponse.status(), equalTo(RestStatus.OK)); assertThat(newResponse.getSearchResponse().getTotalShards(), equalTo(shardLatchArray.length)); assertThat(newResponse.getSearchResponse().getShardFailures().length, equalTo(numFailures)); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index 6ac5598c5d7cf..c84bf5b192bd1 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.search; -import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.search.SearchResponse; @@ -15,14 +14,10 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.aggregations.metrics.InternalMax; import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.search.action.PartialSearchResponse; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; @@ -30,7 +25,6 @@ import org.junit.Before; import java.io.IOException; -import java.util.Collections; import java.util.List; import static java.util.Collections.emptyList; @@ -94,21 +88,19 @@ protected AsyncSearchResponse copyInstance(AsyncSearchResponse instance, Version } static AsyncSearchResponse randomAsyncSearchResponse(String searchId, SearchResponse searchResponse) { - int rand = randomIntBetween(0, 3); + int rand = randomIntBetween(0, 2); switch (rand) { case 0: - return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); + return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean(), + randomBoolean(), randomNonNegativeLong()); case 1: - return new AsyncSearchResponse(searchId, searchResponse, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); + return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse, null, + randomBoolean(), randomBoolean(), randomNonNegativeLong()); case 2: - return new AsyncSearchResponse(searchId, randomPartialSearchResponse(), - randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); - - case 3: - return new AsyncSearchResponse(searchId, randomPartialSearchResponse(), - new ElasticsearchException(new IOException("boum")), randomIntBetween(0, Integer.MAX_VALUE), randomBoolean()); + return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse, + new ElasticsearchException(new IOException("boum")), randomBoolean(), randomBoolean(), randomNonNegativeLong()); default: throw new AssertionError(); @@ -125,26 +117,13 @@ static SearchResponse randomSearchResponse() { successfulShards, skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); } - private static PartialSearchResponse randomPartialSearchResponse() { - int totalShards = randomIntBetween(0, 10000); - if (randomBoolean()) { - return new PartialSearchResponse(totalShards); - } else { - int successfulShards = randomIntBetween(0, totalShards); - int failedShards = totalShards - successfulShards; - TotalHits totalHits = new TotalHits(randomLongBetween(0, Long.MAX_VALUE), randomFrom(TotalHits.Relation.values())); - InternalMax max = new InternalMax("max", 0f, DocValueFormat.RAW, Collections.emptyList(), Collections.emptyMap()); - InternalAggregations aggs = new InternalAggregations(Collections.singletonList(max)); - return new PartialSearchResponse(totalShards, successfulShards, failedShards, totalHits, aggs, randomBoolean()); - } - } - static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchResponse actual) { assertEquals(expected.getId(), actual.getId()); assertEquals(expected.getVersion(), actual.getVersion()); assertEquals(expected.status(), actual.status()); - assertEquals(expected.getPartialResponse(), actual.getPartialResponse()); assertEquals(expected.getFailure() == null, actual.getFailure() == null); - // TODO check equal SearchResponse + assertEquals(expected.isRunning(), actual.isRunning()); + assertEquals(expected.isPartial(), actual.isPartial()); + assertEquals(expected.getStartDate(), actual.getStartDate()); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java new file mode 100644 index 0000000000000..561fbfde236ae --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java @@ -0,0 +1,162 @@ +package org.elasticsearch.xpack.search; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchShard; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; + +public class AsyncSearchTaskTests extends ESTestCase { + private ThreadPool threadPool; + + @Before + public void beforeTest() { + threadPool = new TestThreadPool(getTestName()); + } + + @After + public void afterTest() { + threadPool.shutdownNow(); + } + + public void testWaitForInit() throws InterruptedException { + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", Collections.emptyMap(), Collections.emptyMap(), + new AsyncSearchId("index", "0", new TaskId("node1", 0)), threadPool, null); + int numShards = randomIntBetween(0, 10); + List shards = new ArrayList<>(); + for (int i = 0; i < numShards; i++) { + shards.add(new SearchShard(null, new ShardId("0", "0", 1))); + } + List skippedShards = new ArrayList<>(); + int numSkippedShards = randomIntBetween(0, 10); + for (int i = 0; i < numSkippedShards; i++) { + skippedShards.add(new SearchShard(null, new ShardId("0", "0", 1))); + } + + List threads = new ArrayList<>(); + int numThreads = randomIntBetween(1, 10); + CountDownLatch latch = new CountDownLatch(numThreads); + for (int i = 0; i < numThreads; i++) { + Thread thread = new Thread(() -> task.addCompletionListener(resp -> { + assertThat(numShards+numSkippedShards, equalTo(resp.getSearchResponse().getTotalShards())); + assertThat(numSkippedShards, equalTo(resp.getSearchResponse().getSkippedShards())); + assertThat(0, equalTo(resp.getSearchResponse().getFailedShards())); + latch.countDown(); + }, TimeValue.timeValueMillis(1))); + threads.add(thread); + thread.start(); + } + assertFalse(latch.await(numThreads*2, TimeUnit.MILLISECONDS)); + task.getProgressListener().onListShards(shards, skippedShards, SearchResponse.Clusters.EMPTY, false); + latch.await(); + } + + public void testWithFailure() throws InterruptedException { + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", Collections.emptyMap(), Collections.emptyMap(), + new AsyncSearchId("index", "0", new TaskId("node1", 0)), threadPool, null); + int numShards = randomIntBetween(0, 10); + List shards = new ArrayList<>(); + for (int i = 0; i < numShards; i++) { + shards.add(new SearchShard(null, new ShardId("0", "0", 1))); + } + List skippedShards = new ArrayList<>(); + int numSkippedShards = randomIntBetween(0, 10); + for (int i = 0; i < numSkippedShards; i++) { + skippedShards.add(new SearchShard(null, new ShardId("0", "0", 1))); + } + + List threads = new ArrayList<>(); + int numThreads = randomIntBetween(1, 10); + CountDownLatch latch = new CountDownLatch(numThreads); + for (int i = 0; i < numThreads; i++) { + Thread thread = new Thread(() -> task.addCompletionListener(resp -> { + assertNull(resp.getSearchResponse()); + assertNotNull(resp.getFailure()); + assertTrue(resp.isPartial()); + latch.countDown(); + }, TimeValue.timeValueMillis(1))); + threads.add(thread); + thread.start(); + } + assertFalse(latch.await(numThreads*2, TimeUnit.MILLISECONDS)); + task.getProgressListener().onFailure(new Exception("boom")); + latch.await(); + } + + public void testWaitForCompletion() throws InterruptedException { + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", Collections.emptyMap(), Collections.emptyMap(), + new AsyncSearchId("index", "0", new TaskId("node1", 0)), threadPool, null); + int numShards = randomIntBetween(0, 10); + List shards = new ArrayList<>(); + for (int i = 0; i < numShards; i++) { + shards.add(new SearchShard(null, new ShardId("0", "0", 1))); + } + List skippedShards = new ArrayList<>(); + int numSkippedShards = randomIntBetween(0, 10); + for (int i = 0; i < numSkippedShards; i++) { + skippedShards.add(new SearchShard(null, new ShardId("0", "0", 1))); + } + + int numShardFailures = 0; + task.getProgressListener().onListShards(shards, skippedShards, SearchResponse.Clusters.EMPTY, false); + for (int i = 0; i < numShards; i++) { + task.getProgressListener().onPartialReduce(shards.subList(i, i+1), + new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), null, 0); + assertCompletionListeners(task, numShards+numSkippedShards, numSkippedShards, numShardFailures, true); + } + task.getProgressListener().onReduce(shards, + new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), null, 0); + assertCompletionListeners(task, numShards+numSkippedShards, numSkippedShards, numShardFailures, true); + task.getProgressListener().onResponse(newSearchResponse(numShards+numSkippedShards, numShards, numSkippedShards)); + assertCompletionListeners(task, numShards+numSkippedShards, + numSkippedShards, numShardFailures, false); + threadPool.shutdownNow(); + } + + private SearchResponse newSearchResponse(int totalShards, int successfulShards, int skippedShards) { + InternalSearchResponse response = new InternalSearchResponse(SearchHits.empty(), + InternalAggregations.EMPTY, null, null, false, null, 1); + return new SearchResponse(response, null, totalShards, successfulShards, skippedShards, + 100, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + } + + private void assertCompletionListeners(AsyncSearchTask task, + int expectedTotalShards, + int expectedSkippedShards, + int expectedShardFailures, + boolean isPartial) throws InterruptedException { + List threads = new ArrayList<>(); + int numThreads = randomIntBetween(1, 10); + CountDownLatch latch = new CountDownLatch(numThreads); + for (int i = 0; i < numThreads; i++) { + Thread thread = new Thread(() -> task.addCompletionListener(resp -> { + assertThat(resp.getSearchResponse().getTotalShards(), equalTo(expectedTotalShards)); + assertThat(resp.getSearchResponse().getSkippedShards(), equalTo(expectedSkippedShards)); + assertThat(resp.getSearchResponse().getFailedShards(), equalTo(expectedShardFailures)); + assertThat(resp.isPartial(), equalTo(isPartial)); + latch.countDown(); + }, TimeValue.timeValueMillis(1))); + threads.add(thread); + thread.start(); + } + latch.await(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 620ce1b199037..3a4b5d4c1aeb1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -14,98 +14,92 @@ import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; -import java.util.concurrent.TimeUnit; -import static org.elasticsearch.rest.RestStatus.NOT_MODIFIED; -import static org.elasticsearch.rest.RestStatus.PARTIAL_CONTENT; +import static org.elasticsearch.rest.RestStatus.OK; /** - * A response of a search progress request that contains a non-null {@link PartialSearchResponse} if the request is running or has failed - * before completion, or a final {@link SearchResponse} if the request succeeded. + * A response of an async search request. */ public class AsyncSearchResponse extends ActionResponse implements StatusToXContentObject { @Nullable private final String id; private final int version; - private final SearchResponse response; - private final PartialSearchResponse partialResponse; + private final SearchResponse searchResponse; private final ElasticsearchException failure; private final boolean isRunning; + private final boolean isPartial; - private long startDateMillis; - private long runningTimeMillis; + private final long startDateMillis; - public AsyncSearchResponse(String id, int version, boolean isRunning) { - this(id, null, null, null, version, isRunning); - } - - public AsyncSearchResponse(String id, SearchResponse response, int version, boolean isRunning) { - this(id, null, response, null, version, isRunning); - } - - public AsyncSearchResponse(String id, PartialSearchResponse response, int version, boolean isRunning) { - this(id, response, null, null, version, isRunning); - } - - public AsyncSearchResponse(String id, PartialSearchResponse response, ElasticsearchException failure, int version, boolean isRunning) { - this(id, response, null, failure, version, isRunning); - } - - public AsyncSearchResponse(String id, AsyncSearchResponse clone) { - this(id, clone.partialResponse, clone.response, clone.failure, clone.version, clone.isRunning); - this.startDateMillis = clone.startDateMillis; - this.runningTimeMillis = clone.runningTimeMillis; + /** + * Creates an {@link AsyncSearchResponse} with meta informations that omits + * the search response. + */ + public AsyncSearchResponse(String id, + int version, + boolean isPartial, + boolean isRunning, + long startDateMillis) { + this(id, version, null, null, isPartial, isRunning, startDateMillis); } + /** + * Creates a new {@link AsyncSearchResponse} + * @param id The id of the search for further retrieval, null if not stored. + * @param version The version number of this response. + * @param searchResponse The actual search response. + * @param failure The actual failure if the search failed, null if the search is running + * or completed without failure. + * @param isPartial Whether the searchResponse contains partial results. + * @param isRunning Whether the search is running in the cluster. + * @param startDateMillis The start date of the search in milliseconds since epoch. + */ public AsyncSearchResponse(String id, - PartialSearchResponse partialResponse, - SearchResponse response, - ElasticsearchException failure, int version, - boolean isRunning) { - assert id != null || isRunning == false; + SearchResponse searchResponse, + ElasticsearchException failure, + boolean isPartial, + boolean isRunning, + long startDateMillis) { this.id = id; this.version = version; - this.partialResponse = partialResponse; this.failure = failure; - this.response = response != null ? wrapFinalResponse(response) : null; + this.searchResponse = searchResponse; + this.isPartial = isPartial; this.isRunning = isRunning; + this.startDateMillis = startDateMillis; } public AsyncSearchResponse(StreamInput in) throws IOException { this.id = in.readOptionalString(); this.version = in.readVInt(); - this.partialResponse = in.readOptionalWriteable(PartialSearchResponse::new); this.failure = in.readOptionalWriteable(ElasticsearchException::new); - this.response = in.readOptionalWriteable(SearchResponse::new); + this.searchResponse = in.readOptionalWriteable(SearchResponse::new); + this.isPartial = in.readBoolean(); this.isRunning = in.readBoolean(); this.startDateMillis = in.readLong(); - this.runningTimeMillis = in.readLong(); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeVInt(version); - out.writeOptionalWriteable(partialResponse); out.writeOptionalWriteable(failure); - out.writeOptionalWriteable(response); + out.writeOptionalWriteable(searchResponse); + out.writeBoolean(isPartial); out.writeBoolean(isRunning); out.writeLong(startDateMillis); - out.writeLong(runningTimeMillis); } - public void setTaskInfo(TaskInfo taskInfo) { - this.startDateMillis = taskInfo.getStartTime(); - this.runningTimeMillis = TimeUnit.NANOSECONDS.toMillis(taskInfo.getRunningTimeNanos()); + public AsyncSearchResponse clone(String id) { + return new AsyncSearchResponse(id, version, searchResponse, failure, isPartial, isRunning, startDateMillis); } + /** - * Return the id of the async search request or null if the response - * was cleaned on completion. + * Returns the id of the async search request or null if the response is not stored in the cluster. */ @Nullable public String getId() { @@ -113,54 +107,34 @@ public String getId() { } /** - * Return the version of this response. + * Returns the version of this response. */ public int getVersion() { return version; } /** - * Return true if the request has failed. - */ - public boolean hasFailed() { - return failure != null; - } - - /** - * Return true if a partial response is available. - */ - public boolean hasPartialResponse() { - return partialResponse != null; - } - - /** - * Return true if the final response is available. - */ - public boolean hasResponse() { - return response != null; - } - - /** - * The final {@link SearchResponse} if the request has completed, or null if the - * request is running or failed. + * Returns the current {@link SearchResponse} or null if not available. + * See {@link #isPartial()} to determine whether the response contains partial or complete + * results. */ public SearchResponse getSearchResponse() { - return response; + return searchResponse; } /** - * The {@link PartialSearchResponse} if the request is running or failed, or null - * if the request has completed. + * Returns the failure reason or null if the query is running or completed normally. */ - public PartialSearchResponse getPartialResponse() { - return partialResponse; + public ElasticsearchException getFailure() { + return failure; } /** - * The failure that occurred during the search. + * Returns true if the {@link SearchResponse} contains partial + * results computed from a subset of the total shards. */ - public ElasticsearchException getFailure() { - return failure; + public boolean isPartial() { + return isPartial; } /** @@ -170,12 +144,12 @@ public long getStartDate() { return startDateMillis; } - public long getRunningTime() { - return runningTimeMillis; - } - /** * Whether the search is still running in the cluster. + * A value of false indicates that the response is final even + * if it contains partial results. In such case the failure should indicate + * why the request could not finish and the search response represents the + * last partial results before the failure. */ public boolean isRunning() { return isRunning; @@ -183,12 +157,13 @@ public boolean isRunning() { @Override public RestStatus status() { - if (response == null && partialResponse == null) { - return failure != null ? failure.status() : NOT_MODIFIED; - } else if (response == null) { - return failure != null ? failure.status() : PARTIAL_CONTENT; + if (searchResponse == null || isPartial) { + // shard failures are not considered fatal for partial results so + // we return OK until we get the final response even if we don't have + // a single successful shard. + return failure != null ? failure.status() : OK; } else { - return response.status(); + return searchResponse.status(); } } @@ -199,15 +174,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("id", id); } builder.field("version", version); + builder.field("is_partial", isPartial); builder.field("is_running", isRunning); builder.field("start_date_in_millis", startDateMillis); - builder.field("running_time_in_millis", runningTimeMillis); - if (partialResponse != null) { - builder.field("response", partialResponse); - } else if (response != null) { - builder.field("response", response); - } + builder.field("response", searchResponse); if (failure != null) { builder.startObject("failure"); failure.toXContent(builder, params); @@ -216,15 +187,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } - - private static SearchResponse wrapFinalResponse(SearchResponse response) { - // Adds a partial flag set to false in the xcontent serialization - return new SearchResponse(response) { - @Override - public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("is_partial", false); - return super.innerToXContent(builder, params); - } - }; - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java deleted file mode 100644 index 9cb44310bd872..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/PartialSearchResponse.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.search.action; - -import org.apache.lucene.search.TotalHits; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.xcontent.ToXContentFragment; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.action.RestActions; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.InternalAggregations; - -import java.io.IOException; -import java.util.Objects; - -/** - * A search response that contains partial results. - */ -public class PartialSearchResponse implements ToXContentFragment, Writeable { - private final int totalShards; - private final int successfulShards; - private final int shardFailures; - - private final TotalHits totalHits; - private final InternalAggregations aggregations; - private final boolean isFinalReduce; - - public PartialSearchResponse(int totalShards) { - this(totalShards, 0, 0, null, null, false); - } - - public PartialSearchResponse(int totalShards, int successfulShards, int shardFailures, - TotalHits totalHits, InternalAggregations aggregations, boolean isFinalReduce) { - this.totalShards = totalShards; - this.successfulShards = successfulShards; - this.shardFailures = shardFailures; - this.totalHits = totalHits; - this.aggregations = aggregations; - this.isFinalReduce = isFinalReduce; - } - - public PartialSearchResponse(StreamInput in) throws IOException { - this.totalShards = in.readVInt(); - this.successfulShards = in.readVInt(); - this.shardFailures = in.readVInt(); - this.totalHits = in.readBoolean() ? Lucene.readTotalHits(in) : null; - this.aggregations = in.readOptionalWriteable(InternalAggregations::new); - this.isFinalReduce = in.readBoolean(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(totalShards); - out.writeVInt(successfulShards); - out.writeVInt(shardFailures); - out.writeBoolean(totalHits != null); - if (totalHits != null) { - Lucene.writeTotalHits(out, totalHits); - } - out.writeOptionalWriteable(aggregations); - out.writeBoolean(isFinalReduce); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("is_partial", true); - RestActions.buildBroadcastShardsHeader(builder, params, totalShards, successfulShards, 0, - shardFailures, null); - if (totalHits != null) { - builder.startObject(SearchHits.Fields.TOTAL); - builder.field("value", totalHits.value); - builder.field("relation", totalHits.relation == TotalHits.Relation.EQUAL_TO ? "eq" : "gte"); - builder.endObject(); - } - if (aggregations != null) { - aggregations.toXContent(builder, params); - } - builder.endObject(); - return builder; - } - - /** - * The total number of shards the search should executed on. - */ - public int getTotalShards() { - return totalShards; - } - - /** - * The successful number of shards the search was executed on. - */ - public int getSuccessfulShards() { - return successfulShards; - } - - /** - * The failed number of shards the search was executed on. - */ - public int getShardFailures() { - return shardFailures; - } - - /** - * Return the partial {@link TotalHits} computed from the shards that - * completed the query phase. - */ - public TotalHits getTotalHits() { - return totalHits; - } - - /** - * Return the partial {@link InternalAggregations} computed from the shards that - * completed the query phase. - */ - public InternalAggregations getAggregations() { - return aggregations; - } - - public boolean isFinalReduce() { - return isFinalReduce; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PartialSearchResponse that = (PartialSearchResponse) o; - return totalShards == that.totalShards && - successfulShards == that.successfulShards && - shardFailures == that.shardFailures && - Objects.equals(totalHits, that.totalHits) && - Objects.equals(aggregations, that.aggregations) && - isFinalReduce == that.isFinalReduce; - } - - @Override - public int hashCode() { - return Objects.hash(totalShards, successfulShards, shardFailures, totalHits, aggregations, isFinalReduce); - } -} From dc6daa8101ae4a343ef73bf674bf6c4f0c07a189 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 22 Jan 2020 15:28:13 +0100 Subject: [PATCH 33/61] fix checkstyle --- .../action/search/SearchPhaseControllerTests.java | 6 +++--- .../action/search/SearchProgressActionListenerIT.java | 4 ++-- .../elasticsearch/xpack/search/MutableSearchResponse.java | 1 + .../elasticsearch/xpack/search/AsyncSearchTaskTests.java | 5 +++++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index 3a4d464c8c432..e29b03102ed32 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -781,12 +781,12 @@ public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Except } @Override - public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int version) { - assertEquals(numReduceListener.incrementAndGet(), version); + public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { + assertEquals(numReduceListener.incrementAndGet(), reducePhase); } @Override - public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { + public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { totalHitsListener.set(totalHits); finalAggsListener.set(aggs); numReduceListener.incrementAndGet(); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 7ad6d36b35f7f..427aee8585db2 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -174,12 +174,12 @@ public void onFetchFailure(int shardIndex, Exception exc) { } @Override - public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int version) { + public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { numReduces.incrementAndGet(); } @Override - public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs) { + public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { numReduces.incrementAndGet(); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index 1d4ef11a43279..5b5b8c50eaba1 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; + import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java index 561fbfde236ae..1f0d689916e96 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java @@ -1,3 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ package org.elasticsearch.xpack.search; import org.apache.lucene.search.TotalHits; From 5e410d38268a307f8f3bd454d22a4d4662e6cdfe Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 22 Jan 2020 16:42:42 +0100 Subject: [PATCH 34/61] fix wrong assert --- .../org/elasticsearch/xpack/search/MutableSearchResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index 5b5b8c50eaba1..f01c81173fb92 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -70,7 +70,8 @@ class MutableSearchResponse { synchronized void updatePartialResponse(int successfulShards, SearchResponseSections newSections, boolean isFinalReduce) { failIfFrozen(); if (newSections.getNumReducePhases() < sections.getNumReducePhases()) { - // should never happen since partial response are called under a lock + // should never happen since partial response are updated under a lock + // in the search phase controller throw new IllegalStateException("received partial response out of order: " + newSections.getNumReducePhases() + " < " + sections.getNumReducePhases()); } @@ -87,7 +88,6 @@ synchronized void updatePartialResponse(int successfulShards, SearchResponseSect * shards. */ synchronized void updateFinalResponse(int successfulShards, SearchResponseSections newSections) { - assert newSections.getNumReducePhases() != sections.getNumReducePhases(); failIfFrozen(); ++ version; this.successfulShards = successfulShards; From eb7ba8a23dea75f73ee2f0326087214c81f18f82 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 24 Jan 2020 15:51:25 +0100 Subject: [PATCH 35/61] Expose the logic to cancel task when the rest channel is closed This commit moves the logic that cancels search requests when the rest channel is closed to a generic client that can be used by other APIs. This will be useful for any rest action that wants to cancel the execution of a task if the underlying rest channel is closed by the client before completion. Relates #49931 Relates #50990 Relates #50990 --- ...er.java => RestCancellableNodeClient.java} | 113 +++++++++++------- .../rest/action/search/RestSearchAction.java | 6 +- ...va => RestCancellableNodeClientTests.java} | 46 ++++--- .../elasticsearch/test/ESIntegTestCase.java | 10 +- 4 files changed, 99 insertions(+), 76 deletions(-) rename server/src/main/java/org/elasticsearch/rest/action/{search/HttpChannelTaskHandler.java => RestCancellableNodeClient.java} (53%) rename server/src/test/java/org/elasticsearch/rest/action/{search/HttpChannelTaskHandlerTests.java => RestCancellableNodeClientTests.java} (84%) diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/HttpChannelTaskHandler.java b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java similarity index 53% rename from server/src/main/java/org/elasticsearch/rest/action/search/HttpChannelTaskHandler.java rename to server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java index 5864551854fca..ef296de66b8e1 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/HttpChannelTaskHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestCancellableNodeClient.java @@ -17,54 +17,84 @@ * under the License. */ -package org.elasticsearch.rest.action.search; +package org.elasticsearch.rest.action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; -import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.Client; +import org.elasticsearch.client.FilterClient; +import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; + /** - * This class executes a request and associates the corresponding {@link Task} with the {@link HttpChannel} that it was originated from, - * so that the tasks associated with a certain channel get cancelled when the underlying connection gets closed. + * A {@linkplain Client} that cancels tasks executed locally when the provided {@link HttpChannel} + * is closed before completion. */ -public final class HttpChannelTaskHandler { +public class RestCancellableNodeClient extends FilterClient { + private static final Map httpChannels = new ConcurrentHashMap<>(); - public static final HttpChannelTaskHandler INSTANCE = new HttpChannelTaskHandler(); - //package private for testing - final Map httpChannels = new ConcurrentHashMap<>(); + private final NodeClient client; + private final HttpChannel httpChannel; - private HttpChannelTaskHandler() { + public RestCancellableNodeClient(NodeClient client, HttpChannel httpChannel) { + super(client); + this.client = client; + this.httpChannel = httpChannel; } - void execute(NodeClient client, HttpChannel httpChannel, ActionRequest request, - ActionType actionType, ActionListener listener) { + /** + * Returns the number of channels tracked globally. + */ + public static int getNumChannels() { + return httpChannels.size(); + } + + /** + * Returns the number of tasks tracked globally. + */ + static int getNumTasks() { + return httpChannels.values().stream() + .mapToInt(CloseListener::getNumTasks) + .sum(); + } - CloseListener closeListener = httpChannels.computeIfAbsent(httpChannel, channel -> new CloseListener(client)); + /** + * Returns the number of tasks tracked by the provided {@link HttpChannel}. + */ + static int getNumTasks(HttpChannel channel) { + CloseListener listener = httpChannels.get(channel); + return listener == null ? 0 : listener.getNumTasks(); + } + + @Override + public void doExecute( + ActionType action, Request request, ActionListener listener) { + CloseListener closeListener = httpChannels.computeIfAbsent(httpChannel, channel -> new CloseListener()); TaskHolder taskHolder = new TaskHolder(); - Task task = client.executeLocally(actionType, request, + Task task = client.executeLocally(action, request, new ActionListener<>() { @Override - public void onResponse(Response searchResponse) { + public void onResponse(Response response) { try { closeListener.unregisterTask(taskHolder); } finally { - listener.onResponse(searchResponse); + listener.onResponse(response); } } @@ -77,25 +107,28 @@ public void onFailure(Exception e) { } } }); - closeListener.registerTask(taskHolder, new TaskId(client.getLocalNodeId(), task.getId())); + final TaskId taskId = new TaskId(client.getLocalNodeId(), task.getId()); + closeListener.registerTask(taskHolder, taskId); closeListener.maybeRegisterChannel(httpChannel); } - public int getNumChannels() { - return httpChannels.size(); + private void cancelTask(TaskId taskId) { + CancelTasksRequest req = new CancelTasksRequest() + .setTaskId(taskId) + .setReason("channel closed"); + // force the origin to execute the cancellation as a system user + new OriginSettingClient(client, TASKS_ORIGIN).admin().cluster().cancelTasks(req, ActionListener.wrap(() -> {})); } - final class CloseListener implements ActionListener { - private final Client client; + private class CloseListener implements ActionListener { private final AtomicReference channel = new AtomicReference<>(); - private final Set taskIds = new HashSet<>(); + private final Set tasks = new HashSet<>(); - CloseListener(Client client) { - this.client = client; + CloseListener() { } int getNumTasks() { - return taskIds.size(); + return tasks.size(); } void maybeRegisterChannel(HttpChannel httpChannel) { @@ -111,35 +144,27 @@ void maybeRegisterChannel(HttpChannel httpChannel) { synchronized void registerTask(TaskHolder taskHolder, TaskId taskId) { taskHolder.taskId = taskId; if (taskHolder.completed == false) { - this.taskIds.add(taskId); + this.tasks.add(taskId); } } synchronized void unregisterTask(TaskHolder taskHolder) { if (taskHolder.taskId != null) { - this.taskIds.remove(taskHolder.taskId); + this.tasks.remove(taskHolder.taskId); } taskHolder.completed = true; } @Override - public synchronized void onResponse(Void aVoid) { - //When the channel gets closed it won't be reused: we can remove it from the map and forget about it. - CloseListener closeListener = httpChannels.remove(channel.get()); - assert closeListener != null : "channel not found in the map of tracked channels"; - for (TaskId taskId : taskIds) { - ThreadContext threadContext = client.threadPool().getThreadContext(); - try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { - // we stash any context here since this is an internal execution and should not leak any existing context information - threadContext.markAsSystemContext(); - ContextPreservingActionListener contextPreservingListener = new ContextPreservingActionListener<>( - threadContext.newRestorableContext(false), ActionListener.wrap(r -> {}, e -> {})); - CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); - cancelTasksRequest.setTaskId(taskId); - //We don't wait for cancel tasks to come back. Task cancellation is just best effort. - client.admin().cluster().cancelTasks(cancelTasksRequest, contextPreservingListener); - } + public void onResponse(Void aVoid) { + final List toCancel; + synchronized (this) { + // when the channel gets closed it won't be reused: we can remove it from the map and forget about it. + CloseListener closeListener = httpChannels.remove(channel.get()); + assert closeListener != null : "channel not found in the map of tracked channels"; + toCancel = new ArrayList<>(tasks); } + toCancel.stream().forEach(taskId -> cancelTask(taskId)); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 36a599c6039d3..e33bc0668454f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -21,7 +21,6 @@ import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Booleans; @@ -32,6 +31,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestStatusToXContentListener; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -100,8 +100,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC parseSearchRequest(searchRequest, request, parser, setSize)); return channel -> { - RestStatusToXContentListener listener = new RestStatusToXContentListener<>(channel); - HttpChannelTaskHandler.INSTANCE.execute(client, request.getHttpChannel(), searchRequest, SearchAction.INSTANCE, listener); + RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + cancelClient.execute(SearchAction.INSTANCE, searchRequest, new RestStatusToXContentListener<>(channel)); }; } diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/HttpChannelTaskHandlerTests.java b/server/src/test/java/org/elasticsearch/rest/action/RestCancellableNodeClientTests.java similarity index 84% rename from server/src/test/java/org/elasticsearch/rest/action/search/HttpChannelTaskHandlerTests.java rename to server/src/test/java/org/elasticsearch/rest/action/RestCancellableNodeClientTests.java index 103981abdc41e..8121b31547599 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/search/HttpChannelTaskHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/RestCancellableNodeClientTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.rest.action.search; +package org.elasticsearch.rest.action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; @@ -45,7 +45,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.CountDownLatch; @@ -56,13 +55,13 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -public class HttpChannelTaskHandlerTests extends ESTestCase { +public class RestCancellableNodeClientTests extends ESTestCase { private ThreadPool threadPool; @Before public void createThreadPool() { - threadPool = new TestThreadPool(HttpChannelTaskHandlerTests.class.getName()); + threadPool = new TestThreadPool(RestCancellableNodeClientTests.class.getName()); } @After @@ -77,8 +76,7 @@ public void stopThreadPool() { */ public void testCompletedTasks() throws Exception { try (TestClient testClient = new TestClient(Settings.EMPTY, threadPool, false)) { - HttpChannelTaskHandler httpChannelTaskHandler = HttpChannelTaskHandler.INSTANCE; - int initialHttpChannels = httpChannelTaskHandler.getNumChannels(); + int initialHttpChannels = RestCancellableNodeClient.getNumChannels(); int totalSearches = 0; List> futures = new ArrayList<>(); int numChannels = randomIntBetween(1, 30); @@ -88,8 +86,8 @@ public void testCompletedTasks() throws Exception { totalSearches += numTasks; for (int j = 0; j < numTasks; j++) { PlainListenableActionFuture actionFuture = PlainListenableActionFuture.newListenableFuture(); - threadPool.generic().submit(() -> httpChannelTaskHandler.execute(testClient, channel, new SearchRequest(), - SearchAction.INSTANCE, actionFuture)); + RestCancellableNodeClient client = new RestCancellableNodeClient(testClient, channel); + threadPool.generic().submit(() -> client.execute(SearchAction.INSTANCE, new SearchRequest(), actionFuture)); futures.add(actionFuture); } } @@ -97,10 +95,8 @@ public void testCompletedTasks() throws Exception { future.get(); } //no channels get closed in this test, hence we expect as many channels as we created in the map - assertEquals(initialHttpChannels + numChannels, httpChannelTaskHandler.getNumChannels()); - for (Map.Entry entry : httpChannelTaskHandler.httpChannels.entrySet()) { - assertEquals(0, entry.getValue().getNumTasks()); - } + assertEquals(initialHttpChannels + numChannels, RestCancellableNodeClient.getNumChannels()); + assertEquals(0, RestCancellableNodeClient.getNumTasks()); assertEquals(totalSearches, testClient.searchRequests.get()); } } @@ -110,9 +106,8 @@ public void testCompletedTasks() throws Exception { * removed and all of its corresponding tasks get cancelled. */ public void testCancelledTasks() throws Exception { - try (TestClient testClient = new TestClient(Settings.EMPTY, threadPool, true)) { - HttpChannelTaskHandler httpChannelTaskHandler = HttpChannelTaskHandler.INSTANCE; - int initialHttpChannels = httpChannelTaskHandler.getNumChannels(); + try (TestClient nodeClient = new TestClient(Settings.EMPTY, threadPool, true)) { + int initialHttpChannels = RestCancellableNodeClient.getNumChannels(); int numChannels = randomIntBetween(1, 30); int totalSearches = 0; List channels = new ArrayList<>(numChannels); @@ -121,18 +116,19 @@ public void testCancelledTasks() throws Exception { channels.add(channel); int numTasks = randomIntBetween(1, 30); totalSearches += numTasks; + RestCancellableNodeClient client = new RestCancellableNodeClient(nodeClient, channel); for (int j = 0; j < numTasks; j++) { - httpChannelTaskHandler.execute(testClient, channel, new SearchRequest(), SearchAction.INSTANCE, null); + client.execute(SearchAction.INSTANCE, new SearchRequest(), null); } - assertEquals(numTasks, httpChannelTaskHandler.httpChannels.get(channel).getNumTasks()); + assertEquals(numTasks, RestCancellableNodeClient.getNumTasks(channel)); } - assertEquals(initialHttpChannels + numChannels, httpChannelTaskHandler.getNumChannels()); + assertEquals(initialHttpChannels + numChannels, RestCancellableNodeClient.getNumChannels()); for (TestHttpChannel channel : channels) { channel.awaitClose(); } - assertEquals(initialHttpChannels, httpChannelTaskHandler.getNumChannels()); - assertEquals(totalSearches, testClient.searchRequests.get()); - assertEquals(totalSearches, testClient.cancelledTasks.size()); + assertEquals(initialHttpChannels, RestCancellableNodeClient.getNumChannels()); + assertEquals(totalSearches, nodeClient.searchRequests.get()); + assertEquals(totalSearches, nodeClient.cancelledTasks.size()); } } @@ -144,8 +140,7 @@ public void testCancelledTasks() throws Exception { */ public void testChannelAlreadyClosed() { try (TestClient testClient = new TestClient(Settings.EMPTY, threadPool, true)) { - HttpChannelTaskHandler httpChannelTaskHandler = HttpChannelTaskHandler.INSTANCE; - int initialHttpChannels = httpChannelTaskHandler.getNumChannels(); + int initialHttpChannels = RestCancellableNodeClient.getNumChannels(); int numChannels = randomIntBetween(1, 30); int totalSearches = 0; for (int i = 0; i < numChannels; i++) { @@ -154,12 +149,13 @@ public void testChannelAlreadyClosed() { channel.close(); int numTasks = randomIntBetween(1, 5); totalSearches += numTasks; + RestCancellableNodeClient client = new RestCancellableNodeClient(testClient, channel); for (int j = 0; j < numTasks; j++) { //here the channel will be first registered, then straight-away removed from the map as the close listener is invoked - httpChannelTaskHandler.execute(testClient, channel, new SearchRequest(), SearchAction.INSTANCE, null); + client.execute(SearchAction.INSTANCE, new SearchRequest(), null); } } - assertEquals(initialHttpChannels, httpChannelTaskHandler.getNumChannels()); + assertEquals(initialHttpChannels, RestCancellableNodeClient.getNumChannels()); assertEquals(totalSearches, testClient.searchRequests.get()); assertEquals(totalSearches, testClient.cancelledTasks.size()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index dd4f937039afc..b144cc8621b13 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -113,7 +113,7 @@ import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.action.search.HttpChannelTaskHandler; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.MockSearchService; import org.elasticsearch.search.SearchHit; @@ -511,9 +511,11 @@ private static void clearClusters() throws Exception { restClient.close(); restClient = null; } - assertBusy(() -> assertEquals(HttpChannelTaskHandler.INSTANCE.getNumChannels() + " channels still being tracked in " + - HttpChannelTaskHandler.class.getSimpleName() + " while there should be none", 0, - HttpChannelTaskHandler.INSTANCE.getNumChannels())); + assertBusy(() -> { + int numChannels = RestCancellableNodeClient.getNumChannels(); + assertEquals( numChannels+ " channels still being tracked in " + RestCancellableNodeClient.class.getSimpleName() + + " while there should be none", 0, numChannels); + }); } private void afterInternal(boolean afterClass) throws Exception { From 6ae79030da84602b509e70f291687672c5a8b75a Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 24 Jan 2020 17:54:08 +0100 Subject: [PATCH 36/61] address review --- .../xpack/search/AsyncSearchTask.java | 101 +++++++++++------- .../xpack/search/MutableSearchResponse.java | 4 +- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 5697adc5b9336..fdf3421c86374 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Supplier; @@ -50,7 +51,7 @@ class AsyncSearchTask extends SearchTask { private final List initListeners = new ArrayList<>(); private final Map> completionListeners = new HashMap<>(); - private MutableSearchResponse searchResponse; + private AtomicReference searchResponse; /** * Creates an instance of {@link AsyncSearchTask}. @@ -78,6 +79,7 @@ class AsyncSearchTask extends SearchTask { this.threadPool = threadPool; this.reduceContextSupplier = reduceContextSupplier; this.progressListener = new Listener(); + this.searchResponse = new AtomicReference<>(); setProgressListener(progressListener); } @@ -106,15 +108,22 @@ public SearchProgressActionListener getProgressListener() { * timeout occurs. In such case the consumed {@link AsyncSearchResponse} will contain partial results. */ public void addCompletionListener(Consumer listener, TimeValue waitForCompletion) { - boolean executeImmediatly = false; + boolean executeImmediately = false; + long startTime = threadPool.relativeTimeInMillis(); synchronized (this) { if (hasCompleted) { - executeImmediatly = true; + executeImmediately = true; } else { - addInitListener(() -> internalAddCompletionListener(listener, waitForCompletion)); + addInitListener(() -> { + long elapsedTime = threadPool.relativeTimeInMillis() - startTime; + // subtract the initialization time to the provided waitForCompletion. + TimeValue remainingWaitForCompletion = + TimeValue.timeValueMillis(Math.max(0, waitForCompletion.getMillis() - elapsedTime)); + internalAddCompletionListener(listener, remainingWaitForCompletion); + }); } } - if (executeImmediatly) { + if (executeImmediately) { listener.accept(getResponse()); } } @@ -124,46 +133,51 @@ public void addCompletionListener(Consumer listener, TimeVa * consumer when the task is finished. */ public void addCompletionListener(Consumer listener) { - boolean executeImmediatly = false; + boolean executeImmediately = false; synchronized (this) { if (hasCompleted == false) { - completionListeners.put(completionId++, resp -> listener.accept(getResponse())); + completionListeners.put(completionId++, resp -> listener.accept(resp)); } else { - executeImmediatly = true; + executeImmediately = true; } } - if (executeImmediatly) { + if (executeImmediately) { listener.accept(getResponse()); } } private void internalAddCompletionListener(Consumer listener, TimeValue waitForCompletion) { - boolean executeImmediatly = false; + boolean executeImmediately = false; synchronized (this) { if (hasCompleted == false) { - // ensure that we consumes the listener only once - AtomicBoolean hasRun = new AtomicBoolean(false); - long id = completionId++; - Cancellable cancellable = - threadPool.schedule(() -> { + if (waitForCompletion.getMillis() == 0) { + // can happen if the initialization time was greater than the original waitForCompletion + executeImmediately = true; + } else { + // ensure that we consumes the listener only once + AtomicBoolean hasRun = new AtomicBoolean(false); + long id = completionId++; + Cancellable cancellable = + threadPool.schedule(() -> { + if (hasRun.compareAndSet(false, true)) { + // timeout occurred before completion + removeCompletionListener(id); + listener.accept(getResponse()); + } + }, waitForCompletion, "generic"); + completionListeners.put(id, resp -> { if (hasRun.compareAndSet(false, true)) { - // timeout occurred before completion - removeCompletionListener(id); - listener.accept(getResponse()); + // completion occurred before timeout + cancellable.cancel(); + listener.accept(resp); } - }, waitForCompletion, "generic"); - completionListeners.put(id, resp -> { - if (hasRun.compareAndSet(false, true)) { - // completion occurred before timeout - cancellable.cancel(); - listener.accept(resp); - } - }); + }); + } } else { - executeImmediatly = true; + executeImmediately = true; } } - if (executeImmediatly) { + if (executeImmediately) { listener.accept(getResponse()); } } @@ -177,15 +191,15 @@ private void removeCompletionListener(long id) { } private void addInitListener(Runnable listener) { - boolean executeImmediatly = false; + boolean executeImmediately = false; synchronized (this) { if (hasInitialized) { - executeImmediatly = true; + executeImmediately = true; } else { initListeners.add(listener); } } - if (executeImmediatly) { + if (executeImmediately) { listener.run(); } } @@ -205,6 +219,9 @@ private void executeInitListeners() { private void executeCompletionListeners() { synchronized (this) { + if (hasCompleted) { + return; + } hasCompleted = true; } AsyncSearchResponse finalResponse = getResponse(); @@ -215,49 +232,51 @@ private void executeCompletionListeners() { } private AsyncSearchResponse getResponse() { - assert searchResponse != null; - return searchResponse.toAsyncSearchResponse(this); + assert searchResponse.get() != null; + return searchResponse.get().toAsyncSearchResponse(this); } private class Listener extends SearchProgressActionListener { @Override public void onListShards(List shards, List skipped, Clusters clusters, boolean fetchPhase) { - searchResponse = new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier); + searchResponse.compareAndSet(null, + new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier)); executeInitListeners(); } @Override public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) { - searchResponse.addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget)); + searchResponse.get().addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget)); } @Override public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { - searchResponse.updatePartialResponse(shards.size(), + searchResponse.get().updatePartialResponse(shards.size(), new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs, null, null, false, null, reducePhase), aggs == null); } @Override public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { - searchResponse.updatePartialResponse(shards.size(), + searchResponse.get().updatePartialResponse(shards.size(), new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs, null, null, false, null, reducePhase), true); } @Override public void onResponse(SearchResponse response) { - searchResponse.updateFinalResponse(response.getSuccessfulShards(), response.getInternalResponse()); + searchResponse.get().updateFinalResponse(response.getSuccessfulShards(), response.getInternalResponse()); executeCompletionListeners(); } @Override public void onFailure(Exception exc) { - if (searchResponse == null) { + if (searchResponse.get() == null) { // if the failure occurred before calling onListShards - searchResponse = new MutableSearchResponse(-1, -1, null, reduceContextSupplier); + searchResponse.compareAndSet(null, + new MutableSearchResponse(-1, -1, null, reduceContextSupplier)); } - searchResponse.updateWithFailure(exc); + searchResponse.get().updateWithFailure(exc); executeInitListeners(); executeCompletionListeners(); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index f01c81173fb92..f4a237f76fa22 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -113,7 +113,9 @@ synchronized void updateWithFailure(Exception exc) { * Adds a shard failure concurrently (non-blocking). */ void addShardFailure(int shardIndex, ShardSearchFailure failure) { - failIfFrozen(); + synchronized (this) { + failIfFrozen(); + } shardFailures.set(shardIndex, failure); } From 159ed81b444f7794b7b8b2375718dab36fd43e94 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 24 Jan 2020 17:55:07 +0100 Subject: [PATCH 37/61] plug the automatic cancel of search if the rest channel is closed when submitting the initial async search request --- .../search/RestSubmitAsyncSearchAction.java | 4 +- .../TransportSubmitAsyncSearchAction.java | 115 ++++++++++++------ .../action/SubmitAsyncSearchRequest.java | 8 +- 3 files changed, 91 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java index 7ef20bee03cc4..a9748aba741b2 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestStatusToXContentListener; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; @@ -51,7 +52,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } return channel -> { RestStatusToXContentListener listener = new RestStatusToXContentListener<>(channel); - client.execute(SubmitAsyncSearchAction.INSTANCE, submitRequest, listener); + RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + cancelClient.execute(SubmitAsyncSearchAction.INSTANCE, submitRequest, listener); }; } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 74cf95ffc341d..5b66eb19a9d07 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -8,8 +8,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; @@ -18,6 +20,7 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; @@ -26,6 +29,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportService; @@ -36,6 +40,8 @@ import java.util.Map; import java.util.function.Supplier; +import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; + public class TransportSubmitAsyncSearchAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportSubmitAsyncSearchAction.class); @@ -72,13 +78,16 @@ protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionList return; } final String docID = UUIDs.randomBase64UUID(); - store.ensureAsyncSearchIndex(clusterService.state(), ActionListener.wrap( - indexName -> executeSearch(request, indexName, docID, submitListener), - submitListener::onFailure - )); + CancellableTask submitTask = (CancellableTask) task; + store.ensureAsyncSearchIndex(clusterService.state(), + ActionListener.wrap(indexName -> executeSearch(submitTask, request, indexName, docID, submitListener), + submitListener::onFailure)); } - private void executeSearch(SubmitAsyncSearchRequest submitRequest, String indexName, String docID, + private void executeSearch(CancellableTask submitTask, + SubmitAsyncSearchRequest submitRequest, + String indexName, + String docID, ActionListener submitListener) { final Map originHeaders = nodeClient.threadPool().getThreadContext().getHeaders(); final SearchRequest searchRequest = new SearchRequest(submitRequest.getSearchRequest()) { @@ -91,62 +100,100 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, }; // trigger the async search - AsyncSearchTask task = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); - searchAction.execute(task, searchRequest, task.getProgressListener()); - task.addCompletionListener(searchResponse -> { + AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); + searchAction.execute(searchTask, searchRequest, searchTask.getProgressListener()); + searchTask.addCompletionListener(searchResponse -> { if (searchResponse.isRunning() || submitRequest.isCleanOnCompletion() == false) { // the task is still running and the user cannot wait more so we create // a document for further retrieval try { - store.storeInitialResponse(originHeaders, indexName, docID, searchResponse, - new ActionListener<>() { - @Override - public void onResponse(IndexResponse r) { - // store the final response - task.addCompletionListener(finalResponse -> storeFinalResponse(task, finalResponse)); - submitListener.onResponse(searchResponse); - } + if (submitTask.isCancelled()) { + // the user cancelled the submit so we don't store anything + // and propagate the failure + searchTask.addCompletionListener(finalResponse -> taskManager.unregister(searchTask)); + submitListener.onFailure(new ElasticsearchException(submitTask.getReasonCancelled())); + } else { + store.storeInitialResponse(originHeaders, indexName, docID, searchResponse, + new ActionListener<>() { + @Override + public void onResponse(IndexResponse r) { + // store the final response on completion unless the submit is cancelled + searchTask.addCompletionListener(finalResponse -> { + if (submitTask.isCancelled()) { + onTaskCompletion(submitTask, searchTask, () -> {}); + } else { + storeFinalResponse(submitTask, searchTask, finalResponse); + } + }); + submitListener.onResponse(searchResponse); + } - @Override - public void onFailure(Exception exc) { - // TODO: cancel search - taskManager.unregister(task); - submitListener.onFailure(exc); - } - }); + @Override + public void onFailure(Exception exc) { + onTaskFailure(searchTask, exc.getMessage(), () -> { + searchTask.addCompletionListener(finalResponse -> taskManager.unregister(searchTask)); + submitListener.onFailure(exc); + }); + } + }); + } } catch (Exception exc) { - // TODO: cancel search - taskManager.unregister(task); - submitListener.onFailure(exc); + onTaskFailure(searchTask, exc.getMessage(), () -> { + searchTask.addCompletionListener(finalResponse -> taskManager.unregister(searchTask)); + submitListener.onFailure(exc); + }); } } else { // the task completed within the timeout so the response is sent back to the user // with a null id since nothing was stored on the cluster. - taskManager.unregister(task); + taskManager.unregister(searchTask); submitListener.onResponse(searchResponse.clone(null)); } }, submitRequest.getWaitForCompletion()); } - private void storeFinalResponse(AsyncSearchTask task, AsyncSearchResponse response) { + private void storeFinalResponse(CancellableTask submitTask, AsyncSearchTask searchTask, AsyncSearchResponse response) { try { - store.storeFinalResponse(task.getOriginHeaders(), response, + store.storeFinalResponse(searchTask.getOriginHeaders(), response, new ActionListener<>() { @Override public void onResponse(UpdateResponse updateResponse) { - taskManager.unregister(task); + onTaskCompletion(submitTask, searchTask, () -> {}); } @Override public void onFailure(Exception exc) { logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", - task.getSearchId().getEncoded()), exc); - taskManager.unregister(task); + searchTask.getSearchId().getEncoded()), exc); + onTaskCompletion(submitTask, searchTask, () -> {}); } }); } catch (Exception exc) { - logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", task.getSearchId().getEncoded()), exc); - taskManager.unregister(task); + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", searchTask.getSearchId().getEncoded()), exc); + onTaskCompletion(submitTask, searchTask, () -> {}); + } + } + + private void onTaskFailure(AsyncSearchTask searchTask, String reason, Runnable onFinish) { + CancelTasksRequest req = new CancelTasksRequest() + .setTaskId(new TaskId(nodeClient.getLocalNodeId(), searchTask.getId())) + .setReason(reason); + // force the origin to execute the cancellation as a system user + new OriginSettingClient(nodeClient, TASKS_ORIGIN).admin().cluster().cancelTasks(req, ActionListener.wrap(() -> onFinish.run())); + } + + private void onTaskCompletion(CancellableTask submitTask, AsyncSearchTask searchTask, Runnable onFinish) { + if (submitTask.isCancelled()) { + // the user cancelled the submit so we ensure that there is nothing stored + // in the response index. + store.deleteResult(searchTask.getSearchId(), + ActionListener.wrap(() -> { + taskManager.unregister(searchTask); + onFinish.run(); + })); + } else { + taskManager.unregister(searchTask); + onFinish.run(); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 8942333ac1da6..37ef6e1e7b73f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -115,7 +116,12 @@ public ActionRequestValidationException validate() { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - return new Task(id, type, action, getDescription(), parentTaskId, headers); + return new CancellableTask(id, type, action, "", parentTaskId, headers) { + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + }; } @Override From 19d89f9a8e7905f3f6767b350a8beb7f376bd1e6 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 29 Jan 2020 09:19:30 +0100 Subject: [PATCH 38/61] handle expiration time on a per request basis --- .../xpack/search/AsyncSearchSecurityIT.java | 3 +- .../xpack/search/AsyncSearch.java | 11 +- .../xpack/search/AsyncSearchId.java | 40 ++-- .../search/AsyncSearchReaperExecutor.java | 164 +++++++++++++++ .../xpack/search/AsyncSearchStoreService.java | 125 ++++------- .../xpack/search/AsyncSearchTask.java | 66 +++--- .../search/AsyncSearchTemplateRegistry.java | 10 +- .../xpack/search/MutableSearchResponse.java | 2 +- .../search/RestGetAsyncSearchAction.java | 2 + .../search/RestSubmitAsyncSearchAction.java | 1 + .../TransportDeleteAsyncSearchAction.java | 38 +++- .../search/TransportGetAsyncSearchAction.java | 81 +++++--- .../TransportSubmitAsyncSearchAction.java | 67 +++--- .../xpack/search/AsyncSearchActionTests.java | 4 +- .../xpack/search/AsyncSearchIdTests.java | 17 +- .../search/AsyncSearchIntegTestCase.java | 5 +- .../search/AsyncSearchResponseTests.java | 10 +- .../search/AsyncSearchStoreServiceTests.java | 194 ------------------ .../xpack/search/AsyncSearchTaskTests.java | 15 +- .../search/GetAsyncSearchRequestTests.java | 4 +- .../search/SubmitAsyncSearchRequestTests.java | 8 + .../search/action/AsyncSearchResponse.java | 36 ++-- .../search/action/GetAsyncSearchAction.java | 26 ++- .../action/SubmitAsyncSearchRequest.java | 28 ++- .../resources/async-search-ilm-policy.json | 18 -- .../core/src/main/resources/async-search.json | 6 + .../rest-api-spec/api/async_search.get.json | 4 + .../api/async_search.submit.json | 19 +- 28 files changed, 503 insertions(+), 501 deletions(-) create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java delete mode 100644 x-pack/plugin/core/src/main/resources/async-search-ilm-policy.json diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java index 9f977f560108a..c663c33d26bcf 100644 --- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java +++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java @@ -27,6 +27,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -81,7 +82,7 @@ private void testCase(String user, String other) throws Exception { // other and user cannot access the result from direct get calls AsyncSearchId searchId = AsyncSearchId.decode(id); for (String runAs : new String[] {user, other}) { - exc = expectThrows(ResponseException.class, () -> get(searchId.getIndexName(), searchId.getDocId(), runAs)); + exc = expectThrows(ResponseException.class, () -> get(ASYNC_SEARCH_ALIAS, searchId.getDocId(), runAs)); assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403)); assertThat(exc.getMessage(), containsString("unauthorized")); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index edb67b0fb0614..3b1762c8a832e 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -16,10 +16,13 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; @@ -36,7 +39,7 @@ import java.util.List; import java.util.function.Supplier; -public final class AsyncSearch extends Plugin implements ActionPlugin { +public final class AsyncSearch extends Plugin implements ActionPlugin, PersistentTaskPlugin { @Override public List> getActions() { return Arrays.asList( @@ -71,4 +74,10 @@ public Collection createComponents(Client client, new AsyncSearchTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry); return Collections.emptyList(); } + + @Override + public List> getPersistentTasksExecutor(ClusterService clusterService, ThreadPool threadPool, + Client client, SettingsModule settingsModule) { + return Collections.singletonList(new AsyncSearchReaperExecutor(client, threadPool)); + } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java index 7c5976be57c77..673061292ccfc 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java @@ -16,29 +16,18 @@ import java.util.Base64; import java.util.Objects; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; - /** * A class that contains all information related to a submitted async search. */ class AsyncSearchId { - private final String indexName; private final String docId; private final TaskId taskId; private final String encoded; - AsyncSearchId(String indexName, String docId, TaskId taskId) { - this.indexName = indexName; + AsyncSearchId(String docId, TaskId taskId) { this.docId = docId; this.taskId = taskId; - this.encoded = encode(indexName, docId, taskId); - } - - /** - * The index name where to find the response if the task is not running. - */ - String getIndexName() { - return indexName; + this.encoded = encode(docId, taskId); } /** @@ -67,23 +56,29 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AsyncSearchId searchId = (AsyncSearchId) o; - return indexName.equals(searchId.indexName) && - docId.equals(searchId.docId) && + return docId.equals(searchId.docId) && taskId.equals(searchId.taskId); } @Override public int hashCode() { - return Objects.hash(indexName, docId, taskId); + return Objects.hash(docId, taskId); + } + + @Override + public String toString() { + return "AsyncSearchId{" + + "docId='" + docId + '\'' + + ", taskId=" + taskId + + '}'; } /** * Encode the informations needed to retrieve a async search response * in a base64 encoded string. */ - static String encode(String indexName, String docId, TaskId taskId) { + static String encode(String docId, TaskId taskId) { try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeString(indexName); out.writeString(docId); out.writeString(taskId.toString()); return Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(out.bytes())); @@ -99,20 +94,13 @@ static String encode(String indexName, String docId, TaskId taskId) { static AsyncSearchId decode(String id) { final AsyncSearchId searchId; try (StreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getUrlDecoder().decode(id)))) { - searchId = new AsyncSearchId(in.readString(), in.readString(), new TaskId(in.readString())); + searchId = new AsyncSearchId(in.readString(), new TaskId(in.readString())); if (in.available() > 0) { throw new IllegalArgumentException("invalid id:[" + id + "]"); } } catch (IOException e) { throw new IllegalArgumentException("invalid id:[" + id + "]"); } - validateAsyncSearchId(searchId); return searchId; } - - static void validateAsyncSearchId(AsyncSearchId searchId) { - if (searchId.getIndexName().startsWith(ASYNC_SEARCH_INDEX_PREFIX) == false) { - throw new IllegalArgumentException("invalid id:[" + searchId.getEncoded() + "]"); - } - } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java new file mode 100644 index 0000000000000..d6e4992e46ebb --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.persistent.AllocatedPersistentTask; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; +import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC; +import static org.elasticsearch.threadpool.ThreadPool.Names.SAME; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.EXPIRATION_TIME_FIELD; +import static org.elasticsearch.xpack.search.AsyncSearchReaperExecutor.Params; +import static org.elasticsearch.xpack.search.AsyncSearchStoreService.IS_RUNNING_FIELD; +import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; + +public class AsyncSearchReaperExecutor extends PersistentTasksExecutor { + public static final String NAME = "async_search/reaper"; + + private final Client client; + private final ThreadPool threadPool; + + public AsyncSearchReaperExecutor(Client client, ThreadPool threadPool) { + super(NAME, SAME); + this.client = new OriginSettingClient(client, TASKS_ORIGIN); + this.threadPool = threadPool; + } + + @Override + protected void nodeOperation(AllocatedPersistentTask task, Params params, PersistentTaskState state) { + cleanup(task, client, 100, System.currentTimeMillis()); + } + + public void cleanup(AllocatedPersistentTask task, Client client, int pageSize, long nowInMillis) { + SearchRequest request = new SearchRequest(ASYNC_SEARCH_ALIAS); + request.source().query(QueryBuilders.rangeQuery(EXPIRATION_TIME_FIELD).gt(nowInMillis)); + request.scroll(TimeValue.timeValueMinutes(1)); + client.search(request, nextPageListener(task, pageSize, () -> { + if (task.isCancelled() == false) { + threadPool.schedule(() -> cleanup(task, client, pageSize, System.currentTimeMillis()), + TimeValue.timeValueMinutes(30), GENERIC); + } + })); + } + + private void nextPage(AllocatedPersistentTask task, Client client, String scrollId, int pageSize, Runnable onCompletion) { + if (task.isCancelled()) { + onCompletion.run(); + return; + } + client.searchScroll(new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(1)), + nextPageListener(task, pageSize, onCompletion)); + } + + public void deleteAll(Client client, Collection toCancel, Collection toDelete, ActionListener listener) { + if (toDelete.isEmpty() && toCancel.isEmpty()) { + listener.onResponse(null); + } + CountDown counter = new CountDown(toDelete.size() + 1); + BulkRequest bulkDelete = new BulkRequest(); + for (String id : toDelete) { + bulkDelete.add(new DeleteRequest(ASYNC_SEARCH_ALIAS).id(id)); + } + client.bulk(bulkDelete, ActionListener.wrap(() -> { + if (counter.countDown()) { + listener.onResponse(null); + } + })); + for (String id : toCancel) { + ThreadContext threadContext = threadPool.getThreadContext(); + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { + threadContext.markAsSystemContext(); + client.execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(id), + ActionListener.wrap(() -> { + if (counter.countDown()) { + listener.onResponse(null); + } + })); + } + } + } + + private ActionListener nextPageListener(AllocatedPersistentTask task, int pageSize, Runnable onCompletion) { + return new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (task.isCancelled()) { + onCompletion.run(); + return; + } + final List toDelete = new ArrayList<>(); + final List toCancel = new ArrayList<>(); + for (SearchHit hit : response.getHits()) { + boolean isRunning = (boolean) hit.getSourceAsMap().get(IS_RUNNING_FIELD); + if (isRunning) { + toCancel.add(hit.getId()); + } else { + toDelete.add(hit.getId()); + } + } + deleteAll(client, toDelete, toCancel, ActionListener.wrap(() -> { + if (response.getHits().getHits().length < pageSize) { + onCompletion.run(); + } else { + nextPage(task, client, response.getScrollId(), pageSize, onCompletion); + } + })); + } + + @Override + public void onFailure(Exception exc) { + onCompletion.run(); + } + }; + } + + static class Params implements PersistentTaskParams { + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) { + return builder; + } + + @Override + public void writeTo(StreamOutput out) {} + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 6c2fca9e1e328..31464b8b600de 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -8,12 +8,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; @@ -23,8 +20,6 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -42,13 +37,13 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Base64; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; -import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.INDEX_TEMPLATE_VERSION; -import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_TEMPLATE_NAME; +import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; /** * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the .async-search index. @@ -56,10 +51,11 @@ class AsyncSearchStoreService { private static final Logger logger = LogManager.getLogger(AsyncSearchStoreService.class); - static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; - static final String ASYNC_SEARCH_INDEX_PREFIX = ASYNC_SEARCH_ALIAS + "-"; - static final String RESULT_FIELD = "result"; + static final String ID_FIELD = "id"; static final String HEADERS_FIELD = "headers"; + static final String IS_RUNNING_FIELD = "is_running"; + static final String EXPIRATION_TIME_FIELD = "expiration_time"; + static final String RESULT_FIELD = "result"; private final TaskManager taskManager; private final ThreadContext threadContext; @@ -80,70 +76,23 @@ Client getClient() { return client; } - /** - * Checks if the async-search index exists, and if not, creates it. - * The provided {@link ActionListener} is called with the index name that should - * be used to store the response. - */ - void ensureAsyncSearchIndex(ClusterState state, ActionListener andThen) { - final String initialIndexName = ASYNC_SEARCH_INDEX_PREFIX + "000001"; - final AliasOrIndex current = state.metaData().getAliasAndIndexLookup().get(ASYNC_SEARCH_ALIAS); - final AliasOrIndex initialIndex = state.metaData().getAliasAndIndexLookup().get(initialIndexName); - if (current == null && initialIndex == null) { - // No alias or index exists with the expected names, so create the index with appropriate alias - client.admin().indices().prepareCreate(initialIndexName) - .setWaitForActiveShards(1) - .addAlias(new Alias(ASYNC_SEARCH_ALIAS).writeIndex(true)) - .execute(new ActionListener<>() { - @Override - public void onResponse(CreateIndexResponse response) { - andThen.onResponse(initialIndexName); - } - - @Override - public void onFailure(Exception e) { - if (e instanceof ResourceAlreadyExistsException) { - // The index didn't exist before we made the call, there was probably a race - just ignore this - andThen.onResponse(initialIndexName); - } else { - andThen.onFailure(e); - } - } - }); - } else if (current == null) { - // alias does not exist but initial index does, something is broken - andThen.onFailure(new IllegalStateException("async-search index [" + initialIndexName + - "] already exists but does not have alias [" + ASYNC_SEARCH_ALIAS + "]")); - } else if (current.isAlias()) { - AliasOrIndex.Alias alias = (AliasOrIndex.Alias) current; - if (alias.getWriteIndex() != null) { - // The alias exists and has a write index, so we're good - andThen.onResponse(alias.getWriteIndex().getIndex().getName()); - } else { - // The alias does not have a write index, so we can't index into it - andThen.onFailure(new IllegalStateException("async-search alias [" + ASYNC_SEARCH_ALIAS + "] does not have a write index")); - } - } else if (current.isAlias() == false) { - // This is not an alias, error out - andThen.onFailure(new IllegalStateException("async-search alias [" + ASYNC_SEARCH_ALIAS + - "] already exists as concrete index")); - } else { - logger.error("unexpected IndexOrAlias for [{}]: [{}]", ASYNC_SEARCH_ALIAS, current); - andThen.onFailure(new IllegalStateException("unexpected IndexOrAlias for async-search index")); - assert false : ASYNC_SEARCH_ALIAS + " cannot be both an alias and not an alias simultaneously"; - } + ThreadContext getThreadContext() { + return threadContext; } /** * Store an empty document in the .async-search index that is used * as a place-holder for the future response. */ - void storeInitialResponse(Map headers, String index, String docID, AsyncSearchResponse response, + void storeInitialResponse(Map headers, String docID, AsyncSearchResponse response, ActionListener listener) throws IOException { Map source = new HashMap<>(); - source.put(RESULT_FIELD, encodeResponse(response)); + source.put(ID_FIELD, response.getId()); source.put(HEADERS_FIELD, headers); - IndexRequest request = new IndexRequest(index) + source.put(IS_RUNNING_FIELD, true); + source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime()); + source.put(RESULT_FIELD, encodeResponse(response)); + IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS) .id(docID) .source(source, XContentType.JSON); client.index(request, listener); @@ -152,15 +101,20 @@ void storeInitialResponse(Map headers, String index, String docI /** * Store the final response if the place-holder document is still present (update). */ - void storeFinalResponse(Map headers, AsyncSearchResponse response, - ActionListener listener) throws IOException { + void storeFinalResponse(AsyncSearchResponse response, ActionListener listener) throws IOException { AsyncSearchId searchId = AsyncSearchId.decode(response.getId()); Map source = new HashMap<>(); + source.put(IS_RUNNING_FIELD, true); source.put(RESULT_FIELD, encodeResponse(response)); - source.put(HEADERS_FIELD, headers); - UpdateRequest request = new UpdateRequest().index(searchId.getIndexName()).id(searchId.getDocId()) - .doc(source, XContentType.JSON) - .detectNoop(false); + UpdateRequest request = new UpdateRequest().index(ASYNC_SEARCH_ALIAS).id(searchId.getDocId()) + .doc(source, XContentType.JSON); + client.update(request, listener); + } + + void updateKeepAlive(String docID, long expirationTimeMillis, ActionListener listener) { + Map source = Collections.singletonMap(EXPIRATION_TIME_FIELD, expirationTimeMillis); + UpdateRequest request = new UpdateRequest().index(ASYNC_SEARCH_ALIAS).id(docID) + .doc(source, XContentType.JSON); client.update(request, listener); } @@ -174,10 +128,13 @@ AsyncSearchTask getTask(AsyncSearchId searchId) throws IOException { return null; } - // Check authentication for the user - final Authentication auth = Authentication.getAuthentication(threadContext); - if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { - throw new ResourceNotFoundException(searchId.getEncoded() + " not found"); + logger.info("Is threadContext " + threadContext.isSystemContext()); + if (threadContext.isSystemContext() == false) { + // Check authentication for the user + final Authentication auth = Authentication.getAuthentication(threadContext); + if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { + throw new ResourceNotFoundException(searchId.getEncoded() + " not found"); + } } return searchTask; } @@ -188,7 +145,7 @@ AsyncSearchTask getTask(AsyncSearchId searchId) throws IOException { */ void getResponse(AsyncSearchId searchId, ActionListener listener) { final Authentication current = Authentication.getAuthentication(client.threadPool().getThreadContext()); - GetRequest internalGet = new GetRequest(searchId.getIndexName()) + GetRequest internalGet = new GetRequest(ASYNC_SEARCH_ALIAS) .preference(searchId.getEncoded()) .id(searchId.getDocId()); client.get(internalGet, ActionListener.wrap( @@ -198,12 +155,14 @@ void getResponse(AsyncSearchId searchId, ActionListener lis return; } - // check the authentication of the current user against the user that initiated the async search - @SuppressWarnings("unchecked") - Map headers = (Map) get.getSource().get(HEADERS_FIELD); - if (ensureAuthenticatedUserIsSame(headers, current) == false) { - listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); - return; + if (threadContext.isSystemContext() == false) { + // check the authentication of the current user against the user that initiated the async search + @SuppressWarnings("unchecked") + Map headers = (Map) get.getSource().get(HEADERS_FIELD); + if (ensureAuthenticatedUserIsSame(headers, current) == false) { + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); + return; + } } @SuppressWarnings("unchecked") @@ -215,7 +174,7 @@ void getResponse(AsyncSearchId searchId, ActionListener lis } void deleteResult(AsyncSearchId searchId, ActionListener listener) { - DeleteRequest request = new DeleteRequest(searchId.getIndexName()).id(searchId.getDocId()); + DeleteRequest request = new DeleteRequest(ASYNC_SEARCH_ALIAS).id(searchId.getDocId()); client.delete(request, ActionListener.wrap( resp -> { if (resp.status() == RestStatus.NOT_FOUND) { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index fdf3421c86374..0af3b8b780fb6 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.Scheduler.Cancellable; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; @@ -32,8 +33,6 @@ import java.util.function.Consumer; import java.util.function.Supplier; -import static org.elasticsearch.tasks.TaskId.EMPTY_TASK_ID; - /** * Task that tracks the progress of a currently running {@link SearchRequest}. */ @@ -51,6 +50,8 @@ class AsyncSearchTask extends SearchTask { private final List initListeners = new ArrayList<>(); private final Map> completionListeners = new HashMap<>(); + private long expirationTimeMillis; + private AtomicReference searchResponse; /** @@ -59,6 +60,7 @@ class AsyncSearchTask extends SearchTask { * @param id The id of the task. * @param type The type of the task. * @param action The action name. + * @param parentTaskId The parent task id. * @param originHeaders All the request context headers. * @param taskHeaders The filtered request headers for the task. * @param searchId The {@link AsyncSearchId} of the task. @@ -68,12 +70,13 @@ class AsyncSearchTask extends SearchTask { AsyncSearchTask(long id, String type, String action, + TaskId parentTaskId, Map originHeaders, Map taskHeaders, AsyncSearchId searchId, ThreadPool threadPool, Supplier reduceContextSupplier) { - super(id, type, action, null, EMPTY_TASK_ID, taskHeaders); + super(id, type, action, "async_search", parentTaskId, taskHeaders); this.originHeaders = originHeaders; this.searchId = searchId; this.threadPool = threadPool; @@ -102,6 +105,14 @@ public SearchProgressActionListener getProgressListener() { return progressListener; } + public synchronized void setExpirationTime(long expirationTimeMillis) { + this.expirationTimeMillis = expirationTimeMillis; + } + + public synchronized long getExpirationTime() { + return expirationTimeMillis; + } + /** * Creates a listener that listens for an {@link AsyncSearchResponse} and executes the * consumer when the task is finished or when the provided waitForCompletion @@ -135,10 +146,10 @@ public void addCompletionListener(Consumer listener, TimeVa public void addCompletionListener(Consumer listener) { boolean executeImmediately = false; synchronized (this) { - if (hasCompleted == false) { - completionListeners.put(completionId++, resp -> listener.accept(resp)); - } else { + if (hasCompleted) { executeImmediately = true; + } else { + completionListeners.put(completionId++, resp -> listener.accept(resp)); } } if (executeImmediately) { @@ -149,32 +160,27 @@ public void addCompletionListener(Consumer listener) { private void internalAddCompletionListener(Consumer listener, TimeValue waitForCompletion) { boolean executeImmediately = false; synchronized (this) { - if (hasCompleted == false) { - if (waitForCompletion.getMillis() == 0) { - // can happen if the initialization time was greater than the original waitForCompletion - executeImmediately = true; - } else { - // ensure that we consumes the listener only once - AtomicBoolean hasRun = new AtomicBoolean(false); - long id = completionId++; - Cancellable cancellable = - threadPool.schedule(() -> { - if (hasRun.compareAndSet(false, true)) { - // timeout occurred before completion - removeCompletionListener(id); - listener.accept(getResponse()); - } - }, waitForCompletion, "generic"); - completionListeners.put(id, resp -> { + if (hasCompleted || waitForCompletion.getMillis() == 0) { + executeImmediately = true; + } else { + // ensure that we consumes the listener only once + AtomicBoolean hasRun = new AtomicBoolean(false); + long id = completionId++; + Cancellable cancellable = + threadPool.schedule(() -> { if (hasRun.compareAndSet(false, true)) { - // completion occurred before timeout - cancellable.cancel(); - listener.accept(resp); + // timeout occurred before completion + removeCompletionListener(id); + listener.accept(getResponse()); } - }); - } - } else { - executeImmediately = true; + }, waitForCompletion, "generic"); + completionListeners.put(id, resp -> { + if (hasRun.compareAndSet(false, true)) { + // completion occurred before timeout + cancellable.cancel(); + listener.accept(resp); + } + }); } } if (executeImmediately) { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java index 501d33607fd60..3ea118bbf6ea8 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java @@ -29,8 +29,7 @@ public class AsyncSearchTemplateRegistry extends IndexTemplateRegistry { public static final String INDEX_TEMPLATE_VERSION = "1"; public static final String ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE = "xpack.async-search.template.version"; public static final String ASYNC_SEARCH_TEMPLATE_NAME = "async-search"; - - public static final String ASYNC_SEARCH_POLICY_NAME = "async-search-ilm-policy"; + public static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; public static final IndexTemplateConfig TEMPLATE_ASYNC_SEARCH = new IndexTemplateConfig( ASYNC_SEARCH_TEMPLATE_NAME, @@ -39,11 +38,6 @@ public class AsyncSearchTemplateRegistry extends IndexTemplateRegistry { ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE ); - public static final LifecyclePolicyConfig ASYNC_SEARCH_POLICY = new LifecyclePolicyConfig( - ASYNC_SEARCH_POLICY_NAME, - "/async-search-ilm-policy.json" - ); - public AsyncSearchTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, @@ -59,7 +53,7 @@ protected List getTemplateConfigs() { @Override protected List getPolicyConfigs() { - return Collections.singletonList(ASYNC_SEARCH_POLICY); + return Collections.emptyList(); } @Override diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index f4a237f76fa22..7d0b384efb1af 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -142,7 +142,7 @@ synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task) { resp = null; } return new AsyncSearchResponse(task.getSearchId().getEncoded(), version, resp, failure, isPartial, - frozen == false, task.getStartTime()); + frozen == false, task.getStartTime(), task.getExpirationTime()); } private void failIfFrozen() { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index 1079112e4ca18..4059a9e900213 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -33,7 +33,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli String id = request.param("id"); int lastVersion = request.paramAsInt("last_version", -1); TimeValue waitForCompletion = request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1)); + TimeValue keepAlive = request.paramAsTime("keep_alive", TimeValue.MINUS_ONE); GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(id, waitForCompletion, lastVersion); + get.setKeepAlive(keepAlive); return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java index a9748aba741b2..b54f3138ddd2f 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -45,6 +45,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli parseSearchRequest(submitRequest.getSearchRequest(), request, parser, setSize)); submitRequest.setWaitForCompletion(request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1))); submitRequest.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", true)); + submitRequest.setKeepAlive(request.paramAsTime("keep_alive", submitRequest.getKeepAlive())); ActionRequestValidationException validationException = submitRequest.validate(); if (validationException != null) { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java index 30b95ac16e89c..13c84271ba7bb 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -6,43 +6,71 @@ package org.elasticsearch.xpack.search; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; +import java.io.IOException; + public class TransportDeleteAsyncSearchAction extends HandledTransportAction { + private final ClusterService clusterService; + private final TransportService transportService; private final AsyncSearchStoreService store; @Inject public TransportDeleteAsyncSearchAction(TransportService transportService, ActionFilters actionFilters, + ClusterService clusterService, ThreadPool threadPool, NamedWriteableRegistry registry, Client client) { super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new); this.store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, registry); + this.clusterService = clusterService; + this.transportService = transportService; } @Override protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); - // check if the response can be retrieved by the user (handle security) and then cancel/delete. - store.getResponse(searchId, ActionListener.wrap(res -> cancelTaskAndDeleteResult(searchId, listener), listener::onFailure)); + DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); + if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) { + cancelTaskAndDeleteResult(searchId, listener); + } else { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + transportService.sendRequest(node, DeleteAsyncSearchAction.NAME, request, builder.build(), + new ActionListenerResponseHandler<>(listener, AcknowledgedResponse::new, ThreadPool.Names.SAME)); + } } catch (Exception exc) { listener.onFailure(exc); } } - private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener listener) { - store.getClient().admin().cluster().prepareCancelTasks().setTaskId(searchId.getTaskId()) - .execute(ActionListener.wrap(() -> store.deleteResult(searchId, listener))); + private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener listener) throws IOException { + AsyncSearchTask task = store.getTask(searchId); + if (task != null && task.isCancelled() == false) { + store.getClient().admin().cluster().prepareCancelTasks() + .setTaskId(searchId.getTaskId()) + .execute(ActionListener.wrap(() -> store.deleteResult(searchId, listener))); + } else { + if (store.getThreadContext().isSystemContext()) { + store.deleteResult(searchId, listener); + } else { + // check if the response can be retrieved by the user (handle security) and then delete. + store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResult(searchId, listener), listener::onFailure)); + } + } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 7b2549041b576..bd551881d5c55 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.search; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; @@ -14,6 +15,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequestOptions; @@ -21,7 +23,7 @@ import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; -import java.io.IOException; +import static org.elasticsearch.action.ActionListener.wrap; public class TransportGetAsyncSearchAction extends HandledTransportAction { private final ClusterService clusterService; @@ -45,48 +47,59 @@ public TransportGetAsyncSearchAction(TransportService transportService, protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { try { AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); - if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId())) { - getSearchResponseFromTask(request, searchId, listener); - } else { - TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); - DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); - if (node == null) { - getSearchResponseFromIndex(request, searchId, listener); + DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); + if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) { + if (request.getKeepAlive() != TimeValue.MINUS_ONE) { + long expirationTime = System.currentTimeMillis() + request.getKeepAlive().getMillis(); + store.updateKeepAlive(searchId.getDocId(), expirationTime, + wrap(up -> getSearchResponseFromTask(searchId, request, expirationTime, listener), listener::onFailure)); } else { - transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), - new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); + getSearchResponseFromTask(searchId, request, -1, listener); } + } else { + TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); + request.setKeepAlive(TimeValue.MINUS_ONE); + transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), + new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); } - } catch (IOException e) { - listener.onFailure(e); + } catch (Exception exc) { + listener.onFailure(exc); } } - private void getSearchResponseFromTask(GetAsyncSearchAction.Request request, AsyncSearchId searchId, - ActionListener listener) throws IOException { - final AsyncSearchTask task = store.getTask(searchId); - if (task != null) { - try { - task.addCompletionListener( - response -> { - if (response.getVersion() <= request.getLastVersion()) { - // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), - response.isPartial(), response.isRunning(), response.getStartDate())); - } else { - listener.onResponse(response); - } - }, request.getWaitForCompletion()); - } catch (Exception exc) { - listener.onFailure(exc); + private void getSearchResponseFromTask(AsyncSearchId searchId, GetAsyncSearchAction.Request request, + long expirationTimeMillis, + ActionListener listener) { + try { + final AsyncSearchTask task = store.getTask(searchId); + if (task == null) { + getSearchResponseFromIndex(searchId, request, listener); + return; + } + + if (task.isCancelled()) { + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); + return; + } + + if (expirationTimeMillis != -1) { + task.setExpirationTime(expirationTimeMillis); } - } else { - // Task isn't running - getSearchResponseFromIndex(request, searchId, listener); + task.addCompletionListener(response -> { + if (response.getVersion() <= request.getLastVersion()) { + // return a not-modified response + listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), + response.isPartial(), response.isRunning(), response.getStartTime(), response.getExpirationTime())); + } else { + listener.onResponse(response); + } + }, request.getWaitForCompletion()); + } catch (Exception exc) { + listener.onFailure(exc); } } - private void getSearchResponseFromIndex(GetAsyncSearchAction.Request request, AsyncSearchId searchId, + private void getSearchResponseFromIndex(AsyncSearchId searchId, GetAsyncSearchAction.Request request, ActionListener listener) { store.getResponse(searchId, new ActionListener<>() { @Override @@ -94,7 +107,7 @@ public void onResponse(AsyncSearchResponse response) { if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), - response.isPartial(), false, response.getStartDate())); + response.isPartial(), false, response.getStartTime(), response.getExpirationTime())); } else { listener.onResponse(response); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 5b66eb19a9d07..8ef0fc843ef1e 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -22,7 +22,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -46,15 +45,13 @@ public class TransportSubmitAsyncSearchAction extends HandledTransportAction reduceContextSupplier; private final TransportSearchAction searchAction; private final AsyncSearchStoreService store; @Inject - public TransportSubmitAsyncSearchAction(ClusterService clusterService, - TransportService transportService, + public TransportSubmitAsyncSearchAction(TransportService transportService, ActionFilters actionFilters, NamedWriteableRegistry registry, Client client, @@ -62,7 +59,6 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService, SearchService searchService, TransportSearchAction searchAction) { super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new); - this.clusterService = clusterService; this.threadContext= transportService.getThreadPool().getThreadContext(); this.nodeClient = nodeClient; this.reduceContextSupplier = () -> searchService.createReduceContext(true); @@ -72,38 +68,31 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService, @Override protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener submitListener) { - ActionRequestValidationException exc = request.validate(); - if (exc != null) { - submitListener.onFailure(exc); + ActionRequestValidationException errors = request.validate(); + if (errors != null) { + submitListener.onFailure(errors); return; } - final String docID = UUIDs.randomBase64UUID(); CancellableTask submitTask = (CancellableTask) task; - store.ensureAsyncSearchIndex(clusterService.state(), - ActionListener.wrap(indexName -> executeSearch(submitTask, request, indexName, docID, submitListener), - submitListener::onFailure)); - } - - private void executeSearch(CancellableTask submitTask, - SubmitAsyncSearchRequest submitRequest, - String indexName, - String docID, - ActionListener submitListener) { + final String docID = UUIDs.randomBase64UUID(); final Map originHeaders = nodeClient.threadPool().getThreadContext().getHeaders(); - final SearchRequest searchRequest = new SearchRequest(submitRequest.getSearchRequest()) { + final SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map taskHeaders) { - AsyncSearchId searchId = new AsyncSearchId(indexName, docID, new TaskId(nodeClient.getLocalNodeId(), id)); - return new AsyncSearchTask(id, type, action, originHeaders, taskHeaders, searchId, + AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id)); + return new AsyncSearchTask(id, type, action, parentTaskId, originHeaders, taskHeaders, searchId, nodeClient.threadPool(), reduceContextSupplier); } }; + searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), submitTask.getId())); // trigger the async search AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); searchAction.execute(searchTask, searchRequest, searchTask.getProgressListener()); + long expirationTime = System.currentTimeMillis() + request.getKeepAlive().getMillis(); + searchTask.setExpirationTime(expirationTime); searchTask.addCompletionListener(searchResponse -> { - if (searchResponse.isRunning() || submitRequest.isCleanOnCompletion() == false) { + if (searchResponse.isRunning() || request.isCleanOnCompletion() == false) { // the task is still running and the user cannot wait more so we create // a document for further retrieval try { @@ -113,7 +102,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, searchTask.addCompletionListener(finalResponse -> taskManager.unregister(searchTask)); submitListener.onFailure(new ElasticsearchException(submitTask.getReasonCancelled())); } else { - store.storeInitialResponse(originHeaders, indexName, docID, searchResponse, + store.storeInitialResponse(originHeaders, docID, searchResponse, new ActionListener<>() { @Override public void onResponse(IndexResponse r) { @@ -149,25 +138,24 @@ public void onFailure(Exception exc) { taskManager.unregister(searchTask); submitListener.onResponse(searchResponse.clone(null)); } - }, submitRequest.getWaitForCompletion()); + }, request.getWaitForCompletion()); } private void storeFinalResponse(CancellableTask submitTask, AsyncSearchTask searchTask, AsyncSearchResponse response) { try { - store.storeFinalResponse(searchTask.getOriginHeaders(), response, - new ActionListener<>() { - @Override - public void onResponse(UpdateResponse updateResponse) { - onTaskCompletion(submitTask, searchTask, () -> {}); - } + store.storeFinalResponse(response, new ActionListener<>() { + @Override + public void onResponse(UpdateResponse updateResponse) { + onTaskCompletion(submitTask, searchTask, () -> {}); + } - @Override - public void onFailure(Exception exc) { - logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", - searchTask.getSearchId().getEncoded()), exc); - onTaskCompletion(submitTask, searchTask, () -> {}); - } - }); + @Override + public void onFailure(Exception exc) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", + searchTask.getSearchId().getEncoded()), exc); + onTaskCompletion(submitTask, searchTask, () -> {}); + } + }); } catch (Exception exc) { logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", searchTask.getSearchId().getEncoded()), exc); onTaskCompletion(submitTask, searchTask, () -> {}); @@ -184,8 +172,7 @@ private void onTaskFailure(AsyncSearchTask searchTask, String reason, Runnable o private void onTaskCompletion(CancellableTask submitTask, AsyncSearchTask searchTask, Runnable onFinish) { if (submitTask.isCancelled()) { - // the user cancelled the submit so we ensure that there is nothing stored - // in the response index. + // the user cancelled the submit so we ensure that there is nothing stored in the response index. store.deleteResult(searchTask.getSearchId(), ActionListener.wrap(() -> { taskManager.unregister(searchTask); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index ad1359dfd6a74..018995dcd9905 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -223,9 +223,7 @@ public void testInvalidId() throws Exception { SearchResponseIterator it = assertBlockingIterator(indexName, new SearchSourceBuilder(), randomBoolean() ? 1 : 0, 2); AsyncSearchResponse response = it.next(); - AsyncSearchId original = AsyncSearchId.decode(response.getId()); - String invalid = AsyncSearchId.encode("another_index", original.getDocId(), original.getTaskId()); - ExecutionException exc = expectThrows(ExecutionException.class, () -> getAsyncSearch(invalid)); + ExecutionException exc = expectThrows(ExecutionException.class, () -> getAsyncSearch("invalid")); assertThat(exc.getCause(), instanceOf(IllegalArgumentException.class)); assertThat(exc.getMessage(), containsString("invalid id")); while (it.hasNext()) { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java index 3e22664431de5..ebfa4bfc97190 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIdTests.java @@ -10,14 +10,12 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; - public class AsyncSearchIdTests extends ESTestCase { public void testEncode() { for (int i = 0; i < 10; i++) { - AsyncSearchId instance = new AsyncSearchId(ASYNC_SEARCH_INDEX_PREFIX + randomAlphaOfLengthBetween(5, 20), - UUIDs.randomBase64UUID(), new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong())); - String encoded = AsyncSearchId.encode(instance.getIndexName(), instance.getDocId(), instance.getTaskId()); + AsyncSearchId instance = new AsyncSearchId(UUIDs.randomBase64UUID(), + new TaskId(randomAlphaOfLengthBetween(5, 20), randomNonNegativeLong())); + String encoded = AsyncSearchId.encode(instance.getDocId(), instance.getTaskId()); AsyncSearchId same = AsyncSearchId.decode(encoded); assertEquals(same, instance); @@ -28,16 +26,13 @@ public void testEncode() { } private AsyncSearchId mutate(AsyncSearchId id) { - int rand = randomIntBetween(0, 2); + int rand = randomIntBetween(0, 1); switch (rand) { case 0: - return new AsyncSearchId(randomAlphaOfLength(id.getIndexName().length()+1), id.getDocId(), id.getTaskId()); + return new AsyncSearchId(randomAlphaOfLength(id.getDocId().length()+1), id.getTaskId()); case 1: - return new AsyncSearchId(id.getIndexName(), randomAlphaOfLength(id.getDocId().length()+1), id.getTaskId()); - - case 2: - return new AsyncSearchId(id.getIndexName(), id.getDocId(), + return new AsyncSearchId(id.getDocId(), new TaskId(randomAlphaOfLength(id.getTaskId().getNodeId().length()), randomNonNegativeLong())); default: diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index 36ba9cc2a447d..c764fd3f9a9b1 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -62,6 +62,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -87,7 +88,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { return super.onNodeStopped(nodeName); } }); - ensureGreen(searchId.getIndexName()); + ensureGreen(ASYNC_SEARCH_ALIAS); } protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request) throws ExecutionException, InterruptedException { @@ -110,7 +111,7 @@ protected void ensureTaskRemoval(String id) throws Exception { AsyncSearchId searchId = AsyncSearchId.decode(id); assertBusy(() -> { GetResponse resp = client().prepareGet() - .setIndex(searchId.getIndexName()) + .setIndex(ASYNC_SEARCH_ALIAS) .setId(searchId.getDocId()) .get(); assertFalse(resp.isExists()); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index c84bf5b192bd1..925ce9d7897eb 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -92,15 +92,16 @@ static AsyncSearchResponse randomAsyncSearchResponse(String searchId, SearchResp switch (rand) { case 0: return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), randomBoolean(), - randomBoolean(), randomNonNegativeLong()); + randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong()); case 1: return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse, null, - randomBoolean(), randomBoolean(), randomNonNegativeLong()); + randomBoolean(), randomBoolean(), randomNonNegativeLong(), randomNonNegativeLong()); case 2: return new AsyncSearchResponse(searchId, randomIntBetween(0, Integer.MAX_VALUE), searchResponse, - new ElasticsearchException(new IOException("boum")), randomBoolean(), randomBoolean(), randomNonNegativeLong()); + new ElasticsearchException(new IOException("boum")), randomBoolean(), randomBoolean(), + randomNonNegativeLong(), randomNonNegativeLong()); default: throw new AssertionError(); @@ -124,6 +125,7 @@ static void assertEqualResponses(AsyncSearchResponse expected, AsyncSearchRespon assertEquals(expected.getFailure() == null, actual.getFailure() == null); assertEquals(expected.isRunning(), actual.isRunning()); assertEquals(expected.isPartial(), actual.isPartial()); - assertEquals(expected.getStartDate(), actual.getStartDate()); + assertEquals(expected.getStartTime(), actual.getStartTime()); + assertEquals(expected.getExpirationTime(), actual.getExpirationTime()); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java index 47b21ca403d81..9152fd09f39a9 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java @@ -5,21 +5,10 @@ */ package org.elasticsearch.xpack.search; -import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.LatchedActionListener; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; @@ -39,22 +28,13 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import static java.util.Collections.emptyList; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.awaitLatch; import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses; import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse; import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_ALIAS; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ASYNC_SEARCH_INDEX_PREFIX; import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ensureAuthenticatedUserIsSame; import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.core.IsEqual.equalTo; import static org.mockito.Mockito.mock; public class AsyncSearchStoreServiceTests extends ESTestCase { @@ -90,180 +70,6 @@ public void testEncode() throws IOException { } } - public void testIndexNeedsCreation() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metaData(MetaData.builder()) - .build(); - - client.setVerifier((a, r, l) -> { - assertThat(a, instanceOf(CreateIndexAction.class)); - assertThat(r, instanceOf(CreateIndexRequest.class)); - CreateIndexRequest request = (CreateIndexRequest) r; - assertThat(request.aliases(), hasSize(1)); - request.aliases().forEach(alias -> { - assertThat(alias.name(), equalTo(ASYNC_SEARCH_ALIAS)); - assertTrue(alias.writeIndex()); - }); - return new CreateIndexResponse(true, true, request.index()); - }); - - CountDownLatch latch = new CountDownLatch(1); - store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( - name -> assertThat(name, equalTo(ASYNC_SEARCH_INDEX_PREFIX + "000001")), - ex -> { - logger.error(ex); - fail("should have called onResponse, not onFailure"); - }), latch)); - - awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testIndexProperlyExistsAlready() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metaData(MetaData.builder() - .put(IndexMetaData.builder(ASYNC_SEARCH_INDEX_PREFIX + "000001") - .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)) - .putAlias(AliasMetaData.builder(ASYNC_SEARCH_ALIAS) - .writeIndex(true) - .build()))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( - name -> assertThat(name, equalTo(ASYNC_SEARCH_INDEX_PREFIX + "000001")), - ex -> { - logger.error(ex); - fail("should have called onResponse, not onFailure"); - }), latch)); - - awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testIndexHasNoWriteIndex() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metaData(MetaData.builder() - .put(IndexMetaData.builder(ASYNC_SEARCH_INDEX_PREFIX + "000001") - .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)) - .putAlias(AliasMetaData.builder(ASYNC_SEARCH_ALIAS) - .build())) - .put(IndexMetaData.builder(randomAlphaOfLength(5)) - .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)) - .putAlias(AliasMetaData.builder(ASYNC_SEARCH_ALIAS) - .build()))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( - name -> fail("should have called onFailure, not onResponse"), - ex -> { - assertThat(ex, instanceOf(IllegalStateException.class)); - assertThat(ex.getMessage(), containsString("async-search alias [" + ASYNC_SEARCH_ALIAS + - "] does not have a write index")); - }), latch)); - - awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testIndexNotAlias() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metaData(MetaData.builder() - .put(IndexMetaData.builder(ASYNC_SEARCH_ALIAS) - .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( - name -> fail("should have called onFailure, not onResponse"), - ex -> { - assertThat(ex, instanceOf(IllegalStateException.class)); - assertThat(ex.getMessage(), containsString("async-search alias [" + ASYNC_SEARCH_ALIAS + - "] already exists as concrete index")); - }), latch)); - - awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testIndexCreatedConcurrently() throws InterruptedException { - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metaData(MetaData.builder()) - .build(); - - client.setVerifier((a, r, l) -> { - assertThat(a, instanceOf(CreateIndexAction.class)); - assertThat(r, instanceOf(CreateIndexRequest.class)); - CreateIndexRequest request = (CreateIndexRequest) r; - assertThat(request.aliases(), hasSize(1)); - request.aliases().forEach(alias -> { - assertThat(alias.name(), equalTo(ASYNC_SEARCH_ALIAS)); - assertTrue(alias.writeIndex()); - }); - throw new ResourceAlreadyExistsException("that index already exists"); - }); - - CountDownLatch latch = new CountDownLatch(1); - store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( - name -> assertThat(name, equalTo(ASYNC_SEARCH_INDEX_PREFIX + "000001")), - ex -> { - logger.error(ex); - fail("should have called onResponse, not onFailure"); - }), latch)); - - awaitLatch(latch, 10, TimeUnit.SECONDS); - } - - public void testAliasDoesntExistButIndexDoes() throws InterruptedException { - final String initialIndex = ASYNC_SEARCH_INDEX_PREFIX + "000001"; - ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5))) - .metaData(MetaData.builder() - .put(IndexMetaData.builder(initialIndex) - .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) - .numberOfShards(randomIntBetween(1,10)) - .numberOfReplicas(randomIntBetween(1,10)))) - .build(); - - client.setVerifier((a, r, l) -> { - fail("no client calls should have been made"); - return null; - }); - - CountDownLatch latch = new CountDownLatch(1); - store.ensureAsyncSearchIndex(state, new LatchedActionListener<>(ActionListener.wrap( - name -> { - logger.error(name); - fail("should have called onFailure, not onResponse"); - }, - ex -> { - assertThat(ex, instanceOf(IllegalStateException.class)); - assertThat(ex.getMessage(), containsString("async-search index [" + initialIndex + - "] already exists but does not have alias [" + ASYNC_SEARCH_ALIAS + "]")); - }), latch)); - - awaitLatch(latch, 10, TimeUnit.SECONDS); - } - public void testEnsuredAuthenticatedUserIsSame() throws IOException { Authentication original = new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java index 1f0d689916e96..ca776aa525987 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java @@ -43,8 +43,9 @@ public void afterTest() { } public void testWaitForInit() throws InterruptedException { - AsyncSearchTask task = new AsyncSearchTask(0L, "", "", Collections.emptyMap(), Collections.emptyMap(), - new AsyncSearchId("index", "0", new TaskId("node1", 0)), threadPool, null); + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), + Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)), + threadPool, null); int numShards = randomIntBetween(0, 10); List shards = new ArrayList<>(); for (int i = 0; i < numShards; i++) { @@ -75,8 +76,9 @@ public void testWaitForInit() throws InterruptedException { } public void testWithFailure() throws InterruptedException { - AsyncSearchTask task = new AsyncSearchTask(0L, "", "", Collections.emptyMap(), Collections.emptyMap(), - new AsyncSearchId("index", "0", new TaskId("node1", 0)), threadPool, null); + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), + Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)), + threadPool, null); int numShards = randomIntBetween(0, 10); List shards = new ArrayList<>(); for (int i = 0; i < numShards; i++) { @@ -107,8 +109,9 @@ public void testWithFailure() throws InterruptedException { } public void testWaitForCompletion() throws InterruptedException { - AsyncSearchTask task = new AsyncSearchTask(0L, "", "", Collections.emptyMap(), Collections.emptyMap(), - new AsyncSearchId("index", "0", new TaskId("node1", 0)), threadPool, null); + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), + Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)), + threadPool, null); int numShards = randomIntBetween(0, 10); List shards = new ArrayList<>(); for (int i = 0; i < numShards; i++) { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java index 3e96e738eede1..022fc7300ac3d 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java @@ -25,8 +25,8 @@ protected GetAsyncSearchAction.Request createTestInstance() { } static String randomSearchId() { - return AsyncSearchId.encode(randomRealisticUnicodeOfLengthBetween(2, 20), UUIDs.randomBase64UUID(), - new TaskId(randomAlphaOfLengthBetween(10, 20), randomLongBetween(0, Long.MAX_VALUE))); + return AsyncSearchId.encode(UUIDs.randomBase64UUID(), + new TaskId(randomAlphaOfLengthBetween(10, 20), randomLongBetween(0, Long.MAX_VALUE))); } public void testValidateWaitForCompletion() { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java index 57434edf818ca..939b6b0914a21 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -28,6 +29,13 @@ protected SubmitAsyncSearchRequest createTestInstance() { } else { searchRequest = new SubmitAsyncSearchRequest(); } + if (randomBoolean()) { + searchRequest.setWaitForCompletion(TimeValue.parseTimeValue(randomPositiveTimeValue(), "wait_for_completion")); + } + searchRequest.setCleanOnCompletion(randomBoolean()); + if (randomBoolean()) { + searchRequest.setKeepAlive(TimeValue.parseTimeValue(randomPositiveTimeValue(), "keep_alive")); + } if (randomBoolean()) { searchRequest.getSearchRequest() .indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 3a4b5d4c1aeb1..6d67f71443435 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -31,7 +31,8 @@ public class AsyncSearchResponse extends ActionResponse implements StatusToXCont private final boolean isRunning; private final boolean isPartial; - private final long startDateMillis; + private final long startTimeMillis; + private long expirationTimeMillis = -1; /** * Creates an {@link AsyncSearchResponse} with meta informations that omits @@ -41,8 +42,9 @@ public AsyncSearchResponse(String id, int version, boolean isPartial, boolean isRunning, - long startDateMillis) { - this(id, version, null, null, isPartial, isRunning, startDateMillis); + long startTimeMillis, + long expirationTimeMillis) { + this(id, version, null, null, isPartial, isRunning, startTimeMillis, expirationTimeMillis); } /** @@ -54,7 +56,7 @@ public AsyncSearchResponse(String id, * or completed without failure. * @param isPartial Whether the searchResponse contains partial results. * @param isRunning Whether the search is running in the cluster. - * @param startDateMillis The start date of the search in milliseconds since epoch. + * @param startTimeMillis The start date of the search in milliseconds since epoch. */ public AsyncSearchResponse(String id, int version, @@ -62,14 +64,16 @@ public AsyncSearchResponse(String id, ElasticsearchException failure, boolean isPartial, boolean isRunning, - long startDateMillis) { + long startTimeMillis, + long expirationTimeMillis) { this.id = id; this.version = version; this.failure = failure; this.searchResponse = searchResponse; this.isPartial = isPartial; this.isRunning = isRunning; - this.startDateMillis = startDateMillis; + this.startTimeMillis = startTimeMillis; + this.expirationTimeMillis = expirationTimeMillis; } public AsyncSearchResponse(StreamInput in) throws IOException { @@ -79,7 +83,8 @@ public AsyncSearchResponse(StreamInput in) throws IOException { this.searchResponse = in.readOptionalWriteable(SearchResponse::new); this.isPartial = in.readBoolean(); this.isRunning = in.readBoolean(); - this.startDateMillis = in.readLong(); + this.startTimeMillis = in.readLong(); + this.expirationTimeMillis = in.readLong(); } @Override @@ -90,14 +95,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(searchResponse); out.writeBoolean(isPartial); out.writeBoolean(isRunning); - out.writeLong(startDateMillis); + out.writeLong(startTimeMillis); + out.writeLong(expirationTimeMillis); } public AsyncSearchResponse clone(String id) { - return new AsyncSearchResponse(id, version, searchResponse, failure, isPartial, isRunning, startDateMillis); + return new AsyncSearchResponse(id, version, searchResponse, failure, isPartial, isRunning, startTimeMillis, expirationTimeMillis); } - /** * Returns the id of the async search request or null if the response is not stored in the cluster. */ @@ -140,8 +145,8 @@ public boolean isPartial() { /** * When this response was created as a timestamp in milliseconds since epoch. */ - public long getStartDate() { - return startDateMillis; + public long getStartTime() { + return startTimeMillis; } /** @@ -155,6 +160,10 @@ public boolean isRunning() { return isRunning; } + public long getExpirationTime() { + return expirationTimeMillis; + } + @Override public RestStatus status() { if (searchResponse == null || isPartial) { @@ -176,7 +185,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("version", version); builder.field("is_partial", isPartial); builder.field("is_running", isRunning); - builder.field("start_date_in_millis", startDateMillis); + builder.field("start_time_in_millis", startTimeMillis); + builder.field("expiration_time_in_millis", expirationTimeMillis); builder.field("response", searchResponse); if (failure != null) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java index cc96d8d913300..7ff1deef17172 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -17,6 +17,9 @@ import java.io.IOException; import java.util.Objects; +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest.MIN_KEEP_ALIVE; + public class GetAsyncSearchAction extends ActionType { public static final GetAsyncSearchAction INSTANCE = new GetAsyncSearchAction(); public static final String NAME = "indices:data/read/async_search/get"; @@ -34,6 +37,7 @@ public static class Request extends ActionRequest { private final String id; private final int lastVersion; private final TimeValue waitForCompletion; + private TimeValue keepAlive = TimeValue.MINUS_ONE; /** * Create a new request @@ -64,7 +68,12 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { - return null; + ActionRequestValidationException validationException = null; + if (keepAlive != TimeValue.MINUS_ONE && keepAlive.getMillis() < MIN_KEEP_ALIVE) { + validationException = + addValidationError("keep_alive must be greater than 1 minute, got:" + keepAlive.toString(), validationException); + } + return validationException; } public String getId() { @@ -87,19 +96,28 @@ public TimeValue getWaitForCompletion() { return waitForCompletion; } + public TimeValue getKeepAlive() { + return keepAlive; + } + + public void setKeepAlive(TimeValue keepAlive) { + this.keepAlive = keepAlive; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; return lastVersion == request.lastVersion && - id.equals(request.id) && - waitForCompletion.equals(request.waitForCompletion); + Objects.equals(id, request.id) && + waitForCompletion.equals(request.waitForCompletion) && + keepAlive.equals(request.keepAlive); } @Override public int hashCode() { - return Objects.hash(id, lastVersion, waitForCompletion); + return Objects.hash(id, lastVersion, waitForCompletion, keepAlive); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 37ef6e1e7b73f..d88ca457784b7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -28,8 +28,11 @@ * @see AsyncSearchResponse */ public class SubmitAsyncSearchRequest extends ActionRequest { + public static long MIN_KEEP_ALIVE = TimeValue.timeValueHours(1).millis(); + private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1); private boolean cleanOnCompletion = true; + private TimeValue keepAlive = TimeValue.timeValueDays(5); private final SearchRequest request; @@ -55,6 +58,7 @@ public SubmitAsyncSearchRequest(StreamInput in) throws IOException { this.request = new SearchRequest(in); this.waitForCompletion = in.readTimeValue(); this.cleanOnCompletion = in.readBoolean(); + this.keepAlive = in.readTimeValue(); } @Override @@ -62,6 +66,7 @@ public void writeTo(StreamOutput out) throws IOException { request.writeTo(out); out.writeTimeValue(waitForCompletion); out.writeBoolean(cleanOnCompletion); + out.writeTimeValue(keepAlive); } public SearchRequest getSearchRequest() { @@ -82,12 +87,12 @@ public TimeValue getWaitForCompletion() { return waitForCompletion; } - public void setBatchedReduceSize(int size) { - request.setBatchedReduceSize(size); + public void setKeepAlive(TimeValue keepAlive) { + this.keepAlive = keepAlive; } - public SearchSourceBuilder source() { - return request.source(); + public TimeValue getKeepAlive() { + return keepAlive; } /** @@ -101,6 +106,14 @@ public void setCleanOnCompletion(boolean value) { this.cleanOnCompletion = value; } + public void setBatchedReduceSize(int size) { + request.setBatchedReduceSize(size); + } + + public SearchSourceBuilder source() { + return request.source(); + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = request.validate(); @@ -110,6 +123,10 @@ public ActionRequestValidationException validate() { if (request.isSuggestOnly()) { validationException = addValidationError("suggest-only queries are not supported", validationException); } + if (keepAlive.getMillis() < MIN_KEEP_ALIVE) { + validationException = + addValidationError("keep_alive must be greater than 1 minute, got:" + keepAlive.toString(), validationException); + } return validationException; } @@ -131,11 +148,12 @@ public boolean equals(Object o) { SubmitAsyncSearchRequest request1 = (SubmitAsyncSearchRequest) o; return cleanOnCompletion == request1.cleanOnCompletion && waitForCompletion.equals(request1.waitForCompletion) && + keepAlive.equals(request1.keepAlive) && request.equals(request1.request); } @Override public int hashCode() { - return Objects.hash(waitForCompletion, cleanOnCompletion, request); + return Objects.hash(waitForCompletion, cleanOnCompletion, keepAlive, request); } } diff --git a/x-pack/plugin/core/src/main/resources/async-search-ilm-policy.json b/x-pack/plugin/core/src/main/resources/async-search-ilm-policy.json deleted file mode 100644 index fd46244836cb5..0000000000000 --- a/x-pack/plugin/core/src/main/resources/async-search-ilm-policy.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "phases": { - "hot": { - "actions": { - "rollover": { - "max_size": "50GB", - "max_age": "1d" - } - } - }, - "delete": { - "min_age": "5d", - "actions": { - "delete": {} - } - } - } -} diff --git a/x-pack/plugin/core/src/main/resources/async-search.json b/x-pack/plugin/core/src/main/resources/async-search.json index 9f5c6ea5ce79b..a3b556bab5d2b 100644 --- a/x-pack/plugin/core/src/main/resources/async-search.json +++ b/x-pack/plugin/core/src/main/resources/async-search.json @@ -19,6 +19,12 @@ "type": "object", "enabled": false }, + "is_running": { + "type": "boolean" + }, + "expiration_time":{ + "type": "long" + }, "result": { "type": "object", "enabled": false diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json index c69c30ba0cc5e..8a0b41762ec24 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json @@ -28,6 +28,10 @@ "last_version":{ "type":"number", "description":"Specify the last version returned by a previous call. The request will return 304 (not modified) status if the new version is lower than or equals to the provided one and will remove all details in the response except id and version. (default: -1)" + }, + "keep_alive": { + "type": "time", + "description": "Specify the time that the request should remain reachable in the cluster." } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json index bba7c39723fea..fbe4d42fbc272 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json @@ -37,6 +37,15 @@ "type":"boolean", "description":"Specify whether the response should not be stored in the cluster if it completed within the provided wait_for_completion time (default: true)" }, + "batched_reduce_size":{ + "type":"number", + "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.", + "default":5 + }, + "keep_alive": { + "type": "time", + "description": "Specify the time that the request should remain reachable in the cluster." + }, "analyzer":{ "type":"string", "description":"The analyzer to use for the query string" @@ -204,20 +213,10 @@ "type":"boolean", "description":"Specify if request cache should be used for this request or not, defaults to index level setting" }, - "batched_reduce_size":{ - "type":"number", - "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.", - "default":5 - }, "max_concurrent_shard_requests":{ "type":"number", "description":"The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests", "default":5 - }, - "pre_filter_shard_size":{ - "type":"number", - "description":"A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.", - "default":1 } }, "body":{ From 373af3a943f576a3258807bf96d7cab65101698a Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 31 Jan 2020 17:02:39 +0100 Subject: [PATCH 39/61] use a single replica for the hidden index --- x-pack/plugin/core/src/main/resources/async-search.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/async-search.json b/x-pack/plugin/core/src/main/resources/async-search.json index a3b556bab5d2b..68736e7280304 100644 --- a/x-pack/plugin/core/src/main/resources/async-search.json +++ b/x-pack/plugin/core/src/main/resources/async-search.json @@ -5,8 +5,7 @@ "order": 2147483647, "settings": { "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", + "index.number_of_replicas": 1, "index.lifecycle.name": "async-search", "index.lifecycle.rollover_alias": ".async-search-${xpack.async-search-template.version}", "index.format": 1 From 42371225c72647fc4b0c5fab5cd6c11fa0a9480c Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 4 Feb 2020 11:38:39 +0100 Subject: [PATCH 40/61] iter --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 1 + .../elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java | 2 +- x-pack/plugin/core/src/main/resources/async-search.json | 2 -- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 052cb5ececfcf..934aba4340c05 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1112,6 +1112,7 @@ protected static boolean isXPackTemplate(String name) { case ".logstash-management": case "security_audit_log": case ".slm-history": + case ".async-search": return true; default: return false; diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java index 3ea118bbf6ea8..de3db6d903c3a 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java @@ -28,7 +28,7 @@ public class AsyncSearchTemplateRegistry extends IndexTemplateRegistry { // version 1: initial public static final String INDEX_TEMPLATE_VERSION = "1"; public static final String ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE = "xpack.async-search.template.version"; - public static final String ASYNC_SEARCH_TEMPLATE_NAME = "async-search"; + public static final String ASYNC_SEARCH_TEMPLATE_NAME = ".async-search"; public static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; public static final IndexTemplateConfig TEMPLATE_ASYNC_SEARCH = new IndexTemplateConfig( diff --git a/x-pack/plugin/core/src/main/resources/async-search.json b/x-pack/plugin/core/src/main/resources/async-search.json index 68736e7280304..4f4c67d8f4dc6 100644 --- a/x-pack/plugin/core/src/main/resources/async-search.json +++ b/x-pack/plugin/core/src/main/resources/async-search.json @@ -6,8 +6,6 @@ "settings": { "index.number_of_shards": 1, "index.number_of_replicas": 1, - "index.lifecycle.name": "async-search", - "index.lifecycle.rollover_alias": ".async-search-${xpack.async-search-template.version}", "index.format": 1 }, "mappings": { From 4b53aad70f7841e2b1bea6323390467e485fb47e Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 4 Feb 2020 13:15:37 +0100 Subject: [PATCH 41/61] do not set id twice --- .../elasticsearch/xpack/search/AsyncSearchStoreService.java | 2 -- .../elasticsearch/xpack/search/AsyncSearchActionTests.java | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 31464b8b600de..778d01faf586d 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -51,7 +51,6 @@ class AsyncSearchStoreService { private static final Logger logger = LogManager.getLogger(AsyncSearchStoreService.class); - static final String ID_FIELD = "id"; static final String HEADERS_FIELD = "headers"; static final String IS_RUNNING_FIELD = "is_running"; static final String EXPIRATION_TIME_FIELD = "expiration_time"; @@ -87,7 +86,6 @@ ThreadContext getThreadContext() { void storeInitialResponse(Map headers, String docID, AsyncSearchResponse response, ActionListener listener) throws IOException { Map source = new HashMap<>(); - source.put(ID_FIELD, response.getId()); source.put(HEADERS_FIELD, headers); source.put(IS_RUNNING_FIELD, true); source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime()); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index 018995dcd9905..c309ec60cbc0b 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -55,8 +55,8 @@ public void indexDocuments() throws InterruptedException { for (int i = 0; i < numKeywords; i++) { keywords[i] = randomAlphaOfLengthBetween(10, 20); } + List reqs = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { - List reqs = new ArrayList<>(); float metric = randomFloat(); maxMetric = Math.max(metric, maxMetric); minMetric = Math.min(metric, minMetric); @@ -70,8 +70,8 @@ public void indexDocuments() throws InterruptedException { return v; }); reqs.add(client().prepareIndex(indexName).setSource("terms", keyword, "metric", metric)); - indexRandom(true, true, reqs); } + indexRandom(true, true, reqs); ensureGreen("test-async"); } From 434b4910acb4d13466e4177ad3e21908eab9e794 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 4 Feb 2020 13:20:51 +0100 Subject: [PATCH 42/61] change expectation in test now that we use one replica by default --- .../elasticsearch/xpack/search/AsyncSearchIntegTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index cba1bb4fb438c..532213f346a7f 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -89,7 +89,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { return super.onNodeStopped(nodeName); } }); - ensureGreen(ASYNC_SEARCH_ALIAS); + ensureYellow(ASYNC_SEARCH_ALIAS); } protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request) throws ExecutionException, InterruptedException { From 0f866492807bfb81766703b8003c9c445c855f04 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 4 Feb 2020 15:54:21 +0100 Subject: [PATCH 43/61] =?UTF-8?q?fix=20x-pack=20user=20to=20allow=20restri?= =?UTF-8?q?cted=20indices=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../elasticsearch/xpack/core/security/user/XPackUser.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java index 38c9fe84aa934..10dd3e54d6e03 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java @@ -19,7 +19,10 @@ public class XPackUser extends User { public static final String ROLE_NAME = UsernamesField.XPACK_ROLE; public static final Role ROLE = Role.builder(new RoleDescriptor(ROLE_NAME, new String[] { "all" }, new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder().indices("/@&~(\\.security.*)/").privileges("all").build(), + RoleDescriptor.IndicesPrivileges.builder().indices("/@&~(\\.security.*)/") + .allowRestrictedIndices(true) + .privileges("all") + .build(), RoleDescriptor.IndicesPrivileges.builder().indices(IndexAuditTrailField.INDEX_NAME_PREFIX + "-*") .privileges("read").build() }, From 3b9b89e9b2ff60dc233529be5bf118e96daa572a Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 4 Feb 2020 15:54:30 +0100 Subject: [PATCH 44/61] cleanup --- .../xpack/search/AsyncSearchStoreService.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java index 778d01faf586d..a289e9988d7ac 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java @@ -104,14 +104,17 @@ void storeFinalResponse(AsyncSearchResponse response, ActionListener source = new HashMap<>(); source.put(IS_RUNNING_FIELD, true); source.put(RESULT_FIELD, encodeResponse(response)); - UpdateRequest request = new UpdateRequest().index(ASYNC_SEARCH_ALIAS).id(searchId.getDocId()) + UpdateRequest request = new UpdateRequest() + .index(ASYNC_SEARCH_ALIAS) + .id(searchId.getDocId()) .doc(source, XContentType.JSON); client.update(request, listener); } void updateKeepAlive(String docID, long expirationTimeMillis, ActionListener listener) { Map source = Collections.singletonMap(EXPIRATION_TIME_FIELD, expirationTimeMillis); - UpdateRequest request = new UpdateRequest().index(ASYNC_SEARCH_ALIAS).id(docID) + UpdateRequest request = new UpdateRequest().index(ASYNC_SEARCH_ALIAS) + .id(docID) .doc(source, XContentType.JSON); client.update(request, listener); } @@ -126,7 +129,6 @@ AsyncSearchTask getTask(AsyncSearchId searchId) throws IOException { return null; } - logger.info("Is threadContext " + threadContext.isSystemContext()); if (threadContext.isSystemContext() == false) { // Check authentication for the user final Authentication auth = Authentication.getAuthentication(threadContext); From b52a85a2c62d69692cb48f6d5e41d0662ea120cd Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 4 Feb 2020 22:42:51 +0100 Subject: [PATCH 45/61] iter --- .../xpack/search/TransportSubmitAsyncSearchAction.java | 7 +++++-- .../elasticsearch/xpack/core/security/user/XPackUser.java | 5 +---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 8ef0fc843ef1e..1de9004bae766 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import org.elasticsearch.tasks.CancellableTask; @@ -151,8 +152,10 @@ public void onResponse(UpdateResponse updateResponse) { @Override public void onFailure(Exception exc) { - logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", - searchTask.getSearchId().getEncoded()), exc); + if (exc.getCause() instanceof DocumentMissingException == false) { + logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", + searchTask.getSearchId().getEncoded()), exc); + } onTaskCompletion(submitTask, searchTask, () -> {}); } }); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java index 10dd3e54d6e03..38c9fe84aa934 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java @@ -19,10 +19,7 @@ public class XPackUser extends User { public static final String ROLE_NAME = UsernamesField.XPACK_ROLE; public static final Role ROLE = Role.builder(new RoleDescriptor(ROLE_NAME, new String[] { "all" }, new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder().indices("/@&~(\\.security.*)/") - .allowRestrictedIndices(true) - .privileges("all") - .build(), + RoleDescriptor.IndicesPrivileges.builder().indices("/@&~(\\.security.*)/").privileges("all").build(), RoleDescriptor.IndicesPrivileges.builder().indices(IndexAuditTrailField.INDEX_NAME_PREFIX + "-*") .privileges("read").build() }, From 6a1c98ddd53a988444aded511a43767277c4084e Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 4 Feb 2020 16:07:29 +0100 Subject: [PATCH 46/61] Allow x-pack user to manage async-search indices The x-pack user has all privileges on all indices except security indices and restricted indices. This change adds the authorization for restricted indices in order to allow the system user to manage these indices internally. This will allow the async-search restricted indices to be accessed only with the x-pack user. --- .../xpack/core/security/user/XPackUser.java | 10 ++++++++-- .../xpack/security/user/XPackUserTests.java | 8 +++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java index 38c9fe84aa934..5196f8e663e00 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java @@ -19,9 +19,15 @@ public class XPackUser extends User { public static final String ROLE_NAME = UsernamesField.XPACK_ROLE; public static final Role ROLE = Role.builder(new RoleDescriptor(ROLE_NAME, new String[] { "all" }, new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder().indices("/@&~(\\.security.*)/").privileges("all").build(), + RoleDescriptor.IndicesPrivileges.builder() + .indices("/@&~(\\.security.*)/") + .privileges("all") + // true for async-search indices + .allowRestrictedIndices(true) + .build(), RoleDescriptor.IndicesPrivileges.builder().indices(IndexAuditTrailField.INDEX_NAME_PREFIX + "-*") - .privileges("read").build() + .privileges("read") + .build() }, new String[] { "*" }, MetadataUtils.DEFAULT_RESERVED_METADATA), null).build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java index 6c7fb2abdabf6..d2bb9cb2d65e1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java @@ -28,13 +28,15 @@ public void testXPackUserCanAccessNonSecurityIndices() { assertThat(predicate.test(index), Matchers.is(true)); } - public void testXPackUserCannotAccessRestrictedIndices() { + public void testXPackUserAndRestrictedIndices() { final String action = randomFrom(GetAction.NAME, SearchAction.NAME, IndexAction.NAME); final Predicate predicate = XPackUser.ROLE.indices().allowedIndicesMatcher(action); for (String index : RestrictedIndicesNames.RESTRICTED_NAMES) { - assertThat(predicate.test(index), Matchers.is(false)); + // access should be authorized for async-search indices only + boolean authorize = index.startsWith(RestrictedIndicesNames.ASYNC_SEARCH_PREFIX); + assertThat(predicate.test(index), Matchers.is(authorize)); } - assertThat(predicate.test(RestrictedIndicesNames.ASYNC_SEARCH_PREFIX + randomAlphaOfLengthBetween(0, 2)), Matchers.is(false)); + assertThat(predicate.test(RestrictedIndicesNames.ASYNC_SEARCH_PREFIX + randomAlphaOfLengthBetween(0, 2)), Matchers.is(true)); } public void testXPackUserCanReadAuditTrail() { From 1df46f8922aaf9259b79bf115913ba49fe1e616f Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 5 Feb 2020 18:33:59 +0100 Subject: [PATCH 47/61] another big iter --- x-pack/plugin/async-search/build.gradle | 2 +- .../xpack/search/AsyncSearchSecurityIT.java | 4 +- .../xpack/search/AsyncSearchId.java | 6 +- .../xpack/search/AsyncSearchIndexService.java | 339 ++++++++++++++++++ .../search/AsyncSearchMaintenanceService.java | 119 ++++++ ...syncSearch.java => AsyncSearchPlugin.java} | 20 +- .../search/AsyncSearchReaperExecutor.java | 164 --------- .../xpack/search/AsyncSearchStoreService.java | 254 ------------- .../xpack/search/AsyncSearchTask.java | 125 +++++-- .../search/AsyncSearchTemplateRegistry.java | 63 ---- .../xpack/search/MutableSearchResponse.java | 19 +- .../search/RestGetAsyncSearchAction.java | 26 +- .../search/RestSubmitAsyncSearchAction.java | 23 +- .../TransportDeleteAsyncSearchAction.java | 20 +- .../search/TransportGetAsyncSearchAction.java | 85 +++-- .../TransportSubmitAsyncSearchAction.java | 184 +++++----- .../xpack/search/AsyncSearchActionTests.java | 1 + .../search/AsyncSearchIndexServiceTests.java | 103 ++++++ .../search/AsyncSearchIntegTestCase.java | 21 +- .../search/AsyncSearchStoreServiceTests.java | 161 --------- .../xpack/search/AsyncSearchTaskTests.java | 72 ++-- .../search/GetAsyncSearchRequestTests.java | 13 +- .../search/action/AsyncSearchResponse.java | 57 +-- .../search/action/GetAsyncSearchAction.java | 49 ++- .../action/SubmitAsyncSearchRequest.java | 50 +-- .../core/src/main/resources/async-search.json | 32 -- 26 files changed, 1044 insertions(+), 968 deletions(-) create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java create mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java rename x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/{AsyncSearch.java => AsyncSearchPlugin.java} (82%) delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java delete mode 100644 x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java create mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java delete mode 100644 x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java delete mode 100644 x-pack/plugin/core/src/main/resources/async-search.json diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle index 8aac2b9f885ac..484769069c984 100644 --- a/x-pack/plugin/async-search/build.gradle +++ b/x-pack/plugin/async-search/build.gradle @@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-async-search' description 'A module which allows to track the progress of a search asynchronously.' - classname 'org.elasticsearch.xpack.search.AsyncSearch' + classname 'org.elasticsearch.xpack.search.AsyncSearchPlugin' extendedPlugins = ['x-pack-core'] } archivesBaseName = 'x-pack-async-search' diff --git a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java index c663c33d26bcf..bf063c4f8be0a 100644 --- a/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java +++ b/x-pack/plugin/async-search/qa/security/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSecurityIT.java @@ -27,7 +27,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; +import static org.elasticsearch.xpack.search.AsyncSearchIndexService.INDEX; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -82,7 +82,7 @@ private void testCase(String user, String other) throws Exception { // other and user cannot access the result from direct get calls AsyncSearchId searchId = AsyncSearchId.decode(id); for (String runAs : new String[] {user, other}) { - exc = expectThrows(ResponseException.class, () -> get(ASYNC_SEARCH_ALIAS, searchId.getDocId(), runAs)); + exc = expectThrows(ResponseException.class, () -> get(INDEX, searchId.getDocId(), runAs)); assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(403)); assertThat(exc.getMessage(), containsString("unauthorized")); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java index 673061292ccfc..ef8b47ee00312 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchId.java @@ -45,7 +45,7 @@ TaskId getTaskId() { } /** - * Get the encoded string that represents this search. + * Gets the encoded string that represents this search. */ String getEncoded() { return encoded; @@ -74,7 +74,7 @@ public String toString() { } /** - * Encode the informations needed to retrieve a async search response + * Encodes the informations needed to retrieve a async search response * in a base64 encoded string. */ static String encode(String docId, TaskId taskId) { @@ -88,7 +88,7 @@ static String encode(String docId, TaskId taskId) { } /** - * Decode a base64 encoded string into an {@link AsyncSearchId} that can be used + * Decodes a base64 encoded string into an {@link AsyncSearchId} that can be used * to retrieve the response of an async search. */ static AsyncSearchId decode(String id) { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java new file mode 100644 index 0000000000000..718f1ff009ace --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java @@ -0,0 +1,339 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.security.authc.Authentication; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; + +/** + * A service that exposes the CRUD operations for the async-search index. + */ +class AsyncSearchIndexService { + private static final Logger logger = LogManager.getLogger(AsyncSearchIndexService.class); + + public static final String INDEX = ".async-search"; + + public static final String HEADERS_FIELD = "headers"; + public static final String EXPIRATION_TIME_FIELD = "expiration_time"; + public static final String RESULT_FIELD = "result"; + + public static Settings settings() { + return Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + } + + public static XContentBuilder mappings() throws IOException { + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject(SINGLE_MAPPING_NAME) + .startObject("_meta") + .field("version", Version.CURRENT) + .endObject() + .field("dynamic", "strict") + .startObject("properties") + .startObject(HEADERS_FIELD) + .field("type", "object") + .field("enabled", "false") + .endObject() + .startObject(RESULT_FIELD) + .field("type", "object") + .field("enabled", "false") + .endObject() + .startObject(EXPIRATION_TIME_FIELD) + .field("type", "long") + .endObject() + .endObject() + .endObject() + .endObject(); + return builder; + } + + private final ClusterService clusterService; + private final ThreadContext threadContext; + private final Client client; + private final NamedWriteableRegistry registry; + + AsyncSearchIndexService(ClusterService clusterService, + ThreadContext threadContext, + Client client, + NamedWriteableRegistry registry) { + this.clusterService = clusterService; + this.threadContext = threadContext; + this.client = new OriginSettingClient(client, TASKS_ORIGIN); + this.registry = registry; + } + + /** + * Returns the internal client with origin. + */ + Client getClient() { + return client; + } + + /** + * Creates the index with the expected settings and mappings if it doesn't exist. + */ + void createIndexIfNecessary(ActionListener listener) { + if (clusterService.state().routingTable().hasIndex(AsyncSearchIndexService.INDEX) == false) { + try { + client.admin().indices().prepareCreate(INDEX) + .setSettings(settings()) + .setMapping(mappings()) + .execute(ActionListener.wrap( + resp -> listener.onResponse(null), + exc -> { + if (ExceptionsHelper.unwrapCause(exc) instanceof ResourceAlreadyExistsException) { + listener.onResponse(null); + } else { + logger.error("failed to create async-search index", exc); + listener.onFailure(exc); + } + })); + } catch (Exception exc) { + logger.error("failed to create async-search index", exc); + listener.onFailure(exc); + } + } else { + listener.onResponse(null); + } + } + + /** + * Stores the initial response with the original headers of the authenticated user + * and the expected expiration time. + */ + void storeInitialResponse(String docId, + Map headers, + AsyncSearchResponse response, + ActionListener listener) throws IOException { + Map source = new HashMap<>(); + source.put(HEADERS_FIELD, headers); + source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime()); + source.put(RESULT_FIELD, encodeResponse(response)); + IndexRequest indexRequest = new IndexRequest(INDEX) + .id(docId) + .source(source, XContentType.JSON); + createIndexIfNecessary(ActionListener.wrap(v -> client.index(indexRequest, listener), listener::onFailure)); + } + + /** + * Stores the final response if the place-holder document is still present (update). + */ + void storeFinalResponse(String docId, + AsyncSearchResponse response, + ActionListener listener) throws IOException { + Map source = new HashMap<>(); + source.put(RESULT_FIELD, encodeResponse(response)); + UpdateRequest request = new UpdateRequest() + .index(INDEX) + .id(docId) + .doc(source, XContentType.JSON); + createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure)); + } + + /** + * Updates the expiration time of the provided docId if the place-holder + * document is still present (update). + */ + void updateExpirationTime(String docId, + long expirationTimeMillis, + ActionListener listener) { + Map source = Collections.singletonMap(EXPIRATION_TIME_FIELD, expirationTimeMillis); + UpdateRequest request = new UpdateRequest().index(INDEX) + .id(docId) + .doc(source, XContentType.JSON); + createIndexIfNecessary(ActionListener.wrap(v -> client.update(request, listener), listener::onFailure)); + } + + /** + * Deletes the provided searchId from the index if present. + */ + void deleteResponse(AsyncSearchId searchId, + ActionListener listener) { + DeleteRequest request = new DeleteRequest(INDEX).id(searchId.getDocId()); + createIndexIfNecessary( + ActionListener.wrap(v -> client.delete(request, + ActionListener.wrap( + resp -> { + if (resp.status() == RestStatus.NOT_FOUND) { + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); + } else { + listener.onResponse(new AcknowledgedResponse(true)); + } + }, + exc -> { + logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc); + listener.onFailure(exc); + })), + listener::onFailure)); + } + + /** + * Returns the {@link AsyncSearchTask} if the provided searchId + * is registered in the task manager, null otherwise. + * + * This method throws a {@link ResourceNotFoundException} if the authenticated user + * is not the creator of the original task. + */ + AsyncSearchTask getTask(TaskManager taskManager, AsyncSearchId searchId) throws IOException { + Task task = taskManager.getTask(searchId.getTaskId().getId()); + if (task == null || task instanceof AsyncSearchTask == false) { + return null; + } + AsyncSearchTask searchTask = (AsyncSearchTask) task; + if (searchTask.getSearchId().equals(searchId) == false) { + return null; + } + + // Check authentication for the user + final Authentication auth = Authentication.getAuthentication(threadContext); + if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { + throw new ResourceNotFoundException(searchId.getEncoded() + " not found"); + } + return searchTask; + } + + /** + * Gets the response from the index if present, or delegate a {@link ResourceNotFoundException} + * failure to the provided listener if not. + */ + void getResponse(AsyncSearchId searchId, + ActionListener listener) { + final Authentication current = Authentication.getAuthentication(client.threadPool().getThreadContext()); + GetRequest internalGet = new GetRequest(INDEX) + .preference(searchId.getEncoded()) + .id(searchId.getDocId()); + client.get(internalGet, ActionListener.wrap( + get -> { + if (get.isExists() == false) { + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); + return; + } + + // check the authentication of the current user against the user that initiated the async search + @SuppressWarnings("unchecked") + Map headers = (Map) get.getSource().get(HEADERS_FIELD); + if (ensureAuthenticatedUserIsSame(headers, current) == false) { + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); + return; + } + + @SuppressWarnings("unchecked") + String encoded = (String) get.getSource().get(RESULT_FIELD); + listener.onResponse(encoded != null ? decodeResponse(encoded) : null); + }, + listener::onFailure + )); + } + + /** + * Extracts the authentication from the original headers and checks that it matches + * the current user. This function returns always true if the provided + * headers do not contain any authentication. + */ + boolean ensureAuthenticatedUserIsSame(Map originHeaders, Authentication current) throws IOException { + if (originHeaders == null || originHeaders.containsKey(AUTHENTICATION_KEY) == false) { + // no authorization attached to the original request + return true; + } + if (current == null) { + // origin is an authenticated user but current is not + return false; + } + Authentication origin = Authentication.decode(originHeaders.get(AUTHENTICATION_KEY)); + return ensureAuthenticatedUserIsSame(origin, current); + } + + /** + * Compares the {@link Authentication} that was used to create the {@link AsyncSearchId} with the + * current authentication. + */ + boolean ensureAuthenticatedUserIsSame(Authentication original, Authentication current) { + final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal()); + final boolean sameRealmType; + if (original.getUser().isRunAs()) { + if (current.getUser().isRunAs()) { + sameRealmType = original.getLookedUpBy().getType().equals(current.getLookedUpBy().getType()); + } else { + sameRealmType = original.getLookedUpBy().getType().equals(current.getAuthenticatedBy().getType()); + } + } else if (current.getUser().isRunAs()) { + sameRealmType = original.getAuthenticatedBy().getType().equals(current.getLookedUpBy().getType()); + } else { + sameRealmType = original.getAuthenticatedBy().getType().equals(current.getAuthenticatedBy().getType()); + } + return samePrincipal && sameRealmType; + } + + /** + * Encode the provided response in a binary form using base64 encoding. + */ + String encodeResponse(AsyncSearchResponse response) throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Version.writeVersion(Version.CURRENT, out); + response.writeTo(out); + return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes())); + } + } + + /** + * Decode the provided base-64 bytes into a {@link AsyncSearchResponse}. + */ + AsyncSearchResponse decodeResponse(String value) throws IOException { + try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getDecoder().decode(value)))) { + try (StreamInput in = new NamedWriteableAwareStreamInput(buf, registry)) { + in.setVersion(Version.readVersion(in)); + return new AsyncSearchResponse(in); + } + } + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java new file mode 100644 index 0000000000000..0a0a78ed8b44f --- /dev/null +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.search; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.threadpool.Scheduler; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.xpack.search.AsyncSearchIndexService.EXPIRATION_TIME_FIELD; + +/** + * A service that runs a periodic cleanup over the async-search index. + */ +class AsyncSearchMaintenanceService implements Releasable, ClusterStateListener { + private static final Logger logger = LogManager.getLogger(AsyncSearchMaintenanceService.class); + + private final String localNodeId; + private final ThreadPool threadPool; + private final AsyncSearchIndexService indexService; + private final TimeValue delay; + + private final AtomicBoolean isCleanupRunning = new AtomicBoolean(false); + private final AtomicBoolean isClosed = new AtomicBoolean(false); + private volatile Scheduler.Cancellable cancellable; + + AsyncSearchMaintenanceService(String localNodeId, + ThreadPool threadPool, + AsyncSearchIndexService indexService, + TimeValue delay) { + this.localNodeId = localNodeId; + this.threadPool = threadPool; + this.indexService = indexService; + this.delay = delay; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + final ClusterState state = event.state(); + if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + // Wait until the gateway has recovered from disk. + return; + } + if (state.nodes().getLocalNode().isDataNode()) { + tryStartCleanup(state); + } + } + + void tryStartCleanup(ClusterState state) { + if (isClosed.get()) { + return; + } + IndexRoutingTable indexRouting = state.routingTable().index(AsyncSearchIndexService.INDEX); + if (indexRouting == null) { + if (isCleanupRunning.compareAndSet(true, false)) { + close(); + } + return; + } + String primaryNodeId = indexRouting.shard(0).primaryShard().currentNodeId(); + if (localNodeId.equals(primaryNodeId)) { + if (isCleanupRunning.compareAndSet(false, true)) { + executeNextCleanup(); + } + } else if (isCleanupRunning.compareAndSet(true, false)) { + close(); + } + } + + synchronized void executeNextCleanup() { + if (isClosed.get() == false && isCleanupRunning.get()) { + long nowInMillis = System.currentTimeMillis(); + DeleteByQueryRequest toDelete = new DeleteByQueryRequest() + .setQuery(QueryBuilders.rangeQuery(EXPIRATION_TIME_FIELD).lte(nowInMillis)); + indexService.getClient() + .execute(DeleteByQueryAction.INSTANCE, toDelete, ActionListener.wrap(() -> scheduleNextCleanup())); + } + } + + synchronized void scheduleNextCleanup() { + if (isClosed.get() == false && isCleanupRunning.get()) { + try { + cancellable = threadPool.schedule(this::executeNextCleanup, delay, ThreadPool.Names.GENERIC); + } catch (EsRejectedExecutionException e) { + if (e.isExecutorShutdown()) { + logger.debug("failed to schedule next maintenance task; shutting down", e); + } else { + throw e; + } + } + } + } + + @Override + public void close() { + if (cancellable != null && cancellable.isCancelled() == false) { + cancellable.cancel(); + } + isClosed.compareAndSet(false, true); + } +} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchPlugin.java similarity index 82% rename from x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java rename to x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchPlugin.java index 3b1762c8a832e..096ecb5947531 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchPlugin.java @@ -16,13 +16,11 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; @@ -39,7 +37,7 @@ import java.util.List; import java.util.function.Supplier; -public final class AsyncSearch extends Plugin implements ActionPlugin, PersistentTaskPlugin { +public final class AsyncSearchPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { return Arrays.asList( @@ -71,13 +69,11 @@ public Collection createComponents(Client client, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - new AsyncSearchTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry); - return Collections.emptyList(); - } - - @Override - public List> getPersistentTasksExecutor(ClusterService clusterService, ThreadPool threadPool, - Client client, SettingsModule settingsModule) { - return Collections.singletonList(new AsyncSearchReaperExecutor(client, threadPool)); + AsyncSearchIndexService indexService = + new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, namedWriteableRegistry); + AsyncSearchMaintenanceService maintenanceService = + new AsyncSearchMaintenanceService(nodeEnvironment.nodeId(), threadPool, indexService, TimeValue.timeValueHours(1)); + clusterService.addListener(maintenanceService); + return Collections.singletonList(maintenanceService); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java deleted file mode 100644 index d6e4992e46ebb..0000000000000 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchReaperExecutor.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.search; - -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.OriginSettingClient; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.CountDown; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.persistent.AllocatedPersistentTask; -import org.elasticsearch.persistent.PersistentTaskParams; -import org.elasticsearch.persistent.PersistentTaskState; -import org.elasticsearch.persistent.PersistentTasksExecutor; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; -import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC; -import static org.elasticsearch.threadpool.ThreadPool.Names.SAME; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.EXPIRATION_TIME_FIELD; -import static org.elasticsearch.xpack.search.AsyncSearchReaperExecutor.Params; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.IS_RUNNING_FIELD; -import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; - -public class AsyncSearchReaperExecutor extends PersistentTasksExecutor { - public static final String NAME = "async_search/reaper"; - - private final Client client; - private final ThreadPool threadPool; - - public AsyncSearchReaperExecutor(Client client, ThreadPool threadPool) { - super(NAME, SAME); - this.client = new OriginSettingClient(client, TASKS_ORIGIN); - this.threadPool = threadPool; - } - - @Override - protected void nodeOperation(AllocatedPersistentTask task, Params params, PersistentTaskState state) { - cleanup(task, client, 100, System.currentTimeMillis()); - } - - public void cleanup(AllocatedPersistentTask task, Client client, int pageSize, long nowInMillis) { - SearchRequest request = new SearchRequest(ASYNC_SEARCH_ALIAS); - request.source().query(QueryBuilders.rangeQuery(EXPIRATION_TIME_FIELD).gt(nowInMillis)); - request.scroll(TimeValue.timeValueMinutes(1)); - client.search(request, nextPageListener(task, pageSize, () -> { - if (task.isCancelled() == false) { - threadPool.schedule(() -> cleanup(task, client, pageSize, System.currentTimeMillis()), - TimeValue.timeValueMinutes(30), GENERIC); - } - })); - } - - private void nextPage(AllocatedPersistentTask task, Client client, String scrollId, int pageSize, Runnable onCompletion) { - if (task.isCancelled()) { - onCompletion.run(); - return; - } - client.searchScroll(new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(1)), - nextPageListener(task, pageSize, onCompletion)); - } - - public void deleteAll(Client client, Collection toCancel, Collection toDelete, ActionListener listener) { - if (toDelete.isEmpty() && toCancel.isEmpty()) { - listener.onResponse(null); - } - CountDown counter = new CountDown(toDelete.size() + 1); - BulkRequest bulkDelete = new BulkRequest(); - for (String id : toDelete) { - bulkDelete.add(new DeleteRequest(ASYNC_SEARCH_ALIAS).id(id)); - } - client.bulk(bulkDelete, ActionListener.wrap(() -> { - if (counter.countDown()) { - listener.onResponse(null); - } - })); - for (String id : toCancel) { - ThreadContext threadContext = threadPool.getThreadContext(); - try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) { - threadContext.markAsSystemContext(); - client.execute(DeleteAsyncSearchAction.INSTANCE, new DeleteAsyncSearchAction.Request(id), - ActionListener.wrap(() -> { - if (counter.countDown()) { - listener.onResponse(null); - } - })); - } - } - } - - private ActionListener nextPageListener(AllocatedPersistentTask task, int pageSize, Runnable onCompletion) { - return new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - if (task.isCancelled()) { - onCompletion.run(); - return; - } - final List toDelete = new ArrayList<>(); - final List toCancel = new ArrayList<>(); - for (SearchHit hit : response.getHits()) { - boolean isRunning = (boolean) hit.getSourceAsMap().get(IS_RUNNING_FIELD); - if (isRunning) { - toCancel.add(hit.getId()); - } else { - toDelete.add(hit.getId()); - } - } - deleteAll(client, toDelete, toCancel, ActionListener.wrap(() -> { - if (response.getHits().getHits().length < pageSize) { - onCompletion.run(); - } else { - nextPage(task, client, response.getScrollId(), pageSize, onCompletion); - } - })); - } - - @Override - public void onFailure(Exception exc) { - onCompletion.run(); - } - }; - } - - static class Params implements PersistentTaskParams { - @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) { - return builder; - } - - @Override - public void writeTo(StreamOutput out) {} - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public Version getMinimalSupportedVersion() { - return Version.CURRENT; - } - } -} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java deleted file mode 100644 index a289e9988d7ac..0000000000000 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchStoreService.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.search; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.OriginSettingClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.security.authc.Authentication; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Base64; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; -import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; -import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; - -/** - * A class that encapsulates the logic to store and retrieve {@link AsyncSearchResponse} to/from the .async-search index. - */ -class AsyncSearchStoreService { - private static final Logger logger = LogManager.getLogger(AsyncSearchStoreService.class); - - static final String HEADERS_FIELD = "headers"; - static final String IS_RUNNING_FIELD = "is_running"; - static final String EXPIRATION_TIME_FIELD = "expiration_time"; - static final String RESULT_FIELD = "result"; - - private final TaskManager taskManager; - private final ThreadContext threadContext; - private final Client client; - private final NamedWriteableRegistry registry; - - AsyncSearchStoreService(TaskManager taskManager, ThreadContext threadContext, Client client, NamedWriteableRegistry registry) { - this.taskManager = taskManager; - this.threadContext = threadContext; - this.client = new OriginSettingClient(client, TASKS_ORIGIN); - this.registry = registry; - } - - /** - * Return the internal client with origin. - */ - Client getClient() { - return client; - } - - ThreadContext getThreadContext() { - return threadContext; - } - - /** - * Store an empty document in the .async-search index that is used - * as a place-holder for the future response. - */ - void storeInitialResponse(Map headers, String docID, AsyncSearchResponse response, - ActionListener listener) throws IOException { - Map source = new HashMap<>(); - source.put(HEADERS_FIELD, headers); - source.put(IS_RUNNING_FIELD, true); - source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime()); - source.put(RESULT_FIELD, encodeResponse(response)); - IndexRequest request = new IndexRequest(ASYNC_SEARCH_ALIAS) - .id(docID) - .source(source, XContentType.JSON); - client.index(request, listener); - } - - /** - * Store the final response if the place-holder document is still present (update). - */ - void storeFinalResponse(AsyncSearchResponse response, ActionListener listener) throws IOException { - AsyncSearchId searchId = AsyncSearchId.decode(response.getId()); - Map source = new HashMap<>(); - source.put(IS_RUNNING_FIELD, true); - source.put(RESULT_FIELD, encodeResponse(response)); - UpdateRequest request = new UpdateRequest() - .index(ASYNC_SEARCH_ALIAS) - .id(searchId.getDocId()) - .doc(source, XContentType.JSON); - client.update(request, listener); - } - - void updateKeepAlive(String docID, long expirationTimeMillis, ActionListener listener) { - Map source = Collections.singletonMap(EXPIRATION_TIME_FIELD, expirationTimeMillis); - UpdateRequest request = new UpdateRequest().index(ASYNC_SEARCH_ALIAS) - .id(docID) - .doc(source, XContentType.JSON); - client.update(request, listener); - } - - AsyncSearchTask getTask(AsyncSearchId searchId) throws IOException { - Task task = taskManager.getTask(searchId.getTaskId().getId()); - if (task == null || task instanceof AsyncSearchTask == false) { - return null; - } - AsyncSearchTask searchTask = (AsyncSearchTask) task; - if (searchTask.getSearchId().equals(searchId) == false) { - return null; - } - - if (threadContext.isSystemContext() == false) { - // Check authentication for the user - final Authentication auth = Authentication.getAuthentication(threadContext); - if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { - throw new ResourceNotFoundException(searchId.getEncoded() + " not found"); - } - } - return searchTask; - } - - /** - * Get the response from the .async-search index if present, or delegate a {@link ResourceNotFoundException} - * failure to the provided listener if not. - */ - void getResponse(AsyncSearchId searchId, ActionListener listener) { - final Authentication current = Authentication.getAuthentication(client.threadPool().getThreadContext()); - GetRequest internalGet = new GetRequest(ASYNC_SEARCH_ALIAS) - .preference(searchId.getEncoded()) - .id(searchId.getDocId()); - client.get(internalGet, ActionListener.wrap( - get -> { - if (get.isExists() == false) { - listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); - return; - } - - if (threadContext.isSystemContext() == false) { - // check the authentication of the current user against the user that initiated the async search - @SuppressWarnings("unchecked") - Map headers = (Map) get.getSource().get(HEADERS_FIELD); - if (ensureAuthenticatedUserIsSame(headers, current) == false) { - listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); - return; - } - } - - @SuppressWarnings("unchecked") - String encoded = (String) get.getSource().get(RESULT_FIELD); - listener.onResponse(encoded != null ? decodeResponse(encoded, registry) : null); - }, - listener::onFailure - )); - } - - void deleteResult(AsyncSearchId searchId, ActionListener listener) { - DeleteRequest request = new DeleteRequest(ASYNC_SEARCH_ALIAS).id(searchId.getDocId()); - client.delete(request, ActionListener.wrap( - resp -> { - if (resp.status() == RestStatus.NOT_FOUND) { - listener.onFailure(new ResourceNotFoundException("id [{}] not found", searchId.getEncoded())); - } else { - listener.onResponse(new AcknowledgedResponse(true)); - } - }, - exc -> { - logger.error(() -> new ParameterizedMessage("failed to clean async-search [{}]", searchId.getEncoded()), exc); - listener.onFailure(exc); - }) - ); - } - - /** - * Extracts the authentication from the original headers and checks that it matches - * the current user. This function returns always true if the provided - * headers do not contain any authentication. - */ - static boolean ensureAuthenticatedUserIsSame(Map originHeaders, Authentication current) throws IOException { - if (originHeaders == null || originHeaders.containsKey(AUTHENTICATION_KEY) == false) { - // no authorization attached to the original request - return true; - } - if (current == null) { - // origin is an authenticated user but current is not - return false; - } - Authentication origin = Authentication.decode(originHeaders.get(AUTHENTICATION_KEY)); - return ensureAuthenticatedUserIsSame(origin, current); - } - - /** - * Compares the {@link Authentication} that was used to create the {@link AsyncSearchId} with the - * current authentication. - */ - static boolean ensureAuthenticatedUserIsSame(Authentication original, Authentication current) { - final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal()); - final boolean sameRealmType; - if (original.getUser().isRunAs()) { - if (current.getUser().isRunAs()) { - sameRealmType = original.getLookedUpBy().getType().equals(current.getLookedUpBy().getType()); - } else { - sameRealmType = original.getLookedUpBy().getType().equals(current.getAuthenticatedBy().getType()); - } - } else if (current.getUser().isRunAs()) { - sameRealmType = original.getAuthenticatedBy().getType().equals(current.getLookedUpBy().getType()); - } else { - sameRealmType = original.getAuthenticatedBy().getType().equals(current.getAuthenticatedBy().getType()); - } - return samePrincipal && sameRealmType; - } - - /** - * Encode the provided response in a binary form using base64 encoding. - */ - static String encodeResponse(AsyncSearchResponse response) throws IOException { - try (BytesStreamOutput out = new BytesStreamOutput()) { - Version.writeVersion(Version.CURRENT, out); - response.writeTo(out); - return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes())); - } - } - - /** - * Decode the provided base-64 bytes into a {@link AsyncSearchResponse}. - */ - static AsyncSearchResponse decodeResponse(String value, NamedWriteableRegistry registry) throws IOException { - try (ByteBufferStreamInput buf = new ByteBufferStreamInput(ByteBuffer.wrap(Base64.getDecoder().decode(value)))) { - try (StreamInput in = new NamedWriteableAwareStreamInput(buf, registry)) { - in.setVersion(Version.readVersion(in)); - return new AsyncSearchResponse(in); - } - } - } -} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 0af3b8b780fb6..8f927f3218dfa 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -6,6 +6,9 @@ package org.elasticsearch.xpack.search; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.action.search.SearchProgressActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -13,7 +16,9 @@ import org.elasticsearch.action.search.SearchShard; import org.elasticsearch.action.search.SearchTask; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; @@ -38,6 +43,7 @@ */ class AsyncSearchTask extends SearchTask { private final AsyncSearchId searchId; + private final Client client; private final ThreadPool threadPool; private final Supplier reduceContextSupplier; private final Listener progressListener; @@ -50,7 +56,8 @@ class AsyncSearchTask extends SearchTask { private final List initListeners = new ArrayList<>(); private final Map> completionListeners = new HashMap<>(); - private long expirationTimeMillis; + private volatile long expirationTimeMillis; + private final AtomicBoolean isCancelling = new AtomicBoolean(false); private AtomicReference searchResponse; @@ -71,14 +78,18 @@ class AsyncSearchTask extends SearchTask { String type, String action, TaskId parentTaskId, + TimeValue keepAlive, Map originHeaders, Map taskHeaders, AsyncSearchId searchId, + Client client, ThreadPool threadPool, Supplier reduceContextSupplier) { super(id, type, action, "async_search", parentTaskId, taskHeaders); + this.expirationTimeMillis = getStartTime() + keepAlive.getMillis(); this.originHeaders = originHeaders; this.searchId = searchId; + this.client = client; this.threadPool = threadPool; this.reduceContextSupplier = reduceContextSupplier; this.progressListener = new Listener(); @@ -105,12 +116,41 @@ public SearchProgressActionListener getProgressListener() { return progressListener; } - public synchronized void setExpirationTime(long expirationTimeMillis) { + /** + * Update the expiration time of the (partial) response. + */ + public void setExpirationTime(long expirationTimeMillis) { this.expirationTimeMillis = expirationTimeMillis; } - public synchronized long getExpirationTime() { - return expirationTimeMillis; + /** + * Cancels the running task and its children. + */ + public void cancelTask(Runnable runnable) { + if (isCancelled() == false && isCancelling.compareAndSet(false, true)) { + CancelTasksRequest req = new CancelTasksRequest().setTaskId(searchId.getTaskId()); + client.admin().cluster().cancelTasks(req, new ActionListener<>() { + @Override + public void onResponse(CancelTasksResponse cancelTasksResponse) { + runnable.run(); + } + + @Override + public void onFailure(Exception e) { + // cancelling failed + isCancelling.compareAndSet(true, false); + runnable.run(); + } + }); + } else { + runnable.run(); + } + } + + @Override + protected void onCancelled() { + super.onCancelled(); + isCancelling.compareAndSet(true, false); } /** @@ -118,7 +158,7 @@ public synchronized long getExpirationTime() { * consumer when the task is finished or when the provided waitForCompletion * timeout occurs. In such case the consumed {@link AsyncSearchResponse} will contain partial results. */ - public void addCompletionListener(Consumer listener, TimeValue waitForCompletion) { + public void addCompletionListener(ActionListener listener, TimeValue waitForCompletion) { boolean executeImmediately = false; long startTime = threadPool.relativeTimeInMillis(); synchronized (this) { @@ -126,16 +166,20 @@ public void addCompletionListener(Consumer listener, TimeVa executeImmediately = true; } else { addInitListener(() -> { - long elapsedTime = threadPool.relativeTimeInMillis() - startTime; - // subtract the initialization time to the provided waitForCompletion. - TimeValue remainingWaitForCompletion = - TimeValue.timeValueMillis(Math.max(0, waitForCompletion.getMillis() - elapsedTime)); + final TimeValue remainingWaitForCompletion; + if (waitForCompletion.getMillis() > 0) { + long elapsedTime = threadPool.relativeTimeInMillis() - startTime; + // subtract the initialization time from the provided waitForCompletion. + remainingWaitForCompletion = TimeValue.timeValueMillis(Math.max(0, waitForCompletion.getMillis() - elapsedTime)); + } else { + remainingWaitForCompletion = TimeValue.ZERO; + } internalAddCompletionListener(listener, remainingWaitForCompletion); }); } } if (executeImmediately) { - listener.accept(getResponse()); + listener.onResponse(getResponse()); } } @@ -157,7 +201,7 @@ public void addCompletionListener(Consumer listener) { } } - private void internalAddCompletionListener(Consumer listener, TimeValue waitForCompletion) { + private void internalAddCompletionListener(ActionListener listener, TimeValue waitForCompletion) { boolean executeImmediately = false; synchronized (this) { if (hasCompleted || waitForCompletion.getMillis() == 0) { @@ -166,25 +210,31 @@ private void internalAddCompletionListener(Consumer listene // ensure that we consumes the listener only once AtomicBoolean hasRun = new AtomicBoolean(false); long id = completionId++; - Cancellable cancellable = - threadPool.schedule(() -> { + + final Cancellable cancellable; + try { + cancellable = threadPool.schedule(() -> { if (hasRun.compareAndSet(false, true)) { // timeout occurred before completion removeCompletionListener(id); - listener.accept(getResponse()); + listener.onResponse(getResponse()); } - }, waitForCompletion, "generic"); + }, waitForCompletion, "generic"); + } catch (EsRejectedExecutionException exc) { + listener.onFailure(exc); + return; + } completionListeners.put(id, resp -> { if (hasRun.compareAndSet(false, true)) { // completion occurred before timeout cancellable.cancel(); - listener.accept(resp); + listener.onResponse(resp); } }); } } if (executeImmediately) { - listener.accept(getResponse()); + listener.onResponse(getResponse()); } } @@ -239,24 +289,53 @@ private void executeCompletionListeners() { private AsyncSearchResponse getResponse() { assert searchResponse.get() != null; - return searchResponse.get().toAsyncSearchResponse(this); + return searchResponse.get().toAsyncSearchResponse(this, expirationTimeMillis); + } + + // cancels the task if it expired + private void checkExpiration() { + long now = System.currentTimeMillis(); + if (expirationTimeMillis < now) { + cancelTask(() -> {}); + } } private class Listener extends SearchProgressActionListener { @Override - public void onListShards(List shards, List skipped, Clusters clusters, boolean fetchPhase) { - searchResponse.compareAndSet(null, - new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier)); - executeInitListeners(); + public void onQueryResult(int shardIndex) { + checkExpiration(); + } + + @Override + public void onFetchResult(int shardIndex) { + checkExpiration(); } @Override public void onQueryFailure(int shardIndex, SearchShardTarget shardTarget, Exception exc) { + // best effort to cancel expired tasks + checkExpiration(); searchResponse.get().addShardFailure(shardIndex, new ShardSearchFailure(exc, shardTarget)); } + @Override + public void onFetchFailure(int shardIndex, Exception exc) { + checkExpiration(); + } + + @Override + public void onListShards(List shards, List skipped, Clusters clusters, boolean fetchPhase) { + // best effort to cancel expired tasks + checkExpiration(); + searchResponse.compareAndSet(null, + new MutableSearchResponse(shards.size() + skipped.size(), skipped.size(), clusters, reduceContextSupplier)); + executeInitListeners(); + } + @Override public void onPartialReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { + // best effort to cancel expired tasks + checkExpiration(); searchResponse.get().updatePartialResponse(shards.size(), new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs, null, null, false, null, reducePhase), aggs == null); @@ -264,6 +343,8 @@ public void onPartialReduce(List shards, TotalHits totalHits, Inter @Override public void onReduce(List shards, TotalHits totalHits, InternalAggregations aggs, int reducePhase) { + // best effort to cancel expired tasks + checkExpiration(); searchResponse.get().updatePartialResponse(shards.size(), new InternalSearchResponse(new SearchHits(SearchHits.EMPTY, totalHits, Float.NaN), aggs, null, null, false, null, reducePhase), true); diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java deleted file mode 100644 index de3db6d903c3a..0000000000000 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTemplateRegistry.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.search; - -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.template.IndexTemplateConfig; -import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; -import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; - -import java.util.Collections; -import java.util.List; - -import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; - -/** - * Manage the index template and associated ILM policy for the .async-search index. - */ -public class AsyncSearchTemplateRegistry extends IndexTemplateRegistry { - // history (please add a comment why you increased the version here) - // version 1: initial - public static final String INDEX_TEMPLATE_VERSION = "1"; - public static final String ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE = "xpack.async-search.template.version"; - public static final String ASYNC_SEARCH_TEMPLATE_NAME = ".async-search"; - public static final String ASYNC_SEARCH_ALIAS = ASYNC_SEARCH_TEMPLATE_NAME + "-" + INDEX_TEMPLATE_VERSION; - - public static final IndexTemplateConfig TEMPLATE_ASYNC_SEARCH = new IndexTemplateConfig( - ASYNC_SEARCH_TEMPLATE_NAME, - "/async-search.json", - INDEX_TEMPLATE_VERSION, - ASYNC_SEARCH_TEMPLATE_VERSION_VARIABLE - ); - - public AsyncSearchTemplateRegistry(Settings nodeSettings, - ClusterService clusterService, - ThreadPool threadPool, - Client client, - NamedXContentRegistry xContentRegistry) { - super(nodeSettings, clusterService, threadPool, client, xContentRegistry); - } - - @Override - protected List getTemplateConfigs() { - return Collections.singletonList(TEMPLATE_ASYNC_SEARCH); - } - - @Override - protected List getPolicyConfigs() { - return Collections.emptyList(); - } - - @Override - protected String getOrigin() { - return TASKS_ORIGIN; - } -} diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java index 7d0b384efb1af..203978edea580 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/MutableSearchResponse.java @@ -50,6 +50,14 @@ class MutableSearchResponse { private boolean frozen; + /** + * Creates a new mutable search response. + * + * @param totalShards The number of shards that participate in the request, or -1 to indicate a failure. + * @param skippedShards The number of skipped shards, or -1 to indicate a failure. + * @param clusters The remote clusters statistics. + * @param reduceContextSupplier A supplier to run final reduce on partial aggregations. + */ MutableSearchResponse(int totalShards, int skippedShards, Clusters clusters, Supplier reduceContextSupplier) { this.totalShards = totalShards; this.skippedShards = skippedShards; @@ -120,12 +128,11 @@ void addShardFailure(int shardIndex, ShardSearchFailure failure) { } /** - * Creates an {@link AsyncSearchResponse} based on the current state - * of the mutable response. The final reduction of aggregations is - * executed if needed in the synchronized block to ensure that only one - * can run concurrently. + * Creates an {@link AsyncSearchResponse} based on the current state of the mutable response. + * The final reduce of the aggregations is executed if needed (partial response). + * This method is synchronized to ensure that we don't perform final reduces concurrently. */ - synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task) { + synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task, long expirationTime) { final SearchResponse resp; if (totalShards != -1) { if (sections.aggregations() != null && isFinalReduce == false) { @@ -142,7 +149,7 @@ synchronized AsyncSearchResponse toAsyncSearchResponse(AsyncSearchTask task) { resp = null; } return new AsyncSearchResponse(task.getSearchId().getEncoded(), version, resp, failure, isPartial, - frozen == false, task.getStartTime(), task.getExpirationTime()); + frozen == false, task.getStartTime(), expirationTime); } private void failIfFrozen() { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index 4059a9e900213..7f2cba4164d1e 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -5,16 +5,14 @@ */ package org.elasticsearch.xpack.search; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestStatusToXContentListener; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; -import java.io.IOException; - import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestGetAsyncSearchAction extends BaseRestHandler { @@ -29,13 +27,21 @@ public String getName() { } @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - String id = request.param("id"); - int lastVersion = request.paramAsInt("last_version", -1); - TimeValue waitForCompletion = request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1)); - TimeValue keepAlive = request.paramAsTime("keep_alive", TimeValue.MINUS_ONE); - GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(id, waitForCompletion, lastVersion); - get.setKeepAlive(keepAlive); + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + GetAsyncSearchAction.Request get = new GetAsyncSearchAction.Request(request.param("id")); + if (request.hasParam("wait_for_completion")) { + get.setWaitForCompletion(request.paramAsTime("wait_for_completion", get.getWaitForCompletion())); + } + if (request.hasParam("keep_alive")) { + get.setKeepAlive(request.paramAsTime("keep_alive", get.getKeepAlive())); + } + if (request.hasParam("last_version")) { + get.setLastVersion(request.paramAsInt("last_version", get.getLastVersion())); + } + ActionRequestValidationException validationException = get.validate(); + if (validationException != null) { + throw validationException; + } return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java index b54f3138ddd2f..41cf2b57c9da4 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -7,7 +7,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; @@ -39,22 +38,28 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - SubmitAsyncSearchRequest submitRequest = new SubmitAsyncSearchRequest(); - IntConsumer setSize = size -> submitRequest.getSearchRequest().source().size(size); + SubmitAsyncSearchRequest submit = new SubmitAsyncSearchRequest(); + IntConsumer setSize = size -> submit.getSearchRequest().source().size(size); request.withContentOrSourceParamParserOrNull(parser -> - parseSearchRequest(submitRequest.getSearchRequest(), request, parser, setSize)); - submitRequest.setWaitForCompletion(request.paramAsTime("wait_for_completion", TimeValue.timeValueSeconds(1))); - submitRequest.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", true)); - submitRequest.setKeepAlive(request.paramAsTime("keep_alive", submitRequest.getKeepAlive())); + parseSearchRequest(submit.getSearchRequest(), request, parser, setSize)); - ActionRequestValidationException validationException = submitRequest.validate(); + if (request.hasParam("wait_for_completion")) { + submit.setWaitForCompletion(request.paramAsTime("wait_for_completion", submit.getWaitForCompletion())); + } + if (request.hasParam("keep_alive")) { + submit.setKeepAlive(request.paramAsTime("keep_alive", submit.getKeepAlive())); + } + if (request.hasParam("clean_on_completion")) { + submit.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", submit.isCleanOnCompletion())); + } + ActionRequestValidationException validationException = submit.validate(); if (validationException != null) { throw validationException; } return channel -> { RestStatusToXContentListener listener = new RestStatusToXContentListener<>(channel); RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); - cancelClient.execute(SubmitAsyncSearchAction.INSTANCE, submitRequest, listener); + cancelClient.execute(SubmitAsyncSearchAction.INSTANCE, submit, listener); }; } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java index 13c84271ba7bb..0fd514a744d96 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportDeleteAsyncSearchAction.java @@ -26,7 +26,7 @@ public class TransportDeleteAsyncSearchAction extends HandledTransportAction { private final ClusterService clusterService; private final TransportService transportService; - private final AsyncSearchStoreService store; + private final AsyncSearchIndexService store; @Inject public TransportDeleteAsyncSearchAction(TransportService transportService, @@ -36,7 +36,7 @@ public TransportDeleteAsyncSearchAction(TransportService transportService, NamedWriteableRegistry registry, Client client) { super(DeleteAsyncSearchAction.NAME, transportService, actionFilters, DeleteAsyncSearchAction.Request::new); - this.store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, registry); + this.store = new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, registry); this.clusterService = clusterService; this.transportService = transportService; } @@ -59,18 +59,12 @@ protected void doExecute(Task task, DeleteAsyncSearchAction.Request request, Act } private void cancelTaskAndDeleteResult(AsyncSearchId searchId, ActionListener listener) throws IOException { - AsyncSearchTask task = store.getTask(searchId); - if (task != null && task.isCancelled() == false) { - store.getClient().admin().cluster().prepareCancelTasks() - .setTaskId(searchId.getTaskId()) - .execute(ActionListener.wrap(() -> store.deleteResult(searchId, listener))); + AsyncSearchTask task = store.getTask(taskManager, searchId); + if (task != null) { + task.cancelTask(() -> store.deleteResponse(searchId, listener)); } else { - if (store.getThreadContext().isSystemContext()) { - store.deleteResult(searchId, listener); - } else { - // check if the response can be retrieved by the user (handle security) and then delete. - store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResult(searchId, listener), listener::onFailure)); - } + // check if the response can be retrieved by the user (handle security) and then delete. + store.getResponse(searchId, ActionListener.wrap(res -> store.deleteResponse(searchId, listener), listener::onFailure)); } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index bd551881d5c55..5606f33282ff7 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.search; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; @@ -16,6 +18,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequestOptions; @@ -23,12 +26,11 @@ import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; -import static org.elasticsearch.action.ActionListener.wrap; - public class TransportGetAsyncSearchAction extends HandledTransportAction { + private final Logger logger = LogManager.getLogger(TransportGetAsyncSearchAction.class); private final ClusterService clusterService; private final TransportService transportService; - private final AsyncSearchStoreService store; + private final AsyncSearchIndexService store; @Inject public TransportGetAsyncSearchAction(TransportService transportService, @@ -40,21 +42,30 @@ public TransportGetAsyncSearchAction(TransportService transportService, super(GetAsyncSearchAction.NAME, transportService, actionFilters, GetAsyncSearchAction.Request::new); this.clusterService = clusterService; this.transportService = transportService; - this.store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, registry); + this.store = new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, registry); } @Override protected void doExecute(Task task, GetAsyncSearchAction.Request request, ActionListener listener) { try { + long nowInMillis = System.currentTimeMillis(); AsyncSearchId searchId = AsyncSearchId.decode(request.getId()); DiscoveryNode node = clusterService.state().nodes().get(searchId.getTaskId().getNodeId()); if (clusterService.localNode().getId().equals(searchId.getTaskId().getNodeId()) || node == null) { - if (request.getKeepAlive() != TimeValue.MINUS_ONE) { - long expirationTime = System.currentTimeMillis() + request.getKeepAlive().getMillis(); - store.updateKeepAlive(searchId.getDocId(), expirationTime, - wrap(up -> getSearchResponseFromTask(searchId, request, expirationTime, listener), listener::onFailure)); + if (request.getKeepAlive().getMillis() > 0) { + long expirationTime = nowInMillis + request.getKeepAlive().getMillis(); + store.updateExpirationTime(searchId.getDocId(), expirationTime, + ActionListener.wrap( + p -> getSearchResponseFromTask(searchId, request, nowInMillis, expirationTime, listener), + exc -> { + if (exc.getCause() instanceof DocumentMissingException == false) { + logger.error("failed to retrieve " + searchId.getEncoded(), exc); + } + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); + } + )); } else { - getSearchResponseFromTask(searchId, request, -1, listener); + getSearchResponseFromTask(searchId, request, nowInMillis, -1, listener); } } else { TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); @@ -67,31 +78,35 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action } } - private void getSearchResponseFromTask(AsyncSearchId searchId, GetAsyncSearchAction.Request request, + private void getSearchResponseFromTask(AsyncSearchId searchId, + GetAsyncSearchAction.Request request, + long nowInMillis, long expirationTimeMillis, ActionListener listener) { try { - final AsyncSearchTask task = store.getTask(searchId); + final AsyncSearchTask task = store.getTask(taskManager, searchId); if (task == null) { - getSearchResponseFromIndex(searchId, request, listener); + getSearchResponseFromIndex(searchId, request, nowInMillis, listener); return; } if (task.isCancelled()) { - listener.onFailure(new ResourceNotFoundException(searchId.getEncoded() + " not found")); + listener.onFailure(new ResourceNotFoundException(searchId.getEncoded())); return; } if (expirationTimeMillis != -1) { task.setExpirationTime(expirationTimeMillis); } - task.addCompletionListener(response -> { - if (response.getVersion() <= request.getLastVersion()) { - // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), - response.isPartial(), response.isRunning(), response.getStartTime(), response.getExpirationTime())); - } else { - listener.onResponse(response); + task.addCompletionListener(new ActionListener<>() { + @Override + public void onResponse(AsyncSearchResponse response) { + sendFinalResponse(request, response, nowInMillis, listener); + } + + @Override + public void onFailure(Exception exc) { + listener.onFailure(exc); } }, request.getWaitForCompletion()); } catch (Exception exc) { @@ -99,18 +114,14 @@ private void getSearchResponseFromTask(AsyncSearchId searchId, GetAsyncSearchAct } } - private void getSearchResponseFromIndex(AsyncSearchId searchId, GetAsyncSearchAction.Request request, + private void getSearchResponseFromIndex(AsyncSearchId searchId, + GetAsyncSearchAction.Request request, + long nowInMillis, ActionListener listener) { store.getResponse(searchId, new ActionListener<>() { @Override public void onResponse(AsyncSearchResponse response) { - if (response.getVersion() <= request.getLastVersion()) { - // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), - response.isPartial(), false, response.getStartTime(), response.getExpirationTime())); - } else { - listener.onResponse(response); - } + sendFinalResponse(request, response, nowInMillis, listener); } @Override @@ -119,4 +130,22 @@ public void onFailure(Exception e) { } }); } + + private void sendFinalResponse(GetAsyncSearchAction.Request request, + AsyncSearchResponse response, + long nowInMillis, + ActionListener listener) { + if (response.getExpirationTime() < nowInMillis) { + listener.onFailure(new ResourceNotFoundException(request.getId())); + return; + } + if (response.getVersion() <= request.getLastVersion()) { + // return a not-modified response + listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), + response.isPartial(), false, response.getStartTime(), response.getExpirationTime())); + return; + } + + listener.onResponse(response); + } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 1de9004bae766..025b9638cf7fa 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -8,10 +8,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; @@ -20,17 +18,19 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; @@ -40,8 +40,6 @@ import java.util.Map; import java.util.function.Supplier; -import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; - public class TransportSubmitAsyncSearchAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportSubmitAsyncSearchAction.class); @@ -49,10 +47,11 @@ public class TransportSubmitAsyncSearchAction extends HandledTransportAction reduceContextSupplier; private final TransportSearchAction searchAction; - private final AsyncSearchStoreService store; + private final AsyncSearchIndexService store; @Inject - public TransportSubmitAsyncSearchAction(TransportService transportService, + public TransportSubmitAsyncSearchAction(ClusterService clusterService, + TransportService transportService, ActionFilters actionFilters, NamedWriteableRegistry registry, Client client, @@ -64,7 +63,7 @@ public TransportSubmitAsyncSearchAction(TransportService transportService, this.nodeClient = nodeClient; this.reduceContextSupplier = () -> searchService.createReduceContext(true); this.searchAction = searchAction; - this.store = new AsyncSearchStoreService(taskManager, threadContext, client, registry); + this.store = new AsyncSearchIndexService(clusterService, threadContext, client, registry); } @Override @@ -75,79 +74,106 @@ protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionList return; } CancellableTask submitTask = (CancellableTask) task; - final String docID = UUIDs.randomBase64UUID(); - final Map originHeaders = nodeClient.threadPool().getThreadContext().getHeaders(); - final SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) { + final SearchRequest searchRequest = createSearchRequest(request, submitTask.getId(), request.getKeepAlive()); + AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); + searchAction.execute(searchTask, searchRequest, searchTask.getProgressListener()); + searchTask.addCompletionListener( + new ActionListener<>() { + @Override + public void onResponse(AsyncSearchResponse searchResponse) { + if (searchResponse.isRunning() || request.isCleanOnCompletion() == false) { + // the task is still running and the user cannot wait more so we create + // a document for further retrieval + try { + if (submitTask.isCancelled()) { + // the user cancelled the submit so we don't store anything + // and propagate the failure + Exception cause = new TaskCancelledException(submitTask.getReasonCancelled()); + onFatalFailure(searchTask, cause, false, submitListener); + } else { + final String docId = searchTask.getSearchId().getDocId(); + store.storeInitialResponse(docId, searchTask.getOriginHeaders(), searchResponse, + new ActionListener<>() { + @Override + public void onResponse(IndexResponse r) { + try { + // store the final response on completion unless the submit is cancelled + searchTask.addCompletionListener(finalResponse -> + onFinalResponse(submitTask, searchTask, finalResponse)); + } finally { + submitListener.onResponse(searchResponse); + } + } + + @Override + public void onFailure(Exception exc) { + onFatalFailure(searchTask, exc, searchResponse.isRunning(), submitListener); + } + }); + } + } catch (Exception exc) { + onFatalFailure(searchTask, exc, searchResponse.isRunning(), submitListener); + } + } else { + // the task completed within the timeout so the response is sent back to the user + // with a null id since nothing was stored on the cluster. + taskManager.unregister(searchTask); + submitListener.onResponse(searchResponse.clone(null)); + } + } + + @Override + public void onFailure(Exception exc) { + submitListener.onFailure(exc); + } + }, request.getWaitForCompletion()); + } + + private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, long parentTaskId, TimeValue keepAlive) { + String docID = UUIDs.randomBase64UUID(); + Map originHeaders = nodeClient.threadPool().getThreadContext().getHeaders(); + SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map taskHeaders) { AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id)); - return new AsyncSearchTask(id, type, action, parentTaskId, originHeaders, taskHeaders, searchId, - nodeClient.threadPool(), reduceContextSupplier); + return new AsyncSearchTask(id, type, action, parentTaskId, keepAlive, originHeaders, taskHeaders, searchId, + store.getClient(), nodeClient.threadPool(), reduceContextSupplier); } }; - searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), submitTask.getId())); + searchRequest.setParentTask(new TaskId(nodeClient.getLocalNodeId(), parentTaskId)); + return searchRequest; + } - // trigger the async search - AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); - searchAction.execute(searchTask, searchRequest, searchTask.getProgressListener()); - long expirationTime = System.currentTimeMillis() + request.getKeepAlive().getMillis(); - searchTask.setExpirationTime(expirationTime); - searchTask.addCompletionListener(searchResponse -> { - if (searchResponse.isRunning() || request.isCleanOnCompletion() == false) { - // the task is still running and the user cannot wait more so we create - // a document for further retrieval + private void onFatalFailure(AsyncSearchTask task, Exception error, boolean shouldCancel, ActionListener listener) { + if (shouldCancel) { + task.cancelTask(() -> { try { - if (submitTask.isCancelled()) { - // the user cancelled the submit so we don't store anything - // and propagate the failure - searchTask.addCompletionListener(finalResponse -> taskManager.unregister(searchTask)); - submitListener.onFailure(new ElasticsearchException(submitTask.getReasonCancelled())); - } else { - store.storeInitialResponse(originHeaders, docID, searchResponse, - new ActionListener<>() { - @Override - public void onResponse(IndexResponse r) { - // store the final response on completion unless the submit is cancelled - searchTask.addCompletionListener(finalResponse -> { - if (submitTask.isCancelled()) { - onTaskCompletion(submitTask, searchTask, () -> {}); - } else { - storeFinalResponse(submitTask, searchTask, finalResponse); - } - }); - submitListener.onResponse(searchResponse); - } - - @Override - public void onFailure(Exception exc) { - onTaskFailure(searchTask, exc.getMessage(), () -> { - searchTask.addCompletionListener(finalResponse -> taskManager.unregister(searchTask)); - submitListener.onFailure(exc); - }); - } - }); - } - } catch (Exception exc) { - onTaskFailure(searchTask, exc.getMessage(), () -> { - searchTask.addCompletionListener(finalResponse -> taskManager.unregister(searchTask)); - submitListener.onFailure(exc); - }); + task.addCompletionListener(finalResponse -> taskManager.unregister(task)); + } finally { + listener.onFailure(error); } - } else { - // the task completed within the timeout so the response is sent back to the user - // with a null id since nothing was stored on the cluster. - taskManager.unregister(searchTask); - submitListener.onResponse(searchResponse.clone(null)); + }); + } else { + try { + task.addCompletionListener(finalResponse -> taskManager.unregister(task)); + } finally { + listener.onFailure(error); } - }, request.getWaitForCompletion()); + } } - private void storeFinalResponse(CancellableTask submitTask, AsyncSearchTask searchTask, AsyncSearchResponse response) { + private void onFinalResponse(CancellableTask submitTask, AsyncSearchTask searchTask, AsyncSearchResponse response) { + if (submitTask.isCancelled() || searchTask.isCancelled()) { + // the user cancelled the submit so we ensure that there is nothing stored in the response index. + store.deleteResponse(searchTask.getSearchId(), ActionListener.wrap(() -> taskManager.unregister(searchTask))); + return; + } + try { - store.storeFinalResponse(response, new ActionListener<>() { + store.storeFinalResponse(searchTask.getSearchId().getDocId(), response, new ActionListener<>() { @Override public void onResponse(UpdateResponse updateResponse) { - onTaskCompletion(submitTask, searchTask, () -> {}); + taskManager.unregister(searchTask); } @Override @@ -156,34 +182,12 @@ public void onFailure(Exception exc) { logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", searchTask.getSearchId().getEncoded()), exc); } - onTaskCompletion(submitTask, searchTask, () -> {}); + taskManager.unregister(searchTask); } }); } catch (Exception exc) { logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", searchTask.getSearchId().getEncoded()), exc); - onTaskCompletion(submitTask, searchTask, () -> {}); - } - } - - private void onTaskFailure(AsyncSearchTask searchTask, String reason, Runnable onFinish) { - CancelTasksRequest req = new CancelTasksRequest() - .setTaskId(new TaskId(nodeClient.getLocalNodeId(), searchTask.getId())) - .setReason(reason); - // force the origin to execute the cancellation as a system user - new OriginSettingClient(nodeClient, TASKS_ORIGIN).admin().cluster().cancelTasks(req, ActionListener.wrap(() -> onFinish.run())); - } - - private void onTaskCompletion(CancellableTask submitTask, AsyncSearchTask searchTask, Runnable onFinish) { - if (submitTask.isCancelled()) { - // the user cancelled the submit so we ensure that there is nothing stored in the response index. - store.deleteResult(searchTask.getSearchId(), - ActionListener.wrap(() -> { - taskManager.unregister(searchTask); - onFinish.run(); - })); - } else { taskManager.unregister(searchTask); - onFinish.run(); } } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index c309ec60cbc0b..ce3df2d91dfbf 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -33,6 +33,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; +// TODO: add tests for keepAlive and expiration public class AsyncSearchActionTests extends AsyncSearchIntegTestCase { private String indexName; private int numShards; diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java new file mode 100644 index 0000000000000..afc7627b2e4df --- /dev/null +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIndexServiceTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.search; + +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.user.User; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; + +import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses; +import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse; +import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse; +import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; + +// TODO: test CRUD operations +public class AsyncSearchIndexServiceTests extends ESSingleNodeTestCase { + private AsyncSearchIndexService indexService; + + @Before + public void setup() { + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + TransportService transportService = getInstanceFromNode(TransportService.class); + indexService = new AsyncSearchIndexService(clusterService, transportService.getThreadPool().getThreadContext(), + client(), writableRegistry()); + } + + public void testEncodeSearchResponse() throws IOException { + for (int i = 0; i < 10; i++) { + AsyncSearchResponse response = randomAsyncSearchResponse(randomSearchId(), randomSearchResponse()); + String encoded = indexService.encodeResponse(response); + AsyncSearchResponse same = indexService.decodeResponse(encoded); + assertEqualResponses(response, same); + } + } + + public void testEnsuredAuthenticatedUserIsSame() throws IOException { + Authentication original = + new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null); + Authentication current = randomBoolean() ? original : + new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null); + assertTrue(indexService.ensureAuthenticatedUserIsSame(original, current)); + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + original.writeToContext(threadContext); + assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); + + // original is not set + assertTrue(indexService.ensureAuthenticatedUserIsSame(Collections.emptyMap(), current)); + // current is not set + assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), null)); + + // original user being run as + User user = new User(new User("test", "role"), new User("authenticated", "runas")); + current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), + new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node")); + assertTrue(indexService.ensureAuthenticatedUserIsSame(original, current)); + assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); + + // both user are run as + current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), + new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node")); + Authentication runAs = current; + assertTrue(indexService.ensureAuthenticatedUserIsSame(runAs, current)); + threadContext = new ThreadContext(Settings.EMPTY); + original.writeToContext(threadContext); + assertTrue(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); + + // different authenticated by type + Authentication differentRealmType = + new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", randomAlphaOfLength(5), "node"), null); + threadContext = new ThreadContext(Settings.EMPTY); + original.writeToContext(threadContext); + assertFalse(indexService.ensureAuthenticatedUserIsSame(original, differentRealmType)); + assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), differentRealmType)); + + // wrong user + Authentication differentUser = + new Authentication(new User("test2", "role"), new Authentication.RealmRef("realm", "realm", "node"), null); + assertFalse(indexService.ensureAuthenticatedUserIsSame(original, differentUser)); + + // run as different user + Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")), + new Authentication.RealmRef("realm", "file", "node1"), new Authentication.RealmRef("realm", "file", "node1")); + assertFalse(indexService.ensureAuthenticatedUserIsSame(original, diffRunAs)); + assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), diffRunAs)); + + // run as different looked up by type + Authentication runAsDiffType = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), + new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node")); + assertFalse(indexService.ensureAuthenticatedUserIsSame(original, runAsDiffType)); + assertFalse(indexService.ensureAuthenticatedUserIsSame(threadContext.getHeaders(), runAsDiffType)); + } +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index 532213f346a7f..659c3b140bea3 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.xpack.search; import org.apache.lucene.search.IndexSearcher; @@ -30,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; @@ -63,7 +63,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.search.AsyncSearchTemplateRegistry.ASYNC_SEARCH_ALIAS; +import static org.elasticsearch.xpack.search.AsyncSearchIndexService.INDEX; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -72,7 +72,8 @@ interface SearchResponseIterator extends Iterator, Closeabl @Override protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class, QueryBlockPlugin.class); + return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearchPlugin.class, IndexLifecycle.class, + QueryBlockPlugin.class, ReindexPlugin.class); } /** @@ -89,7 +90,7 @@ public Settings onNodeStopped(String nodeName) throws Exception { return super.onNodeStopped(nodeName); } }); - ensureYellow(ASYNC_SEARCH_ALIAS); + ensureYellow(INDEX); } protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request) throws ExecutionException, InterruptedException { @@ -97,8 +98,7 @@ protected AsyncSearchResponse submitAsyncSearch(SubmitAsyncSearchRequest request } protected AsyncSearchResponse getAsyncSearch(String id) throws ExecutionException, InterruptedException { - return client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(id, TimeValue.MINUS_ONE, -1)).get(); + return client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncSearchAction.Request(id)).get(); } protected AcknowledgedResponse deleteAsyncSearch(String id) throws ExecutionException, InterruptedException { @@ -112,7 +112,7 @@ protected void ensureTaskRemoval(String id) throws Exception { AsyncSearchId searchId = AsyncSearchId.decode(id); assertBusy(() -> { GetResponse resp = client().prepareGet() - .setIndex(ASYNC_SEARCH_ALIAS) + .setIndex(INDEX) .setId(searchId.getDocId()) .get(); assertFalse(resp.isExists()); @@ -159,7 +159,7 @@ protected SearchResponseIterator assertBlockingIterator(String indexName, .sorted(Comparator.comparing(ShardIdLatch::shard)) .toArray(ShardIdLatch[]::new); resetPluginsLatch(shardLatchMap); - request.source().query(new BlockQueryBuilder(shardLatchMap)); + request.getSearchRequest().source().query(new BlockQueryBuilder(shardLatchMap)); final AsyncSearchResponse initial = client().execute(SubmitAsyncSearchAction.INSTANCE, request).get(); @@ -208,8 +208,9 @@ private AsyncSearchResponse doNext() throws Exception { } assertBusy(() -> { AsyncSearchResponse newResp = client().execute(GetAsyncSearchAction.INSTANCE, - new GetAsyncSearchAction.Request(response.getId(), TimeValue.timeValueMillis(10), lastVersion) - ).get(); + new GetAsyncSearchAction.Request(response.getId()) + .setWaitForCompletion(TimeValue.timeValueMillis(10)) + .setLastVersion(lastVersion)).get(); atomic.set(newResp); assertNotEquals(lastVersion, newResp.getVersion()); }); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java deleted file mode 100644 index 9152fd09f39a9..0000000000000 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchStoreServiceTests.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.search; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.TriFunction; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.tasks.TaskManager; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.client.NoOpClient; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.security.authc.Authentication; -import org.elasticsearch.xpack.core.security.user.User; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -import static java.util.Collections.emptyList; -import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.assertEqualResponses; -import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomAsyncSearchResponse; -import static org.elasticsearch.xpack.search.AsyncSearchResponseTests.randomSearchResponse; -import static org.elasticsearch.xpack.search.AsyncSearchStoreService.ensureAuthenticatedUserIsSame; -import static org.elasticsearch.xpack.search.GetAsyncSearchRequestTests.randomSearchId; -import static org.mockito.Mockito.mock; - -public class AsyncSearchStoreServiceTests extends ESTestCase { - private NamedWriteableRegistry namedWriteableRegistry; - private ThreadPool threadPool; - private VerifyingClient client; - private AsyncSearchStoreService store; - - @Before - public void setup() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - List namedWriteables = searchModule.getNamedWriteables(); - namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); - threadPool = new TestThreadPool(this.getClass().getName()); - client = new VerifyingClient(threadPool); - TaskManager taskManager = mock(TaskManager.class); - store = new AsyncSearchStoreService(taskManager, threadPool.getThreadContext(), client, namedWriteableRegistry); - } - - @After - @Override - public void tearDown() throws Exception { - super.tearDown(); - threadPool.shutdownNow(); - } - - public void testEncode() throws IOException { - for (int i = 0; i < 10; i++) { - AsyncSearchResponse response = randomAsyncSearchResponse(randomSearchId(), randomSearchResponse()); - String encoded = AsyncSearchStoreService.encodeResponse(response); - AsyncSearchResponse same = AsyncSearchStoreService.decodeResponse(encoded, namedWriteableRegistry); - assertEqualResponses(response, same); - } - } - - public void testEnsuredAuthenticatedUserIsSame() throws IOException { - Authentication original = - new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null); - Authentication current = randomBoolean() ? original : - new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", "file", "node"), null); - assertTrue(ensureAuthenticatedUserIsSame(original, current)); - ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - original.writeToContext(threadContext); - assertTrue(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); - - // original is not set - assertTrue(ensureAuthenticatedUserIsSame(Collections.emptyMap(), current)); - // current is not set - assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), null)); - - // original user being run as - User user = new User(new User("test", "role"), new User("authenticated", "runas")); - current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), - new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node")); - assertTrue(ensureAuthenticatedUserIsSame(original, current)); - assertTrue(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); - - // both user are run as - current = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), - new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node")); - Authentication runAs = current; - assertTrue(ensureAuthenticatedUserIsSame(runAs, current)); - threadContext = new ThreadContext(Settings.EMPTY); - original.writeToContext(threadContext); - assertTrue(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), current)); - - // different authenticated by type - Authentication differentRealmType = - new Authentication(new User("test", "role"), new Authentication.RealmRef("realm", randomAlphaOfLength(5), "node"), null); - threadContext = new ThreadContext(Settings.EMPTY); - original.writeToContext(threadContext); - assertFalse(ensureAuthenticatedUserIsSame(original, differentRealmType)); - assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), differentRealmType)); - - // wrong user - Authentication differentUser = - new Authentication(new User("test2", "role"), new Authentication.RealmRef("realm", "realm", "node"), null); - assertFalse(ensureAuthenticatedUserIsSame(original, differentUser)); - - // run as different user - Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")), - new Authentication.RealmRef("realm", "file", "node1"), new Authentication.RealmRef("realm", "file", "node1")); - assertFalse(ensureAuthenticatedUserIsSame(original, diffRunAs)); - assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), diffRunAs)); - - // run as different looked up by type - Authentication runAsDiffType = new Authentication(user, new Authentication.RealmRef("realm", "file", "node"), - new Authentication.RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node")); - assertFalse(ensureAuthenticatedUserIsSame(original, runAsDiffType)); - assertFalse(ensureAuthenticatedUserIsSame(threadContext.getHeaders(), runAsDiffType)); - } - - /** - * A client that delegates to a verifying function for action/request/listener - */ - public static class VerifyingClient extends NoOpClient { - - private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { - fail("verifier not set"); - return null; - }; - - VerifyingClient(ThreadPool threadPool) { - super(threadPool); - } - - @Override - @SuppressWarnings("unchecked") - protected void doExecute(ActionType action, - Request request, - ActionListener listener) { - try { - listener.onResponse((Response) verifier.apply(action, request, listener)); - } catch (Exception e) { - listener.onFailure(e); - } - } - - public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { - this.verifier = verifier; - return this; - } - } -} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java index ca776aa525987..1f5f4c406db61 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.search; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchShard; import org.elasticsearch.action.search.ShardSearchFailure; @@ -16,8 +17,10 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.junit.After; import org.junit.Before; @@ -43,9 +46,9 @@ public void afterTest() { } public void testWaitForInit() throws InterruptedException { - AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1), Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)), - threadPool, null); + new NoOpClient(threadPool), threadPool, null); int numShards = randomIntBetween(0, 10); List shards = new ArrayList<>(); for (int i = 0; i < numShards; i++) { @@ -61,11 +64,20 @@ public void testWaitForInit() throws InterruptedException { int numThreads = randomIntBetween(1, 10); CountDownLatch latch = new CountDownLatch(numThreads); for (int i = 0; i < numThreads; i++) { - Thread thread = new Thread(() -> task.addCompletionListener(resp -> { - assertThat(numShards+numSkippedShards, equalTo(resp.getSearchResponse().getTotalShards())); - assertThat(numSkippedShards, equalTo(resp.getSearchResponse().getSkippedShards())); - assertThat(0, equalTo(resp.getSearchResponse().getFailedShards())); - latch.countDown(); + Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() { + @Override + public void onResponse(AsyncSearchResponse resp) { + assertThat(numShards + numSkippedShards, equalTo(resp.getSearchResponse().getTotalShards())); + assertThat(numSkippedShards, equalTo(resp.getSearchResponse().getSkippedShards())); + assertThat(0, equalTo(resp.getSearchResponse().getFailedShards())); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError(e); + + } }, TimeValue.timeValueMillis(1))); threads.add(thread); thread.start(); @@ -76,9 +88,9 @@ public void testWaitForInit() throws InterruptedException { } public void testWithFailure() throws InterruptedException { - AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1), Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)), - threadPool, null); + new NoOpClient(threadPool), threadPool, null); int numShards = randomIntBetween(0, 10); List shards = new ArrayList<>(); for (int i = 0; i < numShards; i++) { @@ -94,11 +106,19 @@ public void testWithFailure() throws InterruptedException { int numThreads = randomIntBetween(1, 10); CountDownLatch latch = new CountDownLatch(numThreads); for (int i = 0; i < numThreads; i++) { - Thread thread = new Thread(() -> task.addCompletionListener(resp -> { - assertNull(resp.getSearchResponse()); - assertNotNull(resp.getFailure()); - assertTrue(resp.isPartial()); - latch.countDown(); + Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() { + @Override + public void onResponse(AsyncSearchResponse resp) { + assertNull(resp.getSearchResponse()); + assertNotNull(resp.getFailure()); + assertTrue(resp.isPartial()); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError(e); + } }, TimeValue.timeValueMillis(1))); threads.add(thread); thread.start(); @@ -109,9 +129,9 @@ public void testWithFailure() throws InterruptedException { } public void testWaitForCompletion() throws InterruptedException { - AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), + AsyncSearchTask task = new AsyncSearchTask(0L, "", "", new TaskId("node1", 0), TimeValue.timeValueHours(1), Collections.emptyMap(), Collections.emptyMap(), new AsyncSearchId("0", new TaskId("node1", 1)), - threadPool, null); + new NoOpClient(threadPool), threadPool, null); int numShards = randomIntBetween(0, 10); List shards = new ArrayList<>(); for (int i = 0; i < numShards; i++) { @@ -155,12 +175,20 @@ private void assertCompletionListeners(AsyncSearchTask task, int numThreads = randomIntBetween(1, 10); CountDownLatch latch = new CountDownLatch(numThreads); for (int i = 0; i < numThreads; i++) { - Thread thread = new Thread(() -> task.addCompletionListener(resp -> { - assertThat(resp.getSearchResponse().getTotalShards(), equalTo(expectedTotalShards)); - assertThat(resp.getSearchResponse().getSkippedShards(), equalTo(expectedSkippedShards)); - assertThat(resp.getSearchResponse().getFailedShards(), equalTo(expectedShardFailures)); - assertThat(resp.isPartial(), equalTo(isPartial)); - latch.countDown(); + Thread thread = new Thread(() -> task.addCompletionListener(new ActionListener<>() { + @Override + public void onResponse(AsyncSearchResponse resp) { + assertThat(resp.getSearchResponse().getTotalShards(), equalTo(expectedTotalShards)); + assertThat(resp.getSearchResponse().getSkippedShards(), equalTo(expectedSkippedShards)); + assertThat(resp.getSearchResponse().getFailedShards(), equalTo(expectedShardFailures)); + assertThat(resp.isPartial(), equalTo(isPartial)); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError(e); + } }, TimeValue.timeValueMillis(1))); threads.add(thread); thread.start(); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java index 022fc7300ac3d..a985e219b18f8 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java @@ -20,8 +20,17 @@ protected Writeable.Reader instanceReader() { @Override protected GetAsyncSearchAction.Request createTestInstance() { - return new GetAsyncSearchAction.Request(randomSearchId(), TimeValue.timeValueMillis(randomIntBetween(1, 10000)), - randomIntBetween(-1, Integer.MAX_VALUE)); + GetAsyncSearchAction.Request req = new GetAsyncSearchAction.Request(randomSearchId()); + if (randomBoolean()) { + req.setWaitForCompletion(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); + } + if (randomBoolean()) { + req.setLastVersion(randomIntBetween(-1, Integer.MAX_VALUE)); + } + if (randomBoolean()) { + req.setKeepAlive(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); + } + return req; } static String randomSearchId() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 6d67f71443435..933af4d3b1662 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -27,16 +27,15 @@ public class AsyncSearchResponse extends ActionResponse implements StatusToXCont private final String id; private final int version; private final SearchResponse searchResponse; - private final ElasticsearchException failure; + private final ElasticsearchException error; private final boolean isRunning; private final boolean isPartial; private final long startTimeMillis; - private long expirationTimeMillis = -1; + private final long expirationTimeMillis; /** - * Creates an {@link AsyncSearchResponse} with meta informations that omits - * the search response. + * Creates an {@link AsyncSearchResponse} with meta-information only (not-modified). */ public AsyncSearchResponse(String id, int version, @@ -49,11 +48,12 @@ public AsyncSearchResponse(String id, /** * Creates a new {@link AsyncSearchResponse} + * * @param id The id of the search for further retrieval, null if not stored. * @param version The version number of this response. * @param searchResponse The actual search response. - * @param failure The actual failure if the search failed, null if the search is running - * or completed without failure. + * @param error The error if the search failed, null if the search is running + * or has completed without failure. * @param isPartial Whether the searchResponse contains partial results. * @param isRunning Whether the search is running in the cluster. * @param startTimeMillis The start date of the search in milliseconds since epoch. @@ -61,14 +61,14 @@ public AsyncSearchResponse(String id, public AsyncSearchResponse(String id, int version, SearchResponse searchResponse, - ElasticsearchException failure, + ElasticsearchException error, boolean isPartial, boolean isRunning, long startTimeMillis, long expirationTimeMillis) { this.id = id; this.version = version; - this.failure = failure; + this.error = error; this.searchResponse = searchResponse; this.isPartial = isPartial; this.isRunning = isRunning; @@ -79,7 +79,7 @@ public AsyncSearchResponse(String id, public AsyncSearchResponse(StreamInput in) throws IOException { this.id = in.readOptionalString(); this.version = in.readVInt(); - this.failure = in.readOptionalWriteable(ElasticsearchException::new); + this.error = in.readOptionalWriteable(ElasticsearchException::new); this.searchResponse = in.readOptionalWriteable(SearchResponse::new); this.isPartial = in.readBoolean(); this.isRunning = in.readBoolean(); @@ -91,7 +91,7 @@ public AsyncSearchResponse(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeVInt(version); - out.writeOptionalWriteable(failure); + out.writeOptionalWriteable(error); out.writeOptionalWriteable(searchResponse); out.writeBoolean(isPartial); out.writeBoolean(isRunning); @@ -100,7 +100,7 @@ public void writeTo(StreamOutput out) throws IOException { } public AsyncSearchResponse clone(String id) { - return new AsyncSearchResponse(id, version, searchResponse, failure, isPartial, isRunning, startTimeMillis, expirationTimeMillis); + return new AsyncSearchResponse(id, version, searchResponse, error, isPartial, isRunning, startTimeMillis, expirationTimeMillis); } /** @@ -120,6 +120,7 @@ public int getVersion() { /** * Returns the current {@link SearchResponse} or null if not available. + * * See {@link #isPartial()} to determine whether the response contains partial or complete * results. */ @@ -128,10 +129,10 @@ public SearchResponse getSearchResponse() { } /** - * Returns the failure reason or null if the query is running or completed normally. + * Returns the failure reason or null if the query is running or has completed normally. */ public ElasticsearchException getFailure() { - return failure; + return error; } /** @@ -142,6 +143,18 @@ public boolean isPartial() { return isPartial; } + /** + * Whether the search is still running in the cluster. + * + * A value of false indicates that the response is final + * even if {@link #isPartial()} returns true. In such case, + * the partial response represents the status of the search before a + * non-recoverable failure. + */ + public boolean isRunning() { + return isRunning; + } + /** * When this response was created as a timestamp in milliseconds since epoch. */ @@ -150,16 +163,8 @@ public long getStartTime() { } /** - * Whether the search is still running in the cluster. - * A value of false indicates that the response is final even - * if it contains partial results. In such case the failure should indicate - * why the request could not finish and the search response represents the - * last partial results before the failure. + * When this response will expired as a timestamp in milliseconds since epoch. */ - public boolean isRunning() { - return isRunning; - } - public long getExpirationTime() { return expirationTimeMillis; } @@ -170,7 +175,7 @@ public RestStatus status() { // shard failures are not considered fatal for partial results so // we return OK until we get the final response even if we don't have // a single successful shard. - return failure != null ? failure.status() : OK; + return error != null ? error.status() : OK; } else { return searchResponse.status(); } @@ -189,9 +194,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("expiration_time_in_millis", expirationTimeMillis); builder.field("response", searchResponse); - if (failure != null) { - builder.startObject("failure"); - failure.toXContent(builder, params); + if (error != null) { + builder.startObject("error"); + error.toXContent(builder, params); builder.endObject(); } builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java index 7ff1deef17172..254fd2f77b9e1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.rest.RestStatus; import java.io.IOException; import java.util.Objects; @@ -35,26 +34,24 @@ public Writeable.Reader getResponseReader() { public static class Request extends ActionRequest { private final String id; - private final int lastVersion; - private final TimeValue waitForCompletion; + private int lastVersion = -1; + private TimeValue waitForCompletion = TimeValue.MINUS_ONE; private TimeValue keepAlive = TimeValue.MINUS_ONE; /** - * Create a new request + * Creates a new request + * * @param id The id of the search progress request. - * @param waitForCompletion The minimum time that the request should wait before returning a partial result. - * @param lastVersion The last version returned by a previous call. */ - public Request(String id, TimeValue waitForCompletion, int lastVersion) { + public Request(String id) { this.id = id; - this.waitForCompletion = waitForCompletion; - this.lastVersion = lastVersion; } public Request(StreamInput in) throws IOException { super(in); this.id = in.readString(); this.waitForCompletion = TimeValue.timeValueMillis(in.readLong()); + this.keepAlive = in.readTimeValue(); this.lastVersion = in.readInt(); } @@ -63,45 +60,61 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeLong(waitForCompletion.millis()); + out.writeTimeValue(keepAlive); out.writeInt(lastVersion); } @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; - if (keepAlive != TimeValue.MINUS_ONE && keepAlive.getMillis() < MIN_KEEP_ALIVE) { + if (keepAlive.getMillis() != -1 && keepAlive.getMillis() < MIN_KEEP_ALIVE) { validationException = addValidationError("keep_alive must be greater than 1 minute, got:" + keepAlive.toString(), validationException); } return validationException; } + /** + * Returns the id of the async search. + */ public String getId() { return id; } /** - * Return the version of the previously retrieved {@link AsyncSearchResponse}. - * Partial hits and aggs are not included in the new response if they match this - * version and the request returns {@link RestStatus#NOT_MODIFIED}. + * Omits the result from the response if the new version is greater than the provided version (not-modified). */ + public Request setLastVersion(int version) { + this.lastVersion = version; + return this; + } + public int getLastVersion() { return lastVersion; } /** - * Return the minimum time that the request should wait for completion. + * Sets the minimum time that the request should wait before returning a partial result (defaults to no wait). */ + public Request setWaitForCompletion(TimeValue timeValue) { + this.waitForCompletion = timeValue; + return this; + } + public TimeValue getWaitForCompletion() { return waitForCompletion; } - public TimeValue getKeepAlive() { - return keepAlive; + /** + * Extends the amount of time after which the result will expire (defaults to no extension). + */ + public Request setKeepAlive(TimeValue timeValue) { + this.keepAlive = timeValue; + return this; } - public void setKeepAlive(TimeValue keepAlive) { - this.keepAlive = keepAlive; + public TimeValue getKeepAlive() { + return keepAlive; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index d88ca457784b7..85aeb4e7bf232 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -57,38 +57,48 @@ public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) { public SubmitAsyncSearchRequest(StreamInput in) throws IOException { this.request = new SearchRequest(in); this.waitForCompletion = in.readTimeValue(); - this.cleanOnCompletion = in.readBoolean(); this.keepAlive = in.readTimeValue(); + this.cleanOnCompletion = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { request.writeTo(out); out.writeTimeValue(waitForCompletion); - out.writeBoolean(cleanOnCompletion); out.writeTimeValue(keepAlive); + out.writeBoolean(cleanOnCompletion); } - public SearchRequest getSearchRequest() { - return request; + /** + * Sets the number of shard results that should be returned to notify search progress (default to 5). + */ + public SubmitAsyncSearchRequest setBatchedReduceSize(int size) { + request.setBatchedReduceSize(size); + return this; + } + + public int getBatchReduceSize() { + return request.getBatchedReduceSize(); } /** - * Sets the minimum time that the request should wait before returning a partial result. + * Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second). */ - public void setWaitForCompletion(TimeValue waitForCompletion) { + public SubmitAsyncSearchRequest setWaitForCompletion(TimeValue waitForCompletion) { this.waitForCompletion = waitForCompletion; + return this; } - /** - * Returns the minimum time that the request should wait before returning a partial result. - */ public TimeValue getWaitForCompletion() { return waitForCompletion; } - public void setKeepAlive(TimeValue keepAlive) { + /** + * Sets the amount of time after which the result will expire (defaults to 5 days). + */ + public SubmitAsyncSearchRequest setKeepAlive(TimeValue keepAlive) { this.keepAlive = keepAlive; + return this; } public TimeValue getKeepAlive() { @@ -96,22 +106,22 @@ public TimeValue getKeepAlive() { } /** - * Should the resource be removed on completion or failure. + * Returns the underlying {@link SearchRequest}. */ - public boolean isCleanOnCompletion() { - return cleanOnCompletion; + public SearchRequest getSearchRequest() { + return request; } - public void setCleanOnCompletion(boolean value) { + /** + * Should the resource be removed on completion or failure (defaults to true). + */ + public SubmitAsyncSearchRequest setCleanOnCompletion(boolean value) { this.cleanOnCompletion = value; + return this; } - public void setBatchedReduceSize(int size) { - request.setBatchedReduceSize(size); - } - - public SearchSourceBuilder source() { - return request.source(); + public boolean isCleanOnCompletion() { + return cleanOnCompletion; } @Override diff --git a/x-pack/plugin/core/src/main/resources/async-search.json b/x-pack/plugin/core/src/main/resources/async-search.json deleted file mode 100644 index 4f4c67d8f4dc6..0000000000000 --- a/x-pack/plugin/core/src/main/resources/async-search.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "index_patterns": [ - ".async-search-${xpack.async-search.template.version}*" - ], - "order": 2147483647, - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 1, - "index.format": 1 - }, - "mappings": { - "_doc": { - "dynamic": "strict", - "properties": { - "headers": { - "type": "object", - "enabled": false - }, - "is_running": { - "type": "boolean" - }, - "expiration_time":{ - "type": "long" - }, - "result": { - "type": "object", - "enabled": false - } - } - } - } -} From 05d9638128523ef37674ebf6a54ab3eac96c1860 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 6 Feb 2020 11:37:15 +0100 Subject: [PATCH 48/61] small fix after merging master --- .../action/search/SearchPhaseController.java | 4 ++-- x-pack/plugin/async-search/build.gradle | 2 +- ...syncSearchPlugin.java => AsyncSearch.java} | 20 ++++++++++++------- .../search/AsyncSearchMaintenanceService.java | 4 +--- .../search/TransportGetAsyncSearchAction.java | 4 +++- .../search/AsyncSearchIntegTestCase.java | 2 +- 6 files changed, 21 insertions(+), 15 deletions(-) rename x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/{AsyncSearchPlugin.java => AsyncSearch.java} (81%) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 5193629baa29f..f26b0fc80ccbe 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -664,9 +664,9 @@ private synchronized void consumeInternal(QuerySearchResult querySearchResult) { } numReducePhases++; index = 1; - if (hasAggs) { + if (hasAggs || hasTopDocs) { progressListener.notifyPartialReduce(progressListener.searchShards(processedShards), - topDocsStats.getTotalHits(), aggsBuffer[0], numReducePhases); + topDocsStats.getTotalHits(), hasAggs ? aggsBuffer[0] : null, numReducePhases); } } final int i = index++; diff --git a/x-pack/plugin/async-search/build.gradle b/x-pack/plugin/async-search/build.gradle index 484769069c984..8aac2b9f885ac 100644 --- a/x-pack/plugin/async-search/build.gradle +++ b/x-pack/plugin/async-search/build.gradle @@ -10,7 +10,7 @@ apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack-async-search' description 'A module which allows to track the progress of a search asynchronously.' - classname 'org.elasticsearch.xpack.search.AsyncSearchPlugin' + classname 'org.elasticsearch.xpack.search.AsyncSearch' extendedPlugins = ['x-pack-core'] } archivesBaseName = 'x-pack-async-search' diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchPlugin.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java similarity index 81% rename from x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchPlugin.java rename to x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index 096ecb5947531..63a0df9515a48 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchPlugin.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -37,7 +38,7 @@ import java.util.List; import java.util.function.Supplier; -public final class AsyncSearchPlugin extends Plugin implements ActionPlugin { +public final class AsyncSearch extends Plugin implements ActionPlugin { @Override public List> getActions() { return Arrays.asList( @@ -69,11 +70,16 @@ public Collection createComponents(Client client, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { - AsyncSearchIndexService indexService = - new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, namedWriteableRegistry); - AsyncSearchMaintenanceService maintenanceService = - new AsyncSearchMaintenanceService(nodeEnvironment.nodeId(), threadPool, indexService, TimeValue.timeValueHours(1)); - clusterService.addListener(maintenanceService); - return Collections.singletonList(maintenanceService); + if (DiscoveryNode.isDataNode(environment.settings())) { + // only data nodes should be eligible to run the maintenance service. + AsyncSearchIndexService indexService = + new AsyncSearchIndexService(clusterService, threadPool.getThreadContext(), client, namedWriteableRegistry); + AsyncSearchMaintenanceService maintenanceService = + new AsyncSearchMaintenanceService(nodeEnvironment.nodeId(), threadPool, indexService, TimeValue.timeValueHours(1)); + clusterService.addListener(maintenanceService); + return Collections.singletonList(maintenanceService); + } else { + return Collections.emptyList(); + } } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java index 0a0a78ed8b44f..1cac9c0eaf458 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchMaintenanceService.java @@ -59,9 +59,7 @@ public void clusterChanged(ClusterChangedEvent event) { // Wait until the gateway has recovered from disk. return; } - if (state.nodes().getLocalNode().isDataNode()) { - tryStartCleanup(state); - } + tryStartCleanup(state); } void tryStartCleanup(ClusterState state) { diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 5606f33282ff7..792d3c0b1b90a 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -69,7 +69,6 @@ protected void doExecute(Task task, GetAsyncSearchAction.Request request, Action } } else { TransportRequestOptions.Builder builder = TransportRequestOptions.builder(); - request.setKeepAlive(TimeValue.MINUS_ONE); transportService.sendRequest(node, GetAsyncSearchAction.NAME, request, builder.build(), new ActionListenerResponseHandler<>(listener, AsyncSearchResponse::new, ThreadPool.Names.SAME)); } @@ -135,10 +134,13 @@ private void sendFinalResponse(GetAsyncSearchAction.Request request, AsyncSearchResponse response, long nowInMillis, ActionListener listener) { + // check if the result has expired if (response.getExpirationTime() < nowInMillis) { listener.onFailure(new ResourceNotFoundException(request.getId())); return; } + + // check last version if (response.getVersion() <= request.getLastVersion()) { // return a not-modified response listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index 659c3b140bea3..41a81eb7dd671 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -72,7 +72,7 @@ interface SearchResponseIterator extends Iterator, Closeabl @Override protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearchPlugin.class, IndexLifecycle.class, + return Arrays.asList(LocalStateCompositeXPackPlugin.class, AsyncSearch.class, IndexLifecycle.class, QueryBlockPlugin.class, ReindexPlugin.class); } From e0e2072158df4850c9fafea961e7e12d8d8a9dfc Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 6 Feb 2020 11:38:46 +0100 Subject: [PATCH 49/61] checkstyle --- .../xpack/search/TransportGetAsyncSearchAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index 792d3c0b1b90a..fa4101ab86661 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; From 33f921e974ac26c98dd492fb1dad0be0af816231 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 6 Feb 2020 12:18:12 +0100 Subject: [PATCH 50/61] restore test --- .../org/elasticsearch/xpack/search/AsyncSearchActionTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index ce3df2d91dfbf..8426bfa3ae7a7 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -122,7 +122,7 @@ public void testMaxMinAggregation() throws Exception { public void testTermsAggregation() throws Exception { int step = numShards > 2 ? randomIntBetween(2, numShards) : 2; - int numFailures = 0;//randomBoolean() ? randomIntBetween(0, numShards) : 0; + int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0; int termsSize = randomIntBetween(1, numKeywords); SearchSourceBuilder source = new SearchSourceBuilder() .aggregation(AggregationBuilders.terms("terms").field("terms.keyword").size(termsSize).shardSize(termsSize*2)); From b903d0e9cf34a0f07bc54c808e71d807e1838190 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 6 Feb 2020 12:37:40 +0100 Subject: [PATCH 51/61] fix more test --- .../xpack/search/AsyncSearchActionTests.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index 8426bfa3ae7a7..df648b662b400 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -21,9 +21,12 @@ import org.junit.Before; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; @@ -52,10 +55,12 @@ public void indexDocuments() throws InterruptedException { createIndex(indexName, Settings.builder().put("index.number_of_shards", numShards).build()); numKeywords = randomIntBetween(1, 100); keywordFreqs = new HashMap<>(); - String[] keywords = new String[numKeywords]; + Set keywordSet = new HashSet<>(); for (int i = 0; i < numKeywords; i++) { - keywords[i] = randomAlphaOfLengthBetween(10, 20); + keywordSet.add(randomAlphaOfLengthBetween(10, 20)); } + numKeywords = keywordSet.size(); + String[] keywords = keywordSet.toArray(String[]::new); List reqs = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { float metric = randomFloat(); @@ -123,9 +128,8 @@ public void testMaxMinAggregation() throws Exception { public void testTermsAggregation() throws Exception { int step = numShards > 2 ? randomIntBetween(2, numShards) : 2; int numFailures = randomBoolean() ? randomIntBetween(0, numShards) : 0; - int termsSize = randomIntBetween(1, numKeywords); SearchSourceBuilder source = new SearchSourceBuilder() - .aggregation(AggregationBuilders.terms("terms").field("terms.keyword").size(termsSize).shardSize(termsSize*2)); + .aggregation(AggregationBuilders.terms("terms").field("terms.keyword").size(numKeywords)); try (SearchResponseIterator it = assertBlockingIterator(indexName, source, numFailures, step)) { AsyncSearchResponse response = it.next(); @@ -137,7 +141,7 @@ public void testTermsAggregation() throws Exception { assertNotNull(response.getSearchResponse().getAggregations().get("terms")); StringTerms terms = response.getSearchResponse().getAggregations().get("terms"); assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0)); - assertThat(terms.getBuckets().size(), lessThanOrEqualTo(termsSize)); + assertThat(terms.getBuckets().size(), lessThanOrEqualTo(numKeywords)); for (InternalTerms.Bucket bucket : terms.getBuckets()) { long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get(); assertThat(bucket.getDocCount(), lessThanOrEqualTo(count)); @@ -152,7 +156,7 @@ public void testTermsAggregation() throws Exception { assertNotNull(response.getSearchResponse().getAggregations().get("terms")); StringTerms terms = response.getSearchResponse().getAggregations().get("terms"); assertThat(terms.getBuckets().size(), greaterThanOrEqualTo(0)); - assertThat(terms.getBuckets().size(), lessThanOrEqualTo(termsSize)); + assertThat(terms.getBuckets().size(), lessThanOrEqualTo(numKeywords)); for (InternalTerms.Bucket bucket : terms.getBuckets()) { long count = keywordFreqs.getOrDefault(bucket.getKeyAsString(), new AtomicInteger(0)).get(); if (numFailures > 0) { From 761274820e18a4d8cf9307a8624f717d8737b1ae Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 6 Feb 2020 12:43:59 +0100 Subject: [PATCH 52/61] iter --- .../org/elasticsearch/xpack/search/AsyncSearchActionTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java index df648b662b400..b7787a68c2c4a 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchActionTests.java @@ -21,7 +21,6 @@ import org.junit.Before; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; From e9b7addbdcab09f6198d68818e06a5645650cbc8 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 5 Mar 2020 14:39:15 +0100 Subject: [PATCH 53/61] address review and plug the new async_search origin --- .../action/search/SearchRequest.java | 2 +- .../action/search/SearchResponse.java | 5 ----- .../support/tasks/TransportTasksAction.java | 7 +------ .../SearchProgressActionListenerIT.java | 4 ++-- .../test/rest/yaml/section/DoSection.java | 1 - x-pack/plugin/async-search/qa/build.gradle | 9 --------- .../plugin/async-search/qa/rest/build.gradle | 6 ++++++ .../xpack/search/AsyncSearch.java | 15 ++++++++++---- .../xpack/search/AsyncSearchIndexService.java | 20 ++++++++++--------- .../xpack/search/AsyncSearchTask.java | 4 ++-- .../search/RestDeleteAsyncSearchAction.java | 13 ++++++++---- .../search/RestGetAsyncSearchAction.java | 13 ++++++++---- .../search/RestSubmitAsyncSearchAction.java | 17 ++++++++++------ .../TransportSubmitAsyncSearchAction.java | 13 ++---------- .../index/RestrictedIndicesNames.java | 2 +- .../xpack/core/security/user/XPackUser.java | 10 ++-------- 16 files changed, 68 insertions(+), 73 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index a719afbdb5ee0..55ecfffcf564b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -560,7 +560,7 @@ public boolean isSuggestOnly() { } @Override - public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { // generating description in a lazy way since source can be quite big return new SearchTask(id, type, action, null, parentTaskId, headers) { @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index fa38ea63bff24..81d61f2996ef4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -108,11 +108,6 @@ public SearchResponse(SearchResponseSections internalResponse, String scrollId, assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards; } - public SearchResponse(SearchResponse clone) { - this(clone.internalResponse, clone.scrollId, clone.totalShards, clone.successfulShards, clone.skippedShards, - clone.tookInMillis, clone.shardFailures, clone.clusters); - } - @Override public RestStatus status() { return RestStatus.status(successfulShards, totalShards, shardFailures); diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index 7cdc9d331ab3a..a20e72d853af5 100644 --- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -99,12 +99,7 @@ protected void doExecute(Task task, TasksRequest request, ActionListener listener) { TasksRequest request = nodeTaskRequest.tasksRequest; List tasks = new ArrayList<>(); - try { - processTasks(request, tasks::add); - } catch (Exception exc) { - listener.onFailure(exc); - return; - } + processTasks(request, tasks::add); if (tasks.isEmpty()) { listener.onResponse(new NodeTasksResponse(clusterService.localNode().getId(), emptyList(), emptyList())); return; diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 427aee8585db2..8da91748038f7 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -196,8 +196,8 @@ public void onFailure(Exception e) { }; client.executeLocally(SearchAction.INSTANCE, new SearchRequest(request) { @Override - public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - SearchTask task = (SearchTask) super.createTask(id, type, action, parentTaskId, headers); + public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + SearchTask task = super.createTask(id, type, action, parentTaskId, headers); task.setProgressListener(listener); return task; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index fa669ec70a7ab..1b588f554fa53 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -345,7 +345,6 @@ private String formatStatusCodeMessage(ClientYamlTestResponse restTestResponse, private static Map>> catches = new HashMap<>(); static { - catches.put("not_modified", tuple("304", equalTo(304))); catches.put("bad_request", tuple("400", equalTo(400))); catches.put("unauthorized", tuple("401", equalTo(401))); catches.put("forbidden", tuple("403", equalTo(403))); diff --git a/x-pack/plugin/async-search/qa/build.gradle b/x-pack/plugin/async-search/qa/build.gradle index d3e95d997c3fb..79ff4091f6d2d 100644 --- a/x-pack/plugin/async-search/qa/build.gradle +++ b/x-pack/plugin/async-search/qa/build.gradle @@ -6,12 +6,3 @@ test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle index 39bfb37b23877..fbe97dcb7aba5 100644 --- a/x-pack/plugin/async-search/qa/rest/build.gradle +++ b/x-pack/plugin/async-search/qa/rest/build.gradle @@ -3,6 +3,12 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' +restResources { + restApi { + includeXpack 'async_search' + } +} + dependencies { testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackModule('async-search'), configuration: 'runtime') diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java index 63a0df9515a48..580f7d8a5560a 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearch.java @@ -39,6 +39,12 @@ import java.util.function.Supplier; public final class AsyncSearch extends Plugin implements ActionPlugin { + private final Settings settings; + + public AsyncSearch(Settings settings) { + this.settings = settings; + } + @Override public List> getActions() { return Arrays.asList( @@ -54,9 +60,9 @@ public List getRestHandlers(Settings settings, RestController restC IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster) { return Arrays.asList( - new RestSubmitAsyncSearchAction(restController), - new RestGetAsyncSearchAction(restController), - new RestDeleteAsyncSearchAction(restController) + new RestSubmitAsyncSearchAction(), + new RestGetAsyncSearchAction(), + new RestDeleteAsyncSearchAction() ); } @@ -69,7 +75,8 @@ public Collection createComponents(Client client, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry) { + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver) { if (DiscoveryNode.isDataNode(environment.settings())) { // only data nodes should be eligible to run the maintenance service. AsyncSearchIndexService indexService = diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java index 718f1ff009ace..b48ad768e1207 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchIndexService.java @@ -39,6 +39,8 @@ import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; +import org.elasticsearch.xpack.core.security.SecurityContext; import java.io.IOException; import java.nio.ByteBuffer; @@ -47,9 +49,9 @@ import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; +import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY; /** @@ -98,8 +100,8 @@ public static XContentBuilder mappings() throws IOException { } private final ClusterService clusterService; - private final ThreadContext threadContext; private final Client client; + private final SecurityContext securityContext; private final NamedWriteableRegistry registry; AsyncSearchIndexService(ClusterService clusterService, @@ -107,8 +109,8 @@ public static XContentBuilder mappings() throws IOException { Client client, NamedWriteableRegistry registry) { this.clusterService = clusterService; - this.threadContext = threadContext; - this.client = new OriginSettingClient(client, TASKS_ORIGIN); + this.securityContext = new SecurityContext(clusterService.getSettings(), threadContext); + this.client = new OriginSettingClient(client, ASYNC_SEARCH_ORIGIN); this.registry = registry; } @@ -160,6 +162,7 @@ void storeInitialResponse(String docId, source.put(EXPIRATION_TIME_FIELD, response.getExpirationTime()); source.put(RESULT_FIELD, encodeResponse(response)); IndexRequest indexRequest = new IndexRequest(INDEX) + .create(true) .id(docId) .source(source, XContentType.JSON); createIndexIfNecessary(ActionListener.wrap(v -> client.index(indexRequest, listener), listener::onFailure)); @@ -226,7 +229,7 @@ void deleteResponse(AsyncSearchId searchId, */ AsyncSearchTask getTask(TaskManager taskManager, AsyncSearchId searchId) throws IOException { Task task = taskManager.getTask(searchId.getTaskId().getId()); - if (task == null || task instanceof AsyncSearchTask == false) { + if (task instanceof AsyncSearchTask == false) { return null; } AsyncSearchTask searchTask = (AsyncSearchTask) task; @@ -235,7 +238,7 @@ AsyncSearchTask getTask(TaskManager taskManager, AsyncSearchId searchId) throws } // Check authentication for the user - final Authentication auth = Authentication.getAuthentication(threadContext); + final Authentication auth = securityContext.getAuthentication(); if (ensureAuthenticatedUserIsSame(searchTask.getOriginHeaders(), auth) == false) { throw new ResourceNotFoundException(searchId.getEncoded() + " not found"); } @@ -248,7 +251,7 @@ AsyncSearchTask getTask(TaskManager taskManager, AsyncSearchId searchId) throws */ void getResponse(AsyncSearchId searchId, ActionListener listener) { - final Authentication current = Authentication.getAuthentication(client.threadPool().getThreadContext()); + final Authentication current = securityContext.getAuthentication(); GetRequest internalGet = new GetRequest(INDEX) .preference(searchId.getEncoded()) .id(searchId.getDocId()); @@ -267,7 +270,6 @@ void getResponse(AsyncSearchId searchId, return; } - @SuppressWarnings("unchecked") String encoded = (String) get.getSource().get(RESULT_FIELD); listener.onResponse(encoded != null ? decodeResponse(encoded) : null); }, @@ -289,7 +291,7 @@ boolean ensureAuthenticatedUserIsSame(Map originHeaders, Authent // origin is an authenticated user but current is not return false; } - Authentication origin = Authentication.decode(originHeaders.get(AUTHENTICATION_KEY)); + Authentication origin = AuthenticationContextSerializer.decode(originHeaders.get(AUTHENTICATION_KEY)); return ensureAuthenticatedUserIsSame(origin, current); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java index 8f927f3218dfa..3a1b4f63f271d 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/AsyncSearchTask.java @@ -136,7 +136,7 @@ public void onResponse(CancelTasksResponse cancelTasksResponse) { } @Override - public void onFailure(Exception e) { + public void onFailure(Exception exc) { // cancelling failed isCancelling.compareAndSet(true, false); runnable.run(); @@ -144,7 +144,7 @@ public void onFailure(Exception e) { }); } else { runnable.run(); - } + } } @Override diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java index fd8dc3aefee02..faab51dc3af73 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestDeleteAsyncSearchAction.java @@ -7,19 +7,24 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler.Route; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.search.action.DeleteAsyncSearchAction; + import java.io.IOException; +import java.util.List; +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.DELETE; public class RestDeleteAsyncSearchAction extends BaseRestHandler { - - public RestDeleteAsyncSearchAction(RestController controller) { - controller.registerHandler(DELETE, "/_async_search/{id}", this); + @Override + public List routes() { + return unmodifiableList(asList( + new Route(DELETE, "/_async_search/{id}"))); } @Override diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index 7f2cba4164d1e..ef832e0ede569 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -8,19 +8,24 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler.Route; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestStatusToXContentListener; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; +import java.util.List; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestGetAsyncSearchAction extends BaseRestHandler { - - public RestGetAsyncSearchAction(RestController controller) { - controller.registerHandler(GET, "/_async_search/{id}", this); + @Override + public List routes() { + return unmodifiableList(asList(new Route(GET, "/_async_search/{id}"))); } + @Override public String getName() { return "async_search_get_action"; diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java index 41cf2b57c9da4..01c792b258598 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -8,7 +8,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler.Route; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestStatusToXContentListener; @@ -18,17 +18,22 @@ import java.io.IOException; import java.util.function.IntConsumer; +import java.util.List; +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.action.search.RestSearchAction.parseSearchRequest; public final class RestSubmitAsyncSearchAction extends BaseRestHandler { - RestSubmitAsyncSearchAction(RestController controller) { - controller.registerHandler(POST, "/_async_search", this); - controller.registerHandler(GET, "/_async_search", this); - controller.registerHandler(POST, "/{index}/_async_search", this); - controller.registerHandler(GET, "/{index}/_async_search", this); + @Override + public List routes() { + return unmodifiableList(asList( + new Route(POST, "/_async_search"), + new Route(GET, "/_async_search"), + new Route(POST, "/{index}/_async_search"), + new Route(GET, "/{index}/_async_search"))); } @Override diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index 025b9638cf7fa..e8bcb848db5a3 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; @@ -24,7 +23,6 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.engine.DocumentMissingException; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; @@ -44,7 +42,6 @@ public class TransportSubmitAsyncSearchAction extends HandledTransportAction reduceContextSupplier; private final TransportSearchAction searchAction; private final AsyncSearchIndexService store; @@ -59,20 +56,14 @@ public TransportSubmitAsyncSearchAction(ClusterService clusterService, SearchService searchService, TransportSearchAction searchAction) { super(SubmitAsyncSearchAction.NAME, transportService, actionFilters, SubmitAsyncSearchRequest::new); - this.threadContext= transportService.getThreadPool().getThreadContext(); this.nodeClient = nodeClient; this.reduceContextSupplier = () -> searchService.createReduceContext(true); this.searchAction = searchAction; - this.store = new AsyncSearchIndexService(clusterService, threadContext, client, registry); + this.store = new AsyncSearchIndexService(clusterService, transportService.getThreadPool().getThreadContext(), client, registry); } @Override protected void doExecute(Task task, SubmitAsyncSearchRequest request, ActionListener submitListener) { - ActionRequestValidationException errors = request.validate(); - if (errors != null) { - submitListener.onFailure(errors); - return; - } CancellableTask submitTask = (CancellableTask) task; final SearchRequest searchRequest = createSearchRequest(request, submitTask.getId(), request.getKeepAlive()); AsyncSearchTask searchTask = (AsyncSearchTask) taskManager.register("transport", SearchAction.INSTANCE.name(), searchRequest); @@ -134,7 +125,7 @@ private SearchRequest createSearchRequest(SubmitAsyncSearchRequest request, long Map originHeaders = nodeClient.threadPool().getThreadContext().getHeaders(); SearchRequest searchRequest = new SearchRequest(request.getSearchRequest()) { @Override - public Task createTask(long id, String type, String action, TaskId parentTaskId, Map taskHeaders) { + public AsyncSearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map taskHeaders) { AsyncSearchId searchId = new AsyncSearchId(docID, new TaskId(nodeClient.getLocalNodeId(), id)); return new AsyncSearchTask(id, type, action, parentTaskId, keepAlive, originHeaders, taskHeaders, searchId, store.getClient(), nodeClient.threadPool(), reduceContextSupplier); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java index 77f6c537b6f81..2d7f660bd188e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java @@ -23,7 +23,7 @@ public final class RestrictedIndicesNames { public static final String SECURITY_TOKENS_ALIAS = ".security-tokens"; // public for tests - public static final String ASYNC_SEARCH_PREFIX = ".async-search-"; + public static final String ASYNC_SEARCH_PREFIX = ".async-search"; private static final Automaton ASYNC_SEARCH_AUTOMATON = Automatons.patterns(ASYNC_SEARCH_PREFIX + "*"); // public for tests diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java index 5196f8e663e00..38c9fe84aa934 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java @@ -19,15 +19,9 @@ public class XPackUser extends User { public static final String ROLE_NAME = UsernamesField.XPACK_ROLE; public static final Role ROLE = Role.builder(new RoleDescriptor(ROLE_NAME, new String[] { "all" }, new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder() - .indices("/@&~(\\.security.*)/") - .privileges("all") - // true for async-search indices - .allowRestrictedIndices(true) - .build(), + RoleDescriptor.IndicesPrivileges.builder().indices("/@&~(\\.security.*)/").privileges("all").build(), RoleDescriptor.IndicesPrivileges.builder().indices(IndexAuditTrailField.INDEX_NAME_PREFIX + "-*") - .privileges("read") - .build() + .privileges("read").build() }, new String[] { "*" }, MetadataUtils.DEFAULT_RESERVED_METADATA), null).build(); From 3e8ba5896af1ab7d6d1fd031f80a2a8f44752c65 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 5 Mar 2020 19:59:30 +0100 Subject: [PATCH 54/61] unused import --- .../main/java/org/elasticsearch/action/search/SearchRequest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 55ecfffcf564b..c8b8981e3218d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -33,7 +33,6 @@ import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import java.io.IOException; From cd6f3f202c999665d02e9ff47891a48b129797b8 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 5 Mar 2020 20:38:20 +0100 Subject: [PATCH 55/61] ensure that the task is unregistered before returning the final response to the user --- .../TransportSubmitAsyncSearchAction.java | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java index e8bcb848db5a3..72e93575c977a 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportSubmitAsyncSearchAction.java @@ -87,12 +87,17 @@ public void onResponse(AsyncSearchResponse searchResponse) { new ActionListener<>() { @Override public void onResponse(IndexResponse r) { - try { - // store the final response on completion unless the submit is cancelled - searchTask.addCompletionListener(finalResponse -> - onFinalResponse(submitTask, searchTask, finalResponse)); - } finally { - submitListener.onResponse(searchResponse); + if (searchResponse.isRunning()) { + try { + // store the final response on completion unless the submit is cancelled + searchTask.addCompletionListener(finalResponse -> + onFinalResponse(submitTask, searchTask, finalResponse, () -> {})); + } finally { + submitListener.onResponse(searchResponse); + } + } else { + onFinalResponse(submitTask, searchTask, searchResponse, + () -> submitListener.onResponse(searchResponse)); } } @@ -153,10 +158,16 @@ private void onFatalFailure(AsyncSearchTask task, Exception error, boolean shoul } } - private void onFinalResponse(CancellableTask submitTask, AsyncSearchTask searchTask, AsyncSearchResponse response) { + private void onFinalResponse(CancellableTask submitTask, + AsyncSearchTask searchTask, + AsyncSearchResponse response, + Runnable nextAction) { if (submitTask.isCancelled() || searchTask.isCancelled()) { // the user cancelled the submit so we ensure that there is nothing stored in the response index. - store.deleteResponse(searchTask.getSearchId(), ActionListener.wrap(() -> taskManager.unregister(searchTask))); + store.deleteResponse(searchTask.getSearchId(), ActionListener.wrap(() -> { + taskManager.unregister(searchTask); + nextAction.run(); + })); return; } @@ -165,6 +176,7 @@ private void onFinalResponse(CancellableTask submitTask, AsyncSearchTask searchT @Override public void onResponse(UpdateResponse updateResponse) { taskManager.unregister(searchTask); + nextAction.run(); } @Override @@ -174,11 +186,13 @@ public void onFailure(Exception exc) { searchTask.getSearchId().getEncoded()), exc); } taskManager.unregister(searchTask); + nextAction.run(); } }); } catch (Exception exc) { logger.error(() -> new ParameterizedMessage("failed to store async-search [{}]", searchTask.getSearchId().getEncoded()), exc); taskManager.unregister(searchTask); + nextAction.run(); } } } From 20aa0fef82b14f0e90732254c77161d42e9999a8 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 5 Mar 2020 21:29:09 +0100 Subject: [PATCH 56/61] checkstyle --- .../action/search/SearchProgressActionListenerIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 8da91748038f7..931fdd506ed97 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESSingleNodeTestCase; From 415c91034674506b915c732881ab6a784e5b9d55 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 10 Mar 2020 00:56:20 +0100 Subject: [PATCH 57/61] address review --- .../action/search/SearchRequest.java | 2 +- .../rest-api-spec/test/search/10_basic.yml | 36 +++++++++++++++- .../search/RestGetAsyncSearchAction.java | 13 +++--- .../search/RestSubmitAsyncSearchAction.java | 16 ++++--- .../search/SubmitAsyncSearchRequestTests.java | 43 +++++++++++++++++++ .../search/action/AsyncSearchResponse.java | 5 ++- .../action/SubmitAsyncSearchRequest.java | 24 +++++++++-- .../rest-api-spec/api/async_search.get.json | 4 ++ 8 files changed, 123 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index c8b8981e3218d..96206aa4bcd58 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -57,7 +57,7 @@ */ public class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable { - private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false")); + public static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false")); public static final int DEFAULT_PRE_FILTER_SHARD_SIZE = 128; public static final int DEFAULT_BATCHED_REDUCE_SIZE = 512; diff --git a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml index e151a204b8a8c..ef457a569ad0e 100644 --- a/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml +++ b/x-pack/plugin/async-search/qa/rest/src/test/resources/rest-api-spec/test/search/10_basic.yml @@ -67,8 +67,6 @@ wait_for_completion: 10s clean_on_completion: false body: - query: - match_all: {} aggs: 1: max: @@ -82,6 +80,28 @@ - match: { response.hits.hits.0._source.max: 1 } - match: { response.aggregations.1.value: 3.0 } + # test with typed_keys: + - do: + async_search.submit: + index: test-* + batched_reduce_size: 2 + wait_for_completion: 10s + clean_on_completion: false + typed_keys: true + body: + aggs: + 1: + max: + field: max + sort: max + + - set: { id: id } + - match: { version: 6 } + - match: { is_partial: false } + - length: { response.hits.hits: 3 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.max#1.value: 3.0 } + - do: async_search.get: id: "$id" @@ -92,6 +112,18 @@ - match: { response.hits.hits.0._source.max: 1 } - match: { response.aggregations.1.value: 3.0 } + # test with typed_keys: + - do: + async_search.get: + id: "$id" + typed_keys: true + + - match: { version: 6 } + - match: { is_partial: false } + - length: { response.hits.hits: 3 } + - match: { response.hits.hits.0._source.max: 1 } + - match: { response.aggregations.max#1.value: 3.0 } + - do: async_search.delete: id: "$id" diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index ef832e0ede569..823f071264d40 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -5,19 +5,19 @@ */ package org.elasticsearch.xpack.search; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestHandler.Route; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestStatusToXContentListener; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import java.util.List; +import java.util.Set; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.search.RestSubmitAsyncSearchAction.RESPONSE_PARAMS; public class RestGetAsyncSearchAction extends BaseRestHandler { @Override @@ -43,10 +43,11 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (request.hasParam("last_version")) { get.setLastVersion(request.paramAsInt("last_version", get.getLastVersion())); } - ActionRequestValidationException validationException = get.validate(); - if (validationException != null) { - throw validationException; - } return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java index 01c792b258598..c1847bca7b69f 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestSubmitAsyncSearchAction.java @@ -5,10 +5,8 @@ */ package org.elasticsearch.xpack.search; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestHandler.Route; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestStatusToXContentListener; @@ -17,6 +15,8 @@ import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; import java.io.IOException; +import java.util.Collections; +import java.util.Set; import java.util.function.IntConsumer; import java.util.List; @@ -27,6 +27,9 @@ import static org.elasticsearch.rest.action.search.RestSearchAction.parseSearchRequest; public final class RestSubmitAsyncSearchAction extends BaseRestHandler { + static final String TYPED_KEYS_PARAM = "typed_keys"; + static final Set RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM); + @Override public List routes() { return unmodifiableList(asList( @@ -57,14 +60,15 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (request.hasParam("clean_on_completion")) { submit.setCleanOnCompletion(request.paramAsBoolean("clean_on_completion", submit.isCleanOnCompletion())); } - ActionRequestValidationException validationException = submit.validate(); - if (validationException != null) { - throw validationException; - } return channel -> { RestStatusToXContentListener listener = new RestStatusToXContentListener<>(channel); RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); cancelClient.execute(SubmitAsyncSearchAction.INSTANCE, submit, listener); }; } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java index 939b6b0914a21..2aae3817205e2 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.search; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.Writeable; @@ -12,9 +13,13 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; import org.elasticsearch.xpack.core.transform.action.AbstractWireSerializingTransformTestCase; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + public class SubmitAsyncSearchRequestTests extends AbstractWireSerializingTransformTestCase { @Override protected Writeable.Reader instanceReader() { @@ -66,4 +71,42 @@ protected SearchSourceBuilder randomSearchSourceBuilder() { } return source; } + + public void testValidateCssMinimizeRoundtrips() { + SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest(); + req.getSearchRequest().setCcsMinimizeRoundtrips(true); + ActionRequestValidationException exc = req.validate(); + assertNotNull(exc); + assertThat(exc.validationErrors().size(), equalTo(1)); + assertThat(exc.validationErrors().get(0), containsString("[ccs_minimize_roundtrips]")); + } + + public void testValidateScroll() { + SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest(); + req.getSearchRequest().scroll(TimeValue.timeValueMinutes(5)); + ActionRequestValidationException exc = req.validate(); + assertNotNull(exc); + assertThat(exc.validationErrors().size(), equalTo(2)); + // request_cache is activated by default + assertThat(exc.validationErrors().get(0), containsString("[request_cache]")); + assertThat(exc.validationErrors().get(1), containsString("[scroll]")); + } + + public void testValidateKeepAlive() { + SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest(); + req.setKeepAlive(TimeValue.timeValueSeconds(randomIntBetween(1, 59))); + ActionRequestValidationException exc = req.validate(); + assertNotNull(exc); + assertThat(exc.validationErrors().size(), equalTo(1)); + assertThat(exc.validationErrors().get(0), containsString("[keep_alive]")); + } + + public void testValidateSuggestOnly() { + SubmitAsyncSearchRequest req = new SubmitAsyncSearchRequest(); + req.getSearchRequest().source(new SearchSourceBuilder().suggest(new SuggestBuilder())); + ActionRequestValidationException exc = req.validate(); + assertNotNull(exc); + assertThat(exc.validationErrors().size(), equalTo(1)); + assertThat(exc.validationErrors().get(0), containsString("suggest")); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 933af4d3b1662..ed27a0e06dee9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -193,7 +193,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("start_time_in_millis", startTimeMillis); builder.field("expiration_time_in_millis", expirationTimeMillis); - builder.field("response", searchResponse); + if (searchResponse != null) { + builder.field("response"); + searchResponse.toXContent(builder, params); + } if (error != null) { builder.startObject("error"); error.toXContent(builder, params); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 85aeb4e7bf232..37f75817755a0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; @@ -21,6 +22,7 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.action.search.SearchRequest.FORMAT_PARAMS; /** * A request to track asynchronously the progress of a search against one or more indices. @@ -28,7 +30,7 @@ * @see AsyncSearchResponse */ public class SubmitAsyncSearchRequest extends ActionRequest { - public static long MIN_KEEP_ALIVE = TimeValue.timeValueHours(1).millis(); + public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis(); private TimeValue waitForCompletion = TimeValue.timeValueSeconds(1); private boolean cleanOnCompletion = true; @@ -128,14 +130,18 @@ public boolean isCleanOnCompletion() { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = request.validate(); if (request.scroll() != null) { - addValidationError("scroll queries are not supported", validationException); + addValidationError("[scroll] queries are not supported", validationException); } if (request.isSuggestOnly()) { validationException = addValidationError("suggest-only queries are not supported", validationException); } if (keepAlive.getMillis() < MIN_KEEP_ALIVE) { validationException = - addValidationError("keep_alive must be greater than 1 minute, got:" + keepAlive.toString(), validationException); + addValidationError("[keep_alive] must be greater than 1 minute, got:" + keepAlive.toString(), validationException); + } + if (request.isCcsMinimizeRoundtrips()) { + validationException = + addValidationError("[ccs_minimize_roundtrips] is not supported on async search queries", validationException); } return validationException; @@ -143,7 +149,7 @@ public ActionRequestValidationException validate() { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - return new CancellableTask(id, type, action, "", parentTaskId, headers) { + return new CancellableTask(id, type, action, toString(), parentTaskId, headers) { @Override public boolean shouldCancelChildrenOnCancellation() { return true; @@ -166,4 +172,14 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(waitForCompletion, cleanOnCompletion, keepAlive, request); } + + @Override + public String toString() { + return "SubmitAsyncSearchRequest{" + + "waitForCompletion=" + waitForCompletion + + ", cleanOnCompletion=" + cleanOnCompletion + + ", keepAlive=" + keepAlive + + ", request=" + request + + '}'; + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json index 8a0b41762ec24..f961404706ead 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json @@ -32,6 +32,10 @@ "keep_alive": { "type": "time", "description": "Specify the time that the request should remain reachable in the cluster." + }, + "typed_keys":{ + "type":"boolean", + "description":"Specify whether aggregation and suggester names should be prefixed by their respective types in the response" } } } From 91699d541b4c6a19fc4a9fde1986f910cd5cc680 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 10 Mar 2020 08:49:07 +0100 Subject: [PATCH 58/61] unused import --- .../xpack/core/search/action/SubmitAsyncSearchRequest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java index 37f75817755a0..a6397cbf08c3b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/SubmitAsyncSearchRequest.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; @@ -22,7 +21,6 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.action.search.SearchRequest.FORMAT_PARAMS; /** * A request to track asynchronously the progress of a search against one or more indices. From e3bea16eddf28384d549eda723c7e37fe019bbb8 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 10 Mar 2020 09:31:41 +0100 Subject: [PATCH 59/61] fix rest API reference to the outdated 304 response status --- .../src/test/resources/rest-api-spec/api/async_search.get.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json index f961404706ead..d0f80f386c2f6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json @@ -27,7 +27,7 @@ }, "last_version":{ "type":"number", - "description":"Specify the last version returned by a previous call. The request will return 304 (not modified) status if the new version is lower than or equals to the provided one and will remove all details in the response except id and version. (default: -1)" + "description":"Specify the last version returned by a previous call. If the new version is lower than or equals to the provided one, all details in the response except id and version are omitted. (default: -1)" }, "keep_alive": { "type": "time", From 87273859030d71b2e507e09433f84d75fdff6bc0 Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 10 Mar 2020 13:26:16 +0100 Subject: [PATCH 60/61] remove last_version parameter --- .../search/RestGetAsyncSearchAction.java | 3 --- .../search/TransportGetAsyncSearchAction.java | 8 -------- .../search/AsyncSearchIntegTestCase.java | 3 +-- .../search/GetAsyncSearchRequestTests.java | 3 --- .../search/action/GetAsyncSearchAction.java | 20 ++----------------- .../rest-api-spec/api/async_search.get.json | 6 +----- .../api/async_search.submit.json | 16 +++++++-------- 7 files changed, 12 insertions(+), 47 deletions(-) diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java index 823f071264d40..8dc29bf071e0d 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/RestGetAsyncSearchAction.java @@ -40,9 +40,6 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (request.hasParam("keep_alive")) { get.setKeepAlive(request.paramAsTime("keep_alive", get.getKeepAlive())); } - if (request.hasParam("last_version")) { - get.setLastVersion(request.paramAsInt("last_version", get.getLastVersion())); - } return channel -> client.execute(GetAsyncSearchAction.INSTANCE, get, new RestStatusToXContentListener<>(channel)); } diff --git a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java index fa4101ab86661..01c807b19aea1 100644 --- a/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java +++ b/x-pack/plugin/async-search/src/main/java/org/elasticsearch/xpack/search/TransportGetAsyncSearchAction.java @@ -139,14 +139,6 @@ private void sendFinalResponse(GetAsyncSearchAction.Request request, return; } - // check last version - if (response.getVersion() <= request.getLastVersion()) { - // return a not-modified response - listener.onResponse(new AsyncSearchResponse(response.getId(), response.getVersion(), - response.isPartial(), false, response.getStartTime(), response.getExpirationTime())); - return; - } - listener.onResponse(response); } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java index 41a81eb7dd671..5c229a5de1d80 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchIntegTestCase.java @@ -209,8 +209,7 @@ private AsyncSearchResponse doNext() throws Exception { assertBusy(() -> { AsyncSearchResponse newResp = client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncSearchAction.Request(response.getId()) - .setWaitForCompletion(TimeValue.timeValueMillis(10)) - .setLastVersion(lastVersion)).get(); + .setWaitForCompletion(TimeValue.timeValueMillis(10))).get(); atomic.set(newResp); assertNotEquals(lastVersion, newResp.getVersion()); }); diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java index a985e219b18f8..5e3ec4ded0ec8 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/GetAsyncSearchRequestTests.java @@ -24,9 +24,6 @@ protected GetAsyncSearchAction.Request createTestInstance() { if (randomBoolean()) { req.setWaitForCompletion(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); } - if (randomBoolean()) { - req.setLastVersion(randomIntBetween(-1, Integer.MAX_VALUE)); - } if (randomBoolean()) { req.setKeepAlive(TimeValue.timeValueMillis(randomIntBetween(1, 10000))); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java index 254fd2f77b9e1..fe4801aab4a10 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/GetAsyncSearchAction.java @@ -34,7 +34,6 @@ public Writeable.Reader getResponseReader() { public static class Request extends ActionRequest { private final String id; - private int lastVersion = -1; private TimeValue waitForCompletion = TimeValue.MINUS_ONE; private TimeValue keepAlive = TimeValue.MINUS_ONE; @@ -52,7 +51,6 @@ public Request(StreamInput in) throws IOException { this.id = in.readString(); this.waitForCompletion = TimeValue.timeValueMillis(in.readLong()); this.keepAlive = in.readTimeValue(); - this.lastVersion = in.readInt(); } @Override @@ -61,7 +59,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeLong(waitForCompletion.millis()); out.writeTimeValue(keepAlive); - out.writeInt(lastVersion); } @Override @@ -81,18 +78,6 @@ public String getId() { return id; } - /** - * Omits the result from the response if the new version is greater than the provided version (not-modified). - */ - public Request setLastVersion(int version) { - this.lastVersion = version; - return this; - } - - public int getLastVersion() { - return lastVersion; - } - /** * Sets the minimum time that the request should wait before returning a partial result (defaults to no wait). */ @@ -122,15 +107,14 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return lastVersion == request.lastVersion && - Objects.equals(id, request.id) && + return Objects.equals(id, request.id) && waitForCompletion.equals(request.waitForCompletion) && keepAlive.equals(request.keepAlive); } @Override public int hashCode() { - return Objects.hash(id, lastVersion, waitForCompletion, keepAlive); + return Objects.hash(id, waitForCompletion, keepAlive); } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json index d0f80f386c2f6..a325c3a025390 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json @@ -25,13 +25,9 @@ "type":"time", "description":"Specify the time that the request should block waiting for the final response (default: 1s)" }, - "last_version":{ - "type":"number", - "description":"Specify the last version returned by a previous call. If the new version is lower than or equals to the provided one, all details in the response except id and version are omitted. (default: -1)" - }, "keep_alive": { "type": "time", - "description": "Specify the time that the request should remain reachable in the cluster." + "description": "Control how long to keep responses (partial or final) for this request alive (default: 5d)" }, "typed_keys":{ "type":"boolean", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json index fbe4d42fbc272..71ccc7ffa866c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json @@ -33,19 +33,15 @@ "type":"time", "description":"Specify the time that the request should block waiting for the final response (default: 1s)" }, - "clean_on_completion":{ - "type":"boolean", - "description":"Specify whether the response should not be stored in the cluster if it completed within the provided wait_for_completion time (default: true)" + "keep_alive": { + "type": "time", + "description": "Optional parameter to update how long to keep responses (partial or final) for this request alive" }, "batched_reduce_size":{ "type":"number", - "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.", + "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available. (default: 5)", "default":5 }, - "keep_alive": { - "type": "time", - "description": "Specify the time that the request should remain reachable in the cluster." - }, "analyzer":{ "type":"string", "description":"The analyzer to use for the query string" @@ -217,6 +213,10 @@ "type":"number", "description":"The number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests", "default":5 + }, + "clean_on_completion":{ + "type":"boolean", + "description":"Control whether the response should not be stored in the cluster if it completed within the provided [wait_for_completion] time (default: true)" } }, "body":{ From 315bb49e75dce07d063701ad46ece58abe48aa7b Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 10 Mar 2020 13:38:24 +0100 Subject: [PATCH 61/61] rephrase rest option after review --- .../test/resources/rest-api-spec/api/async_search.get.json | 6 ++++-- .../resources/rest-api-spec/api/async_search.submit.json | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json index a325c3a025390..f5ea1424756e1 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.get.json @@ -23,11 +23,13 @@ "params":{ "wait_for_completion":{ "type":"time", - "description":"Specify the time that the request should block waiting for the final response (default: 1s)" + "description":"Specify the time that the request should block waiting for the final response", + "default": "1s" }, "keep_alive": { "type": "time", - "description": "Control how long to keep responses (partial or final) for this request alive (default: 5d)" + "description": "Specify the time interval in which the results (partial or final) for this search will be available", + "default": "5d" }, "typed_keys":{ "type":"boolean", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json index 71ccc7ffa866c..3d057e2da0642 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/async_search.submit.json @@ -31,15 +31,16 @@ "params":{ "wait_for_completion":{ "type":"time", - "description":"Specify the time that the request should block waiting for the final response (default: 1s)" + "description":"Specify the time that the request should block waiting for the final response", + "default": "1s" }, "keep_alive": { "type": "time", - "description": "Optional parameter to update how long to keep responses (partial or final) for this request alive" + "description": "Update the time interval in which the results (partial or final) for this search will be available" }, "batched_reduce_size":{ "type":"number", - "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available. (default: 5)", + "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.", "default":5 }, "analyzer":{