Add ability to reindex RED shards

Author: henningandersen

docs/reference/docs/reindex.asciidoc

@@ -1255,3 +1255,24 @@ POST _reindex
 
 <1> `_reindex` defaults to sorting by `_doc` so `random_score` will not have any
 effect unless you override the sort to `_score`.
+
+[float]
+==== Reindexing RED shards
+
+`_reindex` by default fails if a shard is unavailable (red). It is possible to
+reindex only the available shards using:
+
+[source,js]
+----------------------------------------------------------------
+POST _reindex?allow_partial_search_results=true
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "red_twitter"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]

modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java

@@ -64,6 +64,11 @@ protected ReindexRequest buildRequest(RestRequest request) throws IOException {
         if (request.hasParam("scroll")) {
             internal.setScroll(parseTimeValue(request.param("scroll"), "scroll"));
         }
+
+        if (request.hasParam("allow_partial_search_results")) {
+            internal.getSearchRequest().allowPartialSearchResults(request.paramAsBoolean("allow_partial_search_results",false));
+        }
+
         return internal;
     }
 }

modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java

@@ -125,9 +125,13 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query,
             request.addParameter(storedFieldsParamName, fields.toString());
         }
 
+        // allow_partial_results introduced in 6.3, running remote reindex against earlier versions still silently discards RED shards.
         if (remoteVersion.onOrAfter(Version.fromId(6030099))) {
-            // allow_partial_results introduced in 6.3, running remote reindex against earlier versions still silently discards RED shards.
-            request.addParameter("allow_partial_search_results", "false");
+            boolean allowPartialSearchResults = searchRequest.allowPartialSearchResults() == Boolean.TRUE;
+            // be explicit always from 7.5 to stop relying on the default.
+            if (allowPartialSearchResults == false || remoteVersion.onOrAfter(Version.V_7_5_0)) {
+                request.addParameter("allow_partial_search_results", Boolean.toString(allowPartialSearchResults));
+            }
         }
 
         // EMPTY is safe here because we're not calling namedObject

modules/reindex/src/test/java/org/elasticsearch/index/reindex/ClientScrollableHitSourceTests.java

@@ -106,18 +106,26 @@ private void dotestBasicsWithRetry(int retries, int minFailures, int maxFailures
         TaskId parentTask = new TaskId("thenode", randomInt());
         AtomicInteger actualSearchRetries = new AtomicInteger();
         int expectedSearchRetries = 0;
+        SearchRequest searchRequest = new SearchRequest().scroll("1m");
+        boolean allowPartialSearchResults = randomBoolean();
+        if (allowPartialSearchResults || randomBoolean()) {
+            searchRequest.allowPartialSearchResults(allowPartialSearchResults);
+        }
         ClientScrollableHitSource hitSource = new ClientScrollableHitSource(logger, BackoffPolicy.constantBackoff(TimeValue.ZERO, retries),
             threadPool, actualSearchRetries::incrementAndGet, responses::add, failureHandler,
             new ParentTaskAssigningClient(client, parentTask),
-            new SearchRequest().scroll("1m"));
+            searchRequest);
 
         hitSource.start();
         for (int retry = 0; retry < randomIntBetween(minFailures, maxFailures); ++retry) {
             client.fail(SearchAction.INSTANCE, new EsRejectedExecutionException());
             client.awaitOperation();
             ++expectedSearchRetries;
         }
-        client.validateRequest(SearchAction.INSTANCE, (SearchRequest r) -> assertTrue(r.allowPartialSearchResults() == Boolean.FALSE));
+
+        // we explicitly set it on 7.5+, to not rely on the default anymore.
+        client.validateRequest(SearchAction.INSTANCE,
+            (SearchRequest r) -> assertTrue(r.allowPartialSearchResults() == allowPartialSearchResults));
         SearchResponse searchResponse = createSearchResponse();
         client.respond(SearchAction.INSTANCE, searchResponse);
 

modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java

@@ -195,6 +195,22 @@ public void testInitialSearchDisallowPartialResults() {
         assertThat(params.keySet(), not(contains(allowPartialParamName)));
     }
 
+    public void testInitialSearchExplicitlyAllowPartialResults() {
+        final String allowPartialParamName = "allow_partial_search_results";
+
+        BytesReference query = new BytesArray("{}");
+        SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()).allowPartialSearchResults(true);
+
+        // we explicitly set it against 7.5+, to not rely on the default anymore.
+        Version allowVersion = Version.fromId(between(Version.V_7_5_0.id, Version.CURRENT.id));
+        Map<String, String> params = initialSearch(searchRequest, query, allowVersion).getParameters();
+        assertEquals("true", params.get(allowPartialParamName));
+
+        Version notSetVersion = Version.fromId(between(0, Version.V_7_5_0.id-1));
+        params = initialSearch(searchRequest, query, notSetVersion).getParameters();
+        assertThat(params.keySet(), not(contains(allowPartialParamName)));
+    }
+
     private void assertScroll(Version remoteVersion, Map<String, String> params, TimeValue requested) {
         // V_5_0_0
         if (remoteVersion.before(Version.fromId(5000099))) {

modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml

@@ -1,5 +1,49 @@
 ---
-"Response format for search failures":
+"Response format for search failures with allow_partial_search_results=true":
+  - do:
+      indices.create:
+        index: source
+        body:
+          settings:
+            index.number_of_shards: 2
+
+  - do:
+      index:
+        index:   source
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: bad_request
+      reindex:
+        allow_partial_search_results: true
+        body:
+          source:
+            index:   source
+            query:
+              script:
+                script:
+                  lang: painless
+                  source: throw new IllegalArgumentException("Cats!")
+          dest:
+            index:   dest
+  - match: {created: 0}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 0}
+  - match: {failures.0.shard: 0}
+  - match: {failures.0.index:  source}
+  - is_true: failures.0.node
+  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.reason: runtime error}
+  - match: {failures.0.reason.caused_by.type:   illegal_argument_exception}
+  - match: {failures.0.reason.caused_by.reason: Cats!}
+  - gte: { took: 0 }
+
+---
+"Response format for search failures normal":
   - do:
       indices.create:
         index: source

rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json

@@ -52,6 +52,11 @@
       "max_docs":{
         "type":"number",
         "description":"Maximum number of documents to process (default: all documents)"
+      },
+      "allow_partial_search_results":{
+        "type":"boolean",
+        "default":false,
+        "description":"Indicate if an error should be returned if there is a partial search failure or timeout"
       }
     },
     "body":{

server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java

@@ -64,7 +64,7 @@ public ClientScrollableHitSource(Logger logger, BackoffPolicy backoffPolicy, Thr
         super(logger, backoffPolicy, threadPool, countSearchRetry, onResponse, fail);
         this.client = client;
         this.firstSearchRequest = firstSearchRequest;
-        firstSearchRequest.allowPartialSearchResults(false);
+        firstSearchRequest.allowPartialSearchResults(firstSearchRequest.allowPartialSearchResults() == Boolean.TRUE);
     }
 
     @Override