Skip to content

Commit 12c1ecb

Browse files
jasontedorjasontedor
committed
Merge remote-tracking branch 'elastic/master' into os-pretty-name
* elastic/master: (25 commits) Fixes fast vector highlighter docs per issue 24318. (elastic#34190) [ML] Prevent notifications on deletion of a non existent job (elastic#35337) [CCR] Auto follow Coordinator fetch cluster state in system context (elastic#35120) Mute test for elastic#35361 Preserve `date_histogram` format when aggregating on unmapped fields (elastic#35254) Test: Mute failing SSL test Allow unmapped fields in composite aggregations (elastic#35331) [RCI] Add IndexShardOperationPermits.asyncBlockOperations(ActionListener<Releasable>) (elastic#34902) HLRC: reindex API with wait_for_completion false (elastic#35202) Add docs on JNA temp directory not being noexec (elastic#35355) [CCR] Adjust list of dynamic index settings that should be replicated (elastic#35195) Replicate index settings to followers (elastic#35089) Rename RealmConfig.globalSettings() to settings() (elastic#35330) [TEST] Cleanup FileUserPasswdStoreTests (elastic#35329) Scripting: Add back lookup vars in score script (elastic#34833) watcher: Fix integration tests to ensure correct start/stop of Watcher (elastic#35271) Remove ALL shard check in CheckShrinkReadyStep (elastic#35346) Use soft-deleted docs to resolve strategy for engine operation (elastic#35230) [ILM] Check shard and relocation status in AllocationRoutedStep (elastic#35316) Ignore date ranges containing 'now' when pre-processing a percolator query (elastic#35160) ...
2 parents 9001de0 + d00b23c commit 12c1ecb

File tree

108 files changed

+3823
-632
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+3823
-632
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -486,9 +486,18 @@ static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
486486
}
487487

488488
static Request reindex(ReindexRequest reindexRequest) throws IOException {
489+
return prepareReindexRequest(reindexRequest, true);
490+
}
491+
492+
static Request submitReindex(ReindexRequest reindexRequest) throws IOException {
493+
return prepareReindexRequest(reindexRequest, false);
494+
}
495+
496+
private static Request prepareReindexRequest(ReindexRequest reindexRequest, boolean waitForCompletion) throws IOException {
489497
String endpoint = new EndpointBuilder().addPathPart("_reindex").build();
490498
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
491499
Params params = new Params(request)
500+
.withWaitForCompletion(waitForCompletion)
492501
.withRefresh(reindexRequest.isRefresh())
493502
.withTimeout(reindexRequest.getTimeout())
494503
.withWaitForActiveShards(reindexRequest.getWaitForActiveShards())
@@ -897,11 +906,8 @@ Params withDetailed(boolean detailed) {
897906
return this;
898907
}
899908

900-
Params withWaitForCompletion(boolean waitForCompletion) {
901-
if (waitForCompletion) {
902-
return putParam("wait_for_completion", Boolean.TRUE.toString());
903-
}
904-
return this;
909+
Params withWaitForCompletion(Boolean waitForCompletion) {
910+
return putParam("wait_for_completion", waitForCompletion.toString());
905911
}
906912

907913
Params withNodes(String[] nodes) {

‎client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.elasticsearch.client.core.CountResponse;
6161
import org.elasticsearch.client.core.TermVectorsResponse;
6262
import org.elasticsearch.client.core.TermVectorsRequest;
63+
import org.elasticsearch.client.tasks.TaskSubmissionResponse;
6364
import org.elasticsearch.common.CheckedConsumer;
6465
import org.elasticsearch.common.CheckedFunction;
6566
import org.elasticsearch.common.ParseField;
@@ -476,6 +477,20 @@ public final BulkByScrollResponse reindex(ReindexRequest reindexRequest, Request
476477
);
477478
}
478479

480+
/**
481+
* Submits a reindex task.
482+
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html">Reindex API on elastic.co</a>
483+
* @param reindexRequest the request
484+
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
485+
* @return the submission response
486+
* @throws IOException in case there is a problem sending the request or parsing back the response
487+
*/
488+
public final TaskSubmissionResponse submitReindexTask(ReindexRequest reindexRequest, RequestOptions options) throws IOException {
489+
return performRequestAndParseEntity(
490+
reindexRequest, RequestConverters::submitReindex, options, TaskSubmissionResponse::fromXContent, emptySet()
491+
);
492+
}
493+
479494
/**
480495
* Asynchronously executes a reindex request.
481496
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html">Reindex API on elastic.co</a>

‎client/rest-high-level/src/main/java/org/elasticsearch/client/tasks/TaskSubmissionResponse.java

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.client.tasks;
21+
22+
import org.elasticsearch.action.ActionResponse;
23+
import org.elasticsearch.common.ParseField;
24+
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
25+
import org.elasticsearch.common.xcontent.XContentParser;
26+
27+
import java.io.IOException;
28+
import java.util.Objects;
29+
30+
public class TaskSubmissionResponse extends ActionResponse {
31+
32+
private static final ParseField TASK = new ParseField("task");
33+
34+
public static final ConstructingObjectParser<TaskSubmissionResponse, Void> PARSER = new ConstructingObjectParser<>(
35+
"task_submission_response",
36+
true, a -> new TaskSubmissionResponse((String) a[0]));
37+
38+
static {
39+
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TASK);
40+
}
41+
42+
private final String task;
43+
44+
TaskSubmissionResponse(String task) {
45+
this.task = task;
46+
}
47+
48+
/**
49+
* Get the task id
50+
*
51+
* @return the id of the reindex task.
52+
*/
53+
public String getTask() {
54+
return task;
55+
}
56+
57+
@Override
58+
public int hashCode() {
59+
return Objects.hash(task);
60+
}
61+
62+
@Override
63+
public boolean equals(Object other) {
64+
if (this == other) {
65+
return true;
66+
}
67+
if (other == null || getClass() != other.getClass()) {
68+
return false;
69+
}
70+
TaskSubmissionResponse that = (TaskSubmissionResponse) other;
71+
return Objects.equals(task, that.task);
72+
}
73+
74+
public static TaskSubmissionResponse fromXContent(XContentParser parser) throws IOException {
75+
return PARSER.parse(parser, null);
76+
}
77+
78+
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

Lines changed: 0 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,6 @@
6060
import org.elasticsearch.index.reindex.BulkByScrollResponse;
6161
import org.elasticsearch.index.reindex.DeleteByQueryAction;
6262
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
63-
import org.elasticsearch.index.reindex.ReindexAction;
64-
import org.elasticsearch.index.reindex.ReindexRequest;
6563
import org.elasticsearch.index.reindex.UpdateByQueryAction;
6664
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
6765
import org.elasticsearch.rest.RestStatus;
@@ -706,111 +704,6 @@ public void testBulk() throws IOException {
706704
validateBulkResponses(nbItems, errors, bulkResponse, bulkRequest);
707705
}
708706

709-
public void testReindex() throws Exception {
710-
final String sourceIndex = "source1";
711-
final String destinationIndex = "dest";
712-
{
713-
// Prepare
714-
Settings settings = Settings.builder()
715-
.put("number_of_shards", 1)
716-
.put("number_of_replicas", 0)
717-
.build();
718-
createIndex(sourceIndex, settings);
719-
createIndex(destinationIndex, settings);
720-
BulkRequest bulkRequest = new BulkRequest()
721-
.add(new IndexRequest(sourceIndex, "type", "1").source(Collections.singletonMap("foo", "bar"), XContentType.JSON))
722-
.add(new IndexRequest(sourceIndex, "type", "2").source(Collections.singletonMap("foo2", "bar2"), XContentType.JSON))
723-
.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
724-
assertEquals(
725-
RestStatus.OK,
726-
highLevelClient().bulk(
727-
bulkRequest,
728-
RequestOptions.DEFAULT
729-
).status()
730-
);
731-
}
732-
{
733-
// test1: create one doc in dest
734-
ReindexRequest reindexRequest = new ReindexRequest();
735-
reindexRequest.setSourceIndices(sourceIndex);
736-
reindexRequest.setDestIndex(destinationIndex);
737-
reindexRequest.setSourceQuery(new IdsQueryBuilder().addIds("1").types("type"));
738-
reindexRequest.setRefresh(true);
739-
BulkByScrollResponse bulkResponse = execute(reindexRequest, highLevelClient()::reindex, highLevelClient()::reindexAsync);
740-
assertEquals(1, bulkResponse.getCreated());
741-
assertEquals(1, bulkResponse.getTotal());
742-
assertEquals(0, bulkResponse.getDeleted());
743-
assertEquals(0, bulkResponse.getNoops());
744-
assertEquals(0, bulkResponse.getVersionConflicts());
745-
assertEquals(1, bulkResponse.getBatches());
746-
assertTrue(bulkResponse.getTook().getMillis() > 0);
747-
assertEquals(1, bulkResponse.getBatches());
748-
assertEquals(0, bulkResponse.getBulkFailures().size());
749-
assertEquals(0, bulkResponse.getSearchFailures().size());
750-
}
751-
{
752-
// test2: create 1 and update 1
753-
ReindexRequest reindexRequest = new ReindexRequest();
754-
reindexRequest.setSourceIndices(sourceIndex);
755-
reindexRequest.setDestIndex(destinationIndex);
756-
BulkByScrollResponse bulkResponse = execute(reindexRequest, highLevelClient()::reindex, highLevelClient()::reindexAsync);
757-
assertEquals(1, bulkResponse.getCreated());
758-
assertEquals(2, bulkResponse.getTotal());
759-
assertEquals(1, bulkResponse.getUpdated());
760-
assertEquals(0, bulkResponse.getDeleted());
761-
assertEquals(0, bulkResponse.getNoops());
762-
assertEquals(0, bulkResponse.getVersionConflicts());
763-
assertEquals(1, bulkResponse.getBatches());
764-
assertTrue(bulkResponse.getTook().getMillis() > 0);
765-
assertEquals(1, bulkResponse.getBatches());
766-
assertEquals(0, bulkResponse.getBulkFailures().size());
767-
assertEquals(0, bulkResponse.getSearchFailures().size());
768-
}
769-
{
770-
// test reindex rethrottling
771-
ReindexRequest reindexRequest = new ReindexRequest();
772-
reindexRequest.setSourceIndices(sourceIndex);
773-
reindexRequest.setDestIndex(destinationIndex);
774-
775-
// this following settings are supposed to halt reindexing after first document
776-
reindexRequest.setSourceBatchSize(1);
777-
reindexRequest.setRequestsPerSecond(0.00001f);
778-
final CountDownLatch reindexTaskFinished = new CountDownLatch(1);
779-
highLevelClient().reindexAsync(reindexRequest, RequestOptions.DEFAULT, new ActionListener<BulkByScrollResponse>() {
780-
781-
@Override
782-
public void onResponse(BulkByScrollResponse response) {
783-
reindexTaskFinished.countDown();
784-
}
785-
786-
@Override
787-
public void onFailure(Exception e) {
788-
fail(e.toString());
789-
}
790-
});
791-
792-
TaskId taskIdToRethrottle = findTaskToRethrottle(ReindexAction.NAME);
793-
float requestsPerSecond = 1000f;
794-
ListTasksResponse response = execute(new RethrottleRequest(taskIdToRethrottle, requestsPerSecond),
795-
highLevelClient()::reindexRethrottle, highLevelClient()::reindexRethrottleAsync);
796-
assertThat(response.getTasks(), hasSize(1));
797-
assertEquals(taskIdToRethrottle, response.getTasks().get(0).getTaskId());
798-
assertThat(response.getTasks().get(0).getStatus(), instanceOf(RawTaskStatus.class));
799-
assertEquals(Float.toString(requestsPerSecond),
800-
((RawTaskStatus) response.getTasks().get(0).getStatus()).toMap().get("requests_per_second").toString());
801-
reindexTaskFinished.await(2, TimeUnit.SECONDS);
802-
803-
// any rethrottling after the reindex is done performed with the same taskId should result in a failure
804-
response = execute(new RethrottleRequest(taskIdToRethrottle, requestsPerSecond),
805-
highLevelClient()::reindexRethrottle, highLevelClient()::reindexRethrottleAsync);
806-
assertTrue(response.getTasks().isEmpty());
807-
assertFalse(response.getNodeFailures().isEmpty());
808-
assertEquals(1, response.getNodeFailures().size());
809-
assertEquals("Elasticsearch exception [type=resource_not_found_exception, reason=task [" + taskIdToRethrottle + "] is missing]",
810-
response.getNodeFailures().get(0).getCause().getMessage());
811-
}
812-
}
813-
814707
private TaskId findTaskToRethrottle(String actionName) throws IOException {
815708
long start = System.nanoTime();
816709
ListTasksRequest request = new ListTasksRequest();

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.client;
21+
22+
import org.elasticsearch.action.bulk.BulkRequest;
23+
import org.elasticsearch.action.index.IndexRequest;
24+
import org.elasticsearch.action.support.WriteRequest;
25+
import org.elasticsearch.client.tasks.TaskSubmissionResponse;
26+
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.common.xcontent.XContentType;
28+
import org.elasticsearch.index.query.IdsQueryBuilder;
29+
import org.elasticsearch.index.reindex.BulkByScrollResponse;
30+
import org.elasticsearch.index.reindex.ReindexRequest;
31+
import org.elasticsearch.rest.RestStatus;
32+
33+
import java.io.IOException;
34+
import java.util.Collections;
35+
import java.util.function.BooleanSupplier;
36+
37+
public class ReindexIT extends ESRestHighLevelClientTestCase {
38+
39+
public void testReindex() throws IOException {
40+
final String sourceIndex = "source1";
41+
final String destinationIndex = "dest";
42+
{
43+
// Prepare
44+
Settings settings = Settings.builder()
45+
.put("number_of_shards", 1)
46+
.put("number_of_replicas", 0)
47+
.build();
48+
createIndex(sourceIndex, settings);
49+
createIndex(destinationIndex, settings);
50+
BulkRequest bulkRequest = new BulkRequest()
51+
.add(new IndexRequest(sourceIndex, "type", "1").source(Collections.singletonMap("foo", "bar"), XContentType.JSON))
52+
.add(new IndexRequest(sourceIndex, "type", "2").source(Collections.singletonMap("foo2", "bar2"), XContentType.JSON))
53+
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
54+
assertEquals(
55+
RestStatus.OK,
56+
highLevelClient().bulk(
57+
bulkRequest,
58+
RequestOptions.DEFAULT
59+
).status()
60+
);
61+
}
62+
{
63+
// reindex one document with id 1 from source to destination
64+
ReindexRequest reindexRequest = new ReindexRequest();
65+
reindexRequest.setSourceIndices(sourceIndex);
66+
reindexRequest.setDestIndex(destinationIndex);
67+
reindexRequest.setSourceQuery(new IdsQueryBuilder().addIds("1").types("type"));
68+
reindexRequest.setRefresh(true);
69+
70+
BulkByScrollResponse bulkResponse = execute(reindexRequest, highLevelClient()::reindex, highLevelClient()::reindexAsync);
71+
72+
assertEquals(1, bulkResponse.getCreated());
73+
assertEquals(1, bulkResponse.getTotal());
74+
assertEquals(0, bulkResponse.getDeleted());
75+
assertEquals(0, bulkResponse.getNoops());
76+
assertEquals(0, bulkResponse.getVersionConflicts());
77+
assertEquals(1, bulkResponse.getBatches());
78+
assertTrue(bulkResponse.getTook().getMillis() > 0);
79+
assertEquals(1, bulkResponse.getBatches());
80+
assertEquals(0, bulkResponse.getBulkFailures().size());
81+
assertEquals(0, bulkResponse.getSearchFailures().size());
82+
}
83+
}
84+
85+
public void testReindexTask() throws IOException, InterruptedException {
86+
final String sourceIndex = "source123";
87+
final String destinationIndex = "dest2";
88+
{
89+
// Prepare
90+
Settings settings = Settings.builder()
91+
.put("number_of_shards", 1)
92+
.put("number_of_replicas", 0)
93+
.build();
94+
createIndex(sourceIndex, settings);
95+
createIndex(destinationIndex, settings);
96+
BulkRequest bulkRequest = new BulkRequest()
97+
.add(new IndexRequest(sourceIndex, "type", "1").source(Collections.singletonMap("foo", "bar"), XContentType.JSON))
98+
.add(new IndexRequest(sourceIndex, "type", "2").source(Collections.singletonMap("foo2", "bar2"), XContentType.JSON))
99+
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
100+
assertEquals(
101+
RestStatus.OK,
102+
highLevelClient().bulk(
103+
bulkRequest,
104+
RequestOptions.DEFAULT
105+
).status()
106+
);
107+
}
108+
{
109+
ReindexRequest reindexRequest = new ReindexRequest();
110+
reindexRequest.setSourceIndices(sourceIndex);
111+
reindexRequest.setDestIndex(destinationIndex);
112+
reindexRequest.setSourceQuery(new IdsQueryBuilder().addIds("1").types("type"));
113+
reindexRequest.setRefresh(true);
114+
115+
TaskSubmissionResponse reindexSubmission = highLevelClient().submitReindexTask(reindexRequest, RequestOptions.DEFAULT);
116+
117+
BooleanSupplier hasUpgradeCompleted = checkCompletionStatus(reindexSubmission.getTask());
118+
awaitBusy(hasUpgradeCompleted);
119+
}
120+
}
121+
122+
private BooleanSupplier checkCompletionStatus(String taskId) {
123+
return () -> {
124+
try {
125+
Response response = client().performRequest(new Request("GET", "/_tasks/" + taskId));
126+
return (boolean) entityAsMap(response).get("completed");
127+
} catch (IOException e) {
128+
fail(e.getMessage());
129+
return false;
130+
}
131+
};
132+
}
133+
}

0 commit comments

Comments
 (0)