Skip to content

Commit ebe8951

Browse files
pugnascotiaDaveCTurner
pugnascotia
and
DaveCTurner
authored
Implement dangling indices API (#50920)
Part of #48366. Implement an API for listing, importing and deleting dangling indices. Co-authored-by: David Turner <[email protected]>
1 parent 69e1c06 commit ebe8951

File tree

42 files changed

+3102
-95
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+3102
-95
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

+3
Original file line numberDiff line numberDiff line change
@@ -816,6 +816,9 @@ public void testApiNamingConventions() throws Exception {
816816
"cluster.stats",
817817
"cluster.post_voting_config_exclusions",
818818
"cluster.delete_voting_config_exclusions",
819+
"dangling_indices.delete",
820+
"dangling_indices.import",
821+
"dangling_indices.list",
819822
"indices.shard_stores",
820823
"indices.upgrade",
821824
"indices.recovery",

‎qa/smoke-test-http/src/test/java/org/elasticsearch/http/DanglingIndicesRestIT.java

+316
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.http;
21+
22+
import org.elasticsearch.client.Request;
23+
import org.elasticsearch.client.Response;
24+
import org.elasticsearch.client.RestClient;
25+
import org.elasticsearch.common.io.Streams;
26+
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.indices.IndicesService;
28+
import org.elasticsearch.test.ESIntegTestCase;
29+
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
30+
import org.elasticsearch.test.InternalTestCluster;
31+
import org.elasticsearch.test.XContentTestUtils;
32+
33+
import java.io.IOException;
34+
import java.util.ArrayList;
35+
import java.util.Arrays;
36+
import java.util.HashMap;
37+
import java.util.List;
38+
import java.util.Map;
39+
import java.util.concurrent.atomic.AtomicReference;
40+
41+
import static org.elasticsearch.cluster.metadata.IndexGraveyard.SETTING_MAX_TOMBSTONES;
42+
import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING;
43+
import static org.elasticsearch.indices.IndicesService.WRITE_DANGLING_INDICES_INFO_SETTING;
44+
import static org.elasticsearch.rest.RestStatus.ACCEPTED;
45+
import static org.elasticsearch.rest.RestStatus.OK;
46+
import static org.elasticsearch.test.XContentTestUtils.createJsonMapView;
47+
import static org.hamcrest.Matchers.empty;
48+
import static org.hamcrest.Matchers.equalTo;
49+
import static org.hamcrest.Matchers.hasSize;
50+
import static org.hamcrest.Matchers.instanceOf;
51+
import static org.hamcrest.Matchers.is;
52+
53+
/**
54+
* This class tests the dangling indices REST API. These tests are here
55+
* today so they have access to a proper REST client. They cannot be in
56+
* :server:integTest since the REST client needs a proper transport
57+
* implementation, and they cannot be REST tests today since they need to
58+
* restart nodes. Really, though, this test should live elsewhere.
59+
*
60+
* @see org.elasticsearch.action.admin.indices.dangling
61+
*/
62+
@ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST, autoManageMasterNodes = false)
63+
public class DanglingIndicesRestIT extends HttpSmokeTestCase {
64+
private static final String INDEX_NAME = "test-idx-1";
65+
private static final String OTHER_INDEX_NAME = INDEX_NAME + "-other";
66+
67+
private Settings buildSettings(int maxTombstones) {
68+
return Settings.builder()
69+
// Limit the indices kept in the graveyard. This can be set to zero, so that
70+
// when we delete an index, it's definitely considered to be dangling.
71+
.put(SETTING_MAX_TOMBSTONES.getKey(), maxTombstones)
72+
.put(WRITE_DANGLING_INDICES_INFO_SETTING.getKey(), true)
73+
.put(AUTO_IMPORT_DANGLING_INDICES_SETTING.getKey(), false)
74+
.build();
75+
}
76+
77+
/**
78+
* Check that when dangling indices are discovered, then they can be listed via the REST API.
79+
*/
80+
public void testDanglingIndicesCanBeListed() throws Exception {
81+
internalCluster().setBootstrapMasterNodeIndex(1);
82+
internalCluster().startNodes(3, buildSettings(0));
83+
84+
final DanglingIndexDetails danglingIndexDetails = createDanglingIndices(INDEX_NAME);
85+
final String stoppedNodeId = mapNodeNameToId(danglingIndexDetails.stoppedNodeName);
86+
87+
final RestClient restClient = getRestClient();
88+
89+
final Response listResponse = restClient.performRequest(new Request("GET", "/_dangling"));
90+
assertOK(listResponse);
91+
92+
final XContentTestUtils.JsonMapView mapView = createJsonMapView(listResponse.getEntity().getContent());
93+
94+
assertThat(mapView.get("_nodes.total"), equalTo(3));
95+
assertThat(mapView.get("_nodes.successful"), equalTo(3));
96+
assertThat(mapView.get("_nodes.failed"), equalTo(0));
97+
98+
List<Object> indices = mapView.get("dangling_indices");
99+
assertThat(indices, hasSize(1));
100+
101+
assertThat(mapView.get("dangling_indices.0.index_name"), equalTo(INDEX_NAME));
102+
assertThat(mapView.get("dangling_indices.0.index_uuid"), equalTo(danglingIndexDetails.indexToUUID.get(INDEX_NAME)));
103+
assertThat(mapView.get("dangling_indices.0.creation_date_millis"), instanceOf(Long.class));
104+
assertThat(mapView.get("dangling_indices.0.node_ids.0"), equalTo(stoppedNodeId));
105+
}
106+
107+
/**
108+
* Check that dangling indices can be imported.
109+
*/
110+
public void testDanglingIndicesCanBeImported() throws Exception {
111+
internalCluster().setBootstrapMasterNodeIndex(1);
112+
internalCluster().startNodes(3, buildSettings(0));
113+
114+
createDanglingIndices(INDEX_NAME);
115+
116+
final RestClient restClient = getRestClient();
117+
118+
final List<String> danglingIndexIds = listDanglingIndexIds();
119+
assertThat(danglingIndexIds, hasSize(1));
120+
121+
final Request importRequest = new Request("POST", "/_dangling/" + danglingIndexIds.get(0));
122+
importRequest.addParameter("accept_data_loss", "true");
123+
// Ensure this parameter is accepted
124+
importRequest.addParameter("timeout", "20s");
125+
importRequest.addParameter("master_timeout", "20s");
126+
final Response importResponse = restClient.performRequest(importRequest);
127+
assertThat(importResponse.getStatusLine().getStatusCode(), equalTo(ACCEPTED.getStatus()));
128+
129+
final XContentTestUtils.JsonMapView mapView = createJsonMapView(importResponse.getEntity().getContent());
130+
assertThat(mapView.get("acknowledged"), equalTo(true));
131+
132+
assertTrue("Expected dangling index " + INDEX_NAME + " to be recovered", indexExists(INDEX_NAME));
133+
}
134+
135+
/**
136+
* Check that dangling indices can be deleted. Since this requires that
137+
* we add an entry to the index graveyard, the graveyard size must be
138+
* greater than 1. To test deletes, we set the index graveyard size to
139+
* 1, then create two indices and delete them both while one node in
140+
* the cluster is stopped. The deletion of the second pushes the deletion
141+
* of the first out of the graveyard. When the stopped node is resumed,
142+
* only the second index will be found into the graveyard and the the
143+
* other will be considered dangling, and can therefore be listed and
144+
* deleted through the API
145+
*/
146+
public void testDanglingIndicesCanBeDeleted() throws Exception {
147+
internalCluster().setBootstrapMasterNodeIndex(1);
148+
internalCluster().startNodes(3, buildSettings(1));
149+
150+
createDanglingIndices(INDEX_NAME, OTHER_INDEX_NAME);
151+
152+
final RestClient restClient = getRestClient();
153+
154+
final List<String> danglingIndexIds = listDanglingIndexIds();
155+
assertThat(danglingIndexIds, hasSize(1));
156+
157+
final Request deleteRequest = new Request("DELETE", "/_dangling/" + danglingIndexIds.get(0));
158+
deleteRequest.addParameter("accept_data_loss", "true");
159+
// Ensure these parameters is accepted
160+
deleteRequest.addParameter("timeout", "20s");
161+
deleteRequest.addParameter("master_timeout", "20s");
162+
final Response deleteResponse = restClient.performRequest(deleteRequest);
163+
assertThat(deleteResponse.getStatusLine().getStatusCode(), equalTo(ACCEPTED.getStatus()));
164+
165+
final XContentTestUtils.JsonMapView mapView = createJsonMapView(deleteResponse.getEntity().getContent());
166+
assertThat(mapView.get("acknowledged"), equalTo(true));
167+
168+
assertBusy(() -> assertThat("Expected dangling index to be deleted", listDanglingIndexIds(), hasSize(0)));
169+
170+
// The dangling index that we deleted ought to have been removed from disk. Check by
171+
// creating and deleting another index, which creates a new tombstone entry, which should
172+
// not cause the deleted dangling index to be considered "live" again, just because its
173+
// tombstone has been pushed out of the graveyard.
174+
createIndex("additional");
175+
deleteIndex("additional");
176+
assertThat(listDanglingIndexIds(), is(empty()));
177+
}
178+
179+
private List<String> listDanglingIndexIds() throws IOException {
180+
final Response response = getRestClient().performRequest(new Request("GET", "/_dangling"));
181+
assertOK(response);
182+
183+
final XContentTestUtils.JsonMapView mapView = createJsonMapView(response.getEntity().getContent());
184+
185+
assertThat(mapView.get("_nodes.total"), equalTo(3));
186+
assertThat(mapView.get("_nodes.successful"), equalTo(3));
187+
assertThat(mapView.get("_nodes.failed"), equalTo(0));
188+
189+
List<Object> indices = mapView.get("dangling_indices");
190+
191+
List<String> danglingIndexIds = new ArrayList<>();
192+
193+
for (int i = 0; i < indices.size(); i++) {
194+
danglingIndexIds.add(mapView.get("dangling_indices." + i + ".index_uuid"));
195+
}
196+
197+
return danglingIndexIds;
198+
}
199+
200+
private void assertOK(Response response) {
201+
assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus()));
202+
}
203+
204+
/**
205+
* Given a node name, finds the corresponding node ID.
206+
*/
207+
private String mapNodeNameToId(String nodeName) throws IOException {
208+
final Response catResponse = getRestClient().performRequest(new Request("GET", "/_cat/nodes?full_id&h=id,name"));
209+
assertOK(catResponse);
210+
211+
for (String nodeLine : Streams.readAllLines(catResponse.getEntity().getContent())) {
212+
String[] elements = nodeLine.split(" ");
213+
if (elements[1].equals(nodeName)) {
214+
return elements[0];
215+
}
216+
}
217+
218+
throw new AssertionError("Failed to map node name [" + nodeName + "] to node ID");
219+
}
220+
221+
/**
222+
* Helper that creates one or more indices, and importantly,
223+
* checks that they are green before proceeding. This is important
224+
* because the tests in this class stop and restart nodes, assuming
225+
* that each index has a primary or replica shard on every node, and if
226+
* a node is stopped prematurely, this assumption is broken.
227+
*
228+
* @return a mapping from each created index name to its UUID
229+
*/
230+
private Map<String, String> createIndices(String... indices) throws IOException {
231+
assert indices.length > 0;
232+
233+
for (String index : indices) {
234+
String indexSettings = "{"
235+
+ " \"settings\": {"
236+
+ " \"index\": {"
237+
+ " \"number_of_shards\": 1,"
238+
+ " \"number_of_replicas\": 2,"
239+
+ " \"routing\": {"
240+
+ " \"allocation\": {"
241+
+ " \"total_shards_per_node\": 1"
242+
+ " }"
243+
+ " }"
244+
+ " }"
245+
+ " }"
246+
+ "}";
247+
Request request = new Request("PUT", "/" + index);
248+
request.setJsonEntity(indexSettings);
249+
assertOK(getRestClient().performRequest(request));
250+
}
251+
ensureGreen(indices);
252+
253+
final Response catResponse = getRestClient().performRequest(new Request("GET", "/_cat/indices?h=index,uuid"));
254+
assertOK(catResponse);
255+
256+
final Map<String, String> createdIndexIDs = new HashMap<>();
257+
258+
final List<String> indicesAsList = Arrays.asList(indices);
259+
260+
for (String indexLine : Streams.readAllLines(catResponse.getEntity().getContent())) {
261+
String[] elements = indexLine.split(" +");
262+
if (indicesAsList.contains(elements[0])) {
263+
createdIndexIDs.put(elements[0], elements[1]);
264+
}
265+
}
266+
267+
assertThat("Expected to find as many index UUIDs as created indices", createdIndexIDs.size(), equalTo(indices.length));
268+
269+
return createdIndexIDs;
270+
}
271+
272+
private void deleteIndex(String indexName) throws IOException {
273+
Response deleteResponse = getRestClient().performRequest(new Request("DELETE", "/" + indexName));
274+
assertOK(deleteResponse);
275+
}
276+
277+
private DanglingIndexDetails createDanglingIndices(String... indices) throws Exception {
278+
ensureStableCluster(3);
279+
final Map<String, String> indexToUUID = createIndices(indices);
280+
281+
final AtomicReference<String> stoppedNodeName = new AtomicReference<>();
282+
283+
assertBusy(
284+
() -> internalCluster().getInstances(IndicesService.class)
285+
.forEach(indicesService -> assertTrue(indicesService.allPendingDanglingIndicesWritten()))
286+
);
287+
288+
// Restart node, deleting the index in its absence, so that there is a dangling index to recover
289+
internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
290+
291+
@Override
292+
public Settings onNodeStopped(String nodeName) throws Exception {
293+
ensureClusterSizeConsistency();
294+
stoppedNodeName.set(nodeName);
295+
for (String index : indices) {
296+
deleteIndex(index);
297+
}
298+
return super.onNodeStopped(nodeName);
299+
}
300+
});
301+
302+
ensureStableCluster(3);
303+
304+
return new DanglingIndexDetails(stoppedNodeName.get(), indexToUUID);
305+
}
306+
307+
private static class DanglingIndexDetails {
308+
private final String stoppedNodeName;
309+
private final Map<String, String> indexToUUID;
310+
311+
DanglingIndexDetails(String stoppedNodeName, Map<String, String> indexToUUID) {
312+
this.stoppedNodeName = stoppedNodeName;
313+
this.indexToUUID = indexToUUID;
314+
}
315+
}
316+
}

‎rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.delete.json

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"dangling_indices.delete": {
3+
"documentation": {
4+
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-gateway-dangling-indices.html",
5+
"description": "Deletes the specified dangling index"
6+
},
7+
"stability": "stable",
8+
"url": {
9+
"paths": [
10+
{
11+
"path": "/_dangling/{index_uuid}",
12+
"methods": [
13+
"DELETE"
14+
],
15+
"parts": {
16+
"index_uuid": {
17+
"type": "string",
18+
"description": "The UUID of the dangling index"
19+
}
20+
}
21+
}
22+
]
23+
},
24+
"params": {
25+
"accept_data_loss": {
26+
"type": "boolean",
27+
"description": "Must be set to true in order to delete the dangling index"
28+
},
29+
"timeout": {
30+
"type": "time",
31+
"description": "Explicit operation timeout"
32+
},
33+
"master_timeout": {
34+
"type": "time",
35+
"description": "Specify timeout for connection to master"
36+
}
37+
}
38+
}
39+
}

0 commit comments

Comments
 (0)