Skip to content

Commit 4db7fd9

Browse files
tlrxtlrx
authored
Adapt the Recovery API for closed indices (#38421)
This commit adapts the Recovery API to make it work with shards of replicated closed indices. Relates #33888
1 parent 4fd1bb2 commit 4db7fd9

File tree

6 files changed

+156
-19
lines changed

6 files changed

+156
-19
lines changed

‎rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,60 @@
7575
\n
7676
)+
7777
$/
78+
79+
---
80+
"Test cat recovery output for closed index":
81+
- skip:
82+
version: " - 7.99.99"
83+
reason: closed indices are replicated starting version 8.0.0
84+
85+
- do:
86+
indices.create:
87+
index: index2
88+
body:
89+
settings:
90+
index:
91+
number_of_replicas: 0
92+
93+
- do:
94+
indices.close:
95+
index: index2
96+
- is_true: acknowledged
97+
98+
- do:
99+
cluster.health:
100+
index: index2
101+
wait_for_status: green
102+
103+
- do:
104+
cat.recovery:
105+
index: index2
106+
h: i,s,t,ty,st,shost,thost,rep,snap,f,fr,fp,tf,b,br,bp,tb,to,tor,top
107+
108+
- match:
109+
$body: |
110+
/^
111+
(
112+
index2 \s+
113+
\d \s+ # shard
114+
(?:\d+ms|\d+(?:\.\d+)?s) \s+ # time in ms or seconds
115+
existing_store \s+ # source type (always existing_store for closed indices)
116+
done \s+ # stage
117+
[-\w./]+ \s+ # source_host
118+
[-\w./]+ \s+ # target_host
119+
[-\w./]+ \s+ # repository
120+
[-\w./]+ \s+ # snapshot
121+
\d+ \s+ # files
122+
\d+ \s+ # files_recovered
123+
\d+\.\d+% \s+ # files_percent
124+
\d+ \s+ # files_total
125+
\d+ \s+ # bytes
126+
\d+ \s+ # bytes_recovered
127+
\d+\.\d+% \s+ # bytes_percent
128+
\d+ \s+ # bytes_total
129+
0 \s+ # translog_ops (always 0 for closed indices)
130+
0 \s+ # translog_ops_recovered (always 0 for closed indices)
131+
100\.0% # translog_ops_percent (always 100.0% for closed indices)
132+
\n
133+
)+
134+
$/

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,56 @@
4040
- gte: { test_1.shards.0.verify_index.check_index_time_in_millis: 0 }
4141
- gte: { test_1.shards.0.verify_index.total_time_in_millis: 0 }
4242
---
43+
"Indices recovery test for closed index":
44+
- skip:
45+
version: " - 7.99.99"
46+
reason: closed indices are replicated starting version 8.0.0
47+
48+
- do:
49+
indices.create:
50+
index: test_2
51+
body:
52+
settings:
53+
index:
54+
number_of_replicas: 0
55+
56+
- do:
57+
indices.close:
58+
index: test_2
59+
- is_true: acknowledged
60+
61+
- do:
62+
cluster.health:
63+
index: test_2
64+
wait_for_status: green
65+
66+
- do:
67+
indices.recovery:
68+
index: [test_2]
69+
human: true
70+
71+
- match: { test_2.shards.0.type: "EXISTING_STORE" }
72+
- match: { test_2.shards.0.stage: "DONE" }
73+
- match: { test_2.shards.0.primary: true }
74+
- match: { test_2.shards.0.start_time: /^2\d\d\d-.+/ }
75+
- match: { test_2.shards.0.target.ip: /^\d+\.\d+\.\d+\.\d+$/ }
76+
- gte: { test_2.shards.0.index.files.total: 0 }
77+
- gte: { test_2.shards.0.index.files.reused: 0 }
78+
- gte: { test_2.shards.0.index.files.recovered: 0 }
79+
- match: { test_2.shards.0.index.files.percent: /^\d+\.\d\%$/ }
80+
- gte: { test_2.shards.0.index.size.total_in_bytes: 0 }
81+
- gte: { test_2.shards.0.index.size.reused_in_bytes: 0 }
82+
- gte: { test_2.shards.0.index.size.recovered_in_bytes: 0 }
83+
- match: { test_2.shards.0.index.size.percent: /^\d+\.\d\%$/ }
84+
- gte: { test_2.shards.0.index.source_throttle_time_in_millis: 0 }
85+
- gte: { test_2.shards.0.index.target_throttle_time_in_millis: 0 }
86+
- gte: { test_2.shards.0.translog.recovered: 0 }
87+
- gte: { test_2.shards.0.translog.total: 0 }
88+
- gte: { test_2.shards.0.translog.total_on_start: 0 }
89+
- gte: { test_2.shards.0.translog.total_time_in_millis: 0 }
90+
- gte: { test_2.shards.0.verify_index.check_index_time_in_millis: 0 }
91+
- gte: { test_2.shards.0.verify_index.total_time_in_millis: 0 }
92+
---
4393
"Indices recovery test index name not matching":
4494

4595
- do:

‎server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryRequest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.action.admin.indices.recovery;
2121

22+
import org.elasticsearch.action.support.IndicesOptions;
2223
import org.elasticsearch.action.support.broadcast.BroadcastRequest;
2324
import org.elasticsearch.common.Strings;
2425
import org.elasticsearch.common.io.stream.StreamInput;
@@ -47,7 +48,7 @@ public RecoveryRequest() {
4748
* @param indices Comma-separated list of indices about which to gather recovery information
4849
*/
4950
public RecoveryRequest(String... indices) {
50-
super(indices);
51+
super(indices, IndicesOptions.STRICT_EXPAND_OPEN_CLOSED);
5152
}
5253

5354
/**

‎server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,11 @@ protected ShardsIterator shards(ClusterState state, RecoveryRequest request, Str
112112

113113
@Override
114114
protected ClusterBlockException checkGlobalBlock(ClusterState state, RecoveryRequest request) {
115-
return state.blocks().globalBlockedException(ClusterBlockLevel.READ);
115+
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
116116
}
117117

118118
@Override
119119
protected ClusterBlockException checkRequestBlock(ClusterState state, RecoveryRequest request, String[] concreteIndices) {
120-
return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices);
120+
return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices);
121121
}
122122
}

‎server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastRequest.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ protected BroadcastRequest(String[] indices) {
4040
this.indices = indices;
4141
}
4242

43+
protected BroadcastRequest(String[] indices, IndicesOptions indicesOptions) {
44+
this.indices = indices;
45+
this.indicesOptions = indicesOptions;
46+
}
47+
4348
@Override
4449
public String[] indices() {
4550
return indices;

‎server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.elasticsearch.plugins.Plugin;
5555
import org.elasticsearch.snapshots.Snapshot;
5656
import org.elasticsearch.snapshots.SnapshotState;
57+
import org.elasticsearch.test.BackgroundIndexer;
5758
import org.elasticsearch.test.ESIntegTestCase;
5859
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
5960
import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -209,24 +210,34 @@ public void testGatewayRecoveryTestActiveOnly() throws Exception {
209210
}
210211

211212
public void testReplicaRecovery() throws Exception {
212-
logger.info("--> start node A");
213-
String nodeA = internalCluster().startNode();
213+
final String nodeA = internalCluster().startNode();
214+
createIndex(INDEX_NAME, Settings.builder()
215+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, SHARD_COUNT)
216+
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, REPLICA_COUNT)
217+
.build());
218+
ensureGreen(INDEX_NAME);
219+
220+
final int numOfDocs = scaledRandomIntBetween(0, 200);
221+
try (BackgroundIndexer indexer = new BackgroundIndexer(INDEX_NAME, "_doc", client(), numOfDocs)) {
222+
waitForDocs(numOfDocs, indexer);
223+
}
214224

215-
logger.info("--> create index on node: {}", nodeA);
216-
createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
225+
refresh(INDEX_NAME);
226+
assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs);
217227

218-
logger.info("--> start node B");
219-
String nodeB = internalCluster().startNode();
220-
ensureGreen();
228+
final boolean closedIndex = randomBoolean();
229+
if (closedIndex) {
230+
assertAcked(client().admin().indices().prepareClose(INDEX_NAME));
231+
ensureGreen(INDEX_NAME);
232+
}
221233

222234
// force a shard recovery from nodeA to nodeB
223-
logger.info("--> bump replica count");
224-
client().admin().indices().prepareUpdateSettings(INDEX_NAME)
225-
.setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet();
226-
ensureGreen();
235+
final String nodeB = internalCluster().startNode();
236+
assertAcked(client().admin().indices().prepareUpdateSettings(INDEX_NAME)
237+
.setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)));
238+
ensureGreen(INDEX_NAME);
227239

228-
logger.info("--> request recoveries");
229-
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
240+
final RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
230241

231242
// we should now have two total shards, one primary and one replica
232243
List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
@@ -238,14 +249,27 @@ public void testReplicaRecovery() throws Exception {
238249
assertThat(nodeBResponses.size(), equalTo(1));
239250

240251
// validate node A recovery
241-
RecoveryState nodeARecoveryState = nodeAResponses.get(0);
242-
assertRecoveryState(nodeARecoveryState, 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA);
252+
final RecoveryState nodeARecoveryState = nodeAResponses.get(0);
253+
final RecoverySource expectedRecoverySource;
254+
if (closedIndex == false) {
255+
expectedRecoverySource = RecoverySource.EmptyStoreRecoverySource.INSTANCE;
256+
} else {
257+
expectedRecoverySource = RecoverySource.ExistingStoreRecoverySource.INSTANCE;
258+
}
259+
assertRecoveryState(nodeARecoveryState, 0, expectedRecoverySource, true, Stage.DONE, null, nodeA);
243260
validateIndexRecoveryState(nodeARecoveryState.getIndex());
244261

245262
// validate node B recovery
246-
RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
263+
final RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
247264
assertRecoveryState(nodeBRecoveryState, 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeA, nodeB);
248265
validateIndexRecoveryState(nodeBRecoveryState.getIndex());
266+
267+
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
268+
269+
if (closedIndex) {
270+
assertAcked(client().admin().indices().prepareOpen(INDEX_NAME));
271+
}
272+
assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), numOfDocs);
249273
}
250274

251275
@TestLogging(

0 commit comments

Comments
 (0)