Skip to content

Commit 49073dd

Browse files
Fail start on invalid index metadata (#37748)
Node started with node.data=false and node.master=false can no longer start if they have index metadata. This avoids resurrecting old indexes into the cluster and ensures metadata is cleaned out before re-purposing a node that was previously master or data node. Issue #27073
1 parent deafce1 commit 49073dd

File tree

3 files changed

+119
-20
lines changed

3 files changed

+119
-20
lines changed

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,10 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
312312
}
313313

314314
if (DiscoveryNode.isDataNode(settings) == false) {
315+
if (DiscoveryNode.isMasterNode(settings) == false) {
316+
ensureNoIndexMetaData(nodePaths);
317+
}
318+
315319
ensureNoShardData(nodePaths);
316320
}
317321

@@ -1037,37 +1041,59 @@ private static void ensureAtomicMoveSupported(final NodePath[] nodePaths) throws
10371041
}
10381042

10391043
private void ensureNoShardData(final NodePath[] nodePaths) throws IOException {
1040-
List<Path> shardDataPaths = new ArrayList<>();
1044+
List<Path> shardDataPaths = collectIndexSubPaths(nodePaths, this::isShardPath);
1045+
if (shardDataPaths.isEmpty() == false) {
1046+
throw new IllegalStateException("Node is started with "
1047+
+ Node.NODE_DATA_SETTING.getKey()
1048+
+ "=false, but has shard data: "
1049+
+ shardDataPaths);
1050+
}
1051+
}
1052+
1053+
private void ensureNoIndexMetaData(final NodePath[] nodePaths) throws IOException {
1054+
List<Path> indexMetaDataPaths = collectIndexSubPaths(nodePaths, this::isIndexMetaDataPath);
1055+
if (indexMetaDataPaths.isEmpty() == false) {
1056+
throw new IllegalStateException("Node is started with "
1057+
+ Node.NODE_DATA_SETTING.getKey()
1058+
+ "=false and "
1059+
+ Node.NODE_MASTER_SETTING.getKey()
1060+
+ "=false, but has index metadata: "
1061+
+ indexMetaDataPaths);
1062+
}
1063+
}
1064+
1065+
private List<Path> collectIndexSubPaths(NodePath[] nodePaths, Predicate<Path> subPathPredicate) throws IOException {
1066+
List<Path> indexSubPaths = new ArrayList<>();
10411067
for (NodePath nodePath : nodePaths) {
10421068
Path indicesPath = nodePath.indicesPath;
10431069
if (Files.isDirectory(indicesPath)) {
10441070
try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(indicesPath)) {
10451071
for (Path indexPath : indexStream) {
10461072
if (Files.isDirectory(indexPath)) {
10471073
try (Stream<Path> shardStream = Files.list(indexPath)) {
1048-
shardStream.filter(this::isShardPath)
1074+
shardStream.filter(subPathPredicate)
10491075
.map(Path::toAbsolutePath)
1050-
.forEach(shardDataPaths::add);
1076+
.forEach(indexSubPaths::add);
10511077
}
10521078
}
10531079
}
10541080
}
10551081
}
10561082
}
10571083

1058-
if (shardDataPaths.isEmpty() == false) {
1059-
throw new IllegalStateException("Node is started with "
1060-
+ Node.NODE_DATA_SETTING.getKey()
1061-
+ "=false, but has shard data: "
1062-
+ shardDataPaths);
1063-
}
1084+
return indexSubPaths;
10641085
}
10651086

10661087
private boolean isShardPath(Path path) {
10671088
return Files.isDirectory(path)
10681089
&& path.getFileName().toString().chars().allMatch(Character::isDigit);
10691090
}
10701091

1092+
private boolean isIndexMetaDataPath(Path path) {
1093+
return Files.isDirectory(path)
1094+
&& path.getFileName().toString().equals(MetaDataStateFormat.STATE_DIR_NAME);
1095+
}
1096+
10711097
/**
10721098
* Resolve the custom path for a index's shard.
10731099
* Uses the {@code IndexMetaData.SETTING_DATA_PATH} setting to determine

server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,37 @@ public void testStartFailureOnDataForNonDataNode() throws Exception {
4242
).get();
4343
final String indexUUID = resolveIndex(indexName).getUUID();
4444

45+
logger.info("--> restarting the node with node.data=false and node.master=false");
46+
IllegalStateException ex = expectThrows(IllegalStateException.class,
47+
"Node started with node.data=false and node.master=false while having existing index metadata must fail",
48+
() ->
49+
internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
50+
@Override
51+
public Settings onNodeStopped(String nodeName) {
52+
return Settings.builder()
53+
.put(Node.NODE_DATA_SETTING.getKey(), false)
54+
.put(Node.NODE_MASTER_SETTING.getKey(), false)
55+
.build();
56+
}
57+
}));
58+
assertThat(ex.getMessage(), containsString(indexUUID));
59+
assertThat(ex.getMessage(),
60+
startsWith("Node is started with "
61+
+ Node.NODE_DATA_SETTING.getKey()
62+
+ "=false and "
63+
+ Node.NODE_MASTER_SETTING.getKey()
64+
+ "=false, but has index metadata"));
65+
66+
// client() also starts the node
4567
logger.info("--> indexing a simple document");
4668
client().prepareIndex(indexName, "type1", "1").setSource("field1", "value1").get();
4769

48-
logger.info("--> restarting the node with node.data=true");
70+
logger.info("--> restarting the node with node.data=true and node.master=true");
4971
internalCluster().restartRandomDataNode();
5072

5173
logger.info("--> restarting the node with node.data=false");
52-
IllegalStateException ex = expectThrows(IllegalStateException.class,
53-
"Node started with node.data=false and existing shard data must fail",
74+
ex = expectThrows(IllegalStateException.class,
75+
"Node started with node.data=false while having existing shard data must fail",
5476
() ->
5577
internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
5678
@Override

server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -472,45 +472,96 @@ public void testExistingTempFiles() throws IOException {
472472
}
473473
}
474474

475-
public void testEnsureNoShardData() throws IOException {
475+
public void testEnsureNoShardDataOrIndexMetaData() throws IOException {
476476
Settings settings = buildEnvSettings(Settings.EMPTY);
477477
Index index = new Index("test", "testUUID");
478478

479+
// build settings using same path.data as original but with node.data=false and node.master=false
480+
Settings noDataNoMasterSettings = Settings.builder()
481+
.put(settings)
482+
.put(Node.NODE_DATA_SETTING.getKey(), false)
483+
.put(Node.NODE_MASTER_SETTING.getKey(), false)
484+
.build();
485+
486+
// test that we can create data=false and master=false with no meta information
487+
newNodeEnvironment(noDataNoMasterSettings).close();
488+
489+
Path indexPath;
479490
try (NodeEnvironment env = newNodeEnvironment(settings)) {
480491
for (Path path : env.indexPaths(index)) {
481492
Files.createDirectories(path.resolve(MetaDataStateFormat.STATE_DIR_NAME));
482493
}
494+
indexPath = env.indexPaths(index)[0];
483495
}
484496

497+
verifyFailsOnMetaData(noDataNoMasterSettings, indexPath);
498+
485499
// build settings using same path.data as original but with node.data=false
486500
Settings noDataSettings = Settings.builder()
487501
.put(settings)
488502
.put(Node.NODE_DATA_SETTING.getKey(), false).build();
489503

490504
String shardDataDirName = Integer.toString(randomInt(10));
491-
Path shardPath;
492505

493-
// test that we can create data=false env with only meta information
506+
// test that we can create data=false env with only meta information. Also create shard data for following asserts
494507
try (NodeEnvironment env = newNodeEnvironment(noDataSettings)) {
495508
for (Path path : env.indexPaths(index)) {
496509
Files.createDirectories(path.resolve(shardDataDirName));
497510
}
498-
shardPath = env.indexPaths(index)[0];
499511
}
500512

513+
verifyFailsOnShardData(noDataSettings, indexPath, shardDataDirName);
514+
515+
// assert that we get the stricter message on meta-data when both conditions fail
516+
verifyFailsOnMetaData(noDataNoMasterSettings, indexPath);
517+
518+
// build settings using same path.data as original but with node.master=false
519+
Settings noMasterSettings = Settings.builder()
520+
.put(settings)
521+
.put(Node.NODE_MASTER_SETTING.getKey(), false)
522+
.build();
523+
524+
// test that we can create master=false env regardless of data.
525+
newNodeEnvironment(noMasterSettings).close();
526+
527+
// test that we can create data=true, master=true env. Also remove state dir to leave only shard data for following asserts
528+
try (NodeEnvironment env = newNodeEnvironment(settings)) {
529+
for (Path path : env.indexPaths(index)) {
530+
Files.delete(path.resolve(MetaDataStateFormat.STATE_DIR_NAME));
531+
}
532+
}
533+
534+
// assert that we fail on shard data even without the metadata dir.
535+
verifyFailsOnShardData(noDataSettings, indexPath, shardDataDirName);
536+
verifyFailsOnShardData(noDataNoMasterSettings, indexPath, shardDataDirName);
537+
}
538+
539+
private void verifyFailsOnShardData(Settings settings, Path indexPath, String shardDataDirName) {
501540
IllegalStateException ex = expectThrows(IllegalStateException.class,
502541
"Must fail creating NodeEnvironment on a data path that has shard data if node.data=false",
503-
() -> newNodeEnvironment(noDataSettings).close());
542+
() -> newNodeEnvironment(settings).close());
504543

505544
assertThat(ex.getMessage(),
506-
containsString(shardPath.resolve(shardDataDirName).toAbsolutePath().toString()));
545+
containsString(indexPath.resolve(shardDataDirName).toAbsolutePath().toString()));
507546
assertThat(ex.getMessage(),
508547
startsWith("Node is started with "
509548
+ Node.NODE_DATA_SETTING.getKey()
510549
+ "=false, but has shard data"));
550+
}
511551

512-
// test that we can create data=true env
513-
newNodeEnvironment(settings).close();
552+
private void verifyFailsOnMetaData(Settings settings, Path indexPath) {
553+
IllegalStateException ex = expectThrows(IllegalStateException.class,
554+
"Must fail creating NodeEnvironment on a data path that has index meta-data if node.data=false and node.master=false",
555+
() -> newNodeEnvironment(settings).close());
556+
557+
assertThat(ex.getMessage(),
558+
containsString(indexPath.resolve(MetaDataStateFormat.STATE_DIR_NAME).toAbsolutePath().toString()));
559+
assertThat(ex.getMessage(),
560+
startsWith("Node is started with "
561+
+ Node.NODE_DATA_SETTING.getKey()
562+
+ "=false and "
563+
+ Node.NODE_MASTER_SETTING.getKey()
564+
+ "=false, but has index metadata"));
514565
}
515566

516567
/** Converts an array of Strings to an array of Paths, adding an additional child if specified */

0 commit comments

Comments
 (0)