From 51460fc8e5d5367b6944e04d816cd784b9acec94 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Nov 2022 08:18:19 +0000 Subject: [PATCH 1/4] More actionable error for ancient indices We've had a few reports of failed upgrades caused by deleting the last ancient index just before stopping the first node, which may not have allowed enough time for the deletion to complete. Today the resulting error message doesn't say how to address this, which leaves users struggling mid-upgrade. This commit makes the error message more actionable. --- .../elasticsearch/env/NodeEnvironment.java | 20 +++++++++------- .../env/NodeEnvironmentTests.java | 24 ++++++++++++------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 66b62d8dbbb7f..2e14d50223aa0 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -495,9 +495,6 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings /** * Checks to see if we can upgrade to this version based on the existing index state. Upgrading * from older versions can cause irreversible changes if allowed. - * @param logger - * @param dataPaths - * @throws IOException */ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) throws IOException { final Path[] paths = Arrays.stream(dataPaths).map(np -> np.path).toArray(Path[]::new); @@ -520,14 +517,19 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr if (metadata.oldestIndexVersion().isLegacyIndexVersion()) { throw new IllegalStateException( - "cannot upgrade node because incompatible indices created with version [" + "Cannot start this node because it holds metadata for indices created with version [" + metadata.oldestIndexVersion() - + "] exist, while the minimum compatible index version is [" + + "] which which a node of version [" + + Version.CURRENT + + "] is incompatible. Revert this node to version [" + + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion()) + + "] and delete any indices created in versions earlier than [" + Version.CURRENT.minimumIndexCompatibilityVersion() - + "]. " - + "Upgrade your older indices by reindexing them in version [" - + Version.CURRENT.minimumCompatibilityVersion() - + "] first." + + "] before upgrading to version [" + + Version.CURRENT + + "]. If all such indices have already been deleted, revert this node to version [" + + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion()) + + "] and wait for it to join the cluster to clean up any older indices from its metadata." ); } } diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 814cc712a7c6c..33f7f8a3d24d0 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -564,7 +564,8 @@ public void testIndexCompatibilityChecks() throws IOException { } Version oldIndexVersion = Version.fromId(between(1, Version.CURRENT.minimumIndexCompatibilityVersion().id - 1)); - overrideOldestIndexVersion(oldIndexVersion, env.nodeDataPaths()); + Version previousNodeVersion = Version.fromId(between(Version.CURRENT.minimumCompatibilityVersion().id, Version.CURRENT.id - 1)); + overrideOldestIndexVersion(oldIndexVersion, previousNodeVersion, env.nodeDataPaths()); IllegalStateException ex = expectThrows( IllegalStateException.class, @@ -572,15 +573,21 @@ public void testIndexCompatibilityChecks() throws IOException { () -> checkForIndexCompatibility(logger, env.dataPaths()) ); - assertThat(ex.getMessage(), containsString("[" + oldIndexVersion + "] exist")); - assertThat(ex.getMessage(), startsWith("cannot upgrade node because incompatible indices created with version")); + assertThat( + ex.getMessage(), + allOf( + containsString("Cannot start this node"), + containsString("it holds metadata for indices created with version [" + oldIndexVersion + "]"), + containsString("Revert this node to version [" + previousNodeVersion + "]") + ) + ); // This should work - overrideOldestIndexVersion(Version.CURRENT.minimumIndexCompatibilityVersion(), env.nodeDataPaths()); + overrideOldestIndexVersion(Version.CURRENT.minimumIndexCompatibilityVersion(), previousNodeVersion, env.nodeDataPaths()); checkForIndexCompatibility(logger, env.dataPaths()); // Trying to boot with newer version should pass this check - overrideOldestIndexVersion(NodeMetadataTests.tooNewVersion(), env.nodeDataPaths()); + overrideOldestIndexVersion(NodeMetadataTests.tooNewVersion(), previousNodeVersion, env.nodeDataPaths()); checkForIndexCompatibility(logger, env.dataPaths()); // Simulate empty old index version, attempting to upgrade before 7.17 @@ -690,7 +697,8 @@ public NodeEnvironment newNodeEnvironment(String[] dataPaths, String sharedDataP return new NodeEnvironment(build, TestEnvironment.newEnvironment(build)); } - private static void overrideOldestIndexVersion(Version oldVersion, Path... dataPaths) throws IOException { + private static void overrideOldestIndexVersion(Version oldestIndexVersion, Version previousNodeVersion, Path... dataPaths) + throws IOException { for (final Path dataPath : dataPaths) { final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME); if (Files.exists(indexPath)) { @@ -705,8 +713,8 @@ private static void overrideOldestIndexVersion(Version oldVersion, Path... dataP ) ) { final Map commitData = new HashMap<>(userData); - commitData.put(NODE_VERSION_KEY, Integer.toString(Version.CURRENT.minimumCompatibilityVersion().id)); - commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldVersion.id)); + commitData.put(NODE_VERSION_KEY, Integer.toString(previousNodeVersion.id)); + commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id)); indexWriter.setLiveCommitData(commitData.entrySet()); indexWriter.commit(); } From d759a386142ef9bbe89eac595b2b709f05d73e20 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Nov 2022 08:22:53 +0000 Subject: [PATCH 2/4] Update docs/changelog/91243.yaml --- docs/changelog/91243.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/91243.yaml diff --git a/docs/changelog/91243.yaml b/docs/changelog/91243.yaml new file mode 100644 index 0000000000000..ac0433b51769f --- /dev/null +++ b/docs/changelog/91243.yaml @@ -0,0 +1,5 @@ +pr: 91243 +summary: More actionable error for ancient indices +area: Infra/Core +type: enhancement +issues: [] From 71f5870827cc620d44ef3142ebef9f26eadca82e Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Nov 2022 08:55:55 +0000 Subject: [PATCH 3/4] Typo --- server/src/main/java/org/elasticsearch/env/NodeEnvironment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 2e14d50223aa0..76f10f09b5af7 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -519,7 +519,7 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr throw new IllegalStateException( "Cannot start this node because it holds metadata for indices created with version [" + metadata.oldestIndexVersion() - + "] which which a node of version [" + + "] with which a node of version [" + Version.CURRENT + "] is incompatible. Revert this node to version [" + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion()) From 5602f5c2061106f763f7cd921e68850dc963c975 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Nov 2022 13:24:41 +0000 Subject: [PATCH 4/4] s/a/this/ --- server/src/main/java/org/elasticsearch/env/NodeEnvironment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 76f10f09b5af7..e4c42af85af47 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -519,7 +519,7 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr throw new IllegalStateException( "Cannot start this node because it holds metadata for indices created with version [" + metadata.oldestIndexVersion() - + "] with which a node of version [" + + "] with which this node of version [" + Version.CURRENT + "] is incompatible. Revert this node to version [" + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion())