More actionable error for ancient indices

docs/changelog/91243.yaml

@@ -0,0 +1,5 @@
+pr: 91243
+summary: More actionable error for ancient indices
+area: Infra/Core
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -495,9 +495,6 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings
     /**
      * Checks to see if we can upgrade to this version based on the existing index state. Upgrading
      * from older versions can cause irreversible changes if allowed.
-     * @param logger
-     * @param dataPaths
-     * @throws IOException
      */
     static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) throws IOException {
         final Path[] paths = Arrays.stream(dataPaths).map(np -> np.path).toArray(Path[]::new);
@@ -520,14 +517,19 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr
 
         if (metadata.oldestIndexVersion().isLegacyIndexVersion()) {
             throw new IllegalStateException(
-                "cannot upgrade node because incompatible indices created with version ["
+                "Cannot start this node because it holds metadata for indices created with version ["
                     + metadata.oldestIndexVersion()
-                    + "] exist, while the minimum compatible index version is ["
+                    + "] with which this node of version ["
+                    + Version.CURRENT
+                    + "] is incompatible. Revert this node to version ["
+                    + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion())
+                    + "] and delete any indices created in versions earlier than ["
                     + Version.CURRENT.minimumIndexCompatibilityVersion()
-                    + "]. "
-                    + "Upgrade your older indices by reindexing them in version ["
-                    + Version.CURRENT.minimumCompatibilityVersion()
-                    + "] first."
+                    + "] before upgrading to version ["
+                    + Version.CURRENT
+                    + "]. If all such indices have already been deleted, revert this node to version ["
+                    + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion())
+                    + "] and wait for it to join the cluster to clean up any older indices from its metadata."
             );
         }
     }

server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

@@ -564,23 +564,30 @@ public void testIndexCompatibilityChecks() throws IOException {
             }
 
             Version oldIndexVersion = Version.fromId(between(1, Version.CURRENT.minimumIndexCompatibilityVersion().id - 1));
-            overrideOldestIndexVersion(oldIndexVersion, env.nodeDataPaths());
+            Version previousNodeVersion = Version.fromId(between(Version.CURRENT.minimumCompatibilityVersion().id, Version.CURRENT.id - 1));
+            overrideOldestIndexVersion(oldIndexVersion, previousNodeVersion, env.nodeDataPaths());
 
             IllegalStateException ex = expectThrows(
                 IllegalStateException.class,
                 "Must fail the check on index that's too old",
                 () -> checkForIndexCompatibility(logger, env.dataPaths())
             );
 
-            assertThat(ex.getMessage(), containsString("[" + oldIndexVersion + "] exist"));
-            assertThat(ex.getMessage(), startsWith("cannot upgrade node because incompatible indices created with version"));
+            assertThat(
+                ex.getMessage(),
+                allOf(
+                    containsString("Cannot start this node"),
+                    containsString("it holds metadata for indices created with version [" + oldIndexVersion + "]"),
+                    containsString("Revert this node to version [" + previousNodeVersion + "]")
+                )
+            );
 
             // This should work
-            overrideOldestIndexVersion(Version.CURRENT.minimumIndexCompatibilityVersion(), env.nodeDataPaths());
+            overrideOldestIndexVersion(Version.CURRENT.minimumIndexCompatibilityVersion(), previousNodeVersion, env.nodeDataPaths());
             checkForIndexCompatibility(logger, env.dataPaths());
 
             // Trying to boot with newer version should pass this check
-            overrideOldestIndexVersion(NodeMetadataTests.tooNewVersion(), env.nodeDataPaths());
+            overrideOldestIndexVersion(NodeMetadataTests.tooNewVersion(), previousNodeVersion, env.nodeDataPaths());
             checkForIndexCompatibility(logger, env.dataPaths());
 
             // Simulate empty old index version, attempting to upgrade before 7.17
@@ -690,7 +697,8 @@ public NodeEnvironment newNodeEnvironment(String[] dataPaths, String sharedDataP
         return new NodeEnvironment(build, TestEnvironment.newEnvironment(build));
     }
 
-    private static void overrideOldestIndexVersion(Version oldVersion, Path... dataPaths) throws IOException {
+    private static void overrideOldestIndexVersion(Version oldestIndexVersion, Version previousNodeVersion, Path... dataPaths)
+        throws IOException {
         for (final Path dataPath : dataPaths) {
             final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME);
             if (Files.exists(indexPath)) {
@@ -705,8 +713,8 @@ private static void overrideOldestIndexVersion(Version oldVersion, Path... dataP
                         )
                     ) {
                         final Map<String, String> commitData = new HashMap<>(userData);
-                        commitData.put(NODE_VERSION_KEY, Integer.toString(Version.CURRENT.minimumCompatibilityVersion().id));
-                        commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldVersion.id));
+                        commitData.put(NODE_VERSION_KEY, Integer.toString(previousNodeVersion.id));
+                        commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id));
                         indexWriter.setLiveCommitData(commitData.entrySet());
                         indexWriter.commit();
                     }