Prevent non-data nodes to start with dangling indices

elastic#27073

Author: Vladimir Dolzhenko

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -294,6 +294,7 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
             this.locks = nodeLock.locks;
             this.nodePaths = nodeLock.nodePaths;
             this.nodeLockId = nodeLock.nodeId;
+            checkDanglingIndices(settings);
             this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths);
 
             if (logger.isDebugEnabled()) {
@@ -313,6 +314,33 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
         }
     }
 
+    private void checkDanglingIndices(Settings settings) throws IOException {
+        if (DiscoveryNode.isDataNode(settings) == false) {
+            final List<Path> danglingShards = new ArrayList<>();
+            for (final NodePath nodePath : nodePaths) {
+                // path to shard is: <node-id>/indices/<index-uuid>/<shard-id>
+                if (Files.exists(nodePath.indicesPath) && Files.isDirectory(nodePath.indicesPath)) {
+                    try (DirectoryStream<Path> stream = Files.newDirectoryStream(nodePath.indicesPath)) {
+                        for (final Path index : stream) {
+                            try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(index)) {
+                                for (Path path: indexStream) {
+                                    // it's ok for non data node to have _state folder
+                                    if (MetaDataStateFormat.STATE_DIR_NAME.equals(path.getFileName().toString()) == false) {
+                                        danglingShards.add(path.toAbsolutePath());
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if (danglingShards.isEmpty() == false) {
+                throw new IllegalStateException("Non data node cannot have dangling indices,"
+                    + " data shards found: " + danglingShards);
+            }
+        }
+    }
+
     /**
      * Resolve a specific nodes/{node.id} path for the specified path and node lock id.
      *

server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.env;
 
 import org.apache.lucene.index.SegmentInfos;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
@@ -31,6 +32,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
@@ -53,6 +55,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 
 @LuceneTestCase.SuppressFileSystems("ExtrasFS") // TODO: fix test to allow extras
 public class NodeEnvironmentTests extends ESTestCase {
@@ -472,6 +475,41 @@ public void testExistingTempFiles() throws IOException {
         }
     }
 
+    public void testNonDataNodeFailsInPresenceOfDanglingIndices() throws IOException {
+        final Settings settings = buildEnvSettings(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build());
+        final List<String> dataPaths = Environment.PATH_DATA_SETTING.get(settings);
+        final Path nodePath = NodeEnvironment.resolveNodePath(PathUtils.get(dataPaths.get(0)), 0);
+        final Path indicesPath = nodePath.resolve(NodeEnvironment.INDICES_FOLDER);
+        final Path indexPath = indicesPath.resolve(UUIDs.randomBase64UUID());
+        Files.createDirectories(indexPath);
+
+        if (randomBoolean()) {
+            // _state is ok to keep on non-data node
+            Files.createDirectories(indexPath.resolve(MetaDataStateFormat.STATE_DIR_NAME));
+        }
+
+        final boolean allocateShard = randomBoolean();
+        Path shardPath = null;
+        if (allocateShard) {
+            shardPath = indexPath.resolve("0");
+            // allocate shard 0
+            Files.createDirectories(shardPath);
+        }
+
+        try {
+            try (NodeEnvironment env = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings))) {
+                logger.info("indicesPath: {}", env.nodePaths()[0].indicesPath);
+            }
+            assertThat(allocateShard, equalTo(false));
+        } catch (IllegalStateException e) {
+            assertThat(allocateShard, equalTo(true));
+            assertThat(shardPath, notNullValue());
+            assertThat(e.getMessage(),
+                equalTo("Non data node cannot have dangling indices, data shards found: ["
+                    + shardPath.toAbsolutePath() + "]"));
+        }
+    }
+
     /** Converts an array of Strings to an array of Paths, adding an additional child if specified */
     private Path[] stringsToPaths(String[] strings, String additional) {
         Path[] locations = new Path[strings.length];

server/src/test/java/org/elasticsearch/gateway/DanglingIndicesIT.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gateway;
+
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+
+import java.nio.file.Path;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.startsWith;
+
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class DanglingIndicesIT extends ESIntegTestCase {
+
+    public void testDanglingIndices() throws Exception {
+        // replicating the case described in https://github.com/elastic/elasticsearch/issues/27073
+        final String indexName = "simple1";
+
+        final InternalTestCluster cluster = internalCluster();
+        String node1 = cluster.startNode();
+        String node2 = cluster.startNode();
+
+        final boolean allocateOnNode2 = randomBoolean();
+
+        createIndex(indexName,
+            Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                // explicitly specify shard location
+                .put("index.routing.allocation.include._name", allocateOnNode2 ? node2 : node1)
+                .build());
+        ensureGreen(indexName);
+
+        if (randomBoolean()) {
+            index(indexName, "_doc", "1", "f", "f");
+        }
+
+        final NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, node1);
+        final Path[] node1DataPaths = env.nodeDataPaths();
+
+        cluster.stopRandomNode(InternalTestCluster.nameFilter(node1));
+        cluster.stopRandomNode(InternalTestCluster.nameFilter(node2));
+
+        // clean up data path on node1 to trigger dangling index being picked up
+        IOUtils.rm(node1DataPaths);
+
+        // start node that could have dangling index
+        node1 = cluster.startNode();
+        try {
+            node2 = randomBoolean() ? cluster.startMasterOnlyNode() : cluster.startCoordinatingOnlyNode(Settings.EMPTY);
+            assertThat("Node 2 has to fail as it contains shard data",
+                allocateOnNode2, equalTo(false));
+            assertThat("index exists but has to be red",
+                client().admin().cluster().health(Requests.clusterHealthRequest(indexName)).get().getStatus(),
+                equalTo(ClusterHealthStatus.RED));
+        } catch (IllegalStateException e) {
+            assertThat(allocateOnNode2, equalTo(true));
+            assertThat(e.getMessage(), startsWith("Non data node cannot have dangling indices"));
+        }
+    }
+
+}