Deprecate shared and index data path settings (#73178) (#73198)

This commit adds deprecation warnings for use of the path.shared_data
setting as well as the index setting index.data_path.

relates #73168

Author: rjernst

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1221,7 +1221,6 @@ private void createConfiguration() {
         baseConfig.put("path.repo", confPathRepo.toAbsolutePath().toString());
         baseConfig.put("path.data", confPathData.toAbsolutePath().toString());
         baseConfig.put("path.logs", confPathLogs.toAbsolutePath().toString());
-        baseConfig.put("path.shared_data", workingDir.resolve("sharedData").toString());
         baseConfig.put("node.attr.testattr", "test");
         baseConfig.put("node.portsfile", "true");
         baseConfig.put("http.port", httpPort);

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -282,7 +282,7 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
     public static final String SETTING_HISTORY_UUID = "index.history.uuid";
     public static final String SETTING_DATA_PATH = "index.data_path";
     public static final Setting<String> INDEX_DATA_PATH_SETTING =
-        new Setting<>(SETTING_DATA_PATH, "", Function.identity(), Property.IndexScope);
+        new Setting<>(SETTING_DATA_PATH, "", Function.identity(), Property.IndexScope, Property.Deprecated);
     public static final String INDEX_UUID_NA_VALUE = "_na_";
 
     public static final String INDEX_ROUTING_REQUIRE_GROUP_PREFIX = "index.routing.allocation.require";

server/src/main/java/org/elasticsearch/node/Node.java

@@ -325,6 +325,15 @@ protected Node(final Environment initialEnvironment,
                     "version [{}] is a pre-release version of Elasticsearch and is not suitable for production",
                     Build.CURRENT.getQualifiedVersion());
             }
+            if (Environment.PATH_SHARED_DATA_SETTING.exists(tmpSettings)) {
+                // NOTE: this must be done with an explicit check here because the deprecation property on a path setting will
+                // cause ES to fail to start since logging is not yet initialized on first read of the setting
+                deprecationLogger.deprecate(
+                    DeprecationCategory.SETTINGS,
+                    "shared-data-path",
+                    "setting [path.shared_data] is deprecated and will be removed in a future release"
+                );
+            }
 
             if (initialEnvironment.dataFiles().length > 1) {
                 // NOTE: we use initialEnvironment here, but assertEquivalent below ensures the data paths do not change

server/src/test/java/org/elasticsearch/env/EnvironmentTests.java

@@ -260,5 +260,4 @@ private void assertIsAbsolute(final String path) {
     private void assertIsNormalized(final String path) {
         assertThat("path [" + path + "] is not normalized", PathUtils.get(path), equalTo(PathUtils.get(path).normalize()));
     }
-
 }

server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java

@@ -31,6 +31,7 @@
 import java.util.function.Function;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.StringContains.containsString;
 import static org.hamcrest.object.HasToString.hasToString;
 
@@ -629,4 +630,14 @@ public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() {
         assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting));
         assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting));
     }
+
+    public void testCustomDataPathDeprecated() {
+        final Settings settings = Settings.builder()
+            .put(IndexMetadata.INDEX_DATA_PATH_SETTING.getKey(), "my-custom-dir")
+            .build();
+        IndexMetadata metadata = newIndexMeta("test", settings);
+        IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY);
+        assertThat(indexSettings.hasCustomDataPath(), is(true));
+        assertSettingDeprecationsAndWarnings(new Setting[] { IndexMetadata.INDEX_DATA_PATH_SETTING });
+    }
 }

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -727,8 +727,8 @@ public Settings indexSettings() {
         if (numberOfReplicas >= 0) {
             builder.put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
         }
-        // 30% of the time
-        if (randomInt(9) < 3) {
+        // 30% of the time, for non-external clusters
+        if (cluster() instanceof ExternalTestCluster == false && randomInt(9) < 3) {
             final String dataPath = randomAlphaOfLength(10);
             logger.info("using custom data_path for index: [{}]", dataPath);
             builder.put(IndexMetadata.SETTING_DATA_PATH, dataPath);

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -85,7 +85,8 @@ private DeprecationChecks() {
                         XPackSettings.VECTORS_ENABLED),
                     NodeDeprecationChecks::checkMultipleDataPaths,
                     NodeDeprecationChecks::checkDataPathsList,
-                    NodeDeprecationChecks::checkBootstrapSystemCallFilterSetting
+                    NodeDeprecationChecks::checkBootstrapSystemCallFilterSetting,
+                    NodeDeprecationChecks::checkSharedDataPathSetting
                 )
             ).collect(Collectors.toList());
         }
@@ -97,10 +98,10 @@ private DeprecationChecks() {
             IndexDeprecationChecks::chainedMultiFieldsCheck,
             IndexDeprecationChecks::deprecatedDateTimeFormat,
             IndexDeprecationChecks::translogRetentionSettingCheck,
-            IndexDeprecationChecks::fieldNamesDisabledCheck
+            IndexDeprecationChecks::fieldNamesDisabledCheck,
+            IndexDeprecationChecks::checkIndexDataPath
         ));
 
-
     /**
      * helper utility function to reduce repeat of running a specific {@link List} of checks.
      *
@@ -112,4 +113,5 @@ private DeprecationChecks() {
     static <T> List<DeprecationIssue> filterChecks(List<T> checks, Function<T, DeprecationIssue> mapper) {
         return checks.stream().map(mapper).filter(Objects::nonNull).collect(Collectors.toList());
     }
+
 }

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java

@@ -20,6 +20,7 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -263,4 +264,16 @@ static DeprecationIssue translogRetentionSettingCheck(IndexMetadata indexMetadat
         }
         return null;
     }
+
+    static DeprecationIssue checkIndexDataPath(IndexMetadata indexMetadata) {
+        if (IndexMetadata.INDEX_DATA_PATH_SETTING.exists(indexMetadata.getSettings())) {
+            final String message = String.format(Locale.ROOT,
+                "setting [%s] is deprecated and will be removed in a future version", IndexMetadata.INDEX_DATA_PATH_SETTING.getKey());
+            final String url = "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/" +
+                "breaking-changes-7.13.html#deprecate-shared-data-path-setting";
+            final String details = "Found index data path configured. Discontinue use of this setting.";
+            return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details);
+        }
+        return null;
+    }
 }

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java

@@ -409,4 +409,15 @@ static DeprecationIssue checkDataPathsList(Settings nodeSettings, PluginsAndModu
         return null;
     }
 
+    static DeprecationIssue checkSharedDataPathSetting(final Settings settings, final PluginsAndModules pluginsAndModules) {
+        if (Environment.PATH_SHARED_DATA_SETTING.exists(settings)) {
+            final String message = String.format(Locale.ROOT,
+                "setting [%s] is deprecated and will be removed in a future version", Environment.PATH_SHARED_DATA_SETTING.getKey());
+            final String url = "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/" +
+                "breaking-changes-7.13.html#deprecate-shared-data-path-setting";
+            final String details = "Found shared data path configured. Discontinue use of this setting.";
+            return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details);
+        }
+        return null;
+    }
 }

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java

@@ -441,4 +441,19 @@ public void testFieldNamesEnabling() throws IOException {
         assertEquals("The index mapping contains a deprecated `enabled` setting for `_field_names` that should be removed moving foward.",
                 issue.getDetails());
     }
+
+    public void testIndexDataPathSetting() {
+        Settings.Builder settings = settings(Version.CURRENT);
+        settings.put(IndexMetadata.INDEX_DATA_PATH_SETTING.getKey(), createTempDir());
+        IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata));
+        final String expectedUrl =
+            "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-shared-data-path-setting";
+        assertThat(issues, contains(
+            new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
+                "setting [index.data_path] is deprecated and will be removed in a future version",
+                expectedUrl,
+                "Found index data path configured. Discontinue use of this setting."
+        )));
+    }
 }

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java

@@ -574,4 +574,20 @@ public void testNoDataPathsListDefault() {
         final DeprecationIssue issue = NodeDeprecationChecks.checkDataPathsList(settings, null);
         assertThat(issue, nullValue());
     }
+
+    public void testSharedDataPathSetting() {
+        Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir()).build();
+
+        DeprecationIssue issue = NodeDeprecationChecks.checkSharedDataPathSetting(settings, null);
+        final String expectedUrl =
+            "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-shared-data-path-setting";
+        assertThat(issue, equalTo(
+            new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
+                "setting [path.shared_data] is deprecated and will be removed in a future version",
+                expectedUrl,
+                "Found shared data path configured. Discontinue use of this setting."
+            )));
+    }
 }