Add migration tool checks for _field_names disabling (#46972)

This change adds a check to the migration tool that warns about the deprecated
`enabled` setting for the `_field_names` field on 7.x indices and issues a
warning for templates containing this setting, which has been removed
with 8.0.

Relates to #42854, #46681

Author: Christoph Büscher

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java

@@ -12,15 +12,19 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
@@ -91,6 +95,53 @@ static DeprecationIssue checkTemplatesWithTooManyFields(ClusterState state) {
         return null;
     }
 
+    /**
+     * Check templates that use `_field_names` explicitly, which was deprecated in https://github.com/elastic/elasticsearch/pull/42854
+     * and will throw an error on new indices in 8.0
+     */
+    @SuppressWarnings("unchecked")
+    static DeprecationIssue checkTemplatesWithFieldNamesDisabled(ClusterState state) {
+        Set<String> templatesContainingFieldNames = new HashSet<>();
+        state.getMetaData().getTemplates().forEach((templateCursor) -> {
+            String templateName = templateCursor.key;
+            templateCursor.value.getMappings().forEach((mappingCursor) -> {
+                String type = mappingCursor.key;
+                // there should be the type name at this level, but there was a bug where mappings could be stored without a type (#45120)
+                // to make sure, we try to detect this like we try to do in MappingMetaData#sourceAsMap()
+                Map<String, Object> mapping = XContentHelper.convertToMap(mappingCursor.value.compressedReference(), true).v2();
+                if (mapping.size() == 1 && mapping.containsKey(type)) {
+                    // the type name is the root value, reduce it
+                    mapping = (Map<String, Object>) mapping.get(type);
+                }
+                if (mapContainsFieldNamesDisabled(mapping)) {
+                    templatesContainingFieldNames.add(templateName);
+                }
+            });
+        });
+
+        if (templatesContainingFieldNames.isEmpty() == false) {
+            return new DeprecationIssue(DeprecationIssue.Level.WARNING, "Index templates contain _field_names settings.",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling",
+                    "Index templates " + templatesContainingFieldNames + " use the deprecated `enable` setting for the `"
+                            + FieldNamesFieldMapper.NAME + "` field. Using this setting in new index mappings will throw an error "
+                                    + "in the next major version and needs to be removed from existing mappings and templates.");
+        }
+        return null;
+    }
+
+    /**
+     * check for "_field_names" entries in the map that contain another property "enabled" in the sub-map
+     */
+    static boolean mapContainsFieldNamesDisabled(Map<?, ?> map) {
+        Object fieldNamesMapping = map.get(FieldNamesFieldMapper.NAME);
+        if (fieldNamesMapping != null) {
+            if (((Map<?, ?>) fieldNamesMapping).keySet().contains("enabled")) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     static DeprecationIssue checkPollIntervalTooLow(ClusterState state) {
         String pollIntervalString = state.metaData().settings().get(LIFECYCLE_POLL_INTERVAL_SETTING.getKey());
         if (Strings.isNullOrEmpty(pollIntervalString)) {

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -35,7 +35,8 @@ private DeprecationChecks() {
         Collections.unmodifiableList(Arrays.asList(
             ClusterDeprecationChecks::checkUserAgentPipelines,
             ClusterDeprecationChecks::checkTemplatesWithTooManyFields,
-            ClusterDeprecationChecks::checkPollIntervalTooLow
+            ClusterDeprecationChecks::checkPollIntervalTooLow,
+            ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled
         ));
 
 
@@ -51,7 +52,8 @@ private DeprecationChecks() {
             IndexDeprecationChecks::tooManyFieldsCheck,
             IndexDeprecationChecks::chainedMultiFieldsCheck,
             IndexDeprecationChecks::deprecatedDateTimeFormat,
-            IndexDeprecationChecks::translogRetentionSettingCheck
+            IndexDeprecationChecks::translogRetentionSettingCheck,
+            IndexDeprecationChecks::fieldNamesDisabledCheck
         ));
 
     static List<BiFunction<DatafeedConfig, NamedXContentRegistry, DeprecationIssue>> ML_SETTINGS_CHECKS =

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java

@@ -7,6 +7,7 @@
 
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -185,6 +186,21 @@ private static boolean containsChainedMultiFields(Map<?, ?> property) {
         return false;
     }
 
+    /**
+     * warn about existing explicit "_field_names" settings in existing mappings
+     */
+    static DeprecationIssue fieldNamesDisabledCheck(IndexMetaData indexMetaData) {
+        MappingMetaData mapping = indexMetaData.mapping();
+        if ((mapping != null) && ClusterDeprecationChecks.mapContainsFieldNamesDisabled(mapping.getSourceAsMap())) {
+            return new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                    "Index mapping contains explicit `_field_names` enabling settings.",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
+                            "#fieldnames-enabling",
+                    "The index mapping contains a deprecated `enabled` setting for `_field_names` that should be removed moving foward.");
+        }
+        return null;
+    }
+
     private static final Set<String> TYPES_THAT_DONT_COUNT;
     static {
         HashSet<String> typesThatDontCount = new HashSet<>();

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@@ -158,6 +159,84 @@ public void testTemplateWithTooManyFields() throws IOException {
         assertEquals(singletonList(expected), issues);
     }
 
+    public void testTemplatesWithFieldNamesDisabled() throws IOException {
+        XContentBuilder goodMappingBuilder = jsonBuilder();
+        goodMappingBuilder.startObject();
+        {
+            goodMappingBuilder.startObject("_doc");
+            {
+                goodMappingBuilder.startObject("properties");
+                {
+                    addRandomFields(10, goodMappingBuilder);
+                }
+                goodMappingBuilder.endObject();
+            }
+            goodMappingBuilder.endObject();
+        }
+        goodMappingBuilder.endObject();
+        assertFieldNamesEnabledTemplate(goodMappingBuilder, false);
+
+        XContentBuilder badMappingBuilder = jsonBuilder();
+        badMappingBuilder.startObject();
+        {
+            // we currently always store a type level internally
+            badMappingBuilder.startObject("_doc");
+            {
+                badMappingBuilder.startObject(FieldNamesFieldMapper.NAME);
+                {
+                    badMappingBuilder.field("enabled", randomBoolean());
+                }
+                badMappingBuilder.endObject();
+            }
+            badMappingBuilder.endObject();
+        }
+        badMappingBuilder.endObject();
+        assertFieldNamesEnabledTemplate(badMappingBuilder, true);
+
+        // however, there was a bug where mappings could be stored without a type (#45120)
+        // so we also should try to check these cases
+
+        XContentBuilder badMappingWithoutTypeBuilder = jsonBuilder();
+        badMappingWithoutTypeBuilder.startObject();
+        {
+            badMappingWithoutTypeBuilder.startObject(FieldNamesFieldMapper.NAME);
+            {
+                badMappingWithoutTypeBuilder.field("enabled", randomBoolean());
+            }
+            badMappingWithoutTypeBuilder.endObject();
+        }
+        badMappingWithoutTypeBuilder.endObject();
+        assertFieldNamesEnabledTemplate(badMappingWithoutTypeBuilder, true);
+    }
+
+    private void assertFieldNamesEnabledTemplate(XContentBuilder templateBuilder, boolean expectIssue) throws IOException {
+        String badTemplateName = randomAlphaOfLength(5);
+        final ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
+            .metaData(MetaData.builder()
+                .put(IndexTemplateMetaData.builder(badTemplateName)
+                    .patterns(Collections.singletonList(randomAlphaOfLength(5)))
+                    .putMapping("_doc", Strings.toString(templateBuilder))
+                    .build())
+                .build())
+            .build();
+
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(state));
+        if (expectIssue) {
+            assertEquals(1, issues.size());
+            DeprecationIssue issue = issues.get(0);
+            assertEquals(DeprecationIssue.Level.WARNING, issue.getLevel());
+            assertEquals("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling"
+                    , issue.getUrl());
+            assertEquals("Index templates contain _field_names settings.", issue.getMessage());
+            assertEquals("Index templates [" + badTemplateName + "] "
+                    + "use the deprecated `enable` setting for the `" + FieldNamesFieldMapper.NAME +
+                    "` field. Using this setting in new index mappings will throw an error in the next major version and " +
+                    "needs to be removed from existing mappings and templates.", issue.getDetails());
+        } else {
+            assertTrue(issues.isEmpty());
+        }
+    }
+
     public void testPollIntervalTooLow() {
         {
             final String tooLowInterval = randomTimeValue(1, 999, "ms", "micros", "nanos");

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java

@@ -11,10 +11,11 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.joda.JodaDeprecationPatterns;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@@ -27,8 +28,8 @@
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
-import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
 
 public class IndexDeprecationChecksTests extends ESTestCase {
@@ -161,6 +162,7 @@ public void testChainedMultiFields() throws IOException {
             "The names of fields that contain chained multi-fields: [[type: _doc, field: invalid-field]]");
         assertEquals(singletonList(expected), issues);
     }
+
     public void testDefinedPatternsDoNotWarn() throws IOException {
         String simpleMapping = "{\n" +
             "\"properties\" : {\n" +
@@ -412,4 +414,30 @@ public void testDefaultTranslogRetentionSettings() {
         List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
         assertThat(issues, empty());
     }
+
+    public void testFieldNamesEnabling() throws IOException {
+        XContentBuilder xContent = XContentFactory.jsonBuilder().startObject()
+            .startObject(FieldNamesFieldMapper.NAME)
+                .field("enabled", randomBoolean())
+            .endObject()
+        .endObject();
+        String mapping = BytesReference.bytes(xContent).utf8ToString();
+
+        IndexMetaData simpleIndex = IndexMetaData.builder(randomAlphaOfLengthBetween(5, 10))
+                .settings(settings(
+                        VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT)))
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .putMapping("_doc", mapping).build();
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex));
+        assertEquals(1, issues.size());
+
+        DeprecationIssue issue = issues.get(0);
+        assertEquals(DeprecationIssue.Level.WARNING, issue.getLevel());
+        assertEquals("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling"
+                , issue.getUrl());
+        assertEquals("Index mapping contains explicit `_field_names` enabling settings.", issue.getMessage());
+        assertEquals("The index mapping contains a deprecated `enabled` setting for `_field_names` that should be removed moving foward.",
+                issue.getDetails());
+    }
 }