Restore a noop _all metadata field for 6x indices

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml

@@ -67,3 +67,9 @@
                 field3: value
   - match: { hits.total: 1 }
   - match: { hits.hits.0._id: q3 }
+
+---
+"Index with _all is available":
+  - do:
+      indices.get:
+        index: all-index

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml

@@ -203,3 +203,21 @@
       tasks.get:
         wait_for_completion: true
         task_id: $task
+
+---
+"Create an index with _all explicitly disabled":
+  - skip:
+      features: warnings
+  - do:
+      warnings:
+        - "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field."
+      indices.create:
+        index: all-index
+        body:
+          mappings:
+            type:
+              _all:
+                enabled: false
+              properties:
+                field:
+                  type: text

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -125,3 +125,17 @@
         wait_for_completion: true
         task_id: $task_id
   - match: { task.headers.X-Opaque-Id: "Reindexing Again" }
+
+---
+"Index with _all is available":
+  - do:
+      indices.get:
+        index: all-index
+
+  - do:
+      indices.get_mapping:
+        index: all-index
+
+  - is_true: all-index.mappings._all
+  - match: { all-index.mappings._all.enabled: false}
+

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.indices.mapping.get;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -91,7 +92,8 @@ protected ShardsIterator shards(ClusterState state, InternalRequest request) {
     protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) {
         assert shardId != null;
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        Predicate<String> metadataFieldPredicate = indicesService::isMetaDataField;
+        Version indexCreatedVersion = indexService.mapperService().getIndexSettings().getIndexVersionCreated();
+        Predicate<String> metadataFieldPredicate = (f) -> indicesService.isMetaDataField(indexCreatedVersion, f);
         Predicate<String> fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName()));
 
         DocumentMapper mapper = indexService.mapperService().documentMapper();

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java

@@ -83,7 +83,8 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI
         for (String field : fieldNames) {
             MappedFieldType ft = mapperService.fullName(field);
             if (ft != null) {
-                if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) {
+                if (indicesService.isMetaDataField(mapperService.getIndexSettings().getIndexVersionCreated(), field)
+                        || fieldPredicate.test(ft.name())) {
                     FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
                     responseMap.put(field, fieldCap);
                 } else {

server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java

@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Noop mapper that ensures that mappings created in 6x that explicitly disable the _all field
+ * can be restored in this version.
+ *
+ * TODO: Remove in 8
+ */
+public class AllFieldMapper extends MetadataFieldMapper {
+    public static final String NAME = "_all";
+    public static final String CONTENT_TYPE = "_all";
+
+    public static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new AllFieldType();
+
+        static {
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+            FIELD_TYPE.setTokenized(true);
+            FIELD_TYPE.setName(NAME);
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    public static class Builder extends MetadataFieldMapper.Builder<Builder, AllFieldMapper> {
+        private boolean disableExplicit = false;
+
+        public Builder(MappedFieldType existing) {
+            super(NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        private Builder setDisableExplicit() {
+            this.disableExplicit = true;
+            return this;
+        }
+
+        @Override
+        public AllFieldMapper build(BuilderContext context) {
+            return new AllFieldMapper(fieldType, context.indexSettings(), disableExplicit);
+        }
+    }
+
+    public static class TypeParser implements MetadataFieldMapper.TypeParser {
+        @Override
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
+                                                 ParserContext parserContext) throws MapperParsingException {
+            Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String fieldName = entry.getKey();
+                if (fieldName.equals("enabled")) {
+                    boolean enabled = XContentMapValues.nodeBooleanValue(entry.getValue(), "enabled");
+                    if (enabled) {
+                        throw new IllegalArgumentException("[_all] is disabled in this version.");
+                    }
+                    builder.setDisableExplicit();
+                    iterator.remove();
+                }
+            }
+            return builder;
+        }
+
+        @Override
+        public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
+            final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
+            return new AllFieldMapper(indexSettings, Defaults.FIELD_TYPE, false);
+        }
+    }
+
+    static final class AllFieldType extends StringFieldType {
+        AllFieldType() {
+        }
+
+        protected AllFieldType(AllFieldType ref) {
+            super(ref);
+        }
+
+        @Override
+        public MappedFieldType clone() {
+            return new AllFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            return new MatchNoDocsQuery();
+        }
+    }
+
+    private final boolean disableExplicit;
+
+    private AllFieldMapper(Settings indexSettings, MappedFieldType existing, boolean disableExplicit) {
+        this(existing.clone(), indexSettings, disableExplicit);
+    }
+
+    private AllFieldMapper(MappedFieldType fieldType, Settings indexSettings, boolean disableExplicit) {
+        super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings);
+        this.disableExplicit = disableExplicit;
+    }
+
+    @Override
+    public void preParse(ParseContext context) throws IOException {
+    }
+
+    @Override
+    public void postParse(ParseContext context) throws IOException {
+        super.parse(context);
+    }
+
+    @Override
+    public void parse(ParseContext context) throws IOException {
+        // we parse in post parse
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        // noop mapper
+        return;
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
+        if (includeDefaults || disableExplicit) {
+            builder.startObject(CONTENT_TYPE);
+            if (disableExplicit) {
+                builder.field("enabled", false);
+            }
+            builder.endObject();
+        }
+        return builder;
+    }
+}

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -27,6 +27,7 @@
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -73,7 +74,9 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) {
 
             final String type = rootObjectMapper.name();
             final DocumentMapper existingMapper = mapperService.documentMapper(type);
-            final Map<String, TypeParser> metadataMapperParsers = mapperService.mapperRegistry.getMetadataMapperParsers();
+            final Version indexCreatedVersion = mapperService.getIndexSettings().getIndexVersionCreated();
+            final Map<String, TypeParser> metadataMapperParsers =
+                mapperService.mapperRegistry.getMetadataMapperParsers(indexCreatedVersion);
             for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : metadataMapperParsers.entrySet()) {
                 final String name = entry.getKey();
                 final MetadataFieldMapper existingMetadataMapper = existingMapper == null

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java

@@ -65,8 +65,8 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer
         this.similarityService = similarityService;
         this.queryShardContextSupplier = queryShardContextSupplier;
         this.typeParsers = mapperRegistry.getMapperParsers();
-        this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers();
-        indexVersionCreated = indexSettings.getIndexVersionCreated();
+        this.indexVersionCreated = indexSettings.getIndexVersionCreated();
+        this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated);
     }
 
     public Mapper.TypeParser.ParserContext parserContext(String type) {

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -29,6 +29,7 @@
 import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
@@ -1382,8 +1383,8 @@ public Function<String, Predicate<String>> getFieldFilter() {
     /**
      * Returns true if the provided field is a registered metadata field (including ones registered via plugins), false otherwise.
      */
-    public boolean isMetaDataField(String field) {
-        return mapperRegistry.isMetaDataField(field);
+    public boolean isMetaDataField(Version indexCreatedVersion, String field) {
+        return mapperRegistry.isMetaDataField(indexCreatedVersion, field);
     }
 
     /**

server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.indices.mapper;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -36,13 +38,19 @@ public final class MapperRegistry {
 
     private final Map<String, Mapper.TypeParser> mapperParsers;
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
+    private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers6x;
     private final Function<String, Predicate<String>> fieldFilter;
 
 
     public MapperRegistry(Map<String, Mapper.TypeParser> mapperParsers,
             Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers, Function<String, Predicate<String>> fieldFilter) {
         this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers));
         this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers));
+        // add the _all field mapper for indices created in 6x
+        Map<String, MetadataFieldMapper.TypeParser> metadata6x = new LinkedHashMap<>();
+        metadata6x.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser());
+        metadata6x.putAll(metadataMapperParsers);
+        this.metadataMapperParsers6x = Collections.unmodifiableMap(metadata6x);
         this.fieldFilter = fieldFilter;
     }
 
@@ -58,15 +66,15 @@ public Map<String, Mapper.TypeParser> getMapperParsers() {
      * Return a map of the meta mappers that have been registered. The
      * returned map uses the name of the field as a key.
      */
-    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers() {
-        return metadataMapperParsers;
+    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers(Version indexCreatedVersion) {
+        return indexCreatedVersion.onOrAfter(Version.V_7_0_0) ? metadataMapperParsers : metadataMapperParsers6x;
     }
 
     /**
-     * Returns true if the provide field is a registered metadata field, false otherwise
+     * Returns true if the provided field is a registered metadata field, false otherwise
      */
-    public boolean isMetaDataField(String field) {
-        return getMetadataMapperParsers().containsKey(field);
+    public boolean isMetaDataField(Version indexCreatedVersion, String field) {
+        return getMetadataMapperParsers(indexCreatedVersion).containsKey(field);
     }
 
     /**