mappings: update cluster state with type mapping also for failed indexing request

When indexing of a document with a type that is not in the mappings fails,
for example because "dynamic": "strict" but doc contains a new field,
then the type is still created on the node that executed the indexing request.
However, the change was never added to the cluster state.
This commit makes sure mapping updates are always added to the cluster state
even if indexing of a document fails.

closes #8692
relates to #8650

Author: brwe

src/main/java/org/elasticsearch/action/WriteFailureException.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchWrapperException;
+import org.elasticsearch.common.Nullable;
+
+
+public class WriteFailureException extends ElasticsearchException implements ElasticsearchWrapperException {
+    @Nullable
+    private final String mappingTypeToUpdate;
+
+    public WriteFailureException(Throwable cause, String mappingTypeToUpdate) {
+        super(null, cause);
+        assert cause != null;
+        this.mappingTypeToUpdate = mappingTypeToUpdate;
+    }
+
+    public String getMappingTypeToUpdate() {
+        return mappingTypeToUpdate;
+    }
+}

src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

@@ -22,11 +22,11 @@
 import com.google.common.collect.Sets;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchIllegalStateException;
-import org.elasticsearch.ElasticsearchWrapperException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionWriteResponse;
 import org.elasticsearch.action.RoutingMissingException;
+import org.elasticsearch.action.WriteFailureException;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexRequest;
@@ -42,7 +42,6 @@
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.ShardIterator;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
@@ -161,9 +160,9 @@ protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(Clu
                             }
                             ops[requestIndex] = result.op;
                         }
-                    } catch (WriteFailure e) {
-                        if (e.mappingTypeToUpdate != null) {
-                            mappingTypesToUpdate.add(e.mappingTypeToUpdate);
+                    } catch (WriteFailureException e) {
+                        if (e.getMappingTypeToUpdate() != null) {
+                            mappingTypesToUpdate.add(e.getMappingTypeToUpdate());
                         }
                         throw e.getCause();
                     }
@@ -397,17 +396,6 @@ <T extends ActionWriteResponse> T response() {
 
     }
 
-    static class WriteFailure extends ElasticsearchException implements ElasticsearchWrapperException {
-        @Nullable
-        final String mappingTypeToUpdate;
-
-        WriteFailure(Throwable cause, String mappingTypeToUpdate) {
-            super(null, cause);
-            assert cause != null;
-            this.mappingTypeToUpdate = mappingTypeToUpdate;
-        }
-    }
-
     private WriteResult shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, ClusterState clusterState,
                                             IndexShard indexShard, boolean processed) {
 
@@ -457,7 +445,7 @@ private WriteResult shardIndexOperation(BulkShardRequest request, IndexRequest i
             indexRequest.versionType(indexRequest.versionType().versionTypeForReplicationAndRecovery());
             indexRequest.version(version);
         } catch (Throwable t) {
-            throw new WriteFailure(t, mappingTypeToUpdate);
+            throw new WriteFailureException(t, mappingTypeToUpdate);
         }
 
         assert indexRequest.versionType().validateVersionForWrites(indexRequest.version());

src/main/java/org/elasticsearch/action/index/TransportIndexAction.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.RoutingMissingException;
+import org.elasticsearch.action.WriteFailureException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
@@ -40,6 +41,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
@@ -167,7 +169,7 @@ protected ShardIterator shards(ClusterState clusterState, InternalRequest reques
     }
 
     @Override
-    protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) {
+    protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable {
         final IndexRequest request = shardRequest.request;
 
         // validate, if routing is required, that we got routing
@@ -185,39 +187,49 @@ protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(ClusterStat
                 .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl());
         long version;
         boolean created;
-        if (request.opType() == IndexRequest.OpType.INDEX) {
-            Engine.Index index = indexShard.prepareIndex(sourceToParse, request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY, request.canHaveDuplicates());
-            if (index.parsedDoc().mappingsModified()) {
-                mappingUpdatedAction.updateMappingOnMaster(shardRequest.shardId.getIndex(), index.docMapper(), indexService.indexUUID());
-            }
-            indexShard.index(index);
-            version = index.version();
-            created = index.created();
-        } else {
-            Engine.Create create = indexShard.prepareCreate(sourceToParse,
-                    request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY, request.canHaveDuplicates(), request.autoGeneratedId());
-            if (create.parsedDoc().mappingsModified()) {
-                mappingUpdatedAction.updateMappingOnMaster(shardRequest.shardId.getIndex(), create.docMapper(), indexService.indexUUID());
+
+        try {
+            if (request.opType() == IndexRequest.OpType.INDEX) {
+                Engine.Index index = indexShard.prepareIndex(sourceToParse, request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY, request.canHaveDuplicates());
+                if (index.parsedDoc().mappingsModified()) {
+                    mappingUpdatedAction.updateMappingOnMaster(shardRequest.shardId.getIndex(), index.docMapper(), indexService.indexUUID());
+                }
+                indexShard.index(index);
+                version = index.version();
+                created = index.created();
+            } else {
+                Engine.Create create = indexShard.prepareCreate(sourceToParse,
+                        request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY, request.canHaveDuplicates(), request.autoGeneratedId());
+                if (create.parsedDoc().mappingsModified()) {
+                    mappingUpdatedAction.updateMappingOnMaster(shardRequest.shardId.getIndex(), create.docMapper(), indexService.indexUUID());
+                }
+                indexShard.create(create);
+                version = create.version();
+                created = true;
             }
-            indexShard.create(create);
-            version = create.version();
-            created = true;
-        }
-        if (request.refresh()) {
-            try {
-                indexShard.refresh("refresh_flag_index");
-            } catch (Throwable e) {
-                // ignore
+            if (request.refresh()) {
+                try {
+                    indexShard.refresh("refresh_flag_index");
+                } catch (Throwable e) {
+                    // ignore
+                }
             }
-        }
-
-        // update the version on the request, so it will be used for the replicas
-        request.version(version);
-        request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
 
-        assert request.versionType().validateVersionForWrites(request.version());
+            // update the version on the request, so it will be used for the replicas
+            request.version(version);
+            request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
 
-        return new Tuple<>(new IndexResponse(shardRequest.shardId.getIndex(), request.type(), request.id(), version, created), shardRequest.request);
+            assert request.versionType().validateVersionForWrites(request.version());
+            return new Tuple<>(new IndexResponse(shardRequest.shardId.getIndex(), request.type(), request.id(), version, created), shardRequest.request);
+        } catch (WriteFailureException e) {
+            if (e.getMappingTypeToUpdate() != null){
+                DocumentMapper docMapper = indexService.mapperService().documentMapper(e.getMappingTypeToUpdate());
+                if (docMapper != null) {
+                    mappingUpdatedAction.updateMappingOnMaster(indexService.index().name(), docMapper, indexService.indexUUID());
+                }
+            }
+            throw e.getCause();
+        }
     }
 
     @Override

src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java

@@ -117,7 +117,7 @@ protected void doExecute(Request request, ActionListener<Response> listener) {
      * @return  A tuple containing not null values, as first value the result of the primary operation and as second value
      *          the request to be executed on the replica shards.
      */
-    protected abstract Tuple<Response, ReplicaRequest> shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest);
+    protected abstract Tuple<Response, ReplicaRequest> shardOperationOnPrimary(ClusterState clusterState, PrimaryOperationRequest shardRequest)  throws Throwable;
 
     protected abstract void shardOperationOnReplica(ReplicaOperationRequest shardRequest);
 

src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -34,7 +34,7 @@
 import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.optimize.OptimizeRequest;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.action.WriteFailureException;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Booleans;
@@ -64,6 +64,12 @@
 import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy;
 import org.elasticsearch.index.deletionpolicy.SnapshotIndexCommit;
 import org.elasticsearch.index.engine.*;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.EngineClosedException;
+import org.elasticsearch.index.engine.EngineException;
+import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException;
+import org.elasticsearch.index.engine.RefreshFailedEngineException;
+import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.fielddata.ShardFieldData;
@@ -409,8 +415,16 @@ private IndexShardState changeState(IndexShardState newState, String reason) {
     public Engine.Create prepareCreate(SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin, boolean canHaveDuplicates, boolean autoGeneratedId) throws ElasticsearchException {
         long startTime = System.nanoTime();
         Tuple<DocumentMapper, Boolean> docMapper = mapperService.documentMapperWithAutoCreate(source.type());
-        ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper);
-        return new Engine.Create(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, state != IndexShardState.STARTED || canHaveDuplicates, autoGeneratedId);
+        try {
+            ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper);
+            return new Engine.Create(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, state != IndexShardState.STARTED || canHaveDuplicates, autoGeneratedId);
+        } catch (Throwable t) {
+            if (docMapper.v2()) {
+                throw new WriteFailureException(t, docMapper.v1().type());
+            } else {
+                throw t;
+            }
+        }
     }
 
     public ParsedDocument create(Engine.Create create) throws ElasticsearchException {
@@ -434,8 +448,16 @@ public ParsedDocument create(Engine.Create create) throws ElasticsearchException
     public Engine.Index prepareIndex(SourceToParse source, long version, VersionType versionType, Engine.Operation.Origin origin, boolean canHaveDuplicates) throws ElasticsearchException {
         long startTime = System.nanoTime();
         Tuple<DocumentMapper, Boolean> docMapper = mapperService.documentMapperWithAutoCreate(source.type());
-        ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper);
-        return new Engine.Index(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, state != IndexShardState.STARTED || canHaveDuplicates);
+        try {
+            ParsedDocument doc = docMapper.v1().parse(source).setMappingsModified(docMapper);
+            return new Engine.Index(docMapper.v1(), docMapper.v1().uidMapper().term(doc.uid().stringValue()), doc, version, versionType, origin, startTime, state != IndexShardState.STARTED || canHaveDuplicates);
+        } catch (Throwable t) {
+            if (docMapper.v2()) {
+                throw new WriteFailureException(t, docMapper.v1().type());
+            } else {
+                throw t;
+            }
+        }
     }
 
     public ParsedDocument index(Engine.Index index) throws ElasticsearchException {

src/test/java/org/elasticsearch/index/mapper/dynamic/DynamicMappingIntegrationTests.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper.dynamic;
+
+import com.google.common.base.Predicate;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.index.mapper.StrictDynamicMappingException;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.junit.Test;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+
+public class DynamicMappingIntegrationTests extends ElasticsearchIntegrationTest {
+
+    // https://github.com/elasticsearch/elasticsearch/issues/8423#issuecomment-64229717
+    @Test
+    public void testStrictAllMapping() throws Exception {
+        String defaultMapping = jsonBuilder().startObject().startObject("_default_")
+                .field("dynamic", "strict")
+                .endObject().endObject().string();
+        client().admin().indices().prepareCreate("index").addMapping("_default_", defaultMapping).get();
+
+        try {
+            client().prepareIndex("index", "type", "id").setSource("test", "test").get();
+            fail();
+        } catch (StrictDynamicMappingException ex) {
+            // this should not be created dynamically so we expect this exception
+        }
+        awaitBusy(new Predicate<Object>() {
+            @Override
+            public boolean apply(java.lang.Object input) {
+                GetMappingsResponse currentMapping = client().admin().indices().prepareGetMappings("index").get();
+                return currentMapping.getMappings().get("index").get("type") != null;
+            }
+        });
+
+        String docMapping = jsonBuilder().startObject().startObject("type")
+                .startObject("_all")
+                .field("enabled", false)
+                .endObject()
+                .endObject().endObject().string();
+        try {
+            client().admin().indices()
+                    .preparePutMapping("index")
+                    .setType("type")
+                    .setSource(docMapping).get();
+            fail();
+        } catch (Exception e) {
+            // the mapping was created anyway with _all enabled: true, although the index request fails so we expect the update to fail
+        }
+
+        // make sure type was created
+        for (Client client : cluster()) {
+            GetMappingsResponse mapping = client.admin().indices().prepareGetMappings("index").setLocal(true).get();
+            assertNotNull(mapping.getMappings().get("index").get("type"));
+        }
+    }
+}