Do not open indices with broken index settings

- remove the leniency on opening indices with unrecognized settings and
instead such an index will be closed
with the unrecognized settings archived
- do not open any index with archived settings
- users can remove archived settings via the wildcard archived
- return "index_open_exception" 400 status code when trying to open an
index with broken settings

Relates to elastic#26995

Closes elastic#26998

Author: mayya-sharipova

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java

@@ -193,9 +193,10 @@ public ClusterState execute(ClusterState currentState) {
                     // We need to check that this index can be upgraded to the current version
                     indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData, minIndexCompatibilityVersion);
                     try {
-                        indicesService.verifyIndexMetadata(indexMetaData, indexMetaData, "metadata verification");
+                        indicesService.verifyIndexMetadata(indexMetaData, indexMetaData, false);
                     } catch (Exception e) {
-                        throw new IndexOpenException(indexMetaData.getIndex(), "Failed to open index! Failed to verify index " + indexMetaData.getIndex() + e.getMessage());
+                        throw new IndexOpenException(indexMetaData.getIndex(),
+                            "Failed to open index! Failed to verify index " + indexMetaData.getIndex() + e.getMessage());
                     }
 
                     mdBuilder.put(indexMetaData, true);

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -276,18 +276,18 @@ public ClusterState execute(ClusterState currentState) {
                     for (Index index : openIndices) {
                         final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                         final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
-                        indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData, "metadata verification");
+                        indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData, false);
                     }
                     for (Index index : closeIndices) {
                         final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                         final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
                         // Verifies that the current index settings can be updated with the updated dynamic settings.
                         // Ignore archived settings during verification, as closed indexes can have archived settings
-                        indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData, "closed index metadata verification");
+                        indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData, true);
                         // Now check that we can create the index with the updated settings (dynamic and non-dynamic).
                         // This step is mandatory since we allow to update non-dynamic settings on closed indices.
                         // Ignore archived settings during verification, , as closed indexes can have archived settings
-                        indicesService.verifyIndexMetadata(updatedMetaData, updatedMetaData, "closed index metadata verification");
+                        indicesService.verifyIndexMetadata(updatedMetaData, updatedMetaData, true);
                     }
                 } catch (IOException ex) {
                     throw ExceptionsHelper.convertToElastic(ex);

server/src/main/java/org/elasticsearch/gateway/Gateway.java

@@ -124,7 +124,7 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t
                     try {
                         if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
                             // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
-                            indicesService.verifyIndexMetadata(electedIndexMetaData, electedIndexMetaData, "metadata verification");
+                            indicesService.verifyIndexMetadata(electedIndexMetaData, electedIndexMetaData, false);
                         }
                     } catch (Exception e) {
                         final Index electedIndex = electedIndexMetaData.getIndex();

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -405,6 +405,7 @@ public void onStoreClosed(ShardId shardId) {
         final IndexService indexService =
                 createIndexService(
                         "create index",
+                        true,
                         indexMetaData,
                         indicesQueryCache,
                         indicesFieldDataCache,
@@ -427,19 +428,15 @@ public void onStoreClosed(ShardId shardId) {
      * This creates a new IndexService without registering it
      */
     private synchronized IndexService createIndexService(final String reason,
+                                                         final boolean ignoreArchivedSettings,
                                                          IndexMetaData indexMetaData,
                                                          IndicesQueryCache indicesQueryCache,
                                                          IndicesFieldDataCache indicesFieldDataCache,
                                                          List<IndexEventListener> builtInListeners,
                                                          IndexingOperationListener... indexingOperationListeners) throws IOException {
         final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopedSettings);
         // we ignore private settings since they are not registered settings
-        if (reason.equals("metadata verification")) {
-            // don't allow archived settings during metadata verification
-            indexScopedSettings.validate(indexMetaData.getSettings(), true, true, false);
-        } else {
-            indexScopedSettings.validate(indexMetaData.getSettings(), true, true, true);
-        }
+        indexScopedSettings.validate(indexMetaData.getSettings(), true, true, ignoreArchivedSettings);
         logger.debug("creating Index [{}], shards [{}]/[{}] - reason [{}]",
             indexMetaData.getIndex(),
             idxSettings.getNumberOfShards(),
@@ -483,24 +480,24 @@ public synchronized MapperService createIndexMapperService(IndexMetaData indexMe
         return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService);
     }
 
-
     /**
      * This method verifies that the given {@code metaData} holds sane values to create an {@link IndexService}.
      * This method tries to update the meta data of the created {@link IndexService} if the given {@code metaDataUpdate} is different from the given {@code metaData}.
      * This method will throw an exception if the creation or the update fails.
      * The created {@link IndexService} will not be registered and will be closed immediately.
      */
-    public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate, String reason) throws IOException {
+    public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate, boolean ignoreArchivedSettings)
+            throws IOException {
         final List<Closeable> closeables = new ArrayList<>();
         try {
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
             closeables.add(indicesFieldDataCache);
             IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings);
             closeables.add(indicesQueryCache);
             // this will also fail if some plugin fails etc. which is nice since we can verify that early
-            final IndexService service =
-                createIndexService(reason, metaData, indicesQueryCache, indicesFieldDataCache, emptyList());
-            closeables.add(() -> service.close(reason, false));
+            final IndexService service = createIndexService("metadata verification", ignoreArchivedSettings, metaData,
+                indicesQueryCache, indicesFieldDataCache, emptyList());
+            closeables.add(() -> service.close("metadata verification", false));
             service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY);
             if (metaData.equals(metaDataUpdate) == false) {
                 service.updateMetaData(metaDataUpdate);