[7.x][ML] Refactor ML mappings and templates into JSON resources (#51… (#52353)

ML mappings and index templates have so far been created
programmatically. While this had its merits due to static typing,
there is consensus it would be clear to maintain those in json files.
In addition, we are going to adding ILM policies to these indices
and the component for a plugin to register ILM policies is
`IndexTemplateRegistry`. It expects the templates to be in resource
json files.

For the above reasons this commit refactors ML mappings and index
templates into json resource files that are registered via
`MlIndexTemplateRegistry`.

Backport of #51765

Author: dimitris-athanasiou

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
+
+import java.util.Collections;
+
+public class MlConfigIndex {
+
+    private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
+
+    private MlConfigIndex() {}
+
+    public static String mapping() {
+        return mapping(MapperService.SINGLE_MAPPING_NAME);
+    }
+
+    public static String mapping(String mappingType) {
+        return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/config_index_mappings.json",
+            Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType));
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java

@@ -5,50 +5,13 @@
  */
 package org.elasticsearch.xpack.core.ml;
 
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.ml.calendars.Calendar;
-import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
-import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
-
-import java.io.IOException;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
-
 public final class MlMetaIndex {
     /**
      * Where to store the ml info in Elasticsearch - must match what's
      * expected by kibana/engineAPI/app/directives/mlLogUsage.js
      */
     public static final String INDEX_NAME = ".ml-meta";
 
-    private MlMetaIndex() {}
 
-    public static XContentBuilder docMapping() throws IOException {
-        XContentBuilder builder = jsonBuilder();
-        builder.startObject();
-            builder.startObject(SINGLE_MAPPING_NAME);
-                ElasticsearchMappings.addMetaInformation(builder);
-                ElasticsearchMappings.addDefaultMapping(builder);
-                builder.startObject(ElasticsearchMappings.PROPERTIES)
-                    .startObject(Calendar.ID.getPreferredName())
-                        .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
-                    .endObject()
-                    .startObject(Calendar.JOB_IDS.getPreferredName())
-                        .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
-                    .endObject()
-                    .startObject(Calendar.DESCRIPTION.getPreferredName())
-                        .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
-                    .endObject()
-                    .startObject(ScheduledEvent.START_TIME.getPreferredName())
-                        .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
-                    .endObject()
-                    .startObject(ScheduledEvent.END_TIME.getPreferredName())
-                        .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
-                    .endObject()
-                .endObject()
-            .endObject()
-        .endObject();
-        return builder;
-    }
+    private MlMetaIndex() {}
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.ml.annotations;
 
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@@ -15,19 +16,13 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.ml.MachineLearningField;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
 
-import java.io.IOException;
 import java.util.SortedMap;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@@ -40,6 +35,8 @@ public class AnnotationIndex {
     public static final String INDEX_NAME = ".ml-annotations-6";
     public static final String INDEX_PATTERN = ".ml-annotations*";
 
+    private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
+
     /**
      * Create the .ml-annotations index with correct mappings if it does not already
      * exist. This index is read and written by the UI results views, so needs to
@@ -64,39 +61,26 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
             // Create the annotations index if it doesn't exist already.
             if (mlLookup.containsKey(INDEX_NAME) == false) {
 
-                final TimeValue delayedNodeTimeOutSetting;
-                // Whether we are using native process is a good way to detect whether we are in dev / test mode:
-                if (MachineLearningField.AUTODETECT_PROCESS.get(settings)) {
-                    delayedNodeTimeOutSetting = UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(settings);
-                } else {
-                    delayedNodeTimeOutSetting = TimeValue.ZERO;
-                }
-
                 CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
-                try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
-                    createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping);
-                    createIndexRequest.settings(Settings.builder()
-                        .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
-                        .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting));
+                createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping(), XContentType.JSON);
+                createIndexRequest.settings(Settings.builder()
+                    .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1"));
 
-                    executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
-                        ActionListener.<CreateIndexResponse>wrap(
-                            r -> createAliasListener.onResponse(r.isAcknowledged()),
-                            e -> {
-                                // Possible that the index was created while the request was executing,
-                                // so we need to handle that possibility
-                                if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
-                                    // Create the alias
-                                    createAliasListener.onResponse(true);
-                                } else {
-                                    finalListener.onFailure(e);
-                                }
+                executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
+                    ActionListener.<CreateIndexResponse>wrap(
+                        r -> createAliasListener.onResponse(r.isAcknowledged()),
+                        e -> {
+                            // Possible that the index was created while the request was executing,
+                            // so we need to handle that possibility
+                            if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
+                                // Create the alias
+                                createAliasListener.onResponse(true);
+                            } else {
+                                finalListener.onFailure(e);
                             }
-                        ), client.admin().indices()::create);
-                } catch (IOException e) {
-                    finalListener.onFailure(e);
-                }
+                        }
+                    ), client.admin().indices()::create);
                 return;
             }
 
@@ -111,42 +95,8 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
         finalListener.onResponse(false);
     }
 
-    public static XContentBuilder annotationsMapping() throws IOException {
-        XContentBuilder builder = jsonBuilder()
-            .startObject()
-                .startObject(SINGLE_MAPPING_NAME);
-        ElasticsearchMappings.addMetaInformation(builder);
-        builder.startObject(ElasticsearchMappings.PROPERTIES)
-                        .startObject(Annotation.ANNOTATION.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.TEXT)
-                        .endObject()
-                        .startObject(Annotation.CREATE_TIME.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
-                        .endObject()
-                        .startObject(Annotation.CREATE_USERNAME.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
-                        .endObject()
-                        .startObject(Annotation.TIMESTAMP.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
-                        .endObject()
-                        .startObject(Annotation.END_TIMESTAMP.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
-                        .endObject()
-                        .startObject(Job.ID.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
-                        .endObject()
-                        .startObject(Annotation.MODIFIED_TIME.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
-                        .endObject()
-                        .startObject(Annotation.MODIFIED_USERNAME.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
-                        .endObject()
-                        .startObject(Annotation.TYPE.getPreferredName())
-                            .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
-                        .endObject()
-                    .endObject()
-                .endObject()
-            .endObject();
-        return builder;
+    public static String annotationsMapping() {
+        return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json",
+            Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE);
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.ml.job.persistence;
 
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -16,7 +17,9 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -31,6 +34,9 @@ public final class AnomalyDetectorsIndex {
 
     public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000;
 
+    private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
+    private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/";
+
     private AnomalyDetectorsIndex() {
     }
 
@@ -144,4 +150,12 @@ public static void createStateIndexAndAliasIfNecessary(Client client, ClusterSta
         }
     }
 
+    public static String resultsMapping() {
+        return resultsMapping(MapperService.SINGLE_MAPPING_NAME);
+    }
+
+    public static String resultsMapping(String mappingType) {
+        return TemplateUtils.loadTemplate(RESOURCE_PATH + "results_index_mappings.json",
+            Version.CURRENT.toString(), RESULTS_MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType));
+    }
 }