ML: adjusting for backport of #36643

Author: benwtrent

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlUpgradeAction.java

@@ -28,7 +28,8 @@
 import java.util.Objects;
 
 
-public class MlUpgradeAction extends Action<AcknowledgedResponse> {
+public class MlUpgradeAction extends Action<MlUpgradeAction.Request, AcknowledgedResponse, MlUpgradeAction.RequestBuilder> {
+
     public static final MlUpgradeAction INSTANCE = new MlUpgradeAction();
     public static final String NAME = "cluster:admin/xpack/ml/upgrade";
 
@@ -41,6 +42,11 @@ public AcknowledgedResponse newResponse() {
         return new AcknowledgedResponse();
     }
 
+    @Override
+    public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new RequestBuilder(client, this);
+    }
+
     public static class Request extends MasterNodeReadRequest<Request> implements ToXContentObject {
 
         private static final ParseField REINDEX_BATCH_SIZE = new ParseField("reindex_batch_size");
@@ -60,6 +66,7 @@ public static class Request extends MasterNodeReadRequest<Request> implements To
 
         // for serialization
         public Request() {
+            super();
         }
 
         public Request(StreamInput in) throws IOException {
@@ -73,6 +80,12 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeInt(reindexBatchSize);
         }
 
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            reindexBatchSize = in.readInt();
+        }
+
         public String[] indices() {
             return new String[]{INDEX};
         }
@@ -152,8 +165,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder<Request, AcknowledgedResponse, RequestBuilder> {
 
-        public RequestBuilder(ElasticsearchClient client) {
-            super(client, INSTANCE, new Request());
+        public RequestBuilder(ElasticsearchClient client, MlUpgradeAction action) {
+            super(client, action, new Request());
         }
     }
 

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlUpgradeIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.reindex.ReindexAction;
@@ -88,7 +89,7 @@ public void testMigrationWhenItIsNotNecessary() throws Exception {
         assertThat(getTotalDocCount(job3Index), equalTo(job3Total));
 
         ClusterState state = admin().cluster().state(new ClusterStateRequest()).actionGet().getState();
-        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
         String[] indices = indexNameExpressionResolver.concreteIndexNames(state,
             IndicesOptions.strictExpandOpenAndForbidClosed(),
             AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*");
@@ -120,7 +121,7 @@ public void testMigration() throws Exception {
         long job2Total = getJobResultsCount(job2.getId());
         long job3Total = getJobResultsCount(job3.getId());
 
-        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
 
         ResultsIndexUpgradeService resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver,
             ThreadPool.Names.SAME,
@@ -203,7 +204,7 @@ public void testMigrationWithManuallyCreatedIndexThatNeedsMigrating() throws Exc
         String manuallyCreatedIndex = job1Index + "-" + Version.CURRENT.major;
         client().admin().indices().prepareCreate(manuallyCreatedIndex).execute().actionGet();
 
-        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
 
         ResultsIndexUpgradeService resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver,
             ThreadPool.Names.SAME,
@@ -257,7 +258,7 @@ public void testMigrationWithExistingIndexWithData() throws Exception {
         String alreadyMigratedWriteIndex = job1Index + "-" + Version.CURRENT.major;
         client().admin().indices().prepareCreate(alreadyMigratedWriteIndex).execute().actionGet();
 
-        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
 
         ResultsIndexUpgradeService resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver,
             ThreadPool.Names.SAME,
@@ -315,7 +316,7 @@ private long getTotalDocCount(String indexName) {
             .setTrackTotalHits(true)
             .setQuery(QueryBuilders.matchAllQuery())
             .execute().actionGet();
-        return searchResponse.getHits().getTotalHits().value;
+        return searchResponse.getHits().getTotalHits();
     }
 
     private long getJobResultsCount(String jobId) {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/ResultsIndexUpgradeService.java

@@ -63,7 +63,7 @@ public class ResultsIndexUpgradeService {
 
     // Adjust the following constants as necessary for various versions and backports.
     private static final int INDEX_VERSION = Version.CURRENT.major;
-    private static final Version MIN_REQUIRED_VERSION = Version.CURRENT.minimumCompatibilityVersion();
+    private static final Version MIN_REQUIRED_VERSION = Version.V_5_6_0;
 
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final Predicate<IndexMetaData> shouldUpgrade;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlUpgradeAction.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -33,11 +34,11 @@ public class TransportMlUpgradeAction
     private final ResultsIndexUpgradeService resultsIndexUpgradeService;
 
     @Inject
-    public TransportMlUpgradeAction(TransportService transportService, ClusterService clusterService,
+    public TransportMlUpgradeAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                     ThreadPool threadPool, ActionFilters actionFilters, Client client,
                                     IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(MlUpgradeAction.NAME, transportService, clusterService, threadPool,
-            actionFilters, MlUpgradeAction.Request::new, indexNameExpressionResolver);
+        super(settings, MlUpgradeAction.NAME, transportService, clusterService, threadPool,
+            actionFilters, indexNameExpressionResolver, MlUpgradeAction.Request::new);
         this.client = client;
         this.resultsIndexUpgradeService = new ResultsIndexUpgradeService(indexNameExpressionResolver,
             executor(),

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestUpgradeMlAction.java

@@ -5,9 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.rest.results;
 
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -28,18 +26,10 @@
 
 public class RestUpgradeMlAction extends BaseRestHandler {
 
-    private static final DeprecationLogger deprecationLogger =
-        new DeprecationLogger(LogManager.getLogger(RestUpgradeMlAction.class));
-
     public RestUpgradeMlAction(Settings settings, RestController controller) {
         super(settings);
-        controller.registerWithDeprecatedHandler(
-            POST,
-            MachineLearning.BASE_PATH + "_upgrade",
-            this,
-            POST,
-            MachineLearning.PRE_V7_BASE_PATH + "_upgrade",
-            deprecationLogger);
+        controller.registerHandler(POST, MachineLearning.BASE_PATH + "_upgrade", this);
+        controller.registerHandler(POST,MachineLearning.V7_BASE_PATH + "_upgrade", this);
     }
 
     @Override

x-pack/plugin/src/test/resources/rest-api-spec/api/ml.upgrade.json renamed to x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.ml.upgrade.json

@@ -1,10 +1,10 @@
 {
-  "ml.upgrade": {
+  "xpack.ml.upgrade": {
     "documentation": "TODO",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/_upgrade",
-      "paths": [ "/_ml/_upgrade" ],
+      "path": "/_xpack/ml/_upgrade",
+      "paths": [ "/_xpack/ml/_upgrade" ],
       "params": {
         "wait_for_completion": {
           "type": "boolean",

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/ml_upgrade.yml

@@ -4,7 +4,7 @@ setup:
   - do:
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_job:
+      xpack.ml.put_job:
         job_id: jobs-upgrade-results
         body:  >
           {
@@ -43,7 +43,7 @@ setup:
 ---
 "Upgrade results when there is nothing to upgrade":
   - do:
-      ml.upgrade:
+      xpack.ml.upgrade:
         wait_for_completion: true
 
   - match: { acknowledged: true }
@@ -56,7 +56,7 @@ setup:
 ---
 "Upgrade results when there is nothing to upgrade not waiting for results":
   - do:
-      ml.upgrade:
+      xpack.ml.upgrade:
         wait_for_completion: false
 
   - match: {task: '/.+:\d+/'}

x-pack/qa/rolling-upgrade/build.gradle

@@ -295,8 +295,17 @@ subprojects {
               systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster'
           }
       }
+      def toBlackList = []
+      // If we are upgrading on a minor version, these tests fail
+      // They only work if we are upgrading between major versions
+      if (versionParts[0].equals("6")) {
+        toBlackList << 'upgraded_cluster/80_ml_results_upgrade/Migrate results data to latest index binary version'
+      }
       if (version.before('6.1.0') || version.onOrAfter('6.3.0')) {
-        systemProperty 'tests.rest.blacklist', '/30_ml_jobs_crud/Test model memory limit is updated'
+        toBlackList << '/30_ml_jobs_crud/Test model memory limit is updated'
+      }
+      if (!toBlackList.empty) {
+        systemProperty 'tests.rest.blacklist', toBlackList.join(',')
       }
     }
 

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_ml_results_upgrade.yml

@@ -1,11 +1,11 @@
 ---
 "Verify jobs exist":
   - do:
-      ml.get_jobs:
+      xpack.ml.get_jobs:
         job_id: old-cluster-job-to-upgrade
   - match: { count: 1 }
 
   - do:
-      ml.get_jobs:
+      xpack.ml.get_jobs:
         job_id: old-cluster-job-to-upgrade-custom
   - match: { count: 1 }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_ml_results_upgrade.yml

@@ -2,7 +2,7 @@
 "Put job on the old cluster and post some data":
 
   - do:
-      ml.put_job:
+      xpack.ml.put_job:
         job_id: old-cluster-job-to-upgrade
         body:  >
           {
@@ -23,11 +23,11 @@
   - match: { job_id: old-cluster-job-to-upgrade }
 
   - do:
-      ml.open_job:
+      xpack.ml.open_job:
         job_id: old-cluster-job-to-upgrade
 
   - do:
-      ml.post_data:
+      xpack.ml.post_data:
         job_id: old-cluster-job-to-upgrade
         body:
           - airline: AAL
@@ -41,11 +41,11 @@
   - match: { processed_record_count: 2 }
 
   - do:
-      ml.close_job:
+      xpack.ml.close_job:
         job_id: old-cluster-job-to-upgrade
 
   - do:
-      ml.get_buckets:
+      xpack.ml.get_buckets:
         job_id: old-cluster-job-to-upgrade
   - match: { count: 1 }
 
@@ -59,7 +59,7 @@
 ---
 "Put job on the old cluster with a custom index":
   - do:
-      ml.put_job:
+      xpack.ml.put_job:
         job_id: old-cluster-job-to-upgrade-custom
         body:  >
           {
@@ -81,11 +81,11 @@
   - match: { job_id: old-cluster-job-to-upgrade-custom }
 
   - do:
-      ml.open_job:
+      xpack.ml.open_job:
         job_id: old-cluster-job-to-upgrade-custom
 
   - do:
-      ml.post_data:
+      xpack.ml.post_data:
         job_id: old-cluster-job-to-upgrade-custom
         body:
         - airline: AAL
@@ -103,11 +103,11 @@
   - match: { processed_record_count: 3 }
 
   - do:
-      ml.close_job:
+      xpack.ml.close_job:
         job_id: old-cluster-job-to-upgrade-custom
 
   - do:
-      ml.get_buckets:
+      xpack.ml.get_buckets:
         job_id: old-cluster-job-to-upgrade-custom
   - match: { count: 3 }