[ML] adding for_export flag for ml plugin GET resource APIs (elastic#63092)

This adds the new `for_export` flag to the following APIs:

- GET _ml/anomaly_detection/<job_id>
- GET _ml/datafeeds/<datafeed_id>
- GET _ml/data_frame/analytics/<analytics_id>

The flag is designed for cloning or exporting configuration objects to later be put into the same cluster or a separate cluster.

The following fields are not returned in the objects:

- any field that is not user settable (e.g. version, create_time)
- any field that is a calculated default value (e.g. datafeed chunking_config)
- any field that would effectively require changing to be of use (e.g. datafeed job_id)
- any field that is automatically set via another Elastic stack process (e.g. anomaly job custom_settings.created_by)

closes elastic#63055

Author: benwtrent

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -121,7 +121,10 @@ static Request getJob(GetJobRequest getJobRequest) {
 
         RequestConverters.Params params = new RequestConverters.Params();
         if (getJobRequest.getAllowNoMatch() != null) {
-            params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch()));
+            params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch()));
+        }
+        if (getJobRequest.getForExport() != null) {
+            params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport()));
         }
         request.addParameters(params.asMap());
         return request;
@@ -270,6 +273,9 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) {
             params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(),
                     Boolean.toString(getDatafeedRequest.getAllowNoMatch()));
         }
+        if (getDatafeedRequest.getForExport() != null) {
+            params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport()));
+        }
         request.addParameters(params.asMap());
         return request;
     }
@@ -645,7 +651,10 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) {
             }
         }
         if (getRequest.getAllowNoMatch() != null) {
-            params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch()));
+            params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch()));
+        }
+        if (getRequest.getForExport() != null) {
+            params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport()));
         }
         request.addParameters(params.asMap());
         return request;

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java

@@ -23,7 +23,6 @@
 import org.elasticsearch.client.ValidationException;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseField;
 
 import java.util.Arrays;
 import java.util.List;
@@ -32,11 +31,13 @@
 
 public class GetDataFrameAnalyticsRequest implements Validatable {
 
-    public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+    public static final String ALLOW_NO_MATCH = "allow_no_match";
+    public static final String FOR_EXPORT = "for_export";
 
     private final List<String> ids;
     private Boolean allowNoMatch;
     private PageParams pageParams;
+    private Boolean forExport;
 
     /**
      * Helper method to create a request that will get ALL Data Frame Analytics
@@ -58,6 +59,22 @@ public Boolean getAllowNoMatch() {
         return allowNoMatch;
     }
 
+    /**
+     * Setting this flag to `true` removes certain fields from the configuration on retrieval.
+     *
+     * This is useful when getting the configuration and wanting to put it in another cluster.
+     *
+     * Default value is false.
+     * @param forExport Boolean value indicating if certain fields should be removed
+     */
+    public void setForExport(boolean forExport) {
+        this.forExport = forExport;
+    }
+
+    public Boolean getForExport() {
+        return forExport;
+    }
+
     /**
      * Whether to ignore if a wildcard expression matches no data frame analytics.
      *
@@ -94,11 +111,12 @@ public boolean equals(Object o) {
         GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o;
         return Objects.equals(ids, other.ids)
             && Objects.equals(allowNoMatch, other.allowNoMatch)
+            && Objects.equals(forExport, other.forExport)
             && Objects.equals(pageParams, other.pageParams);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(ids, allowNoMatch, pageParams);
+        return Objects.hash(ids, allowNoMatch, forExport, pageParams);
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java

@@ -42,10 +42,12 @@ public class GetDatafeedRequest extends ActionRequest implements ToXContentObjec
 
     public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids");
     public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+    public static final String FOR_EXPORT = "for_export";
 
     private static final String ALL_DATAFEEDS = "_all";
     private final List<String> datafeedIds;
     private Boolean allowNoMatch;
+    private Boolean forExport;
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<GetDatafeedRequest, Void> PARSER = new ConstructingObjectParser<>(
@@ -101,14 +103,30 @@ public Boolean getAllowNoMatch() {
         return allowNoMatch;
     }
 
+    /**
+     * Setting this flag to `true` removes certain fields from the configuration on retrieval.
+     *
+     * This is useful when getting the configuration and wanting to put it in another cluster.
+     *
+     * Default value is false.
+     * @param forExport Boolean value indicating if certain fields should be removed
+     */
+    public void setForExport(boolean forExport) {
+        this.forExport = forExport;
+    }
+
+    public Boolean getForExport() {
+        return forExport;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         return null;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(datafeedIds, allowNoMatch);
+        return Objects.hash(datafeedIds, forExport, allowNoMatch);
     }
 
     @Override
@@ -123,7 +141,8 @@ public boolean equals(Object other) {
 
         GetDatafeedRequest that = (GetDatafeedRequest) other;
         return Objects.equals(datafeedIds, that.datafeedIds) &&
-            Objects.equals(allowNoMatch, that.allowNoMatch);
+            Objects.equals(allowNoMatch, that.allowNoMatch) &&
+            Objects.equals(forExport, that.forExport);
     }
 
     @Override

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java

@@ -43,10 +43,12 @@ public class GetJobRequest extends ActionRequest implements ToXContentObject {
 
     public static final ParseField JOB_IDS = new ParseField("job_ids");
     public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+    public static final String FOR_EXPORT = "for_export";
 
     private static final String ALL_JOBS = "_all";
     private final List<String> jobIds;
     private Boolean allowNoMatch;
+    private Boolean forExport;
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<GetJobRequest, Void> PARSER = new ConstructingObjectParser<>(
@@ -101,14 +103,30 @@ public Boolean getAllowNoMatch() {
         return allowNoMatch;
     }
 
+    /**
+     * Setting this flag to `true` removes certain fields from the configuration on retrieval.
+     *
+     * This is useful when getting the configuration and wanting to put it in another cluster.
+     *
+     * Default value is false.
+     * @param forExport Boolean value indicating if certain fields should be removed
+     */
+    public void setForExport(boolean forExport) {
+        this.forExport = forExport;
+    }
+
+    public Boolean getForExport() {
+        return forExport;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         return null;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(jobIds, allowNoMatch);
+        return Objects.hash(jobIds, forExport, allowNoMatch);
     }
 
     @Override
@@ -123,6 +141,7 @@ public boolean equals(Object other) {
 
         GetJobRequest that = (GetJobRequest) other;
         return Objects.equals(jobIds, that.jobIds) &&
+            Objects.equals(forExport, that.forExport) &&
             Objects.equals(allowNoMatch, that.allowNoMatch);
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -343,6 +343,7 @@ public void testGetJob() throws Exception {
             // tag::get-job-request
             GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1>
             request.setAllowNoMatch(true); // <2>
+            request.setForExport(false); // <3>
             // end::get-job-request
 
             // tag::get-job-execute
@@ -839,6 +840,7 @@ public void testGetDatafeed() throws Exception {
             // tag::get-datafeed-request
             GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1>
             request.setAllowNoMatch(true); // <2>
+            request.setForExport(false); // <3>
             // end::get-datafeed-request
 
             // tag::get-datafeed-execute
@@ -2864,6 +2866,7 @@ public void testGetDataFrameAnalytics() throws Exception {
         {
             // tag::get-data-frame-analytics-request
             GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1>
+            request.setForExport(false); // <2>
             // end::get-data-frame-analytics-request
 
             // tag::get-data-frame-analytics-execute

docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc

@@ -23,6 +23,9 @@ IDs, or the special wildcard `_all` to get all {dfanalytics-jobs}.
 include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
 <1> Constructing a new GET request referencing an existing {dfanalytics-job}
+<2> Optional boolean value for requesting the {dfanalytics-job} in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the {dfanalytics-job} is created are removed.
 
 include::../execution.asciidoc[]
 

docs/java-rest/high-level/ml/get-datafeed.asciidoc

@@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request]
 contain wildcards.
 <2> Whether to ignore if a wildcard expression matches no datafeeds.
  (This includes `_all` string or when no datafeeds have been specified).
+<3> Optional boolean value for requesting the datafeed in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the datafeed is created are removed.
 
 [id="{upid}-{api}-response"]
 ==== Get datafeeds response

docs/java-rest/high-level/ml/get-job.asciidoc

@@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request]
 wildcards.
 <2> Whether to ignore if a wildcard expression matches no {anomaly-jobs}.
  (This includes `_all` string or when no jobs have been specified).
+<3> Optional boolean value for requesting the job in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the job is created are removed.
 
 [id="{upid}-{api}-response"]
 ==== Get {anomaly-jobs} response

docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc

@@ -57,6 +57,10 @@ all {dfeeds}.
 (Optional, boolean)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds]
 
+`for_export`::
+(Optional, boolean)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
+
 [[ml-get-datafeed-results]]
 == {api-response-body-title}
 

docs/reference/ml/anomaly-detection/apis/get-job.asciidoc

@@ -50,6 +50,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul
 (Optional, boolean)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs]
 
+`for_export`::
+(Optional, boolean)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
+
 [[ml-get-job-results]]
 == {api-response-body-title}
 

docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc

@@ -70,6 +70,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from]
 (Optional, integer) 
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size]
 
+`for_export`::
+(Optional, boolean)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
+
 [role="child_attributes"]
 [[ml-get-dfanalytics-results]]
 == {api-response-body-title}

docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc

@@ -67,9 +67,7 @@ Specifies whether the included model definition should be returned as a JSON map
 
 `for_export`::
 (Optional, boolean)
-Indicates if certain fields should be removed from the model configuration on
-retrieval. This allows the model to be in an acceptable format to be retrieved
-and then added to another cluster. Default is false.
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
 
 `from`::
 (Optional, integer)

docs/reference/ml/ml-shared.asciidoc

@@ -673,6 +673,12 @@ The number of individual forecasts currently available for the job. A value of
 `1` or more indicates that forecasts exist.
 end::forecast-total[]
 
+tag::for-export[]
+Indicates if certain fields should be removed from the configuration on
+retrieval. This allows the configuration to be in an acceptable format to be retrieved
+and then added to another cluster. Default is false.
+end::for-export[]
+
 tag::frequency[]
 The interval at which scheduled queries are made while the {dfeed} runs in real
 time. The default value is either the bucket span for short bucket spans, or,