[ML] Allow datafeed and job configs for datafeed preview API (#70836) (#70927)

Previously, a datafeed and job must already exist for the `_preview` API to work.

With this change, users can get an accurate preview of the data that will be sent to the anomaly detection job
without creating either of them.

closes #70264

Author: benwtrent

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -326,14 +326,18 @@ static Request getDatafeedStats(GetDatafeedStatsRequest getDatafeedStatsRequest)
         return request;
     }
 
-    static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) {
-        String endpoint = new EndpointBuilder()
+    static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) throws IOException {
+        EndpointBuilder builder = new EndpointBuilder()
             .addPathPartAsIs("_ml")
-            .addPathPartAsIs("datafeeds")
-            .addPathPart(previewDatafeedRequest.getDatafeedId())
-            .addPathPartAsIs("_preview")
-            .build();
-        return new Request(HttpGet.METHOD_NAME, endpoint);
+            .addPathPartAsIs("datafeeds");
+        String endpoint = previewDatafeedRequest.getDatafeedId() != null ?
+            builder.addPathPart(previewDatafeedRequest.getDatafeedId()).addPathPartAsIs("_preview").build() :
+            builder.addPathPartAsIs("_preview").build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        if (previewDatafeedRequest.getDatafeedId() == null) {
+            request.setEntity(createEntity(previewDatafeedRequest, REQUEST_BODY_CONTENT_TYPE));
+        }
+        return request;
     }
 
     static Request deleteForecast(DeleteForecastRequest deleteForecastRequest) {

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java

@@ -10,6 +10,9 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.client.ml.job.config.Job;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -24,18 +27,34 @@
  */
 public class PreviewDatafeedRequest extends ActionRequest implements ToXContentObject {
 
+    private static final ParseField DATAFEED_CONFIG = new ParseField("datafeed_config");
+    private static final ParseField JOB_CONFIG = new ParseField("job_config");
+
     public static final ConstructingObjectParser<PreviewDatafeedRequest, Void> PARSER = new ConstructingObjectParser<>(
-        "open_datafeed_request", true, a -> new PreviewDatafeedRequest((String) a[0]));
+        "preview_datafeed_request",
+        a -> new PreviewDatafeedRequest((String) a[0], (DatafeedConfig.Builder) a[1], (Job.Builder) a[2]));
 
     static {
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), DatafeedConfig.ID);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.ID);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.PARSER, DATAFEED_CONFIG);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Job.PARSER, JOB_CONFIG);
     }
 
     public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
 
     private final String datafeedId;
+    private final DatafeedConfig datafeedConfig;
+    private final Job jobConfig;
+
+    private PreviewDatafeedRequest(@Nullable String datafeedId,
+                                   @Nullable DatafeedConfig.Builder datafeedConfig,
+                                   @Nullable Job.Builder jobConfig) {
+        this.datafeedId = datafeedId;
+        this.datafeedConfig = datafeedConfig == null ? null : datafeedConfig.build();
+        this.jobConfig = jobConfig == null ? null : jobConfig.build();
+    }
 
     /**
      * Create a new request with the desired datafeedId
@@ -44,12 +63,33 @@ public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws
      */
     public PreviewDatafeedRequest(String datafeedId) {
         this.datafeedId = Objects.requireNonNull(datafeedId, "[datafeed_id] must not be null");
+        this.datafeedConfig = null;
+        this.jobConfig = null;
+    }
+
+    /**
+     * Create a new request to preview the provided datafeed config and optional job config
+     * @param datafeedConfig The datafeed to preview
+     * @param jobConfig The associated job config (required if the datafeed does not refer to an existing job)
+     */
+    public PreviewDatafeedRequest(DatafeedConfig datafeedConfig, Job jobConfig) {
+        this.datafeedId = null;
+        this.datafeedConfig = datafeedConfig;
+        this.jobConfig = jobConfig;
     }
 
     public String getDatafeedId() {
         return datafeedId;
     }
 
+    public DatafeedConfig getDatafeedConfig() {
+        return datafeedConfig;
+    }
+
+    public Job getJobConfig() {
+        return jobConfig;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         return null;
@@ -58,7 +98,15 @@ public ActionRequestValidationException validate() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId);
+        if (datafeedId != null) {
+            builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId);
+        }
+        if (datafeedConfig != null) {
+            builder.field(DATAFEED_CONFIG.getPreferredName(), datafeedConfig);
+        }
+        if (jobConfig != null) {
+            builder.field(JOB_CONFIG.getPreferredName(), jobConfig);
+        }
         builder.endObject();
         return builder;
     }
@@ -70,7 +118,7 @@ public String toString() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(datafeedId);
+        return Objects.hash(datafeedId, datafeedConfig, jobConfig);
     }
 
     @Override
@@ -84,6 +132,8 @@ public boolean equals(Object other) {
         }
 
         PreviewDatafeedRequest that = (PreviewDatafeedRequest) other;
-        return Objects.equals(datafeedId, that.datafeedId);
+        return Objects.equals(datafeedId, that.datafeedId)
+            && Objects.equals(datafeedConfig, that.datafeedConfig)
+            && Objects.equals(jobConfig, that.jobConfig);
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java

@@ -91,6 +91,7 @@
 import org.elasticsearch.client.ml.job.config.AnalysisConfig;
 import org.elasticsearch.client.ml.job.config.Detector;
 import org.elasticsearch.client.ml.job.config.Job;
+import org.elasticsearch.client.ml.job.config.JobTests;
 import org.elasticsearch.client.ml.job.config.JobUpdate;
 import org.elasticsearch.client.ml.job.config.JobUpdateTests;
 import org.elasticsearch.client.ml.job.config.MlFilter;
@@ -390,11 +391,24 @@ public void testGetDatafeedStats() {
         assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match"));
     }
 
-    public void testPreviewDatafeed() {
+    public void testPreviewDatafeed() throws IOException {
         PreviewDatafeedRequest datafeedRequest = new PreviewDatafeedRequest("datafeed_1");
         Request request = MLRequestConverters.previewDatafeed(datafeedRequest);
-        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
         assertEquals("/_ml/datafeeds/" + datafeedRequest.getDatafeedId() + "/_preview", request.getEndpoint());
+        assertThat(request.getEntity(), is(nullValue()));
+
+        datafeedRequest = new PreviewDatafeedRequest(
+            DatafeedConfigTests.createRandom(),
+            randomBoolean() ? null : JobTests.createRandomizedJob()
+        );
+        request = MLRequestConverters.previewDatafeed(datafeedRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/datafeeds/_preview", request.getEndpoint());
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
+            PreviewDatafeedRequest parsedDatafeedRequest = PreviewDatafeedRequest.PARSER.apply(parser, null);
+            assertThat(parsedDatafeedRequest, equalTo(datafeedRequest));
+        }
     }
 
     public void testDeleteForecast() {

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.client.ml;
 
 import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests;
+import org.elasticsearch.client.ml.job.config.JobTests;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
@@ -17,7 +18,9 @@ public class PreviewDatafeedRequestTests extends AbstractXContentTestCase<Previe
 
     @Override
     protected PreviewDatafeedRequest createTestInstance() {
-        return new PreviewDatafeedRequest(DatafeedConfigTests.randomValidDatafeedId());
+        return randomBoolean() ?
+            new PreviewDatafeedRequest(DatafeedConfigTests.randomValidDatafeedId()) :
+            new PreviewDatafeedRequest(DatafeedConfigTests.createRandom(), randomBoolean() ? null : JobTests.createRandomizedJob());
     }
 
     @Override
@@ -27,6 +30,6 @@ protected PreviewDatafeedRequest doParseInstance(XContentParser parser) throws I
 
     @Override
     protected boolean supportsUnknownFields() {
-        return true;
+        return false;
     }
 }

docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc

@@ -13,7 +13,13 @@ Previews a {dfeed}.
 [[ml-preview-datafeed-request]]
 == {api-request-title}
 
-`GET _ml/datafeeds/<datafeed_id>/_preview`
+`GET _ml/datafeeds/<datafeed_id>/_preview` +
+
+`POST _ml/datafeeds/<datafeed_id>/_preview` +
+
+`GET _ml/datafeeds/_preview` +
+
+`POST _ml/datafeeds/_preview`
 
 [[ml-preview-datafeed-prereqs]]
 == {api-prereq-title}
@@ -25,9 +31,10 @@ Previews a {dfeed}.
 [[ml-preview-datafeed-desc]]
 == {api-description-title}
 
-The preview {dfeeds} API returns the first "page" of results from the `search`
-that is created by using the current {dfeed} settings. This preview shows the
-structure of the data that will be passed to the anomaly detection engine.
+The preview {dfeeds} API returns the first "page" of search results from a 
+{dfeed}. You can preview an existing {dfeed} or provide configuration details
+for the {dfeed} and {anomaly-job} in the API. The preview shows the structure of 
+the data that will be passed to the anomaly detection engine.
 
 IMPORTANT: When {es} {security-features} are enabled, the {dfeed} query is
 previewed using the credentials of the user calling the preview {dfeed} API.
@@ -43,12 +50,32 @@ supply the credentials.
 == {api-path-parms-title}
 
 `<datafeed_id>`::
-(Required, string)
+(Optional, string)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id]
++
+NOTE: If you provide the `<datafeed_id>` as a path parameter, you cannot
+provide {dfeed} or {anomaly-job} configuration details in the request body.
+
+[[ml-preview-datafeed-request-body]]
+== {api-request-body-title}
+
+`datafeed_config`::
+(Optional, object) The {dfeed} definition to preview. For valid definitions, see
+the <<ml-put-datafeed-request-body,create {dfeeds} API>>.
+
+`job_config`::
+(Optional, object) The configuration details for the {anomaly-job} that is
+associated with the {dfeed}. If the `datafeed_config` object does not include a
+`job_id` that references an existing {anomaly-job}, you must supply this
+`job_config` object. If you include both a `job_id` and a `job_config`, the
+latter information is used. You cannot specify a `job_config` object unless you also supply a `datafeed_config` object. For valid definitions, see the
+<<ml-put-job-request-body,create {anomaly-jobs} API>>.
 
 [[ml-preview-datafeed-example]]
 == {api-examples-title}
 
+This is an example of providing the ID of an existing {dfeed}:
+
 [source,console]
 --------------------------------------------------
 GET _ml/datafeeds/datafeed-high_sum_total_sales/_preview
@@ -87,3 +114,88 @@ The data that is returned for this example is as follows:
   ...
 ]
 ----
+
+The following example provides {dfeed} and {anomaly-job} configuration 
+details in the API:
+
+[source,console]
+--------------------------------------------------
+POST _ml/datafeeds/_preview
+{
+  "datafeed_config": {
+    "indices" : [
+      "kibana_sample_data_ecommerce"
+    ],
+    "query" : {
+      "bool" : {
+        "filter" : [
+          {
+            "term" : {
+              "_index" : "kibana_sample_data_ecommerce"
+            }
+          }
+        ]
+      }
+    },
+    "scroll_size" : 1000
+  },
+  "job_config": {
+    "description" : "Find customers spending an unusually high amount in an hour",
+    "analysis_config" : {
+      "bucket_span" : "1h",
+      "detectors" : [
+        {
+          "detector_description" : "High total sales",
+          "function" : "high_sum",
+          "field_name" : "taxful_total_price",
+          "over_field_name" : "customer_full_name.keyword"
+        }
+      ],
+      "influencers" : [
+        "customer_full_name.keyword",
+        "category.keyword"
+      ]
+    },
+    "analysis_limits" : {
+      "model_memory_limit" : "10mb"
+    },
+    "data_description" : {
+      "time_field" : "order_date",
+      "time_format" : "epoch_ms"
+    }
+  }
+}
+--------------------------------------------------
+// TEST[skip:set up Kibana sample data]
+
+The data that is returned for this example is as follows:
+
+[source,console-result]
+----
+[
+  {
+    "order_date" : 1574294659000,
+    "category.keyword" : "Men's Clothing",
+    "customer_full_name.keyword" : "Sultan Al Benson",
+    "taxful_total_price" : 35.96875
+  },
+  {
+    "order_date" : 1574294918000,
+    "category.keyword" : [
+      "Women's Accessories",
+      "Women's Clothing"
+    ],
+    "customer_full_name.keyword" : "Pia Webb",
+    "taxful_total_price" : 83.0
+  },
+  {
+    "order_date" : 1574295782000,
+    "category.keyword" : [
+      "Women's Accessories",
+      "Women's Shoes"
+    ],
+    "customer_full_name.keyword" : "Brigitte Graham",
+    "taxful_total_price" : 72.0
+  }
+]
+----