[ML] Allow datafeed and job configs for datafeed preview API (elastic#70836)

Previously, a datafeed and job must already exist for the `_preview` API to work.

With this change, users can get an accurate preview of the data that will be sent to the anomaly detection job
without creating either of them.

closes elastic#70264

Author: benwtrent

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -326,14 +326,18 @@ static Request getDatafeedStats(GetDatafeedStatsRequest getDatafeedStatsRequest)
         return request;
     }
 
-    static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) {
-        String endpoint = new EndpointBuilder()
+    static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) throws IOException {
+        EndpointBuilder builder = new EndpointBuilder()
             .addPathPartAsIs("_ml")
-            .addPathPartAsIs("datafeeds")
-            .addPathPart(previewDatafeedRequest.getDatafeedId())
-            .addPathPartAsIs("_preview")
-            .build();
-        return new Request(HttpGet.METHOD_NAME, endpoint);
+            .addPathPartAsIs("datafeeds");
+        String endpoint = previewDatafeedRequest.getDatafeedId() != null ?
+            builder.addPathPart(previewDatafeedRequest.getDatafeedId()).addPathPartAsIs("_preview").build() :
+            builder.addPathPartAsIs("_preview").build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        if (previewDatafeedRequest.getDatafeedId() == null) {
+            request.setEntity(createEntity(previewDatafeedRequest, REQUEST_BODY_CONTENT_TYPE));
+        }
+        return request;
     }
 
     static Request deleteForecast(DeleteForecastRequest deleteForecastRequest) {

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java

@@ -10,6 +10,9 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.client.ml.job.config.Job;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -24,18 +27,34 @@
  */
 public class PreviewDatafeedRequest extends ActionRequest implements ToXContentObject {
 
+    private static final ParseField DATAFEED_CONFIG = new ParseField("datafeed_config");
+    private static final ParseField JOB_CONFIG = new ParseField("job_config");
+
     public static final ConstructingObjectParser<PreviewDatafeedRequest, Void> PARSER = new ConstructingObjectParser<>(
-        "open_datafeed_request", true, a -> new PreviewDatafeedRequest((String) a[0]));
+        "preview_datafeed_request",
+        a -> new PreviewDatafeedRequest((String) a[0], (DatafeedConfig.Builder) a[1], (Job.Builder) a[2]));
 
     static {
-        PARSER.declareString(ConstructingObjectParser.constructorArg(), DatafeedConfig.ID);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.ID);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.PARSER, DATAFEED_CONFIG);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Job.PARSER, JOB_CONFIG);
     }
 
     public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
 
     private final String datafeedId;
+    private final DatafeedConfig datafeedConfig;
+    private final Job jobConfig;
+
+    private PreviewDatafeedRequest(@Nullable String datafeedId,
+                                   @Nullable DatafeedConfig.Builder datafeedConfig,
+                                   @Nullable Job.Builder jobConfig) {
+        this.datafeedId = datafeedId;
+        this.datafeedConfig = datafeedConfig == null ? null : datafeedConfig.build();
+        this.jobConfig = jobConfig == null ? null : jobConfig.build();
+    }
 
     /**
      * Create a new request with the desired datafeedId
@@ -44,12 +63,33 @@ public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws
      */
     public PreviewDatafeedRequest(String datafeedId) {
         this.datafeedId = Objects.requireNonNull(datafeedId, "[datafeed_id] must not be null");
+        this.datafeedConfig = null;
+        this.jobConfig = null;
+    }
+
+    /**
+     * Create a new request to preview the provided datafeed config and optional job config
+     * @param datafeedConfig The datafeed to preview
+     * @param jobConfig The associated job config (required if the datafeed does not refer to an existing job)
+     */
+    public PreviewDatafeedRequest(DatafeedConfig datafeedConfig, Job jobConfig) {
+        this.datafeedId = null;
+        this.datafeedConfig = datafeedConfig;
+        this.jobConfig = jobConfig;
     }
 
     public String getDatafeedId() {
         return datafeedId;
     }
 
+    public DatafeedConfig getDatafeedConfig() {
+        return datafeedConfig;
+    }
+
+    public Job getJobConfig() {
+        return jobConfig;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         return null;
@@ -58,7 +98,15 @@ public ActionRequestValidationException validate() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId);
+        if (datafeedId != null) {
+            builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId);
+        }
+        if (datafeedConfig != null) {
+            builder.field(DATAFEED_CONFIG.getPreferredName(), datafeedConfig);
+        }
+        if (jobConfig != null) {
+            builder.field(JOB_CONFIG.getPreferredName(), jobConfig);
+        }
         builder.endObject();
         return builder;
     }
@@ -70,7 +118,7 @@ public String toString() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(datafeedId);
+        return Objects.hash(datafeedId, datafeedConfig, jobConfig);
     }
 
     @Override
@@ -84,6 +132,8 @@ public boolean equals(Object other) {
         }
 
         PreviewDatafeedRequest that = (PreviewDatafeedRequest) other;
-        return Objects.equals(datafeedId, that.datafeedId);
+        return Objects.equals(datafeedId, that.datafeedId)
+            && Objects.equals(datafeedConfig, that.datafeedConfig)
+            && Objects.equals(jobConfig, that.jobConfig);
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java

@@ -91,6 +91,7 @@
 import org.elasticsearch.client.ml.job.config.AnalysisConfig;
 import org.elasticsearch.client.ml.job.config.Detector;
 import org.elasticsearch.client.ml.job.config.Job;
+import org.elasticsearch.client.ml.job.config.JobTests;
 import org.elasticsearch.client.ml.job.config.JobUpdate;
 import org.elasticsearch.client.ml.job.config.JobUpdateTests;
 import org.elasticsearch.client.ml.job.config.MlFilter;
@@ -390,11 +391,24 @@ public void testGetDatafeedStats() {
         assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match"));
     }
 
-    public void testPreviewDatafeed() {
+    public void testPreviewDatafeed() throws IOException {
         PreviewDatafeedRequest datafeedRequest = new PreviewDatafeedRequest("datafeed_1");
         Request request = MLRequestConverters.previewDatafeed(datafeedRequest);
-        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
         assertEquals("/_ml/datafeeds/" + datafeedRequest.getDatafeedId() + "/_preview", request.getEndpoint());
+        assertThat(request.getEntity(), is(nullValue()));
+
+        datafeedRequest = new PreviewDatafeedRequest(
+            DatafeedConfigTests.createRandom(),
+            randomBoolean() ? null : JobTests.createRandomizedJob()
+        );
+        request = MLRequestConverters.previewDatafeed(datafeedRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/datafeeds/_preview", request.getEndpoint());
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
+            PreviewDatafeedRequest parsedDatafeedRequest = PreviewDatafeedRequest.PARSER.apply(parser, null);
+            assertThat(parsedDatafeedRequest, equalTo(datafeedRequest));
+        }
     }
 
     public void testDeleteForecast() {

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.client.ml;
 
 import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests;
+import org.elasticsearch.client.ml.job.config.JobTests;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
@@ -17,7 +18,9 @@ public class PreviewDatafeedRequestTests extends AbstractXContentTestCase<Previe
 
     @Override
     protected PreviewDatafeedRequest createTestInstance() {
-        return new PreviewDatafeedRequest(DatafeedConfigTests.randomValidDatafeedId());
+        return randomBoolean() ?
+            new PreviewDatafeedRequest(DatafeedConfigTests.randomValidDatafeedId()) :
+            new PreviewDatafeedRequest(DatafeedConfigTests.createRandom(), randomBoolean() ? null : JobTests.createRandomizedJob());
     }
 
     @Override
@@ -27,6 +30,6 @@ protected PreviewDatafeedRequest doParseInstance(XContentParser parser) throws I
 
     @Override
     protected boolean supportsUnknownFields() {
-        return true;
+        return false;
     }
 }

docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc

@@ -13,7 +13,13 @@ Previews a {dfeed}.
 [[ml-preview-datafeed-request]]
 == {api-request-title}
 
-`GET _ml/datafeeds/<datafeed_id>/_preview`
+`GET _ml/datafeeds/<datafeed_id>/_preview` +
+
+`POST _ml/datafeeds/<datafeed_id>/_preview` +
+
+`GET _ml/datafeeds/_preview` +
+
+`POST _ml/datafeeds/_preview`
 
 [[ml-preview-datafeed-prereqs]]
 == {api-prereq-title}
@@ -25,9 +31,10 @@ Previews a {dfeed}.
 [[ml-preview-datafeed-desc]]
 == {api-description-title}
 
-The preview {dfeeds} API returns the first "page" of results from the `search`
-that is created by using the current {dfeed} settings. This preview shows the
-structure of the data that will be passed to the anomaly detection engine.
+The preview {dfeeds} API returns the first "page" of search results from a 
+{dfeed}. You can preview an existing {dfeed} or provide configuration details
+for the {dfeed} and {anomaly-job} in the API. The preview shows the structure of 
+the data that will be passed to the anomaly detection engine.
 
 IMPORTANT: When {es} {security-features} are enabled, the {dfeed} query is
 previewed using the credentials of the user calling the preview {dfeed} API.
@@ -43,12 +50,32 @@ supply the credentials.
 == {api-path-parms-title}
 
 `<datafeed_id>`::
-(Required, string)
+(Optional, string)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id]
++
+NOTE: If you provide the `<datafeed_id>` as a path parameter, you cannot
+provide {dfeed} or {anomaly-job} configuration details in the request body.
+
+[[ml-preview-datafeed-request-body]]
+== {api-request-body-title}
+
+`datafeed_config`::
+(Optional, object) The {dfeed} definition to preview. For valid definitions, see
+the <<ml-put-datafeed-request-body,create {dfeeds} API>>.
+
+`job_config`::
+(Optional, object) The configuration details for the {anomaly-job} that is
+associated with the {dfeed}. If the `datafeed_config` object does not include a
+`job_id` that references an existing {anomaly-job}, you must supply this
+`job_config` object. If you include both a `job_id` and a `job_config`, the
+latter information is used. You cannot specify a `job_config` object unless you also supply a `datafeed_config` object. For valid definitions, see the
+<<ml-put-job-request-body,create {anomaly-jobs} API>>.
 
 [[ml-preview-datafeed-example]]
 == {api-examples-title}
 
+This is an example of providing the ID of an existing {dfeed}:
+
 [source,console]
 --------------------------------------------------
 GET _ml/datafeeds/datafeed-high_sum_total_sales/_preview
@@ -87,3 +114,88 @@ The data that is returned for this example is as follows:
   ...
 ]
 ----
+
+The following example provides {dfeed} and {anomaly-job} configuration 
+details in the API:
+
+[source,console]
+--------------------------------------------------
+POST _ml/datafeeds/_preview
+{
+  "datafeed_config": {
+    "indices" : [
+      "kibana_sample_data_ecommerce"
+    ],
+    "query" : {
+      "bool" : {
+        "filter" : [
+          {
+            "term" : {
+              "_index" : "kibana_sample_data_ecommerce"
+            }
+          }
+        ]
+      }
+    },
+    "scroll_size" : 1000
+  },
+  "job_config": {
+    "description" : "Find customers spending an unusually high amount in an hour",
+    "analysis_config" : {
+      "bucket_span" : "1h",
+      "detectors" : [
+        {
+          "detector_description" : "High total sales",
+          "function" : "high_sum",
+          "field_name" : "taxful_total_price",
+          "over_field_name" : "customer_full_name.keyword"
+        }
+      ],
+      "influencers" : [
+        "customer_full_name.keyword",
+        "category.keyword"
+      ]
+    },
+    "analysis_limits" : {
+      "model_memory_limit" : "10mb"
+    },
+    "data_description" : {
+      "time_field" : "order_date",
+      "time_format" : "epoch_ms"
+    }
+  }
+}
+--------------------------------------------------
+// TEST[skip:set up Kibana sample data]
+
+The data that is returned for this example is as follows:
+
+[source,console-result]
+----
+[
+  {
+    "order_date" : 1574294659000,
+    "category.keyword" : "Men's Clothing",
+    "customer_full_name.keyword" : "Sultan Al Benson",
+    "taxful_total_price" : 35.96875
+  },
+  {
+    "order_date" : 1574294918000,
+    "category.keyword" : [
+      "Women's Accessories",
+      "Women's Clothing"
+    ],
+    "customer_full_name.keyword" : "Pia Webb",
+    "taxful_total_price" : 83.0
+  },
+  {
+    "order_date" : 1574295782000,
+    "category.keyword" : [
+      "Women's Accessories",
+      "Women's Shoes"
+    ],
+    "customer_full_name.keyword" : "Brigitte Graham",
+    "taxful_total_price" : 72.0
+  }
+]
+----