Merge remote-tracking branch 'origin/master' into slm-retention

Author: dakrone

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.client.ml.datafeed;
 
-import org.elasticsearch.client.ml.job.config.Job;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -57,7 +56,6 @@ public class DatafeedUpdate implements ToXContentObject {
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DatafeedConfig.ID);
 
-        PARSER.declareString(Builder::setJobId, Job.ID);
         PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDEXES);
         PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDICES);
         PARSER.declareString((builder, val) -> builder.setQueryDelay(
@@ -88,7 +86,6 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti
     }
 
     private final String id;
-    private final String jobId;
     private final TimeValue queryDelay;
     private final TimeValue frequency;
     private final List<String> indices;
@@ -99,11 +96,10 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti
     private final ChunkingConfig chunkingConfig;
     private final DelayedDataCheckConfig delayedDataCheckConfig;
 
-    private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List<String> indices, BytesReference query,
+    private DatafeedUpdate(String id, TimeValue queryDelay, TimeValue frequency, List<String> indices, BytesReference query,
                            BytesReference aggregations, List<SearchSourceBuilder.ScriptField> scriptFields, Integer scrollSize,
                            ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig) {
         this.id = id;
-        this.jobId = jobId;
         this.queryDelay = queryDelay;
         this.frequency = frequency;
         this.indices = indices;
@@ -126,7 +122,6 @@ public String getId() {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(DatafeedConfig.ID.getPreferredName(), id);
-        addOptionalField(builder, Job.ID, jobId);
         if (queryDelay != null) {
             builder.field(DatafeedConfig.QUERY_DELAY.getPreferredName(), queryDelay.getStringRep());
         }
@@ -162,10 +157,6 @@ private void addOptionalField(XContentBuilder builder, ParseField field, Object
         }
     }
 
-    public String getJobId() {
-        return jobId;
-    }
-
     public TimeValue getQueryDelay() {
         return queryDelay;
     }
@@ -228,7 +219,6 @@ public boolean equals(Object other) {
         DatafeedUpdate that = (DatafeedUpdate) other;
 
         return Objects.equals(this.id, that.id)
-            && Objects.equals(this.jobId, that.jobId)
             && Objects.equals(this.frequency, that.frequency)
             && Objects.equals(this.queryDelay, that.queryDelay)
             && Objects.equals(this.indices, that.indices)
@@ -247,7 +237,7 @@ public boolean equals(Object other) {
      */
     @Override
     public int hashCode() {
-        return Objects.hash(id, jobId, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields,
+        return Objects.hash(id, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields,
             chunkingConfig, delayedDataCheckConfig);
     }
 
@@ -258,7 +248,6 @@ public static Builder builder(String id) {
     public static class Builder {
 
         private String id;
-        private String jobId;
         private TimeValue queryDelay;
         private TimeValue frequency;
         private List<String> indices;
@@ -275,7 +264,6 @@ public Builder(String id) {
 
         public Builder(DatafeedUpdate config) {
             this.id = config.id;
-            this.jobId = config.jobId;
             this.queryDelay = config.queryDelay;
             this.frequency = config.frequency;
             this.indices = config.indices;
@@ -287,11 +275,6 @@ public Builder(DatafeedUpdate config) {
             this.delayedDataCheckConfig = config.delayedDataCheckConfig;
         }
 
-        public Builder setJobId(String jobId) {
-            this.jobId = jobId;
-            return this;
-        }
-
         public Builder setIndices(List<String> indices) {
             this.indices = indices;
             return this;
@@ -364,7 +347,7 @@ public Builder setDelayedDataCheckConfig(DelayedDataCheckConfig delayedDataCheck
         }
 
         public DatafeedUpdate build() {
-            return new DatafeedUpdate(id, jobId, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize,
+            return new DatafeedUpdate(id, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize,
                 chunkingConfig, delayedDataCheckConfig);
         }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -463,31 +463,6 @@ public void testUpdateDatafeed() throws Exception {
         assertThat(datafeedUpdate.getScrollSize(), equalTo(updatedDatafeed.getScrollSize()));
     }
 
-    public void testUpdateDatafeed_UpdatingJobIdIsDeprecated() throws Exception {
-        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
-
-        String jobId = randomValidJobId();
-        Job job = buildJob(jobId);
-        execute(new PutJobRequest(job), machineLearningClient::putJob, machineLearningClient::putJobAsync);
-
-        String anotherJobId = randomValidJobId();
-        Job anotherJob = buildJob(anotherJobId);
-        execute(new PutJobRequest(anotherJob), machineLearningClient::putJob, machineLearningClient::putJobAsync);
-
-        String datafeedId = "datafeed-" + jobId;
-        DatafeedConfig datafeedConfig = DatafeedConfig.builder(datafeedId, jobId).setIndices("some_data_index").build();
-        execute(new PutDatafeedRequest(datafeedConfig), machineLearningClient::putDatafeed, machineLearningClient::putDatafeedAsync);
-
-        DatafeedUpdate datafeedUpdateWithChangedJobId = DatafeedUpdate.builder(datafeedId).setJobId(anotherJobId).build();
-        WarningFailureException exception = expectThrows(
-            WarningFailureException.class,
-            () -> execute(
-                new UpdateDatafeedRequest(datafeedUpdateWithChangedJobId),
-                machineLearningClient::updateDatafeed,
-                machineLearningClient::updateDatafeedAsync));
-        assertThat(exception.getResponse().getWarnings(), contains("The ability to update a datafeed's job_id is deprecated."));
-    }
-
     public void testGetDatafeed() throws Exception {
         String jobId1 = "test-get-datafeed-job-1";
         String jobId2 = "test-get-datafeed-job-2";

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdateTests.java

@@ -34,9 +34,6 @@ public class DatafeedUpdateTests extends AbstractXContentTestCase<DatafeedUpdate
 
     public static DatafeedUpdate createRandom() {
         DatafeedUpdate.Builder builder = new DatafeedUpdate.Builder(DatafeedConfigTests.randomValidDatafeedId());
-        if (randomBoolean()) {
-            builder.setJobId(randomAlphaOfLength(10));
-        }
         if (randomBoolean()) {
             builder.setQueryDelay(TimeValue.timeValueMillis(randomIntBetween(1, Integer.MAX_VALUE)));
         }

docs/painless/painless-guide/painless-datetime.asciidoc

@@ -541,7 +541,7 @@ document is most commonly accessible through an input called `doc`.
 +
 [source,Painless]
 ----
-def input = doc['input_datetime'].value;
+ZonedDateTime input = doc['input_datetime'].value;
 String output = input.format(DateTimeFormatter.ISO_INSTANT); <1>
 ----
 <1> Note the use of a built-in DateTimeFormatter.
@@ -584,8 +584,8 @@ if (doc.containsKey('start') && doc.containsKey('end')) { <1>
 
     if (doc['start'].size() > 0 && doc['end'].size() > 0) { <2>
 
-        def start = doc['start'].value;
-        def end = doc['end'].value;
+        ZonedDateTime start = doc['start'].value;
+        ZonedDateTime end = doc['end'].value;
         long differenceInMillis = ChronoUnit.MILLIS.between(start, end);
 
         // handle difference in times
@@ -660,7 +660,7 @@ preferred as there is no need to parse it for comparision.
 [source,Painless]
 ----
 long now = params['now'];
-def inputDateTime = doc['input_datetime'];
+ZonedDateTime inputDateTime = doc['input_datetime'];
 long millisDateTime = zdt.toInstant().toEpochMilli();
 long elapsedTime = now - millisDateTime;
 ----
@@ -712,9 +712,194 @@ long elapsedTime = now - millisDateTime;
 String nowString = params['now'];
 ZonedDateTime nowZdt = ZonedDateTime.parse(datetime); <1>
 long now = ZonedDateTime.toInstant().toEpochMilli();
-def inputDateTime = doc['input_datetime'];
+ZonedDateTime inputDateTime = doc['input_datetime'];
 long millisDateTime = zdt.toInstant().toEpochMilli();
 long elapsedTime = now - millisDateTime;
 ----
 <1> Note this parses the same string datetime every time the script runs. Use a
 numeric datetime to avoid a significant performance hit.
+
+==== Datetime Examples in Contexts
+
+===== Load the Example Data
+
+Run the following curl commands to load the data necessary for the context
+examples into an Elasticsearch cluster:
+
+. Create {ref}/mapping.html[mappings] for the sample data.
++
+[source,js]
+----
+PUT /messages
+{
+    "mappings": {
+        "properties": {
+            "priority": {
+                "type": "integer"
+            },
+            "datetime": {
+                "type": "date"
+            },
+            "message": {
+                "type": "text"
+            }
+        }
+    }
+}
+----
++
+// CONSOLE
++
+. Load the sample data.
++
+[source,js]
+----
+POST /_bulk
+{ "index" : { "_index" : "messages", "_id" : "1" } }
+{ "priority": 1, "datetime": "2019-07-17T12:13:14Z", "message": "m1" }
+{ "index" : { "_index" : "messages", "_id" : "2" } }
+{ "priority": 1, "datetime": "2019-07-24T01:14:59Z", "message": "m2" }
+{ "index" : { "_index" : "messages", "_id" : "3" } }
+{ "priority": 2, "datetime": "1983-10-14T00:36:42Z", "message": "m3" }
+{ "index" : { "_index" : "messages", "_id" : "4" } }
+{ "priority": 3, "datetime": "1983-10-10T02:15:15Z", "message": "m4" }
+{ "index" : { "_index" : "messages", "_id" : "5" } }
+{ "priority": 3, "datetime": "1983-10-10T17:18:19Z", "message": "m5" }
+{ "index" : { "_index" : "messages", "_id" : "6" } }
+{ "priority": 1, "datetime": "2019-08-03T17:19:31Z", "message": "m6" }
+{ "index" : { "_index" : "messages", "_id" : "7" } }
+{ "priority": 3, "datetime": "2019-08-04T17:20:00Z", "message": "m7" }
+{ "index" : { "_index" : "messages", "_id" : "8" } }
+{ "priority": 2, "datetime": "2019-08-04T18:01:01Z", "message": "m8" }
+{ "index" : { "_index" : "messages", "_id" : "9" } }
+{ "priority": 3, "datetime": "1983-10-10T19:00:45Z", "message": "m9" }
+{ "index" : { "_index" : "messages", "_id" : "10" } }
+{ "priority": 2, "datetime": "2019-07-23T23:39:54Z", "message": "m10" }
+----
++
+// CONSOLE
+// TEST[continued]
+
+===== Day-of-the-Week Bucket Aggregation Example
+
+The following example uses a
+{ref}/search-aggregations-bucket-terms-aggregation.html#search-aggregations-bucket-terms-aggregation-script[terms aggregation]
+as part of the
+<<painless-bucket-script-agg-context, bucket script aggregation context>> to
+display the number of messages from each day-of-the-week.
+
+[source,js]
+----
+GET /messages/_search?pretty=true
+{
+    "aggs": {
+        "day-of-week-count": {
+            "terms": {
+                "script": "return doc[\"datetime\"].value.getDayOfWeekEnum();"
+            }
+        }
+    }
+}
+----
+// CONSOLE
+// TEST[continued]
+
+===== Morning/Evening Bucket Aggregation Example
+
+The following example uses a
+{ref}/search-aggregations-bucket-terms-aggregation.html#search-aggregations-bucket-terms-aggregation-script[terms aggregation]
+as part of the
+<<painless-bucket-script-agg-context, bucket script aggregation context>> to
+display the number of messages received in the morning versus the evening.
+
+[source,js]
+----
+GET /messages/_search?pretty=true
+{
+    "aggs": {
+        "am-pm-count": {
+            "terms": {
+                "script": "return doc[\"datetime\"].value.getHour() < 12 ? \"AM\" : \"PM\";"
+            }
+        }
+    }
+}
+----
+// CONSOLE
+// TEST[continued]
+
+===== Age of a Message Script Field Example
+
+The following example uses a
+{ref}/search-request-script-fields.html[script field] as part of the
+<<painless-field-context, field context>> to display the elapsed time between
+"now" and when a message was received.
+
+[source,js]
+----
+GET /_search?pretty=true
+{
+    "query" : {
+        "match_all": {}
+    },
+    "script_fields" : {
+        "message_age" : {
+            "script" : {
+                "source": "ZonedDateTime now = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params[\"now\"]), ZoneId.of(\"Z\")); ZonedDateTime mdt = doc[\"datetime\"].value; String age; long years = mdt.until(now, ChronoUnit.YEARS); age = years + \"Y \"; mdt = mdt.plusYears(years); long months = mdt.until(now, ChronoUnit.MONTHS); age += months + \"M \"; mdt = mdt.plusMonths(months); long days = mdt.until(now, ChronoUnit.DAYS); age += days + \"D \"; mdt = mdt.plusDays(days); long hours = mdt.until(now, ChronoUnit.HOURS); age += hours + \"h \"; mdt = mdt.plusHours(hours); long minutes = mdt.until(now, ChronoUnit.MINUTES); age += minutes + \"m \"; mdt = mdt.plusMinutes(minutes); long seconds = mdt.until(now, ChronoUnit.SECONDS); age += hours + \"s\"; return age;",
+                "params": {
+                    "now": 1574005645830
+                }
+            }
+        }
+    }
+}
+----
+// CONSOLE
+// TEST[continued]
+
+The following shows the script broken into multiple lines:
+
+[source,Painless]
+----
+ZonedDateTime now = ZonedDateTime.ofInstant(
+        Instant.ofEpochMilli(params['now']), ZoneId.of('Z')); <1>
+ZonedDateTime mdt = doc['datetime'].value; <2>
+
+String age;
+
+long years = mdt.until(now, ChronoUnit.YEARS); <3>
+age = years + 'Y '; <4>
+mdt = mdt.plusYears(years); <5>
+
+long months = mdt.until(now, ChronoUnit.MONTHS);
+age += months + 'M ';
+mdt = mdt.plusMonths(months);
+
+long days = mdt.until(now, ChronoUnit.DAYS);
+age += days + 'D ';
+mdt = mdt.plusDays(days);
+
+long hours = mdt.until(now, ChronoUnit.HOURS);
+age += hours + 'h ';
+mdt = mdt.plusHours(hours);
+
+long minutes = mdt.until(now, ChronoUnit.MINUTES);
+age += minutes + 'm ';
+mdt = mdt.plusMinutes(minutes);
+
+long seconds = mdt.until(now, ChronoUnit.SECONDS);
+age += hours + 's';
+
+return age; <6>
+----
+<1> Parse the datetime "now" as input from the user-defined params.
+<2> Store the datetime the message was received as a `ZonedDateTime`.
+<3> Find the difference in years between "now" and the datetime the message was
+received.
+<4> Add the difference in years later returned in the format
+`Y <years> ...` for the age of a message.
+<5> Add the years so only the remainder of the months, days, etc. remain as the
+difference between "now" and the datetime the message was received. Repeat this
+pattern until the desired granularity is reached (seconds in this example).
+<6> Return the age of the message in the format
+`Y <years> M <months> D <days> h <hours> m <minutes> s <seconds>`.

docs/reference/mapping/types.asciidoc

@@ -54,6 +54,8 @@ string::         <<text,`text`>> and <<keyword,`keyword`>>
 
 <<flattened>>:: Allows an entire JSON object to be indexed as a single field.
 
+<<shape>>:: `shape` for arbitrary cartesian geometries.
+
 [float]
 [[types-array-handling]]
 === Arrays
@@ -121,4 +123,6 @@ include::types/sparse-vector.asciidoc[]
 
 include::types/text.asciidoc[]
 
-include::types/token-count.asciidoc[]
+include::types/token-count.asciidoc[]
+
+include::types/shape.asciidoc[]