Merge branch 'master' into feature/runtime_fields

Author: javanna

build.gradle

@@ -174,8 +174,8 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/59293" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

docs/plugins/analysis-icu.asciidoc

@@ -30,7 +30,7 @@ include::install_remove.asciidoc[]
 ==== ICU Analyzer
 
 The `icu_analyzer` analyzer performs basic normalization, tokenization and character folding, using the
-`icu_normalizer` char filter, `icu_tokenizer` and `icu_normalizer` token filter
+`icu_normalizer` char filter, `icu_tokenizer` and `icu_folding` token filter
 
 The following parameters are accepted:
 

docs/reference/how-to/indexing-speed.asciidoc

@@ -62,7 +62,7 @@ gets indexed and when it becomes visible, increasing the
 
 If you have a large amount of data that you want to load all at once into
 Elasticsearch, it may be beneficial to set `index.number_of_replicas` to `0` in
-order to speep up indexing. Having no replicas means that losing a single node
+order to speed up indexing. Having no replicas means that losing a single node
 may incur data loss, so it is important that the data lives elsewhere so that
 this initial load can be retried in case of an issue. Once the initial load is
 finished, you can set `index.number_of_replicas` back to its original value.

docs/reference/indices/get-data-stream.asciidoc

@@ -85,9 +85,10 @@ GET /_data_stream/my-data-stream
 ==== {api-path-parms-title}
 
 `<data-stream>`::
-(Required, string)
-Name of the data stream to retrieve.
-Wildcard (`*`) expressions are supported.
+(Optional, string)
+Comma-separated list of data stream names used to limit the request. Wildcard
+(`*`) expressions are supported. If omitted, all data streams will be
+returned.
 
 [role="child_attributes"]
 [[get-data-stream-api-response-body]]

docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc

@@ -45,11 +45,11 @@ cluster, as well as the cluster state.  You can change this behavior by
 specifying a list of data streams and indices to back up in the body of the
 snapshot request.
 
-NOTE: You must register a snapshot before performing snapshot and restore operations. Use the <<put-snapshot-repo-api,put snapshot repository API>> to register new repositories and update existing ones.
+NOTE: You must register a snapshot repository before performing snapshot and restore operations. Use the <<put-snapshot-repo-api,put snapshot repository API>> to register new repositories and update existing ones.
 
 The snapshot process is incremental. When creating a snapshot, {es} analyzes the list of files that are already stored in the repository and copies only files that were created or changed since the last snapshot. This process allows multiple snapshots to be preserved in the repository in a compact form.
 
-The snapshot process is executed in non-blocking fashion, so all indexing and searching operations can run concurrently against the data stream or index that {es} is snapshotting. Only one snapshot process can run in the cluster at any time.
+The snapshot process is executed in non-blocking fashion, so all indexing and searching operations can run concurrently against the data stream or index that {es} is snapshotting.
 
 A snapshot represents a point-in-time view of the moment when the snapshot was created. No records that were added to a data stream or index after the snapshot process started will be present in the snapshot.
 
@@ -124,9 +124,6 @@ If `true`, allows taking a partial snapshot of indices with unavailable shards.
 If `true`, the request returns a response when the snapshot is complete.
 If `false`, the request returns a response when the snapshot initializes.
 Defaults to `false`.
-+
-NOTE: During snapshot initialization, information about all
-previous snapshots is loaded into memory. In large repositories, this load time can cause requests to take several seconds (or even minutes) to return a response, even if the `wait_for_completion` parameter is `false`.
 
 [[create-snapshot-api-example]]
 ==== {api-examples-title}

docs/reference/transform/painless-examples.asciidoc

@@ -106,7 +106,7 @@ You can retrieve the last value in a similar way:
 
 [discrete]
 [[painless-time-features]]
-==== Getting time features as scripted fields
+==== Getting time features by using aggregations
 
 This snippet shows how to extract time based features by using Painless in a 
 {transform}. The snippet uses an index where `@timestamp` is defined as a `date` 
@@ -115,37 +115,39 @@ type field.
 [source,js]
 --------------------------------------------------
 "aggregations": {
-  "script_fields": {
-      "hour_of_day": { <1>
-        "script": {
-          "lang": "painless",
-          "source": """
-            ZonedDateTime date =  doc['@timestamp'].value; <2>
-            return date.getHour(); <3>
-          """
-        }
-      },
-      "month_of_year": { <4>
-        "script": {
-          "lang": "painless",
-          "source": """
-            ZonedDateTime date =  doc['@timestamp'].value; <5>
-            return date.getMonthValue(); <6>
-          """
-        }
+  "avg_hour_of_day": { <1>
+    "avg":{
+      "script": { <2>
+        "source": """
+          ZonedDateTime date =  doc['@timestamp'].value; <3>
+          return date.getHour(); <4>
+        """
       }
-    },
-  ...
+    }  
+  },
+  "avg_month_of_year": { <5>
+    "avg":{
+      "script": { <6> 
+        "source": """
+          ZonedDateTime date =  doc['@timestamp'].value; <7>
+          return date.getMonthValue(); <8>
+        """
+      }
+    }
+  },
+ ...
 }
 --------------------------------------------------
 // NOTCONSOLE
 
-<1> Contains the Painless script that returns the hour of the day.
-<2> Sets `date` based on the timestamp of the document.
-<3> Returns the hour value from `date`.
-<4> Contains the Painless script that returns the month of the year.
-<5> Sets `date` based on the timestamp of the document.
-<6> Returns the month value from `date`.
+<1> Name of the aggregation.
+<2> Contains the Painless script that returns the hour of the day.
+<3> Sets `date` based on the timestamp of the document.
+<4> Returns the hour value from `date`.
+<5> Name of the aggregation.
+<6> Contains the Painless script that returns the month of the year.
+<7> Sets `date` based on the timestamp of the document.
+<8> Returns the month value from `date`.
 
 
 [discrete]

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java

@@ -210,6 +210,12 @@ public void declareLong(BiConsumer<Value, Long> consumer, ParseField field) {
         declareField(consumer, p -> p.longValue(), field, ValueType.LONG);
     }
 
+    public void declareLongOrNull(BiConsumer<Value, Long> consumer, long nullValue, ParseField field) {
+        // Using a method reference here angers some compilers
+        declareField(consumer, p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? nullValue : p.longValue(),
+            field, ValueType.LONG_OR_NULL);
+    }
+
     public void declareInt(BiConsumer<Value, Integer> consumer, ParseField field) {
         // Using a method reference here angers some compilers
         declareField(consumer, p -> p.intValue(), field, ValueType.INT);

modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/50_data_streams.yml

@@ -19,23 +19,17 @@
 
 package org.elasticsearch.upgrades;
 
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
-import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -49,7 +43,6 @@
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Collection;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -616,7 +609,7 @@ void assertTotalHits(int expectedTotalHits, Map<?, ?> response) {
         assertEquals(response.toString(), expectedTotalHits, actualTotalHits);
     }
 
-    int extractTotalHits(Map<?, ?> response) {
+    static int extractTotalHits(Map<?, ?> response) {
         return (Integer) XContentMapValues.extractValue("hits.total.value", response);
     }
 
@@ -1392,63 +1385,12 @@ public void testResize() throws Exception {
         }
     }
 
-    private void assertNumHits(String index, int numHits, int totalShards) throws IOException {
+    public static void assertNumHits(String index, int numHits, int totalShards) throws IOException {
         Map<String, Object> resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
         assertNoFailures(resp);
         assertThat(XContentMapValues.extractValue("_shards.total", resp), equalTo(totalShards));
         assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards));
         assertThat(extractTotalHits(resp), equalTo(numHits));
     }
 
-    @SuppressWarnings("unchecked")
-    public void testDataStreams() throws Exception {
-        assumeTrue("no data streams in versions before " + Version.V_7_9_0, getOldClusterVersion().onOrAfter(Version.V_7_9_0));
-        if (isRunningAgainstOldCluster()) {
-            String mapping = "{\n" +
-                "      \"properties\": {\n" +
-                "        \"@timestamp\": {\n" +
-                "          \"type\": \"date\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }";
-            Template template = new Template(null, new CompressedXContent(mapping), null);
-            createComposableTemplate(client(), "dst", "ds", template);
-
-            Request indexRequest = new Request("POST", "/ds/_doc/1?op_type=create&refresh");
-            XContentBuilder builder = JsonXContent.contentBuilder().startObject()
-                .field("f", "v")
-                .field("@timestamp", new Date())
-                .endObject();
-            indexRequest.setJsonEntity(Strings.toString(builder));
-            assertOK(client().performRequest(indexRequest));
-        }
-
-        Request getDataStream = new Request("GET", "/_data_stream/ds");
-        Response response = client().performRequest(getDataStream);
-        assertOK(response);
-        List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
-        assertEquals(1, dataStreams.size());
-        Map<String, Object> ds = (Map<String, Object>) dataStreams.get(0);
-        List<Map<String, String>> indices = (List<Map<String, String>>) ds.get("indices");
-        assertEquals("ds", ds.get("name"));
-        assertEquals(1, indices.size());
-        assertEquals(DataStream.getDefaultBackingIndexName("ds", 1), indices.get(0).get("index_name"));
-        assertNumHits("ds", 1, 1);
-    }
-
-    private static void createComposableTemplate(RestClient client, String templateName, String indexPattern, Template template)
-        throws IOException {
-        XContentBuilder builder = jsonBuilder();
-        template.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        StringEntity templateJSON = new StringEntity(
-            String.format(Locale.ROOT, "{\n" +
-                "  \"index_patterns\": \"%s\",\n" +
-                "  \"data_stream\": { \"timestamp_field\": \"@timestamp\" },\n" +
-                "  \"template\": %s\n" +
-                "}", indexPattern, Strings.toString(builder)),
-            ContentType.APPLICATION_JSON);
-        Request createIndexTemplateRequest = new Request("PUT", "_index_template/" + templateName);
-        createIndexTemplateRequest.setEntity(templateJSON);
-        client.performRequest(createIndexTemplateRequest);
-    }
 }