Merge remote-tracking branch 'elastic/master' into geoshape-doc-values

Author: talevy

.ci/java-versions.properties

@@ -8,4 +8,5 @@ ES_BUILD_JAVA=openjdk12
 ES_RUNTIME_JAVA=openjdk11
 GRADLE_TASK=build
 
-
+# Workaround for https://github.com/gradle/gradle/issues/11426
+OPENSHIFT_IP=0.0.0.0

build.gradle

@@ -32,7 +32,6 @@ import org.gradle.util.GradleVersion
 import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 
 plugins {
-  id 'com.gradle.build-scan' version '2.4.2'
   id 'lifecycle-base'
   id 'elasticsearch.global-build-info'
   id "com.diffplug.gradle.spotless" version "3.24.2" apply false

buildSrc/build.gradle

@@ -29,7 +29,7 @@ group = 'org.elasticsearch.gradle'
 
 String minimumGradleVersion = file('src/main/resources/minimumGradleVersion').text.trim()
 if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion)) {
-  throw new GradleException("Gradle ${minimumGradleVersion}+ is required to build elasticsearch")
+   throw new GradleException("Gradle ${minimumGradleVersion}+ is required to build elasticsearch")
 }
 
 if (project == rootProject) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -257,7 +257,6 @@ class BuildPlugin implements Plugin<Project> {
                 rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
                     List<String> messages = []
                     Map<Integer, List<Task>> requiredJavaVersions = (Map<Integer, List<Task>>) extraProperties.get('requiredJavaVersions')
-                    task.logger.warn(requiredJavaVersions.toString())
                     for (Map.Entry<Integer, List<Task>> entry : requiredJavaVersions) {
                         if (BuildParams.javaVersions.any { it.version == entry.key }) {
                             continue

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -212,7 +212,7 @@ private Object dependencyNotation(Project project, ElasticsearchDistribution dis
         }
 
         if (distribution.getType() == Type.INTEG_TEST_ZIP) {
-            return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion();
+            return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
         }
 
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -38,6 +38,7 @@ public class TestClustersPlugin implements Plugin<Project> {
 
     private static final String LIST_TASK_NAME = "listTestClusters";
     public static final String EXTENSION_NAME = "testClusters";
+    private static final String REGISTRY_EXTENSION_NAME = "testClustersRegistry";
 
     private static final Logger logger =  Logging.getLogger(TestClustersPlugin.class);
 
@@ -56,9 +57,9 @@ public void apply(Project project) {
         // provide a task to be able to list defined clusters.
         createListClustersTask(project, container);
 
-        if (project.getRootProject().getExtensions().findByName("testClusters") == null) {
+        if (project.getRootProject().getExtensions().findByName(REGISTRY_EXTENSION_NAME) == null) {
             TestClustersRegistry registry = project.getRootProject().getExtensions()
-                .create("testClusters", TestClustersRegistry.class);
+                .create(REGISTRY_EXTENSION_NAME, TestClustersRegistry.class);
 
             // When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
             // that are defined in the build script and the ones that will actually be used in this invocation of gradle

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-5.6.4
+6.0.1

buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java renamed to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/BuildParams.java

@@ -84,7 +84,6 @@ public static JavaVersion getRuntimeJavaVersion() {
         return value(runtimeJavaVersion);
     }
 
-    @ExecutionTime
     public static Boolean isInFipsJvm() {
         return value(inFipsJvm);
     }
@@ -155,6 +154,9 @@ public void reset() {
                 .filter(f -> Modifier.isStatic(f.getModifiers()))
                 .forEach(f -> {
                     try {
+                        // Since we are mutating private static fields from a public static inner class we need to suppress
+                        // accessibility controls here.
+                        f.setAccessible(true);
                         f.set(null, null);
                     } catch (IllegalAccessException e) {
                         throw new RuntimeException(e);

buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java renamed to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/info/JavaHome.java

@@ -16,6 +16,9 @@ repositories {
   maven {
     name = "local-test"
     url = file("sample_jars/build/testrepo")
+    metadataSources {
+      artifact()
+    }
   }
   jcenter()
 }

buildSrc/src/testKit/thirdPartyAudit/build.gradle

@@ -11,6 +11,7 @@ jts               = 1.15.0
 # you should also inspect that version to see if it can be advanced along with
 # the com.maxmind.geoip2:geoip2 dependency
 jackson           = 2.8.11
+jacksondatabind   = 2.8.11.4
 snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0

buildSrc/version.properties

@@ -34,9 +34,9 @@
 /**
  * Defines the fields and functions used in the analysis. A combination of <code>field_name</code>,
  * <code>by_field_name</code> and <code>over_field_name</code> can be used depending on the specific
- * function chosen. For more information see
- * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-job-resource.html#ml-detectorconfig">configuring
- * detectors</a> and <a href="https://www.elastic.co/guide/en/elastic-stack-overview/current/ml-functions.html">detector functions</a>.
+ * function chosen. For more information see the
+ * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html">create anomaly detection
+ * jobs API</a> and <a href="https://www.elastic.co/guide/en/elastic-stack-overview/current/ml-functions.html">detector functions</a>.
  */
 public class Detector implements ToXContentObject {
 

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Detector.java

@@ -51,16 +51,8 @@ import java.util.regex.Pattern
  *    dpkg -c path/to/elasticsearch.deb
  */
 
-buildscript {
-  repositories {
-    maven {
-      name "gradle-plugins"
-      url "https://plugins.gradle.org/m2/"
-    }
-  }
-  dependencies {
-    classpath 'com.netflix.nebula:gradle-ospackage-plugin:4.7.1'
-  }
+plugins {
+  id "nebula.ospackage-base" version "8.0.3"
 }
 
 void addProcessFilesTask(String type, boolean oss, boolean jdk) {
@@ -115,10 +107,12 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
     arch(type == 'deb' ? 'amd64' : 'X86_64')
     // Follow elasticsearch's file naming convention
     String jdkString = jdk ? "" : "no-jdk-"
-    archiveName "${packageName}-${project.version}-${jdkString}${archString}.${type}"
-
     String prefix = "${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
     destinationDir = file("${prefix}/build/distributions")
+
+    // SystemPackagingTask overrides default archive task convention mappings, but doesn't provide a setter so we have to override the convention mapping itself
+    conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDir}/${packageName}-${project.version}-${jdkString}${archString}.${type}")) }
+
     String packagingFiles = "build/packaging/${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}"
 
     String scripts = "${packagingFiles}/scripts"
@@ -157,7 +151,9 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
       eachFile { FileCopyDetails fcp ->
         String[] segments = fcp.relativePath.segments
         for (int i = segments.length - 2; i > 2; --i) {
-          directory('/' + segments[0..i].join('/'), 0755)
+          if (type == 'rpm') {
+            directory('/' + segments[0..i].join('/'), 0755)
+          }
           if (segments[-2] == 'bin' || segments[-1] == 'jspawnhelper') {
             fcp.mode = 0755
           } else {

distribution/packages/build.gradle

@@ -29,6 +29,7 @@ buildRestTests.expectedUnconvertedCandidates = [
   'reference/ml/anomaly-detection/apis/get-category.asciidoc',
   'reference/ml/anomaly-detection/apis/get-influencer.asciidoc',
   'reference/ml/anomaly-detection/apis/get-job-stats.asciidoc',
+  'reference/ml/anomaly-detection/apis/get-job.asciidoc',
   'reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc',
   'reference/ml/anomaly-detection/apis/get-record.asciidoc',
   'reference/ml/anomaly-detection/apis/get-snapshot.asciidoc',

docs/build.gradle

@@ -25,7 +25,7 @@ Painless also supports the `for in` syntax from Groovy:
 
 [source,painless]
 ---------------------------------------------------------
-for (item : list) {
+for (def item : list) {
   ...
 }
 ---------------------------------------------------------

docs/painless/painless-lang-spec/painless-statements.asciidoc

@@ -549,7 +549,7 @@ the reindex returned a `noop` value for `ctx.op`.
 
 `version_conflicts`::
 
-{integer)The number of version conflicts that reindex hit.
+(integer) The number of version conflicts that reindex hits.
 
 `retries`::
 

docs/reference/docs/reindex.asciidoc

@@ -283,7 +283,7 @@ If you have a lot of documents to index, you can submit them in batches with
 the {ref}/docs-bulk.html[bulk API]. Using bulk to batch document
 operations is significantly faster than submitting requests individually as it minimizes network roundtrips. 
 
-The optimal batch size depends a number of factors: the document size and complexity, the indexing and search load, and the resources available to your cluster. A good place to start is with batches of 1,000 to 5,000 documents
+The optimal batch size depends on a number of factors: the document size and complexity, the indexing and search load, and the resources available to your cluster. A good place to start is with batches of 1,000 to 5,000 documents
 and a total payload between 5MB and 15MB. From there, you can experiment
 to find the sweet spot.
 

docs/reference/getting-started.asciidoc

@@ -10,7 +10,7 @@ Performs a synced flush on one or more indices.
 --------------------------------------------------
 POST /twitter/_flush/synced
 --------------------------------------------------
-// TEST[setup:twitter]
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 
 
 [[synced-flush-api-request]]
@@ -80,7 +80,7 @@ section of the shard stats returned by the <<indices-stats,indices stats>> API:
 --------------------------------------------------
 GET /twitter/_stats?filter_path=**.commit&level=shards <1>
 --------------------------------------------------
-// TEST[s/^/PUT twitter\nPOST twitter\/_flush\/synced\n/]
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 
 <1> `filter_path` is used to reduce the verbosity of the response, but is entirely optional
 
@@ -116,10 +116,7 @@ The API returns the following response:
    }
 }
 --------------------------------------------------
-// TESTRESPONSE[s/"id" : "3M3zkw2GHMo2Y4h4\/KFKCg=="/"id": $body.indices.twitter.shards.0.0.commit.id/]
-// TESTRESPONSE[s/"translog_uuid" : "hnOG3xFcTDeoI_kvvvOdNA"/"translog_uuid": $body.indices.twitter.shards.0.0.commit.user_data.translog_uuid/]
-// TESTRESPONSE[s/"history_uuid" : "XP7KDJGiS1a2fHYiFL5TXQ"/"history_uuid": $body.indices.twitter.shards.0.0.commit.user_data.history_uuid/]
-// TESTRESPONSE[s/"sync_id" : "AVvFY-071siAOuFGEO9P"/"sync_id": $body.indices.twitter.shards.0.0.commit.user_data.sync_id/]
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 <1> the `sync id` marker
 
 NOTE: The `sync_id` marker is removed as soon as the shard is flushed again, and
@@ -172,7 +169,7 @@ A replica shard failed to sync-flush.
 ----
 POST /kimchy/_flush/synced
 ----
-// TEST[s/^/PUT kimchy\n/]
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 
 
 [[synced-flush-api-multi-ex]]
@@ -182,8 +179,7 @@ POST /kimchy/_flush/synced
 --------------------------------------------------
 POST /kimchy,elasticsearch/_flush/synced
 --------------------------------------------------
-// TEST[s/^/PUT elasticsearch\n/]
-// TEST[continued]
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 
 
 [[synced-flush-api-all-ex]]
@@ -193,7 +189,7 @@ POST /kimchy,elasticsearch/_flush/synced
 --------------------------------------------------
 POST /_flush/synced
 --------------------------------------------------
-// TEST[setup:twitter]
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 
 The response contains details about how many shards were successfully
 sync-flushed and information about any failure.
@@ -217,12 +213,12 @@ successfully sync-flushed:
    }
 }
 --------------------------------------------------
-// TESTRESPONSE[s/"successful": 2/"successful": 1/]
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 
 The following response indicates one shard group failed
 due to pending operations:
 
-[source,js]
+[source,console-result]
 --------------------------------------------------
 {
    "_shards": {
@@ -243,13 +239,13 @@ due to pending operations:
    }
 }
 --------------------------------------------------
-// NOTCONSOLE
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]
 
 Sometimes the failures are specific to a shard replica. The copies that failed
 will not be eligible for fast recovery but those that succeeded still will be.
 This case is reported as follows:
 
-[source,js]
+[source,console-result]
 --------------------------------------------------
 {
    "_shards": {
@@ -278,4 +274,4 @@ This case is reported as follows:
    }
 }
 --------------------------------------------------
-// NOTCONSOLE
+// TEST[skip: Synced flush can conflict with scheduled flushes in doc tests]

docs/reference/indices/synced-flush.asciidoc

@@ -25,6 +25,7 @@ uncompressed. The `ingest-geoip` config directory is located at `$ES_CONFIG/inge
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip module ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files.
 | `properties`           | no        | [`continent_name`, `country_iso_code`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+| `first_only`           | no        | `true`                                                                             | If `true` only first found geoip data will be returned, even if `field` contains array
 |======
 
 *Depends on what is available in `database_file`:

docs/reference/ingest/processors/geoip.asciidoc

@@ -60,25 +60,15 @@ results the job might have recently produced or might produce in the future.
 ==== {api-path-parms-title}
 
 `<job_id>`::
-  (Required, string) Identifier for the {anomaly-job}. It can be a job
-  identifier, a group name, or a wildcard expression.
+(Required, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildcard]
 
 [[ml-close-job-query-parms]]
 ==== {api-query-parms-title}
 
 `allow_no_jobs`::
-  (Optional, boolean) Specifies what to do when the request:
-+
---
-* Contains wildcard expressions and there are no jobs that match.
-* Contains the `_all` string or no identifiers and there are no matches.
-* Contains wildcard expressions and there are only partial matches.
-
-The default value is `true`, which returns an empty `jobs` array 
-when there are no matches and the subset of results when there are partial 
-matches. If this parameter is `false`, the request returns a `404` status code
-when there are no matches or only partial matches.
---
+(Optional, boolean)
+include::{docdir}/ml/ml-shared.asciidoc[tag=allow-no-jobs]
 
 `force`::
   (Optional, boolean) Use to close a failed job, or to forcefully close a job

docs/reference/ml/anomaly-detection/apis/close-job.asciidoc

@@ -50,7 +50,7 @@ A {dfeed} resource has the following properties:
 `script_fields`::
   (object) Specifies scripts that evaluate custom expressions and returns
   script fields to the {dfeed}.
-  The <<ml-detectorconfig,detector configuration objects>> in a job can contain
+  The detector configuration objects in a job can contain
   functions that use these script fields.
   For more information, see
   {stack-ov}/ml-configuring-transform.html[Transforming Data With Script Fields].

docs/reference/ml/anomaly-detection/apis/datafeedresource.asciidoc

@@ -39,7 +39,8 @@ separated list.
 ==== {api-path-parms-title}
 
 `<job_id>`::
-  (Required, string) Identifier for the {anomaly-job}.
+(Required, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
 
 [[ml-delete-job-query-parms]]
 ==== {api-query-parms-title}

docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc

@@ -37,7 +37,8 @@ opened again before analyzing further data.
 ==== {api-path-parms-title}
 
 `<job_id>`::
-(string) Required. Identifier for the job.
+(Required, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
 
 [[ml-flush-job-query-parms]]
 ==== {api-query-parms-title}

docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc

@@ -29,15 +29,16 @@ See {stack-ov}/ml-overview.html#ml-forecasting[Forecasting the future].
 ===============================
 
 * If you use an `over_field_name` property in your job, you cannot create a
-forecast. For more information about this property, see <<ml-job-resource>>.
+forecast. For more information about this property, see <<ml-put-job>>.
 * The job must be open when you create a forecast. Otherwise, an error occurs.
 ===============================
 
 [[ml-forecast-path-parms]]
 ==== {api-path-parms-title}
 
 `<job_id>`::
-  (Required, string) Identifier for the job.
+(Required, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
 
 [[ml-forecast-request-body]]
 ==== {api-request-body-title}

docs/reference/ml/anomaly-detection/apis/forecast.asciidoc

@@ -36,7 +36,8 @@ bucket.
 ==== {api-path-parms-title}
 
 `<job_id>`::
-  (Required, string) Identifier for the {anomaly-job}.
+(Required, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
 
 `<timestamp>`::
   (Optional, string) The timestamp of a single bucket result. If you do not

docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc

@@ -35,7 +35,8 @@ For more information about categories, see
 ==== {api-path-parms-title}
 
 `<job_id>`::
-  (Required, string) Identifier for the {anomaly-job}.
+(Required, string)
+include::{docdir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
 
 `<category_id>`::
   (Optional, long) Identifier for the category. If you do not specify this