Merge branch 'master' into ccr

* master:
  [ML] Fix master node deadlock during ML daily maintenance (#31836)
  Build: Switch integ-test-zip to OSS-only (#31866)
  SQL: Remove restriction for single column grouping (#31818)
  Build: Fix detection of Eclipse Compiler Server (#31838)
  Docs: Inconsistency between description and example (#31858)
  Re-enable bwc tests now that #29538 has been backported and 6.x intake build succeeded.
  QA: build improvements related to SQL projects (#31862)
  [Docs] Add clarification to analysis example (#31826)
  Check timeZone() argument in AbstractSqlQueryRequest (#31822)
  SQL: Fix incorrect HAVING equality (#31820)
  Smaller aesthetic fixes to InternalTestCluster (#31831)
  [Docs] Clarify accepted sort case (#31605)
  Temporarily disable bwc test in order to backport #29538
  Remove obsolete parameters from analyze rest spec (#31795)
  [Docs] Fix wrong link in Korean analyzer docs (#31815)
  Fix profiling of ordered terms aggs (#31814)
  Properly mute test involving JDK11 closes #31739
  Do not return all indices if a specific alias is requested via get aliases api. (#29538)
  Get snapshot rest client cleanups (#31740)
  Docs: Explain _bulk?refresh shard targeting
  Fix handling of points_only with term strategy in geo_shape (#31766)

Author: dnhatn

build.gradle

@@ -125,7 +125,10 @@ Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectE
 allprojects {
   project.ext {
     // for ide hacks...
-    isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
+    isEclipse = System.getProperty("eclipse.launcher") != null ||   // Detects gradle launched from Eclipse's IDE
+            System.getProperty("eclipse.application") != null ||    // Detects gradle launched from the Eclipse compiler server
+            gradle.startParameter.taskNames.contains('eclipse') ||  // Detects gradle launched from the command line to do eclipse stuff
+            gradle.startParameter.taskNames.contains('cleanEclipse')
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
 
     // for BWC testing
@@ -446,7 +449,7 @@ allprojects {
 
   File licenseHeaderFile;
   String prefix = ':x-pack';
- 
+
   if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     prefix = prefix.replace(':', '_')
   }
@@ -455,7 +458,7 @@ allprojects {
   } else {
       licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/oss-license-header.txt')
   }
-  
+
   String lineSeparator = Os.isFamily(Os.FAMILY_WINDOWS) ? '\\\\r\\\\n' : '\\\\n'
   String licenseHeader = licenseHeaderFile.getText('UTF-8').replace(System.lineSeparator(), lineSeparator)
   task copyEclipseSettings(type: Copy) {

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -2122,13 +2122,21 @@ public void testGetSnapshots() {
         getSnapshotsRequest.snapshots(Arrays.asList(snapshot1, snapshot2).toArray(new String[0]));
         setRandomMasterTimeout(getSnapshotsRequest, expectedParams);
 
-        boolean ignoreUnavailable = randomBoolean();
-        getSnapshotsRequest.ignoreUnavailable(ignoreUnavailable);
-        expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable));
+        if (randomBoolean()) {
+            boolean ignoreUnavailable = randomBoolean();
+            getSnapshotsRequest.ignoreUnavailable(ignoreUnavailable);
+            expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable));
+        } else {
+            expectedParams.put("ignore_unavailable", Boolean.FALSE.toString());
+        }
 
-        boolean verbose = randomBoolean();
-        getSnapshotsRequest.verbose(verbose);
-        expectedParams.put("verbose", Boolean.toString(verbose));
+        if (randomBoolean()) {
+            boolean verbose = randomBoolean();
+            getSnapshotsRequest.verbose(verbose);
+            expectedParams.put("verbose", Boolean.toString(verbose));
+        } else {
+            expectedParams.put("verbose", Boolean.TRUE.toString());
+        }
 
         Request request = RequestConverters.getSnapshots(getSnapshotsRequest);
         assertThat(endpoint, equalTo(request.getEndpoint()));

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java

@@ -48,7 +48,10 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotShardFailure;
+import org.elasticsearch.snapshots.SnapshotState;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -496,7 +499,14 @@ public void testSnapshotGetSnapshots() throws IOException {
         // end::get-snapshots-execute
 
         // tag::get-snapshots-response
-        List<SnapshotInfo> snapshotsInfos = response.getSnapshots(); // <1>
+        List<SnapshotInfo> snapshotsInfos = response.getSnapshots();
+        SnapshotInfo snapshotInfo = snapshotsInfos.get(0);
+        RestStatus restStatus = snapshotInfo.status(); // <1>
+        SnapshotId snapshotId = snapshotInfo.snapshotId(); // <2>
+        SnapshotState snapshotState = snapshotInfo.state(); // <3>
+        List<SnapshotShardFailure> snapshotShardFailures = snapshotInfo.shardFailures(); // <4>
+        long startTime = snapshotInfo.startTime(); // <5>
+        long endTime = snapshotInfo.endTime(); // <6>
         // end::get-snapshots-response
         assertEquals(1, snapshotsInfos.size());
     }

distribution/archives/build.gradle

@@ -102,7 +102,7 @@ Closure commonZipConfig = {
 
 task buildIntegTestZip(type: Zip) {
   configure(commonZipConfig)
-  with archiveFiles(transportModulesFiles, 'zip', false)
+  with archiveFiles(transportModulesFiles, 'zip', true)
 }
 
 task buildZip(type: Zip) {
@@ -193,7 +193,7 @@ subprojects {
     onlyIf toolExists
     doLast {
       final String licenseFilename
-      if (project.name.contains('oss-')) {
+      if (project.name.contains('oss-') || project.name == 'integ-test-zip') {
         licenseFilename = "APACHE-LICENSE-2.0.txt"
       } else {
         licenseFilename = "ELASTIC-LICENSE.txt"

docs/java-rest/high-level/indices/analyze.asciidoc

@@ -56,7 +56,7 @@ You can analyze text using the mappings for a particular field in an index:
 include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-field-request]
 ---------------------------------------------------
 
-==== Optional arguemnts
+==== Optional arguments
 The following arguments can also optionally be provided:
 
 ["source","java",subs="attributes,callouts,macros"]

docs/java-rest/high-level/snapshot/get_snapshots.asciidoc

@@ -93,11 +93,16 @@ argument.
 [[java-rest-high-snapshot-get-snapshots-response]]
 ==== Get Snapshots Response
 
-Use the `GetSnapshotsResponse` to retrieve information about the evaluated
-request:
+The returned `GetSnapshotsResponse` allows the retrieval of information about the requested
+snapshots:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
 include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-response]
 --------------------------------------------------
-<1> Indicates the node has started the request.
+<1> The REST status of a snapshot
+<2> The snapshot id
+<3> The current state of the snapshot
+<4> Information about failures that occurred during the shard snapshot process.
+<5> The snapshot start time
+<6> The snapshot end time

docs/plugins/analysis-nori.asciidoc

@@ -268,7 +268,7 @@ Which responds with:
 
 The `nori_part_of_speech` token filter removes tokens that match a set of
 part-of-speech tags. The list of supported tags and their meanings can be found here:
-{lucene_version_path}/org/apache/lucene/analysis/ko/POS.Tag.html[Part of speech tags]
+{lucene-core-javadoc}/../analyzers-nori/org/apache/lucene/analysis/ko/POS.Tag.html[Part of speech tags]
 
 It accepts the following setting:
 

docs/reference/analysis.asciidoc

@@ -13,15 +13,18 @@ defined per index.
 [float]
 == Index time analysis
 
-For instance at index time, the built-in <<english-analyzer,`english`>> _analyzer_ would
-convert this sentence:
+For instance, at index time the built-in <<english-analyzer,`english`>> _analyzer_ 
+will first convert the sentence:
 
 [source,text]
 ------
 "The QUICK brown foxes jumped over the lazy dog!"
 ------
 
-into these terms, which would be added to the inverted index.
+into distinct tokens. It will then lowercase each token, remove frequent
+stopwords ("the") and reduce the terms to their word stems (foxes -> fox,
+jumped -> jump, lazy -> lazi). In the end, the following terms will be added
+to the inverted index:
 
 [source,text]
 ------

docs/reference/docs/bulk.asciidoc

@@ -230,6 +230,13 @@ example.
 Control when the changes made by this request are visible to search. See
 <<docs-refresh,refresh>>.
 
+NOTE: Only the shards that receive the bulk request will be affected by
+`refresh`. Imagine a `_bulk?refresh=wait_for` request with three
+documents in it that happen to be routed to different shards in an index
+with five shards. The request will only wait for those three shards to
+refresh. The other two shards of that make up the index do not
+participate in the `_bulk` request at all.
+
 [float]
 [[bulk-update]]
 === Update

docs/reference/indices/aliases.asciidoc

@@ -486,7 +486,7 @@ The rest endpoint is: `/{index}/_alias/{alias}`.
 [float]
 ==== Examples:
 
-All aliases for the index users:
+All aliases for the index `logs_20162801`:
 
 [source,js]
 --------------------------------------------------

docs/reference/search/request/rescore.asciidoc

@@ -15,8 +15,8 @@ Currently the rescore API has only one implementation: the query
 rescorer, which uses a query to tweak the scoring. In the future,
 alternative rescorers may be made available, for example, a pair-wise rescorer.
 
-NOTE: An error will be thrown if an explicit <<search-request-sort,`sort`>> (other than `_score`)
-is provided with a `rescore` query.
+NOTE: An error will be thrown if an explicit <<search-request-sort,`sort`>> 
+(other than `_score` in descending order) is provided with a `rescore` query.
 
 NOTE: when exposing pagination to your users, you should not change
 `window_size` as you step through each page (by passing different

docs/reference/sql/language/syntax/select.asciidoc

@@ -177,6 +177,13 @@ And grouping by column expression (typically used along-side an alias):
 include-tagged::{sql-specs}/docs.csv-spec[groupByExpression]
 ----
 
+Or a mixture of the above:
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs.csv-spec[groupByMulti]
+----
+
+
 When a `GROUP BY` clause is used in a `SELECT`, _all_ output expressions must be either aggregate functions or expressions used for grouping or derivatives of (otherwise there would be more than one possible value to return for each ungrouped column).
 
 To wit:

plugins/repository-hdfs/build.gradle

@@ -228,7 +228,7 @@ if (rootProject.ext.compilerJavaVersion.isJava11()) {
     ].join(',')
   }
 }
-if (rootProject.ext.compilerJavaVersion.isJava11()) {
+if (rootProject.ext.runtimeJavaVersion.isJava11() || rootProject.ext.compilerJavaVersion.isJava11()) {
   // TODO remove when: https://github.com/elastic/elasticsearch/issues/31498
   integTestHa.enabled = false
 }

rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json

@@ -15,16 +15,6 @@
         "index": {
           "type" : "string",
           "description" : "The name of the index to scope the operation"
-        },
-        "prefer_local": {
-          "type" : "boolean",
-          "description" : "With `true`, specify that a local shard should be used if available, with `false`, use a random shard (default: true)"
-        },
-        "format": {
-          "type": "enum",
-          "options" : ["detailed","text"],
-          "default": "detailed",
-          "description": "Format of the output"
         }
       }
     },

server/src/main/java/org/elasticsearch/action/AliasesRequest.java

@@ -33,9 +33,11 @@ public interface AliasesRequest extends IndicesRequest.Replaceable {
     String[] aliases();
 
     /**
-     * Sets the array of aliases that the action relates to
+     * Replaces current aliases with the provided aliases.
+     *
+     * Sometimes aliases expressions need to be resolved to concrete aliases prior to executing the transport action.
      */
-    AliasesRequest aliases(String... aliases);
+    void replaceAliases(String... aliases);
 
     /**
      * Returns true if wildcards expressions among aliases should be resolved, false otherwise

server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java

@@ -302,7 +302,6 @@ public AliasActions index(String index) {
         /**
          * Aliases to use with this action.
          */
-        @Override
         public AliasActions aliases(String... aliases) {
             if (type == AliasActions.Type.REMOVE_INDEX) {
                 throw new IllegalArgumentException("[aliases] is unsupported for [" + type + "]");
@@ -428,6 +427,11 @@ public String[] aliases() {
             return aliases;
         }
 
+        @Override
+        public void replaceAliases(String... aliases) {
+            this.aliases = aliases;
+        }
+
         @Override
         public boolean expandAliasesWildcards() {
             //remove operations support wildcards among aliases, add operations don't

server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.action.admin.indices.alias.get;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.AliasesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -32,15 +33,12 @@ public class GetAliasesRequest extends MasterNodeReadRequest<GetAliasesRequest>
 
     private String[] indices = Strings.EMPTY_ARRAY;
     private String[] aliases = Strings.EMPTY_ARRAY;
-
     private IndicesOptions indicesOptions = IndicesOptions.strictExpand();
+    private String[] originalAliases = Strings.EMPTY_ARRAY;
 
-    public GetAliasesRequest(String[] aliases) {
+    public GetAliasesRequest(String... aliases) {
         this.aliases = aliases;
-    }
-
-    public GetAliasesRequest(String alias) {
-        this.aliases = new String[]{alias};
+        this.originalAliases = aliases;
     }
 
     public GetAliasesRequest() {
@@ -51,6 +49,9 @@ public GetAliasesRequest(StreamInput in) throws IOException {
         indices = in.readStringArray();
         aliases = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
+        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
+            originalAliases = in.readStringArray();
+        }
     }
 
     @Override
@@ -59,6 +60,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(indices);
         out.writeStringArray(aliases);
         indicesOptions.writeIndicesOptions(out);
+        if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
+            out.writeStringArray(originalAliases);
+        }
     }
 
     @Override
@@ -67,9 +71,9 @@ public GetAliasesRequest indices(String... indices) {
         return this;
     }
 
-    @Override
     public GetAliasesRequest aliases(String... aliases) {
         this.aliases = aliases;
+        this.originalAliases = aliases;
         return this;
     }
 
@@ -88,6 +92,18 @@ public String[] aliases() {
         return aliases;
     }
 
+    @Override
+    public void replaceAliases(String... aliases) {
+        this.aliases = aliases;
+    }
+
+    /**
+     * Returns the aliases as was originally specified by the user
+     */
+    public String[] getOriginalAliases() {
+        return originalAliases;
+    }
+
     @Override
     public boolean expandAliasesWildcards() {
         return true;

server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Collections;
 import java.util.List;
 
 public class TransportGetAliasesAction extends TransportMasterNodeReadAction<GetAliasesRequest, GetAliasesResponse> {
@@ -62,7 +63,24 @@ protected GetAliasesResponse newResponse() {
     @Override
     protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener<GetAliasesResponse> listener) {
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
-        ImmutableOpenMap<String, List<AliasMetaData>> result = state.metaData().findAliases(request.aliases(), concreteIndices);
-        listener.onResponse(new GetAliasesResponse(result));
+        ImmutableOpenMap<String, List<AliasMetaData>> aliases = state.metaData().findAliases(request.aliases(), concreteIndices);
+        listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases)));
     }
+
+    /**
+     * Fills alias result with empty entries for requested indices when no specific aliases were requested.
+     */
+    static ImmutableOpenMap<String, List<AliasMetaData>> postProcess(GetAliasesRequest request, String[] concreteIndices,
+                                                                     ImmutableOpenMap<String, List<AliasMetaData>> aliases) {
+        boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0;
+        ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder(aliases);
+        for (String index : concreteIndices) {
+            if (aliases.get(index) == null && noAliasesSpecified) {
+                List<AliasMetaData> previous = mapBuilder.put(index, Collections.emptyList());
+                assert previous == null;
+            }
+        }
+        return mapBuilder.build();
+    }
+
 }