Merge remote-tracking branch 'remotes/origin/master' into lazy_repo_init

Author: Vladimir Dolzhenko

.ci/matrix-build-javas.yml

@@ -7,3 +7,4 @@
 
 ES_BUILD_JAVA:
   - java10
+  - java11

.ci/matrix-runtime-javas.yml

@@ -8,3 +8,4 @@
 ES_RUNTIME_JAVA:
   - java8
   - java10
+  - java11

build.gradle

@@ -125,7 +125,10 @@ Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectE
 allprojects {
   project.ext {
     // for ide hacks...
-    isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
+    isEclipse = System.getProperty("eclipse.launcher") != null ||   // Detects gradle launched from Eclipse's IDE
+            System.getProperty("eclipse.application") != null ||    // Detects gradle launched from the Eclipse compiler server
+            gradle.startParameter.taskNames.contains('eclipse') ||  // Detects gradle launched from the command line to do eclipse stuff
+            gradle.startParameter.taskNames.contains('cleanEclipse')
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
 
     // for BWC testing
@@ -171,7 +174,7 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when commiting bwc changes */
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
@@ -445,12 +448,19 @@ allprojects {
   }
 
   File licenseHeaderFile;
-  if (eclipse.project.name.startsWith(':x-pack')) {
+  String prefix = ':x-pack';
+
+  if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+    prefix = prefix.replace(':', '_')
+  }
+  if (eclipse.project.name.startsWith(prefix)) {
       licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/elastic-license-header.txt')
   } else {
       licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/oss-license-header.txt')
   }
-  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace('\n', '\\\\n')
+
+  String lineSeparator = Os.isFamily(Os.FAMILY_WINDOWS) ? '\\\\r\\\\n' : '\\\\n'
+  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace(System.lineSeparator(), lineSeparator)
   task copyEclipseSettings(type: Copy) {
     // TODO: "package this up" for external builds
     from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings')

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -691,6 +691,7 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
+            systemProperty 'es.scripting.exception_for_missing_value', 'true'
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -546,9 +546,15 @@ class VagrantTestPlugin implements Plugin<Project> {
                 javaPackagingTest.command = 'ssh'
                 javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"']
             } else {
+                // powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however
+                // remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user
+                // without administrator elevation), which we need to do for these tests. passing --elevated runs the session
+                // as a scheduled job locally on the vm as a true administrator to get around this limitation
+                //
+                // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
+                // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
                 javaPackagingTest.command = 'winrm'
-                // winrm commands run as administrator
-                javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
+                javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
             }
 
             TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)

buildSrc/src/main/resources/forbidden/jdk-signatures.txt

@@ -88,7 +88,8 @@ java.lang.Thread#getAllStackTraces()
 
 @defaultMessage Stopping threads explicitly leads to inconsistent states. Use interrupt() instead.
 java.lang.Thread#stop()
-java.lang.Thread#stop(java.lang.Throwable)
+# uncomment when https://github.com/elastic/elasticsearch/issues/31715 is fixed
+# java.lang.Thread#stop(java.lang.Throwable)
 
 @defaultMessage Please do not terminate the application
 java.lang.System#exit(int)

client/rest-high-level/build.gradle

@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.precommit.PrecommitTasks
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -18,29 +16,86 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.XmlProvider
+import org.gradle.api.publish.maven.MavenPublication
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://plugins.gradle.org/m2/'
+    }
+  }
+  dependencies {
+    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
+  }
+}
+
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
+apply plugin: 'com.github.johnrengelman.shadow'
 
 group = 'org.elasticsearch.client'
 archivesBaseName = 'elasticsearch-rest-high-level-client'
 
 publishing {
-    publications {
-        nebula {
-            artifactId = archivesBaseName
+  publications {
+    nebula(MavenPublication) {
+      artifact shadowJar
+      artifactId = archivesBaseName
+      /*
+       * Configure the pom to include the "shadow" as compile dependencies
+       * because that is how we're using them but remove all other dependencies
+       * because they've been shaded into the jar.
+       */
+      pom.withXml { XmlProvider xml ->
+        Node root = xml.asNode()
+        root.remove(root.dependencies)
+        Node dependenciesNode = root.appendNode('dependencies')
+        project.configurations.shadow.allDependencies.each {
+          if (false == it instanceof SelfResolvingDependency) {
+            Node dependencyNode = dependenciesNode.appendNode('dependency')
+            dependencyNode.appendNode('groupId', it.group)
+            dependencyNode.appendNode('artifactId', it.name)
+            dependencyNode.appendNode('version', it.version)
+            dependencyNode.appendNode('scope', 'compile')
+          }
         }
+      }
     }
+  }
+}
+
+/*
+ * We need somewhere to configure dependencies that we don't wish to shade
+ * into the high level REST client. The shadow plugin creates a "shadow"
+ * configuration which is *almost* exactly that. It is never bundled into
+ * the shaded jar but is used for main source compilation. Unfortunately,
+ * by default it is not used for *test* source compilation and isn't used
+ * in tests at all. This change makes it available for test compilation.
+ * A change below makes it available for testing.
+ */
+sourceSets {
+  test {
+    compileClasspath += configurations.shadow
+  }
 }
 
 dependencies {
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
-  compile "org.elasticsearch.plugin:parent-join-client:${version}"
-  compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
-  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
-  compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
+  /*
+   * Everything in the "shadow" configuration is *not* copied into the
+   * shadowJar.
+   */
+  shadow "org.elasticsearch:elasticsearch:${version}"
+  shadow "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  shadow "org.elasticsearch.plugin:parent-join-client:${version}"
+  shadow "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+  shadow "org.elasticsearch.plugin:rank-eval-client:${version}"
+  shadow "org.elasticsearch.plugin:lang-mustache-client:${version}"
+  compile project(':x-pack:protocol')
 
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
@@ -63,3 +118,48 @@ forbiddenApisMain {
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
 }
+
+shadowJar {
+  classifier = null
+  mergeServiceFiles()
+}
+
+// We don't need normal jar, we use shadow jar instead
+jar.enabled = false
+assemble.dependsOn shadowJar
+
+javadoc {
+  /*
+   * Bundle all of the javadoc from all of the shaded projects into this one
+   * so we don't *have* to publish javadoc for all of the "client" jars.
+   */
+  configurations.compile.dependencies.all { Dependency dep ->
+    Project p = dependencyToProject(dep)
+    if (p != null) {
+      evaluationDependsOn(p.path)
+      source += p.sourceSets.main.allJava
+    }
+  }
+}
+
+/*
+ * Use the jar for testing so we have tests of the bundled jar.
+ * Use the "shadow" configuration for testing because we need things
+ * in it.
+ */
+test {
+  classpath -= compileJava.outputs.files
+  classpath -= configurations.compile
+  classpath -= configurations.runtime
+  classpath += configurations.shadow
+  classpath += shadowJar.outputs.files
+  dependsOn shadowJar
+}
+integTestRunner {
+  classpath -= compileJava.outputs.files
+  classpath -= configurations.compile
+  classpath -= configurations.runtime
+  classpath += configurations.shadow
+  classpath += shadowJar.outputs.files
+  dependsOn shadowJar
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -23,6 +23,8 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -37,6 +39,7 @@
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
@@ -435,6 +438,34 @@ public void getSettingsAsync(GetSettingsRequest getSettingsRequest, RequestOptio
             GetSettingsResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Retrieve information about one or more indexes
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-index.html">
+     * Indices Get Index API on elastic.co</a>
+     * @param getIndexRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetIndexResponse get(GetIndexRequest getIndexRequest, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(getIndexRequest, RequestConverters::getIndex, options,
+            GetIndexResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Retrieve information about one or more indexes
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-index.html">
+     * Indices Get Index API on elastic.co</a>
+     * @param getIndexRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getAsync(GetIndexRequest getIndexRequest, RequestOptions options,
+                         ActionListener<GetIndexResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(getIndexRequest, RequestConverters::getIndex, options,
+            GetIndexResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Force merge one or more indices using the Force Merge API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html">
@@ -752,4 +783,32 @@ public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest,
         restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, RequestConverters::getTemplates,
             options, GetIndexTemplatesResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Calls the analyze API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-analyze.html">Analyze API on elastic.co</a>
+     *
+     * @param request   the request
+     * @param options   the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public AnalyzeResponse analyze(AnalyzeRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::analyze, options,
+            AnalyzeResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously calls the analyze API
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-analyze.html">Analyze API on elastic.co</a>
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void analyzeAsync(AnalyzeRequest request, RequestOptions options,
+                             ActionListener<AnalyzeResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::analyze, options,
+            AnalyzeResponse::fromXContent, listener, emptySet());
+    }
 }