Merge branch 'master' into repository-encrypted-client-side

Author: albertzaharovits

build.gradle

@@ -45,6 +45,7 @@ apply from: 'gradle/build-scan.gradle'
 apply from: 'gradle/build-complete.gradle'
 apply from: 'gradle/runtime-jdk-provision.gradle'
 apply from: 'gradle/ide.gradle'
+apply from: 'gradle/forbidden-dependencies.gradle'
 apply from: 'gradle/formatting.gradle'
 
 // common maven publishing configuration

buildSrc/formatterConfig.xml

@@ -47,7 +47,7 @@
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position" value="true"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
-        <setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="80"/>
+        <setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator" value="true"/>
@@ -132,7 +132,7 @@
         <setting id="org.eclipse.jdt.core.formatter.alignment_for_type_parameters" value="48"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
         <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="48"/>
-        <setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="64"/>
+        <setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
         <setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_loops" value="16"/>
         <setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
         <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -284,6 +284,7 @@ class BuildPlugin implements Plugin<Project> {
         project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
         project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
         project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
+        project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
     }
 
     /** Adds repositories used by ES dependencies */

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -131,7 +131,8 @@ class PluginBuildPlugin implements Plugin<Project> {
         }
         createIntegTestTask(project)
         createBundleTasks(project, extension)
-        project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
+        project.configurations.getByName('default')
+            .extendsFrom(project.configurations.getByName('runtimeClasspath'))
         // allow running ES with this plugin in the foreground of a build
         project.tasks.register('run', RunTask) {
             dependsOn(project.tasks.bundlePlugin)
@@ -210,7 +211,7 @@ class PluginBuildPlugin implements Plugin<Project> {
              * that shadow jar.
              */
             from { project.plugins.hasPlugin(ShadowPlugin) ? project.shadowJar : project.jar }
-            from project.configurations.runtime - project.configurations.compileOnly
+            from project.configurations.runtimeClasspath - project.configurations.compileOnly
             // extra files for the plugin to go into the zip
             from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging
             from('src/main') {

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -28,6 +28,7 @@ import org.elasticsearch.gradle.util.Util
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
+import org.gradle.api.file.FileCollection
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.quality.Checkstyle
 import org.gradle.api.tasks.TaskProvider
@@ -142,6 +143,19 @@ class PrecommitTasks {
         ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
         project.tasks.withType(CheckForbiddenApis).configureEach {
             dependsOn(buildResources)
+
+            //  use the runtime classpath if we have it, but some qa projects don't have one...
+            if (name.endsWith('Test')) {
+                FileCollection runtime = project.sourceSets.test.runtimeClasspath
+                if (runtime != null) {
+                    classpath = runtime.plus(project.sourceSets.test.compileClasspath)
+                }
+            } else {
+                FileCollection runtime = project.sourceSets.main.runtimeClasspath
+                if (runtime != null) {
+                    classpath = runtime.plus(project.sourceSets.main.compileClasspath)
+                }
+            }
             targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
             if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
                 project.logger.warn(

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java

@@ -210,8 +210,7 @@ private void failIfAnyMissing(String item, Boolean exists, String type) {
     }
 
     private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolean> notices, Set<File> shaFiles)
-        throws NoSuchAlgorithmException,
-        IOException {
+        throws NoSuchAlgorithmException, IOException {
         for (File dependency : dependencies) {
             String jarName = dependency.getName();
             String depName = regex.matcher(jarName).replaceFirst("");

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -388,7 +388,7 @@ private Set<String> runJdkJarHellCheck() throws IOException {
     }
 
     private Configuration getRuntimeConfiguration() {
-        Configuration runtime = getProject().getConfigurations().findByName("runtime");
+        Configuration runtime = getProject().getConfigurations().findByName("runtimeClasspath");
         if (runtime == null) {
             return getProject().getConfigurations().getByName("testCompile");
         }

distribution/bwc/build.gradle

@@ -194,6 +194,10 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased
         if (gradle.startParameter.isOffline()) {
           args "--offline"
         }
+        String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
+        if (buildCacheUrl) {
+          args "-Dorg.elasticsearch.build.cache.url=${buildCacheUrl}"
+        }
 
         args "-Dbuild.snapshot=true"
         args "-Dscan.tag.NESTED"

distribution/tools/keystore-cli/build.gradle

@@ -24,5 +24,5 @@ dependencies {
   compileOnly project(":libs:elasticsearch-cli")
   testCompile project(":test:framework")
   testCompile 'com.google.jimfs:jimfs:1.1'
-  testCompile 'com.google.guava:guava:18.0'
+  testRuntimeOnly 'com.google.guava:guava:18.0'
 }

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -124,9 +124,7 @@ public static void main(final String[] args) throws InterruptedException, IOExce
     }
 
     private List<String> jvmOptions(final Path config, final String esJavaOpts, final Map<String, String> substitutions)
-        throws InterruptedException,
-        IOException,
-        JvmOptionsFileParserException {
+        throws InterruptedException, IOException, JvmOptionsFileParserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
 

distribution/tools/plugin-cli/build.gradle

@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.info.BuildParams
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -30,7 +28,7 @@ dependencies {
   compile "org.bouncycastle:bc-fips:1.0.1"
   testCompile project(":test:framework")
   testCompile 'com.google.jimfs:jimfs:1.1'
-  testCompile 'com.google.guava:guava:18.0'
+  testRuntimeOnly 'com.google.guava:guava:18.0'
 }
 
 dependencyLicenses {

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -330,8 +330,7 @@ private String getElasticUrl(
         final boolean isSnapshot,
         final String pluginId,
         final String platform
-    ) throws IOException,
-        UserException {
+    ) throws IOException, UserException {
         final String baseUrl;
         if (isSnapshot && stagingHash == null) {
             throw new UserException(
@@ -506,9 +505,7 @@ private Path downloadAndValidate(
         final Path tmpDir,
         final boolean officialPlugin,
         boolean isBatch
-    ) throws IOException,
-        PGPException,
-        UserException {
+    ) throws IOException, PGPException, UserException {
         Path zip = downloadZip(terminal, urlString, tmpDir, isBatch);
         pathsToDeleteOnShutdown.add(zip);
         String checksumUrlString = urlString + ".sha512";

docs/build.gradle

@@ -539,6 +539,41 @@ for (int i = 0; i < 100; i++) {
             {"load_time": "$value"}"""
 }
 
+// Used by t_test aggregations
+buildRestTests.setups['node_upgrade'] = '''
+  - do:
+        indices.create:
+          index: node_upgrade
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+            mappings:
+              properties:
+                name:
+                  type: keyword
+                startup_time_before:
+                  type: long
+                startup_time_after:
+                  type: long
+  - do:
+        bulk:
+          index: node_upgrade
+          refresh: true
+          body: |
+            {"index":{}}
+            {"name": "A", "startup_time_before": 102, "startup_time_after": 89}
+            {"index":{}}
+            {"name": "B", "startup_time_before": 99, "startup_time_after": 93}
+            {"index":{}}
+            {"name": "C", "startup_time_before": 111, "startup_time_after": 72}
+            {"index":{}}
+            {"name": "D", "startup_time_before": 97, "startup_time_after": 98}
+            {"index":{}}
+            {"name": "E", "startup_time_before": 101, "startup_time_after": 102}
+            {"index":{}}
+            {"name": "F", "startup_time_before": 99, "startup_time_after": 98}'''
+
 // Used by iprange agg
 buildRestTests.setups['iprange'] = '''
   - do:

docs/reference/aggregations/metrics.asciidoc

@@ -49,7 +49,7 @@ include::metrics/median-absolute-deviation-aggregation.asciidoc[]
 
 include::metrics/boxplot-aggregation.asciidoc[]
 
-
+include::metrics/t-test-aggregation.asciidoc[]
 
 
 

docs/reference/aggregations/metrics/t-test-aggregation.asciidoc

@@ -0,0 +1,114 @@
+[role="xpack"]
+[testenv="basic"]
+[[search-aggregations-metrics-ttest-aggregation]]
+=== TTest Aggregation
+
+A `t_test` metrics aggregation that performs a statistical hypothesis test in which the test statistic follows a Student's t-distribution
+under the null hypothesis on numeric values extracted from the aggregated documents or generated by provided scripts. In practice, this
+will tell you if the difference between two population means are statistically significant and did not occur by chance alone.
+
+==== Syntax
+
+A `t_test` aggregation looks like this in isolation:
+
+[source,js]
+--------------------------------------------------
+{
+    "t_test": {
+        "a": "value_before",
+        "b": "value_after",
+        "type": "paired"
+    }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Assuming that we have a record of node start up times before and after upgrade, let's look at a t-test to see if upgrade affected
+the node start up time in a meaningful way.
+
+[source,console]
+--------------------------------------------------
+GET node_upgrade/_search
+{
+    "size": 0,
+    "aggs" : {
+        "startup_time_ttest" : {
+            "t_test" : {
+                "a" : {"field": "startup_time_before"}, <1>
+                "b" : {"field": "startup_time_after"}, <2>
+                "type": "paired" <3>
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TEST[setup:node_upgrade]
+<1> The field `startup_time_before` must be a numeric field
+<2> The field `startup_time_after` must be a numeric field
+<3> Since we have data from the same nodes, we are using paired t-test.
+
+The response will return the p-value or probability value for the test. It is the probability of obtaining results at least as extreme as
+the result processed by the aggregation, assuming that the null hypothesis is correct (which means there is no difference between
+population means). Smaller p-value means the null hypothesis is more likely to be incorrect and population means are indeed different.
+
+[source,console-result]
+--------------------------------------------------
+{
+    ...
+
+   "aggregations": {
+      "startup_time_ttest": {
+         "value":  0.1914368843365979 <1>
+      }
+   }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+<1> The p-value.
+
+==== T-Test Types
+
+The `t_test` aggregation supports unpaired and paired two-sample t-tests. The type of the test can be specified using the `type` parameter:
+
+`"type": "paired"`:: performs paired t-test
+`"type": "homoscedastic"`:: performs two-sample equal variance test
+`"type": "heteroscedastic"`:: performs two-sample unequal variance test (this is default)
+
+==== Script
+
+The `t_test` metric supports scripting.  For example, if we need to adjust out load times for the before values, we could use
+a script to recalculate them on-the-fly:
+
+[source,console]
+--------------------------------------------------
+GET node_upgrade/_search
+{
+    "size": 0,
+    "aggs" : {
+        "startup_time_ttest" : {
+            "t_test" : {
+                "a": {
+                    "script" : {
+                        "lang": "painless",
+                        "source": "doc['startup_time_before'].value - params.adjustment", <1>
+                        "params" : {
+                            "adjustment" : 10   <2>
+                        }
+                    }
+                },
+                "b": {
+                    "field": "startup_time_after" <3>
+                },
+                "type": "paired"
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TEST[setup:node_upgrade]
+
+<1> The `field` parameter is replaced with a `script` parameter, which uses the
+script to generate values which percentiles are calculated on
+<2> Scripting supports parameterized input just like any other script
+<3> We can mix scripts and fields
+

docs/reference/analysis/normalizers.asciidoc

@@ -13,11 +13,13 @@ following: `arabic_normalization`, `asciifolding`, `bengali_normalization`,
 `persian_normalization`, `scandinavian_folding`, `serbian_normalization`,
 `sorani_normalization`, `uppercase`.
 
+Elasticsearch ships with a `lowercase` built-in normalizer. For other forms of
+normalization a custom configuration is required.
+
 [float]
 === Custom normalizers
 
-Elasticsearch does not ship with built-in normalizers so far, so the only way
-to get one is by building a custom one. Custom normalizers take a list of char
+Custom normalizers take a list of
 <<analysis-charfilters, character filters>> and a list of
 <<analysis-tokenfilters,token filters>>.
 

docs/reference/autoscaling/apis/autoscaling-apis.asciidoc

@@ -11,9 +11,11 @@ You can use the following APIs to perform autoscaling operations.
 
 * <<autoscaling-get-autoscaling-decision,Get autoscaling decision>>
 * <<autoscaling-delete-autoscaling-policy,Delete autoscaling policy>>
+* <<autoscaling-get-autoscaling-policy,Get autoscaling policy>>
 * <<autoscaling-put-autoscaling-policy,Put autoscaling policy>>
 
 // top-level
 include::get-autoscaling-decision.asciidoc[]
 include::delete-autoscaling-policy.asciidoc[]
+include::get-autoscaling-policy.asciidoc[]
 include::put-autoscaling-policy.asciidoc[]