Merge branch 'master' into unknown-or-invalid-settings-updates

* master: (28 commits)
  Maybe die before failing engine (elastic#28973)
  Remove special handling for _all in nodes info
  Remove Booleans use from XContent and ToXContent (elastic#28768)
  Update Gradle Testing Docs (elastic#28970)
  Make primary-replica resync failures less lenient (elastic#28534)
  Remove temporary file 10_basic.yml~
  Use different pipeline id in test. (pipelines do not get removed between tests extending from ESIntegTestCase)
  Use fixture to test the repository-gcs plugin (elastic#28788)
  Use String.join() to describe a list of tasks (elastic#28941)
  Fixed incorrect test try-catch statement
  Plugins: Consolidate plugin and module loading code (elastic#28815)
  percolator: Take `matchAllDocs` and `verified` of the sub result into account when analyzing a function_score query.
  Build: Remove rest tests on archive distribution projects (elastic#28952)
  Remove FastStringReader in favor of vanilla StringReader (elastic#28944)
  Remove FastCharArrayReader and FastCharArrayWriter (elastic#28951)
  Continue registering pipelines after one pipeline parse failure. (elastic#28752)
  Build: Fix ability to ignore when no tests are run (elastic#28930)
  [rest-api-spec] update doc link for /_rank_eval
  Switch XContentBuilder from BytesStreamOutput to ByteArrayOutputStream (elastic#28945)
  Factor UnknownNamedObjectException into its own class (elastic#28931)
  ...

Author: jasontedor

TESTING.asciidoc

@@ -270,14 +270,14 @@ The REST tests are run automatically when executing the "./gradlew check" comman
 REST tests use the following command:
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:integ-test-zip:integTest   \
+./gradlew :distribution:archives:integ-test-zip:integTest   \
   -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT"
 ---------------------------------------------------------------------------
 
 A specific test case can be run with
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:integ-test-zip:integTest \
+./gradlew :distribution:archives:integ-test-zip:integTest \
   -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT" \
   -Dtests.method="test {p0=cat.shards/10_basic/Help}"
 ---------------------------------------------------------------------------
@@ -407,7 +407,7 @@ destructive. When working with a single package it's generally faster to run its
 tests in a tighter loop than Gradle provides. In one window:
 
 --------------------------------
-./gradlew :distribution:rpm:assemble
+./gradlew :distribution:packages:rpm:assemble
 --------------------------------
 
 and in another window:

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy

@@ -100,12 +100,15 @@ class TestProgressLogger implements AggregatedEventListener {
 
     @Subscribe
     void onQuit(AggregatedQuitEvent e) throws IOException {
-        suiteLogger.completed()
-        testLogger.completed()
-        for (ProgressLogger slaveLogger : slaveLoggers) {
-            slaveLogger.completed()
+        // if onStart was never called (eg no matching tests), suiteLogger and all the other loggers will be null
+        if (suiteLogger != null) {
+            suiteLogger.completed()
+            testLogger.completed()
+            for (ProgressLogger slaveLogger : slaveLoggers) {
+                slaveLogger.completed()
+            }
+            parentProgressLogger.completed()
         }
-        parentProgressLogger.completed()
     }
 
     @Subscribe

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -558,7 +558,7 @@ class BuildPlugin implements Plugin<Project> {
         return {
             jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', 'auto')
-            ifNoTests 'fail'
+            ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
@@ -582,8 +582,6 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
-            // default test sysprop values
-            systemProperty 'tests.ifNoTests', 'fail'
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {

buildSrc/src/main/resources/forbidden/es-server-signatures.txt

@@ -29,8 +29,6 @@ java.util.concurrent.Executors#privilegedThreadFactory()
 java.lang.Character#codePointBefore(char[],int) @ Implicit start offset is error-prone when the char[] is a buffer and the first chars are random chars
 java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars
 
-java.io.StringReader#<init>(java.lang.String) @ Use FastStringReader instead
-
 @defaultMessage Reference management is tricky, leave it to SearcherManager
 org.apache.lucene.index.IndexReader#decRef()
 org.apache.lucene.index.IndexReader#incRef()

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java

@@ -129,6 +129,7 @@ public void testSearch() throws Exception {
             SearchRequest searchRequest = new SearchRequest(); // <1>
             SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // <2>
             searchSourceBuilder.query(QueryBuilders.matchAllQuery()); // <3>
+            searchRequest.source(searchSourceBuilder); // <4>
             // end::search-request-basic
         }
         {

distribution/archives/build.gradle

@@ -21,6 +21,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.apache.tools.ant.filters.FixCrLfFilter
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.EmptyDirTask
+import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 
@@ -125,19 +126,39 @@ subprojects {
   artifacts {
     'default' buildDist
   }
+
+  // sanity checks if a archives can be extracted
+  File extractionDir = new File(buildDir, 'extracted')
+  task testExtraction(type: LoggedExec) {
+    dependsOn buildDist
+    doFirst {
+      project.delete(extractionDir)
+      extractionDir.mkdirs()
+    }
+  }
+  if (project.name.contains('zip')) {
+    testExtraction {
+      onlyIf { new File('/bin/unzip').exists() || new File('/usr/bin/unzip').exists() || new File('/usr/local/bin/unzip').exists() }
+      commandLine 'unzip', "${-> buildDist.outputs.files.singleFile}", '-d', extractionDir
+    }
+  } else { // tar
+    testExtraction {
+      onlyIf { new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }
+      commandLine 'tar', '-xvzf', "${-> buildDist.outputs.files.singleFile}", '-C', extractionDir
+    }
+  }
+  check.dependsOn testExtraction
 }
 
 /*****************************************************************************
  *                            Rest test config                               *
  *****************************************************************************/
-subprojects {
+configure(subprojects.findAll { it.name == 'integ-test-zip' }) {
   apply plugin: 'elasticsearch.standalone-rest-test'
   apply plugin: 'elasticsearch.rest-test'
 
-  if (project.name == 'integ-test-zip') {
-    integTest {
-      includePackaged true
-    }
+  integTest {
+    includePackaged true
   }
 
   integTestCluster {

distribution/archives/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java

@@ -18,6 +18,7 @@ include-tagged::{doc-tests}/SearchDocumentationIT.java[search-request-basic]
 <1> Creates the `SeachRequest`. Without arguments this runs against all indices.
 <2> Most search parameters are added to the `SearchSourceBuilder`. It offers setters for everything that goes into the search request body.
 <3> Add a `match_all` query to the `SearchSourceBuilder`.
+<4> Add the `SearchSourceBuilder` to the `SeachRequest`.
 
 ===== Optional arguments
 

distribution/archives/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml

@@ -20,8 +20,8 @@ A `bucket_sort` aggregation looks like this in isolation:
 {
     "bucket_sort": {
         "sort": [
-            {"sort_field_1": {"order": "asc"},<1>
-            {"sort_field_2": {"order": "desc"},
+            {"sort_field_1": {"order": "asc"}},<1>
+            {"sort_field_2": {"order": "desc"}},
             "sort_field_3"
         ],
         "from": 1,

distribution/archives/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java

@@ -18,10 +18,14 @@
  */
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptEngine;
@@ -33,6 +37,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.function.Consumer;
 import java.util.function.Function;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -64,6 +69,66 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         }
     }
 
+    public void testScriptDisabled() throws Exception {
+        String pipelineIdWithoutScript = randomAlphaOfLengthBetween(5, 10);
+        String pipelineIdWithScript = pipelineIdWithoutScript + "_script";
+        internalCluster().startNode();
+
+        BytesReference pipelineWithScript = new BytesArray("{\n" +
+            "  \"processors\" : [\n" +
+            "      {\"script\" : {\"lang\": \"" + MockScriptEngine.NAME + "\", \"source\": \"my_script\"}}\n" +
+            "  ]\n" +
+            "}");
+        BytesReference pipelineWithoutScript = new BytesArray("{\n" +
+            "  \"processors\" : [\n" +
+            "      {\"set\" : {\"field\": \"y\", \"value\": 0}}\n" +
+            "  ]\n" +
+            "}");
+
+        Consumer<String> checkPipelineExists = (id) -> assertThat(client().admin().cluster().prepareGetPipeline(id)
+                .get().pipelines().get(0).getId(), equalTo(id));
+
+        client().admin().cluster().preparePutPipeline(pipelineIdWithScript, pipelineWithScript, XContentType.JSON).get();
+        client().admin().cluster().preparePutPipeline(pipelineIdWithoutScript, pipelineWithoutScript, XContentType.JSON).get();
+
+        checkPipelineExists.accept(pipelineIdWithScript);
+        checkPipelineExists.accept(pipelineIdWithoutScript);
+
+
+        internalCluster().stopCurrentMasterNode();
+        internalCluster().startNode(Settings.builder().put("script.allowed_types", "none"));
+
+        checkPipelineExists.accept(pipelineIdWithoutScript);
+        checkPipelineExists.accept(pipelineIdWithScript);
+
+        client().prepareIndex("index", "doc", "1")
+            .setSource("x", 0)
+            .setPipeline(pipelineIdWithoutScript)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+            () -> client().prepareIndex("index", "doc", "2")
+                .setSource("x", 0)
+                .setPipeline(pipelineIdWithScript)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .get());
+        assertThat(exception.getHeaderKeys(), equalTo(Sets.newHashSet("processor_type")));
+        assertThat(exception.getHeader("processor_type"), equalTo(Arrays.asList("unknown")));
+        assertThat(exception.getRootCause().getMessage(),
+            equalTo("pipeline with id [" + pipelineIdWithScript + "] could not be loaded, caused by " +
+                "[ElasticsearchParseException[Error updating pipeline with id [" + pipelineIdWithScript + "]]; " +
+                "nested: ElasticsearchException[java.lang.IllegalArgumentException: cannot execute [inline] scripts]; " +
+                "nested: IllegalArgumentException[cannot execute [inline] scripts];; " +
+                "ElasticsearchException[java.lang.IllegalArgumentException: cannot execute [inline] scripts]; " +
+                "nested: IllegalArgumentException[cannot execute [inline] scripts];; java.lang.IllegalArgumentException: " +
+                "cannot execute [inline] scripts]"));
+
+        Map<String, Object> source = client().prepareGet("index", "doc", "1").get().getSource();
+        assertThat(source.get("x"), equalTo(0));
+        assertThat(source.get("y"), equalTo(0));
+    }
+
     public void testPipelineWithScriptProcessorThatHasStoredScript() throws Exception {
         internalCluster().startNode();
 

distribution/archives/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml

@@ -21,11 +21,11 @@
 import com.github.mustachejava.Mustache;
 import com.github.mustachejava.MustacheFactory;
 
+import java.io.StringReader;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.common.io.FastStringReader;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.Script;
@@ -65,7 +65,7 @@ public <T> T compile(String templateName, String templateSource, ScriptContext<T
             throw new IllegalArgumentException("mustache engine does not know how to handle context [" + context.name + "]");
         }
         final MustacheFactory factory = createMustacheFactory(options);
-        Reader reader = new FastStringReader(templateSource);
+        Reader reader = new StringReader(templateSource);
         Mustache template = factory.compile(reader, "query-template");
         TemplateScript.Factory compiled = params -> new MustacheExecutableScript(template, params);
         return context.factoryClazz.cast(compiled);