Merge branch 'master' into peer-recovery-retention-leases

Author: DaveCTurner

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -324,9 +324,9 @@ class VagrantTestPlugin implements Plugin<Project> {
                      fi
                      
                      if [ -z "\$SYSTEM_JAVA_HOME" ]; then
-                       export SYSTEM_JAVA_HOME="${-> convertPath(project, linuxSystemJdk.toString()) }"
+                       export SYSTEM_JAVA_HOME="${-> convertLinuxPath(project, linuxSystemJdk.toString()) }"
                      fi
-                     "${-> convertPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
+                     "${-> convertLinuxPath(project, linuxGradleJdk.toString()) }"/bin/java -cp "\$PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner "\${test_args[@]}"
                      """
         }
         Task createWindowsRunnerScript = project.tasks.create('createWindowsRunnerScript', FileContentsTask) {
@@ -335,14 +335,20 @@ class VagrantTestPlugin implements Plugin<Project> {
             // the use of $args rather than param() here is deliberate because the syntax for array (multivalued) parameters is likely
             // a little trappy for those unfamiliar with powershell
             contents """\
-                     if (\$args.Count -eq 0) {
-                       \$testArgs = @("${-> project.extensions.esvagrant.testClass}")
-                     } else {
-                       \$testArgs = \$args
+                     try {
+                         if (\$args.Count -eq 0) {
+                           \$testArgs = @("${-> project.extensions.esvagrant.testClass}")
+                         } else {
+                           \$testArgs = \$args
+                         }
+                         \$Env:SYSTEM_JAVA_HOME = "${-> convertWindowsPath(project, windowsSystemJdk.toString()) }"
+                         & "${-> convertWindowsPath(project, windowsGradleJdk.toString()) }/bin/java" -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
+                         exit \$LASTEXITCODE
+                     } catch {
+                         # catch if we have a failure to even run the script at all above, equivalent to set -e, sort of
+                         echo "\$_.Exception.Message"
+                         exit 1
                      }
-                     \$Env:SYSTEM_JAVA_HOME = "${-> convertPath(project, windowsSystemJdk.toString()) }"
-                     & "${-> convertPath(project, windowsGradleJdk.toString()) }"/bin/java -cp "\$Env:PACKAGING_TESTS/*" org.elasticsearch.packaging.VMTestRunner @testArgs
-                     exit \$LASTEXITCODE
                      """
         }
 
@@ -578,7 +584,7 @@ class VagrantTestPlugin implements Plugin<Project> {
 
             if (LINUX_BOXES.contains(box)) {
                 Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) {
-                    remoteCommand "export SYSTEM_JAVA_HOME=\"${-> convertPath(project, linuxSystemJdk.toString())}\"; " + BATS_TEST_COMMAND
+                    remoteCommand "export SYSTEM_JAVA_HOME=\"${-> convertLinuxPath(project, linuxSystemJdk.toString())}\"; " + BATS_TEST_COMMAND
                     boxName box
                     environmentVars vagrantEnvVars
                     dependsOn up, setupPackagingTest, linuxSystemJdk
@@ -625,7 +631,7 @@ class VagrantTestPlugin implements Plugin<Project> {
                 // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225
                 // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html
                 javaPackagingTest.command = 'winrm'
-                javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"']
+                javaPackagingTest.args = ['--elevated', '--command', '& "$Env:PACKAGING_TESTS/run-tests.ps1"; exit $LASTEXITCODE']
             }
 
             TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
@@ -658,7 +664,10 @@ class VagrantTestPlugin implements Plugin<Project> {
     }
 
     // convert the given path from an elasticsearch repo path to a VM path
-    private String convertPath(Project project, String path) {
+    private String convertLinuxPath(Project project, String path) {
         return "/elasticsearch/" + project.rootDir.toPath().relativize(Paths.get(path));
     }
+    private String convertWindowsPath(Project project, String path) {
+        return "C:\\elasticsearch\\" + project.rootDir.toPath().relativize(Paths.get(path)).toString().replace('/', '\\');
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameRequestConverters.java

@@ -37,7 +37,9 @@
 
 import static org.elasticsearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE;
 import static org.elasticsearch.client.RequestConverters.createEntity;
+import static org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest.FORCE;
 import static org.elasticsearch.client.dataframe.GetDataFrameTransformRequest.ALLOW_NO_MATCH;
+import static org.elasticsearch.client.dataframe.PutDataFrameTransformRequest.DEFER_VALIDATION;
 
 final class DataFrameRequestConverters {
 
@@ -50,6 +52,9 @@ static Request putDataFrameTransform(PutDataFrameTransformRequest putRequest) th
                 .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         request.setEntity(createEntity(putRequest, REQUEST_BODY_CONTENT_TYPE));
+        if (putRequest.getDeferValidation() != null) {
+            request.addParameter(DEFER_VALIDATION, Boolean.toString(putRequest.getDeferValidation()));
+        }
         return request;
     }
 
@@ -71,12 +76,16 @@ static Request getDataFrameTransform(GetDataFrameTransformRequest getRequest) {
         return request;
     }
 
-    static Request deleteDataFrameTransform(DeleteDataFrameTransformRequest request) {
+    static Request deleteDataFrameTransform(DeleteDataFrameTransformRequest deleteRequest) {
         String endpoint = new RequestConverters.EndpointBuilder()
                 .addPathPartAsIs("_data_frame", "transforms")
-                .addPathPart(request.getId())
+                .addPathPart(deleteRequest.getId())
                 .build();
-        return new Request(HttpDelete.METHOD_NAME, endpoint);
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+        if (deleteRequest.getForce() != null) {
+            request.addParameter(FORCE, Boolean.toString(deleteRequest.getForce()));
+        }
+        return request;
     }
 
     static Request startDataFrameTransform(StartDataFrameTransformRequest startRequest) {

client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DeleteDataFrameTransformRequest.java

@@ -31,7 +31,10 @@
  */
 public class DeleteDataFrameTransformRequest implements Validatable {
 
+    public static final String FORCE = "force";
+
     private final String id;
+    private Boolean force;
 
     public DeleteDataFrameTransformRequest(String id) {
         this.id = id;
@@ -41,6 +44,14 @@ public String getId() {
         return id;
     }
 
+    public Boolean getForce() {
+        return force;
+    }
+
+    public void setForce(boolean force) {
+        this.force = force;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         if (id == null) {
@@ -54,7 +65,7 @@ public Optional<ValidationException> validate() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id);
+        return Objects.hash(id, force);
     }
 
     @Override
@@ -67,6 +78,6 @@ public boolean equals(Object obj) {
             return false;
         }
         DeleteDataFrameTransformRequest other = (DeleteDataFrameTransformRequest) obj;
-        return Objects.equals(id, other.id);
+        return Objects.equals(id, other.id) && Objects.equals(force, other.force);
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequest.java

@@ -31,7 +31,9 @@
 
 public class PutDataFrameTransformRequest implements ToXContentObject, Validatable {
 
+    public static final String DEFER_VALIDATION = "defer_validation";
     private final DataFrameTransformConfig config;
+    private Boolean deferValidation;
 
     public PutDataFrameTransformRequest(DataFrameTransformConfig config) {
         this.config = config;
@@ -41,6 +43,19 @@ public DataFrameTransformConfig getConfig() {
         return config;
     }
 
+    public Boolean getDeferValidation() {
+        return deferValidation;
+    }
+
+    /**
+     * Indicates if deferrable validations should be skipped until the transform starts
+     *
+     * @param deferValidation {@code true} will cause validations to be deferred
+     */
+    public void setDeferValidation(boolean deferValidation) {
+        this.deferValidation = deferValidation;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         ValidationException validationException = new ValidationException();

client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java

@@ -50,6 +50,8 @@
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
 
 public class DataFrameRequestConvertersTests extends ESTestCase {
 
@@ -66,14 +68,17 @@ public void testPutDataFrameTransform() throws IOException {
         PutDataFrameTransformRequest putRequest = new PutDataFrameTransformRequest(
                 DataFrameTransformConfigTests.randomDataFrameTransformConfig());
         Request request = DataFrameRequestConverters.putDataFrameTransform(putRequest);
-
+        assertThat(request.getParameters(), not(hasKey("defer_validation")));
         assertEquals(HttpPut.METHOD_NAME, request.getMethod());
         assertThat(request.getEndpoint(), equalTo("/_data_frame/transforms/" + putRequest.getConfig().getId()));
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
             DataFrameTransformConfig parsedConfig = DataFrameTransformConfig.PARSER.apply(parser, null);
             assertThat(parsedConfig, equalTo(putRequest.getConfig()));
         }
+        putRequest.setDeferValidation(true);
+        request = DataFrameRequestConverters.putDataFrameTransform(putRequest);
+        assertThat(request.getParameters(), hasEntry("defer_validation", Boolean.toString(putRequest.getDeferValidation())));
     }
 
     public void testDeleteDataFrameTransform() {
@@ -82,6 +87,13 @@ public void testDeleteDataFrameTransform() {
 
         assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
         assertThat(request.getEndpoint(), equalTo("/_data_frame/transforms/foo"));
+
+        assertThat(request.getParameters(), not(hasKey("force")));
+
+        deleteRequest.setForce(true);
+        request = DataFrameRequestConverters.deleteDataFrameTransform(deleteRequest);
+
+        assertThat(request.getParameters(), hasEntry("force", "true"));
     }
 
     public void testStartDataFrameTransform() {

client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

@@ -180,6 +180,22 @@ public void testCreateDelete() throws IOException {
         assertThat(deleteError.getMessage(), containsString("Transform with id [test-crud] could not be found"));
     }
 
+    public void testCreateDeleteWithDefer() throws IOException {
+        String sourceIndex = "missing-source-index";
+
+        String id = "test-with-defer";
+        DataFrameTransformConfig transform = validDataFrameTransformConfig(id, sourceIndex, "pivot-dest");
+        DataFrameClient client = highLevelClient().dataFrame();
+        PutDataFrameTransformRequest request = new PutDataFrameTransformRequest(transform);
+        request.setDeferValidation(true);
+        AcknowledgedResponse ack = execute(request, client::putDataFrameTransform, client::putDataFrameTransformAsync);
+        assertTrue(ack.isAcknowledged());
+
+        ack = execute(new DeleteDataFrameTransformRequest(transform.getId()), client::deleteDataFrameTransform,
+            client::deleteDataFrameTransformAsync);
+        assertTrue(ack.isAcknowledged());
+    }
+
     public void testGetTransform() throws IOException {
         String sourceIndex = "transform-source";
         createIndex(sourceIndex);

client/rest-high-level/src/test/java/org/elasticsearch/client/SyncedFlushResponseTests.java

@@ -191,7 +191,7 @@ TestPlan createTestPlan() throws IOException {
                             );
                         } else {
                             successful++;
-                            shardResponses.put(shardRouting, new SyncedFlushService.ShardSyncedFlushResponse());
+                            shardResponses.put(shardRouting, new SyncedFlushService.ShardSyncedFlushResponse((String) null));
                         }
                     }
                     shardsResults.add(new ShardsSyncedFlushResult(shardId, "_sync_id_" + shard, replicas + 1, shardResponses));

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java

@@ -166,6 +166,7 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
             // tag::put-data-frame-transform-request
             PutDataFrameTransformRequest request =
                     new PutDataFrameTransformRequest(transformConfig); // <1>
+            request.setDeferValidation(false); // <2>
             // end::put-data-frame-transform-request
 
             // tag::put-data-frame-transform-execute
@@ -374,6 +375,7 @@ public void testDeleteDataFrameTransform() throws IOException, InterruptedExcept
             // tag::delete-data-frame-transform-request
             DeleteDataFrameTransformRequest request =
                     new DeleteDataFrameTransformRequest("mega-transform"); // <1>
+            request.setForce(false); // <2>
             // end::delete-data-frame-transform-request
 
             // tag::delete-data-frame-transform-execute

client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java

@@ -134,7 +134,7 @@ private static SSLContext getSslContext() throws Exception {
      * 12.0.1 so we pin to TLSv1.2 when running on an earlier JDK
      */
     private static String getProtocol() {
-        String version = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("java.specification.version"));
+        String version = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("java.version"));
         String[] components = version.split("\\.");
         if (components.length > 0) {
             final int major = Integer.valueOf(components[0]);

distribution/src/bin/elasticsearch-service.bat

@@ -106,6 +106,10 @@ if exist "%JAVA_HOME%\bin\server\jvm.dll" (
 )
 
 :foundJVM
+if not defined ES_TMPDIR (
+  for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set ES_TMPDIR=%%a
+)
+
 set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 
 if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -500,17 +500,26 @@ private Path downloadAndValidate(
             }
         }
 
-        try {
-            final byte[] zipBytes = Files.readAllBytes(zip);
-            final String actualChecksum = MessageDigests.toHexString(MessageDigest.getInstance(digestAlgo).digest(zipBytes));
-            if (expectedChecksum.equals(actualChecksum) == false) {
-                throw new UserException(
+        // read the bytes of the plugin zip in chunks to avoid out of memory errors
+        try (InputStream zis = Files.newInputStream(zip)) {
+            try {
+                final MessageDigest digest = MessageDigest.getInstance(digestAlgo);
+                final byte[] bytes = new byte[8192];
+                int read;
+                while ((read = zis.read(bytes)) != -1) {
+                    assert read > 0 : read;
+                    digest.update(bytes, 0, read);
+                }
+                final String actualChecksum = MessageDigests.toHexString(digest.digest());
+                if (expectedChecksum.equals(actualChecksum) == false) {
+                    throw new UserException(
                         ExitCodes.IO_ERROR,
                         digestAlgo + " mismatch, expected " + expectedChecksum + " but got " + actualChecksum);
+                }
+            } catch (final NoSuchAlgorithmException e) {
+                // this should never happen as we are using SHA-1 and SHA-512 here
+                throw new AssertionError(e);
             }
-        } catch (final NoSuchAlgorithmException e) {
-            // this should never happen as we are using SHA-1 and SHA-512 here
-            throw new AssertionError(e);
         }
 
         if (officialPlugin) {

docs/java-rest/high-level/dataframe/delete_data_frame.asciidoc

@@ -16,6 +16,9 @@ A +{request}+ object requires a non-null `id`.
 include-tagged::{doc-tests-file}[{api}-request]
 ---------------------------------------------------
 <1> Constructing a new request referencing an existing {dataframe-transform}
+<2> Sets the optional argument `force`. When `true`, the {dataframe-transform}
+is deleted regardless of its current state. The default value is `false`,
+meaning that only `stopped` {dataframe-transforms} can be deleted.
 
 include::../execution.asciidoc[]
 

docs/java-rest/high-level/dataframe/put_data_frame.asciidoc

@@ -20,6 +20,10 @@ A +{request}+ requires the following argument:
 include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
 <1> The configuration of the {dataframe-transform} to create
+<2> Whether or not to wait to run deferrable validations until `_start` is called.
+This option should be used with care as the created {dataframe-transform} will run
+with the privileges of the user creating it. Meaning, if they do not have privileges,
+such an error will not be visible until `_start` is called.
 
 [id="{upid}-{api}-config"]
 ==== Data Frame Transform Configuration

docs/reference/data-frames/apis/delete-transform.asciidoc

@@ -34,6 +34,13 @@ see {stack-ov}/security-privileges.html[Security privileges] and
 `<data_frame_transform_id>`::
   (Required, string) Identifier for the {dataframe-transform}.
 
+[[delete-data-frame-transform-query-parms]]
+==== {api-query-parms-title}
+
+`force`::
+(Optional, boolean) When `true`, the {dataframe-transform} is deleted regardless of its
+current state. The default value is `false`, meaning that the {dataframe-transform} must be
+`stopped` before it can be deleted.
 
 [[delete-data-frame-transform-examples]]
 ==== {api-examples-title}