Run keystore management tests on docker distros

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -21,17 +21,13 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import org.apache.http.client.fluent.Request;
-import org.elasticsearch.packaging.util.Distribution;
-import org.elasticsearch.packaging.util.Docker.DockerShell;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Platforms;
 import org.elasticsearch.packaging.util.ServerUtils;
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -43,21 +39,17 @@
 import java.util.stream.Collectors;
 
 import static java.nio.file.attribute.PosixFilePermissions.fromString;
-import static org.elasticsearch.packaging.util.Docker.assertPermissionsAndOwnership;
 import static org.elasticsearch.packaging.util.Docker.copyFromContainer;
-import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded;
 import static org.elasticsearch.packaging.util.Docker.existsInContainer;
 import static org.elasticsearch.packaging.util.Docker.getContainerLogs;
 import static org.elasticsearch.packaging.util.Docker.getImageLabels;
 import static org.elasticsearch.packaging.util.Docker.getJson;
 import static org.elasticsearch.packaging.util.Docker.mkDirWithPrivilegeEscalation;
-import static org.elasticsearch.packaging.util.Docker.removeContainer;
 import static org.elasticsearch.packaging.util.Docker.rmDirWithPrivilegeEscalation;
 import static org.elasticsearch.packaging.util.Docker.runContainer;
 import static org.elasticsearch.packaging.util.Docker.runContainerExpectingFailure;
 import static org.elasticsearch.packaging.util.Docker.verifyContainerInstallation;
 import static org.elasticsearch.packaging.util.Docker.waitForElasticsearch;
-import static org.elasticsearch.packaging.util.Docker.waitForPathToExist;
 import static org.elasticsearch.packaging.util.FileMatcher.p600;
 import static org.elasticsearch.packaging.util.FileMatcher.p660;
 import static org.elasticsearch.packaging.util.FileUtils.append;
@@ -77,25 +69,15 @@
 import static org.junit.Assume.assumeTrue;
 
 public class DockerTests extends PackagingTestCase {
-    protected DockerShell sh;
     private Path tempDir;
 
     @BeforeClass
     public static void filterDistros() {
-        assumeTrue("only Docker", distribution.packaging == Distribution.Packaging.DOCKER);
-
-        ensureImageIsLoaded(distribution);
-    }
-
-    @AfterClass
-    public static void cleanup() {
-        // runContainer also calls this, so we don't need this method to be annotated as `@After`
-        removeContainer();
+        assumeTrue("only Docker", distribution().isDocker());
     }
 
     @Before
     public void setupTest() throws IOException {
-        sh = new DockerShell();
         installation = runContainer(distribution());
         tempDir = Files.createTempDirectory(getTempDir(), DockerTests.class.getSimpleName());
     }
@@ -136,44 +118,10 @@ public void test020PluginsListWithNoPlugins() {
         assertThat("Expected no plugins to be listed", r.stdout, emptyString());
     }
 
-    /**
-     * Check that a keystore can be manually created using the provided CLI tool.
-     */
-    public void test040CreateKeystoreManually() throws InterruptedException {
-        final Installation.Executables bin = installation.executables();
-
-        final Path keystorePath = installation.config("elasticsearch.keystore");
-
-        waitForPathToExist(keystorePath);
-
-        // Move the auto-created one out of the way, or else the CLI prompts asks us to confirm
-        sh.run("mv " + keystorePath + " " + keystorePath + ".bak");
-
-        sh.run(bin.keystoreTool + " create");
-
-        final Result r = sh.run(bin.keystoreTool + " list");
-        assertThat(r.stdout, containsString("keystore.seed"));
-    }
-
-    /**
-     * Check that the default keystore is automatically created
-     */
-    public void test041AutoCreateKeystore() throws Exception {
-        final Path keystorePath = installation.config("elasticsearch.keystore");
-
-        waitForPathToExist(keystorePath);
-
-        assertPermissionsAndOwnership(keystorePath, p660);
-
-        final Installation.Executables bin = installation.executables();
-        final Result result = sh.run(bin.keystoreTool + " list");
-        assertThat(result.stdout, containsString("keystore.seed"));
-    }
-
     /**
      * Check that the JDK's cacerts file is a symlink to the copy provided by the operating system.
      */
-    public void test042JavaUsesTheOsProvidedKeystore() {
+    public void test040JavaUsesTheOsProvidedKeystore() {
         final String path = sh.run("realpath jdk/lib/security/cacerts").stdout;
 
         assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts"));
@@ -182,7 +130,7 @@ public void test042JavaUsesTheOsProvidedKeystore() {
     /**
      * Checks that there are Amazon trusted certificates in the cacaerts keystore.
      */
-    public void test043AmazonCaCertsAreInTheKeystore() {
+    public void test041AmazonCaCertsAreInTheKeystore() {
         final boolean matches = sh.run("jdk/bin/keytool -cacerts -storepass changeit -list | grep trustedCertEntry").stdout.lines()
             .anyMatch(line -> line.contains("amazonrootca"));
 
@@ -338,7 +286,7 @@ public void test081ConfigurePasswordThroughEnvironmentVariableFile() throws Exce
     /**
      * Check that environment variables cannot be used with _FILE environment variables.
      */
-    public void test081CannotUseEnvVarsAndFiles() throws Exception {
+    public void test082CannotUseEnvVarsAndFiles() throws Exception {
         final String optionsFilename = "esJavaOpts.txt";
 
         // ES_JAVA_OPTS_FILE
@@ -369,7 +317,7 @@ public void test081CannotUseEnvVarsAndFiles() throws Exception {
      * Check that when populating environment variables by setting variables with the suffix "_FILE",
      * the files' permissions are checked.
      */
-    public void test082EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws Exception {
+    public void test083EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws Exception {
         final String optionsFilename = "esJavaOpts.txt";
 
         // ES_JAVA_OPTS_FILE
@@ -395,7 +343,7 @@ public void test082EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws
      * Check that environment variables are translated to -E options even for commands invoked under
      * `docker exec`, where the Docker image's entrypoint is not executed.
      */
-    public void test83EnvironmentVariablesAreRespectedUnderDockerExec() {
+    public void test084EnvironmentVariablesAreRespectedUnderDockerExec() {
         // This test relies on a CLI tool attempting to connect to Elasticsearch, and the
         // tool in question is only in the default distribution.
         assumeTrue(distribution.isDefault());
@@ -565,7 +513,6 @@ public void test120DockerLogsIncludeElasticsearchLogs() throws Exception {
     /**
      * Check that the Java process running inside the container has the expect PID, UID and username.
      */
-    @Ignore /* Ignored for feature branch, awaits fix: https://github.com/elastic/elasticsearch/issues/49469 */
     public void test130JavaHasCorrectPidAndOwnership() {
         final List<String> processes = sh.run("ps -o pid,uid,user -C java").stdout.lines().skip(1).collect(Collectors.toList());
 

qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.packaging.test;
 
 import org.elasticsearch.packaging.util.Distribution;
+import org.elasticsearch.packaging.util.Docker;
 import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Platforms;
@@ -35,6 +36,8 @@
 import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER;
 import static org.elasticsearch.packaging.util.Archives.installArchive;
 import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation;
+import static org.elasticsearch.packaging.util.Docker.assertPermissionsAndOwnership;
+import static org.elasticsearch.packaging.util.Docker.waitForPathToExist;
 import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
 import static org.elasticsearch.packaging.util.FileMatcher.file;
 import static org.elasticsearch.packaging.util.FileMatcher.p660;
@@ -45,7 +48,6 @@
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
@@ -54,7 +56,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
 
     private static final String PASSWORD_ERROR_MESSAGE = "Provided keystore password was incorrect";
 
-    /** We need an initially installed package */
+    /** Test initial archive state */
     public void test10InstallArchiveDistribution() throws Exception {
         assumeTrue(distribution().isArchive());
 
@@ -63,11 +65,12 @@ public void test10InstallArchiveDistribution() throws Exception {
 
         final Installation.Executables bin = installation.executables();
         Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool.toString() + " has-passwd");
-        assertThat("has-passwd should fail", r.exitCode, not(is(0)));
-        assertThat("has-passwd should fail", r.stderr, containsString("ERROR: Elasticsearch keystore not found"));
+        assertFalse("has-passwd should fail", r.isSuccess());
+        assertThat("has-passwd should indicate missing keystore",
+            r.stderr, containsString("ERROR: Elasticsearch keystore not found"));
     }
 
-    /** We need an initially installed package */
+    /** Test initial package state */
     public void test11InstallPackageDistribution() throws Exception {
         assumeTrue(distribution().isPackage());
 
@@ -78,11 +81,34 @@ public void test11InstallPackageDistribution() throws Exception {
 
         final Installation.Executables bin = installation.executables();
         Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool.toString() + " has-passwd");
-        assertThat("has-passwd should fail", r.exitCode, not(is(0)));
-        assertThat("has-passwd should fail", r.stderr, containsString("ERROR: Keystore is not password-protected"));
+        assertFalse("has-passwd should fail", r.isSuccess());
+        assertThat("has-passwd should indicate unprotected keystore",
+            r.stderr, containsString("ERROR: Keystore is not password-protected"));
+        Shell.Result r2 = bin.keystoreTool.run("list");
+        assertThat(r2.stdout, containsString("keystore.seed"));
+    }
+
+    /** Test initial Docker state */
+    public void test12InstallDockerDistribution() throws Exception {
+        assumeTrue(distribution().isDocker());
+
+        installation = Docker.runContainer(distribution());
+
+        try {
+            waitForPathToExist(installation.config("elasticsearch.keystore"));
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+
+        final Installation.Executables bin = installation.executables();
+        Shell.Result r = sh.runIgnoreExitCode(bin.keystoreTool.toString() + " has-passwd");
+        assertFalse("has-passwd should fail", r.isSuccess());
+        assertThat("has-passwd should indicate unprotected keystore",
+            r.stdout, containsString("ERROR: Keystore is not password-protected"));
+        Shell.Result r2 = bin.keystoreTool.run("list");
+        assertThat(r2.stdout, containsString("keystore.seed"));
     }
 
-    @Ignore /* Ignored for feature branch, awaits fix: https://github.com/elastic/elasticsearch/issues/49469 */
     public void test20CreateKeystoreManually() throws Exception {
         rmKeystoreIfExists();
         createKeystore();
@@ -95,7 +121,7 @@ public void test20CreateKeystoreManually() throws Exception {
     }
 
     public void test30AutoCreateKeystore() throws Exception {
-        assumeTrue("RPMs and Debs install a keystore file", distribution.isArchive());
+        assumeTrue("Packages and docker are installed with a keystore file", distribution.isArchive());
         rmKeystoreIfExists();
 
         startElasticsearch();
@@ -239,12 +265,31 @@ private void createKeystore() throws Exception {
         Platforms.onWindows(() -> {
             sh.chown(keystore);
         });
+
+        if (distribution().isDocker()) {
+            try {
+                waitForPathToExist(keystore);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
     }
 
     private void rmKeystoreIfExists() {
         Path keystore = installation.config("elasticsearch.keystore");
-        if (Files.exists(keystore)) {
-            FileUtils.rm(keystore);
+        if (distribution().isDocker()) {
+            try {
+                waitForPathToExist(keystore);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+
+            // Move the auto-created one out of the way, or else the CLI prompts asks us to confirm
+            sh.run("rm " + keystore);
+        } else {
+            if (Files.exists(keystore)) {
+                FileUtils.rm(keystore);
+            }
         }
     }
 
@@ -267,7 +312,7 @@ private void assertPasswordProtectedKeystore() {
         assertThat("keystore should be password protected", r.exitCode, is(0));
     }
 
-    private void verifyKeystorePermissions() throws Exception {
+    private void verifyKeystorePermissions() {
         Path keystore = installation.config("elasticsearch.keystore");
         switch (distribution.packaging) {
             case TAR:
@@ -279,7 +324,7 @@ private void verifyKeystorePermissions() throws Exception {
                 assertThat(keystore, file(File, "root", "elasticsearch", p660));
                 break;
             case DOCKER:
-                // TODO #49469
+                assertPermissionsAndOwnership(keystore, p660);
                 break;
             default:
                 throw new IllegalStateException("Unknown Elasticsearch packaging type.");

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.packaging.util.Platforms;
 import org.elasticsearch.packaging.util.Shell;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -49,6 +50,8 @@
 import java.nio.file.Paths;
 
 import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
+import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded;
+import static org.elasticsearch.packaging.util.Docker.removeContainer;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.junit.Assume.assumeFalse;
@@ -117,9 +120,23 @@ public static void cleanup() throws Exception {
 
     @BeforeClass
     public static void createShell() throws Exception {
-        sh = new Shell();
+        if (distribution().isDocker()) {
+            ensureImageIsLoaded(distribution);
+            sh = new Docker.DockerShell();
+        } else {
+            sh = new Shell();
+        }
+    }
+
+    @AfterClass
+    public static void cleanupDocker() {
+        if (distribution().isDocker()) {
+            // runContainer also calls this, so we don't need this method to be annotated as `@After`
+            removeContainer();
+        }
     }
 
+
     @Before
     public void setup() throws Exception {
         assumeFalse(failed); // skip rest of tests once one fails

qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java

@@ -35,7 +35,7 @@ public class Cleanup {
 
     private static final List<String> ELASTICSEARCH_FILES_LINUX = Arrays.asList(
         "/usr/share/elasticsearch",
-        "/etc/elasticsearch/elasticsearch-keystore",
+        "/etc/elasticsearch/elasticsearch.keystore",
         "/etc/elasticsearch",
         "/var/lib/elasticsearch",
         "/var/log/elasticsearch",

qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java

@@ -64,6 +64,10 @@ public boolean isPackage() {
         return packaging == Packaging.RPM || packaging == Packaging.DEB;
     }
 
+    public boolean isDocker() {
+        return packaging == Packaging.DOCKER;
+    }
+
     public enum Packaging {
 
         TAR(".tar.gz", Platforms.LINUX || Platforms.DARWIN),

qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java

@@ -178,9 +178,9 @@ public static void waitForElasticsearchToStart() {
                 // Give the container a chance to crash out
                 Thread.sleep(1000);
 
-                psOutput = dockerShell.run("ps -w ax").stdout;
+                psOutput = dockerShell.run("ps -ww ax").stdout;
 
-                if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java")) {
+                if (psOutput.contains("org.elasticsearch.bootstrap.Elasticsearch")) {
                     isElasticsearchRunning = true;
                     break;
                 }