Migrate systemd packaging tests from bats to java (elastic#39954)

* initial work

* further testing

* migrated

* fix build

* test

* passing test with systemd

* fix compilation

* failing new test

* all tests passing systemd

* testcases in package test

* passing all tests for systemd

* limits are applied for systemd only

* code review followup

* is enabled return false

* missing assertions

* json log name

Author: pgomulka

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/PackageTestCase.java

@@ -20,31 +20,45 @@
 package org.elasticsearch.packaging.test;
 
 import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+import org.apache.http.client.fluent.Request;
+import org.elasticsearch.packaging.util.FileUtils;
 import org.elasticsearch.packaging.util.Shell;
 import org.elasticsearch.packaging.util.Shell.Result;
+import org.hamcrest.CoreMatchers;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.FileUtils.assertPathsDontExist;
+import static org.elasticsearch.packaging.util.FileUtils.assertPathsExist;
+import static org.elasticsearch.packaging.util.FileUtils.cp;
+import static org.elasticsearch.packaging.util.FileUtils.fileWithGlobExist;
+import static org.elasticsearch.packaging.util.FileUtils.mkdir;
 import static org.elasticsearch.packaging.util.FileUtils.mv;
+import static org.elasticsearch.packaging.util.FileUtils.rm;
+import static org.elasticsearch.packaging.util.FileUtils.slurp;
 import static org.elasticsearch.packaging.util.Packages.SYSTEMD_SERVICE;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
 import static org.elasticsearch.packaging.util.Packages.install;
 import static org.elasticsearch.packaging.util.Packages.remove;
+import static org.elasticsearch.packaging.util.Packages.restartElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.startElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.stopElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
+import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.elasticsearch.packaging.util.ServerUtils.runElasticsearchTests;
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.containsString;
@@ -55,42 +69,50 @@
 
 @TestCaseOrdering(TestCaseOrdering.AlphabeticOrder.class)
 public abstract class PackageTestCase extends PackagingTestCase {
+    private Shell sh;
 
     @Before
     public void onlyCompatibleDistributions() {
         assumeTrue("only compatible distributions", distribution().packaging.compatible);
+        sh = newShell();
     }
 
     public void test10InstallPackage() throws IOException {
         assertRemoved(distribution());
         installation = install(distribution());
         assertInstalled(distribution());
-        verifyPackageInstallation(installation, distribution(), newShell());
+        verifyPackageInstallation(installation, distribution(), sh);
     }
 
     public void test20PluginsCommandWhenNoPlugins() {
         assumeThat(installation, is(notNullValue()));
 
-        assertThat(newShell().run(installation.bin("elasticsearch-plugin") + " list").stdout, isEmptyString());
+        assertThat(sh.run(installation.bin("elasticsearch-plugin") + " list").stdout, isEmptyString());
     }
 
-    public void test30InstallDoesNotStartServer() {
+    public void test30DaemonIsNotEnabledOnRestart() {
+        if (isSystemd()) {
+            sh.run("systemctl daemon-reload");
+            String isEnabledOutput = sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").stdout.trim();
+            assertThat(isEnabledOutput, equalTo("disabled"));
+        }
+    }
+
+    public void test31InstallDoesNotStartServer() {
         assumeThat(installation, is(notNullValue()));
 
-        assertThat(newShell().run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch")));
+        assertThat(sh.run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch")));
     }
 
     public void assertRunsWithJavaHome() throws IOException {
-        Shell sh = newShell();
-
         String systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim();
         byte[] originalEnvFile = Files.readAllBytes(installation.envFile);
         try {
             Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8),
                 StandardOpenOption.APPEND);
-            startElasticsearch();
+            startElasticsearch(sh);
             runElasticsearchTests();
-            stopElasticsearch();
+            stopElasticsearch(sh);
         } finally {
             Files.write(installation.envFile, originalEnvFile);
         }
@@ -99,7 +121,7 @@ public void assertRunsWithJavaHome() throws IOException {
         assertThat(new String(Files.readAllBytes(log), StandardCharsets.UTF_8), containsString(systemJavaHome));
     }
 
-    public void test31JavaHomeOverride() throws IOException {
+    public void test32JavaHomeOverride() throws IOException {
         assumeThat(installation, is(notNullValue()));
         // we always run with java home when no bundled jdk is included, so this test would be repetitive
         assumeThat(distribution().hasJdk, is(true));
@@ -121,11 +143,20 @@ public void test42BundledJdkRemoved() throws IOException {
     }
 
     public void test40StartServer() throws IOException {
+        String start = sh.runIgnoreExitCode("date ").stdout.trim();
         assumeThat(installation, is(notNullValue()));
 
-        startElasticsearch();
+        startElasticsearch(sh);
+
+        String journalEntries = sh.runIgnoreExitCode("journalctl _SYSTEMD_UNIT=elasticsearch.service " +
+            "--since \"" + start + "\" --output cat | wc -l").stdout.trim();
+        assertThat(journalEntries, equalTo("0"));
+
+        assertPathsExist(installation.pidDir.resolve("elasticsearch.pid"));
+        assertPathsExist(installation.logs.resolve("elasticsearch_server.json"));
+
         runElasticsearchTests();
-        verifyPackageInstallation(installation, distribution(), newShell()); // check startup script didn't change permissions
+        verifyPackageInstallation(installation, distribution(), sh); // check startup script didn't change permissions
     }
 
     public void test50Remove() {
@@ -134,7 +165,6 @@ public void test50Remove() {
         remove(distribution());
 
         // removing must stop the service
-        final Shell sh = newShell();
         assertThat(sh.run("ps aux").stdout, not(containsString("org.elasticsearch.bootstrap.Elasticsearch")));
 
         if (isSystemd()) {
@@ -184,9 +214,160 @@ public void test60Reinstall() throws IOException {
 
         installation = install(distribution());
         assertInstalled(distribution());
-        verifyPackageInstallation(installation, distribution(), newShell());
+        verifyPackageInstallation(installation, distribution(), sh);
 
         remove(distribution());
         assertRemoved(distribution());
     }
+
+    public void test70RestartServer() throws IOException {
+        try {
+            installation = install(distribution());
+            assertInstalled(distribution());
+
+            startElasticsearch(sh);
+            restartElasticsearch(sh);
+            runElasticsearchTests();
+            stopElasticsearch(sh);
+        } finally {
+            cleanup();
+        }
+    }
+
+
+    public void test72TestRuntimeDirectory() throws IOException {
+        try {
+            installation = install(distribution());
+            FileUtils.rm(installation.pidDir);
+            startElasticsearch(sh);
+            assertPathsExist(installation.pidDir);
+            stopElasticsearch(sh);
+        } finally {
+            cleanup();
+        }
+    }
+
+    public void test73gcLogsExist() throws IOException {
+        installation = install(distribution());
+        startElasticsearch(sh);
+        // it can be gc.log or gc.log.0.current
+        assertThat(installation.logs, fileWithGlobExist("gc.log*"));
+        stopElasticsearch(sh);
+    }
+
+    // TEST CASES FOR SYSTEMD ONLY
+
+
+    /**
+     * # Simulates the behavior of a system restart:
+     * # the PID directory is deleted by the operating system
+     * # but it should not block ES from starting
+     * # see https://github.com/elastic/elasticsearch/issues/11594
+     */
+    public void test80DeletePID_DIRandRestart() throws IOException {
+        assumeTrue(isSystemd());
+
+        rm(installation.pidDir);
+
+        sh.run("systemd-tmpfiles --create");
+
+        startElasticsearch(sh);
+
+        final Path pidFile = installation.pidDir.resolve("elasticsearch.pid");
+
+        assertTrue(Files.exists(pidFile));
+
+        stopElasticsearch(sh);
+    }
+
+    public void test81CustomPathConfAndJvmOptions() throws IOException {
+        assumeTrue(isSystemd());
+
+        assumeThat(installation, is(notNullValue()));
+        assertPathsExist(installation.envFile);
+
+        stopElasticsearch(sh);
+
+        // The custom config directory is not under /tmp or /var/tmp because
+        // systemd's private temp directory functionally means different
+        // processes can have different views of what's in these directories
+        String temp = sh.runIgnoreExitCode("mktemp -p /etc -d").stdout.trim();
+        final Path tempConf = Paths.get(temp);
+
+        try {
+            mkdir(tempConf);
+            cp(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml"));
+            cp(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties"));
+
+            // we have to disable Log4j from using JMX lest it will hit a security
+            // manager exception before we have configured logging; this will fail
+            // startup since we detect usages of logging before it is configured
+            final String jvmOptions =
+                "-Xms512m\n" +
+                    "-Xmx512m\n" +
+                    "-Dlog4j2.disable.jmx=true\n";
+            append(tempConf.resolve("jvm.options"), jvmOptions);
+
+            sh.runIgnoreExitCode("chown -R elasticsearch:elasticsearch " + tempConf);
+
+            final Shell serverShell = newShell();
+            cp(installation.envFile, tempConf.resolve("elasticsearch.bk"));//backup
+            append(installation.envFile, "ES_PATH_CONF=" + tempConf + "\n");
+            append(installation.envFile, "ES_JAVA_OPTS=-XX:-UseCompressedOops");
+
+            startElasticsearch(serverShell);
+
+            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            assertThat(nodesResponse, CoreMatchers.containsString("\"heap_init_in_bytes\":536870912"));
+            assertThat(nodesResponse, CoreMatchers.containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
+
+            stopElasticsearch(serverShell);
+
+        } finally {
+            rm(installation.envFile);
+            cp(tempConf.resolve("elasticsearch.bk"), installation.envFile);
+            rm(tempConf);
+            cleanup();
+        }
+    }
+
+    public void test82SystemdMask() throws IOException {
+        try {
+            assumeTrue(isSystemd());
+
+            sh.run("systemctl mask systemd-sysctl.service");
+
+            installation = install(distribution());
+
+            sh.run("systemctl unmask systemd-sysctl.service");
+        } finally {
+            cleanup();
+        }
+    }
+
+    public void test83serviceFileSetsLimits() throws IOException {
+        // Limits are changed on systemd platforms only
+        assumeTrue(isSystemd());
+
+        installation = install(distribution());
+
+        startElasticsearch(sh);
+
+        final Path pidFile = installation.pidDir.resolve("elasticsearch.pid");
+        assertTrue(Files.exists(pidFile));
+        String pid = slurp(pidFile).trim();
+        String maxFileSize = sh.run("cat /proc/%s/limits | grep \"Max file size\" | awk '{ print $4 }'", pid).stdout.trim();
+        assertThat(maxFileSize, equalTo("unlimited"));
+
+        String maxProcesses = sh.run("cat /proc/%s/limits | grep \"Max processes\" | awk '{ print $3 }'", pid).stdout.trim();
+        assertThat(maxProcesses, equalTo("4096"));
+
+        String maxOpenFiles = sh.run("cat /proc/%s/limits | grep \"Max open files\" | awk '{ print $4 }'", pid).stdout.trim();
+        assertThat(maxOpenFiles, equalTo("65535"));
+
+        String maxAddressSpace = sh.run("cat /proc/%s/limits | grep \"Max address space\" | awk '{ print $4 }'", pid).stdout.trim();
+        assertThat(maxAddressSpace, equalTo("unlimited"));
+
+        stopElasticsearch(sh);
+    }
 }

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java

@@ -20,6 +20,8 @@
 package org.elasticsearch.packaging.util;
 
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.hamcrest.FeatureMatcher;
+import org.hamcrest.Matcher;
 
 import java.io.BufferedWriter;
 import java.io.IOException;
@@ -34,9 +36,11 @@
 import java.nio.file.attribute.PosixFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.core.IsNot.not;
 import static org.hamcrest.text.IsEmptyString.isEmptyOrNullString;
 import static org.junit.Assert.assertFalse;
@@ -69,6 +73,15 @@ public static void rm(Path... paths) {
         }
     }
 
+    public static Path mktempDir(Path path) {
+        try {
+            return Files.createTempDirectory(path,"tmp");
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
     public static Path mkdir(Path path) {
         try {
             return Files.createDirectories(path);
@@ -176,6 +189,20 @@ public static void assertPathsExist(Path... paths) {
         Arrays.stream(paths).forEach(path -> assertTrue(path + " should exist", Files.exists(path)));
     }
 
+    public static Matcher<Path> fileWithGlobExist(String glob) throws IOException {
+        return new FeatureMatcher<Path,Iterable<Path>>(not(emptyIterable()),"File with pattern exist", "file with pattern"){
+
+            @Override
+            protected Iterable<Path> featureValueOf(Path actual) {
+                try {
+                    return Files.newDirectoryStream(actual,glob);
+                } catch (IOException e) {
+                    return Collections.emptyList();
+                }
+            }
+        };
+    }
+
     public static void assertPathsDontExist(Path... paths) {
         Arrays.stream(paths).forEach(path -> assertFalse(path + " should not exist", Files.exists(path)));
     }