[7.x] Test modifications for FIPS 140 mode (#51832)

buildSrc/src/main/resources/fips_java.security

@@ -1,6 +1,7 @@
 security.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider
 security.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider fips:BCFIPS
 security.provider.3=SUN
+security.provider.4=SunJGSS
 securerandom.source=file:/dev/urandom
 securerandom.strongAlgorithms=NativePRNGBlocking:SUN,DRBG:SUN
 securerandom.drbg.config=

plugins/ingest-attachment/build.gradle

@@ -88,8 +88,11 @@ thirdPartyAudit {
   ignoreMissingClasses()
 }
 
-jarHell.onlyIf {
+if (BuildParams.inFipsJvm) {
   // FIPS JVM includes many classes from bouncycastle which count as jar hell for the third party audit,
   // rather than provide a long list of exclusions, disable the check on FIPS.
-  BuildParams.inFipsJvm == false
+  jarHell.enabled = false
+  test.enabled = false
+  integTest.enabled = false;
+  testingConventions.enabled = false;
 }

qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -35,6 +35,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -58,13 +59,15 @@
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.startsWith;
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
 
 public class KeystoreManagementTests extends PackagingTestCase {
 
     public static final String ERROR_INCORRECT_PASSWORD = "Provided keystore password was incorrect";
+    public static final String ERROR_CORRUPTED_KEYSTORE = "Keystore has been corrupted or tampered with";
     public static final String ERROR_KEYSTORE_NOT_PASSWORD_PROTECTED = "ERROR: Keystore is not password-protected";
     public static final String ERROR_KEYSTORE_NOT_FOUND = "ERROR: Elasticsearch keystore not found";
 
@@ -174,7 +177,7 @@ public void test41WrongKeystorePasswordOnStandardInput() {
         assertPasswordProtectedKeystore();
 
         Shell.Result result = startElasticsearchStandardInputPassword("wrong");
-        assertElasticsearchFailure(result, ERROR_INCORRECT_PASSWORD, null);
+        assertElasticsearchFailure(result, Arrays.asList(ERROR_INCORRECT_PASSWORD, ERROR_CORRUPTED_KEYSTORE), null);
     }
 
     @Ignore /* Ignored for feature branch, awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
@@ -210,7 +213,7 @@ public void test43WrongKeystorePasswordOnTty() throws Exception {
 
         Shell.Result result = startElasticsearchTtyPassword("wrong");
         // error will be on stdout for "expect"
-        assertThat(result.stdout, containsString(ERROR_INCORRECT_PASSWORD));
+        assertThat(result.stdout, anyOf(containsString(ERROR_INCORRECT_PASSWORD), containsString(ERROR_CORRUPTED_KEYSTORE)));
     }
 
     /**
@@ -279,7 +282,7 @@ public void test51WrongKeystorePasswordFromFile() throws Exception {
 
             Packages.JournaldWrapper journaldWrapper = new Packages.JournaldWrapper(sh);
             Shell.Result result = runElasticsearchStartCommand();
-            assertElasticsearchFailure(result, ERROR_INCORRECT_PASSWORD, journaldWrapper);
+            assertElasticsearchFailure(result, Arrays.asList(ERROR_INCORRECT_PASSWORD, ERROR_CORRUPTED_KEYSTORE), journaldWrapper);
         } finally {
             sh.run("sudo systemctl unset-environment ES_KEYSTORE_PASSPHRASE_FILE");
         }

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -34,6 +34,8 @@
 import org.elasticsearch.packaging.util.Packages;
 import org.elasticsearch.packaging.util.Platforms;
 import org.elasticsearch.packaging.util.Shell;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.Matcher;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -48,13 +50,16 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
 
 import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
 import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded;
 import static org.elasticsearch.packaging.util.Docker.removeContainer;
 import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.anyOf;
 import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 
@@ -311,23 +316,28 @@ public Shell.Result startElasticsearchTtyPassword(String password) throws Except
         return Archives.startElasticsearchWithTty(installation, sh, password);
     }
 
-
     public void assertElasticsearchFailure(Shell.Result result, String expectedMessage, Packages.JournaldWrapper journaldWrapper) {
+        assertElasticsearchFailure(result, Collections.singletonList(expectedMessage), journaldWrapper);
+    }
 
+    public void assertElasticsearchFailure(Shell.Result result, List<String> expectedMessages, Packages.JournaldWrapper journaldWrapper) {
+        @SuppressWarnings("unchecked")
+        Matcher<String>[] stringMatchers = expectedMessages.stream().map(CoreMatchers::containsString).toArray(Matcher[]::new);
         if (Files.exists(installation.logs.resolve("elasticsearch.log"))) {
 
             // If log file exists, then we have bootstrapped our logging and the
             // error should be in the logs
             assertThat(installation.logs.resolve("elasticsearch.log"), fileExists());
             String logfile = FileUtils.slurp(installation.logs.resolve("elasticsearch.log"));
-            assertThat(logfile, containsString(expectedMessage));
+
+            assertThat(logfile, anyOf(stringMatchers));
 
         } else if (distribution().isPackage() && Platforms.isSystemd()) {
 
             // For systemd, retrieve the error from journalctl
             assertThat(result.stderr, containsString("Job for elasticsearch.service failed"));
             Shell.Result error = journaldWrapper.getLogs();
-            assertThat(error.stdout, containsString(expectedMessage));
+            assertThat(error.stdout, anyOf(stringMatchers));
 
         } else if (Platforms.WINDOWS) {
 
@@ -338,12 +348,12 @@ public void assertElasticsearchFailure(Shell.Result result, String expectedMessa
             sh.runIgnoreExitCode("Get-EventSubscriber | " +
                 "where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |" +
                 "Unregister-EventSubscriber -Force");
-            assertThat(FileUtils.slurp(Archives.getPowershellErrorPath(installation)), containsString(expectedMessage));
+            assertThat(FileUtils.slurp(Archives.getPowershellErrorPath(installation)), anyOf(stringMatchers));
 
         } else {
 
             // Otherwise, error should be on shell stderr
-            assertThat(result.stderr, containsString(expectedMessage));
+            assertThat(result.stderr, anyOf(stringMatchers));
         }
     }
 

qa/smoke-test-plugins/build.gradle

@@ -18,6 +18,7 @@
  */
 
 import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.info.BuildParams
 
 apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
@@ -27,6 +28,10 @@ int pluginsCount = 0
 
 testClusters.integTest {
   project(':plugins').getChildProjects().each { pluginName, pluginProject ->
+    if (BuildParams.inFipsJvm && pluginName == "ingest-attachment"){
+      //Do not attempt to install ingest-attachment in FIPS 140 as it is not supported (it depends on non-FIPS BouncyCastle
+      return
+    }
     plugin file(pluginProject.tasks.bundlePlugin.archiveFile)
     tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin
     pluginsCount += 1

server/src/test/java/org/elasticsearch/common/settings/AddFileKeyStoreCommandTests.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.env.Environment;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
 public class AddFileKeyStoreCommandTests extends KeyStoreCommandTestCase {
@@ -192,7 +193,17 @@ public void testIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongkeystorepassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo", file.toString()));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 
     public void testAddToUnprotectedKeystore() throws Exception {

server/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.env.Environment;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasToString;
 
@@ -57,7 +58,17 @@ public void testInvalidPassphrease() throws Exception {
         terminal.addSecretInput("thewrongpassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo2"));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
 
     }
 

server/src/test/java/org/elasticsearch/common/settings/ChangeKeyStorePasswordCommandTests.java

@@ -26,6 +26,7 @@
 
 import java.util.Map;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
 public class ChangeKeyStorePasswordCommandTests extends KeyStoreCommandTestCase {
@@ -90,6 +91,16 @@ public void testChangeKeyStorePasswordWrongExistingPassword() throws Exception {
         // We'll only be prompted once (for the old password)
         UserException e = expectThrows(UserException.class, this::execute);
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 }

server/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java

@@ -60,6 +60,7 @@
 import java.util.List;
 import java.util.Locale;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -111,9 +112,21 @@ public void testDecryptKeyStoreWithWrongPassword() throws Exception {
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
         keystore.save(env.configFile(), new char[0]);
         final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configFile());
-        final SecurityException exception = expectThrows(SecurityException.class,
-            () -> loadedkeystore.decrypt(new char[]{'i', 'n', 'v', 'a', 'l', 'i', 'd'}));
-        assertThat(exception.getMessage(), containsString("Provided keystore password was incorrect"));
+        final SecurityException exception = expectThrows(
+            SecurityException.class,
+            () -> loadedkeystore.decrypt(new char[] { 'i', 'n', 'v', 'a', 'l', 'i', 'd' })
+        );
+        if (inFipsJvm()) {
+            assertThat(
+                exception.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(exception.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 
     public void testCannotReadStringFromClosedKeystore() throws Exception {

server/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.env.Environment;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
 public class ListKeyStoreCommandTests extends KeyStoreCommandTestCase {
@@ -76,7 +77,17 @@ public void testListWithIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongkeystorepassword");
         UserException e = expectThrows(UserException.class, this::execute);
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
     }
 
     public void testListWithUnprotectedKeystore() throws Exception {

server/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java

@@ -27,6 +27,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 
 public class RemoveSettingKeyStoreCommandTests extends KeyStoreCommandTestCase {
@@ -93,7 +94,18 @@ public void testRemoveWithIncorrectPassword() throws Exception {
         terminal.addSecretInput("thewrongpassword");
         UserException e = expectThrows(UserException.class, () -> execute("foo"));
         assertEquals(e.getMessage(), ExitCodes.DATA_ERROR, e.exitCode);
-        assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        if (inFipsJvm()) {
+            assertThat(
+                e.getMessage(),
+                anyOf(
+                    containsString("Provided keystore password was incorrect"),
+                    containsString("Keystore has been corrupted or tampered with")
+                )
+            );
+        } else {
+            assertThat(e.getMessage(), containsString("Provided keystore password was incorrect"));
+        }
+
     }
 
     public void testRemoveFromUnprotectedKeystore() throws Exception {