Remove BouncyCastle dependency from runtime (#32193)

* Remove BouncyCastle dependency from runtime

This commit introduces a new gradle  project that contains
 the classes that have a dependency on BouncyCastle. For 
the default distribution, It builds  a jar from those and
 in puts it in a subdirectory of lib
 (/tools/security-cli) along with the BouncyCastle jars. 
This directory is then passed in the
ES_ADDITIONAL_CLASSPATH_DIRECTORIES of the CLI tools 
that use these classes.

BouncyCastle is removed as a runtime dependency (remains
as a compileOnly one) from x-pack core and x-pack security.

Author: jkakavas

distribution/archives/build.gradle

@@ -49,7 +49,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean os
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
-        with libFiles
+        with libFiles(oss)
       }
       into('config') {
         dirMode 0750

distribution/build.gradle

@@ -227,16 +227,24 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     /*****************************************************************************
      *                   Common files in all distributions                       *
      *****************************************************************************/
-    libFiles = copySpec {
-      // delay by using closures, since they have not yet been configured, so no jar task exists yet
-      from { project(':server').jar }
-      from { project(':server').configurations.runtime }
-      from { project(':libs:plugin-classloader').jar }
-      from { project(':distribution:tools:java-version-checker').jar }
-      from { project(':distribution:tools:launchers').jar }
-      into('tools/plugin-cli') {
-        from { project(':distribution:tools:plugin-cli').jar }
-        from { project(':distribution:tools:plugin-cli').configurations.runtime }
+    libFiles = { oss ->
+      copySpec {
+        // delay by using closures, since they have not yet been configured, so no jar task exists yet
+        from { project(':server').jar }
+        from { project(':server').configurations.runtime }
+        from { project(':libs:plugin-classloader').jar }
+        from { project(':distribution:tools:java-version-checker').jar }
+        from { project(':distribution:tools:launchers').jar }
+        into('tools/plugin-cli') {
+          from { project(':distribution:tools:plugin-cli').jar }
+          from { project(':distribution:tools:plugin-cli').configurations.runtime }
+        }
+        if (oss == false) {
+          into('tools/security-cli') {
+            from { project(':x-pack:plugin:security:cli').jar }
+            from { project(':x-pack:plugin:security:cli').configurations.compile }
+          }
+        }
       }
     }
 

distribution/packages/build.gradle

@@ -126,7 +126,7 @@ Closure commonPackageConfig(String type, boolean oss) {
       }
       into('lib') {
         with copySpec {
-          with libFiles
+          with libFiles(oss)
           // we need to specify every intermediate directory so we iterate through the parents; duplicate calls with the same part are fine
           eachFile { FileCopyDetails fcp ->
             String[] segments = fcp.relativePath.segments

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -57,6 +57,7 @@
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.isEmptyString;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeThat;
 import static org.junit.Assume.assumeTrue;
@@ -302,5 +303,26 @@ public void test80RelativePathConf() throws IOException {
         }
     }
 
+    public void test90SecurityCliPackaging() {
+        assumeThat(installation, is(notNullValue()));
+
+        final Installation.Executables bin = installation.executables();
+        final Shell sh = new Shell();
+
+        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+            assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli")));
+            Platforms.onLinux(() -> {
+                final Result result = sh.run(bin.elasticsearchCertutil + " help");
+                assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack"));
+            });
+
+            Platforms.onWindows(() -> {
+                final Result result = sh.run(bin.elasticsearchCertutil + " help");
+                assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack"));
+            });
+        } else if (distribution().equals(Distribution.OSS_TAR) || distribution().equals(Distribution.OSS_ZIP)) {
+            assertFalse(Files.exists(installation.lib.resolve("tools").resolve("security-cli")));
+        }
+    }
 
 }

qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java

@@ -101,6 +101,7 @@ public class Executables {
         public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin");
         public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore");
         public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog");
+        public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil");
 
         private Path platformExecutable(String name) {
             final String platformExecutableName = Platforms.WINDOWS

x-pack/plugin/core/build.gradle

@@ -20,7 +20,6 @@ esplugin {
 }
 
 dependencyLicenses {
-  mapping from: /bc.*/, to: 'bouncycastle'
   mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
 }
@@ -38,8 +37,6 @@ dependencies {
 
     // security deps
     compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
     compile project(path: ':modules:transport-netty4', configuration: 'runtime')
 
     testCompile 'org.elasticsearch:securemock:1.2'
@@ -116,6 +113,7 @@ task testJar(type: Jar) {
     appendix 'test'
     from sourceSets.test.output
 }
+
 artifacts {
     // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
     archives jar

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java

@@ -63,7 +63,7 @@ static Path resolvePath(String path, @Nullable Environment environment) {
         return PathUtils.get(path).normalize();
     }
 
-    static KeyStore readKeyStore(Path path, String type, char[] password)
+    public static KeyStore readKeyStore(Path path, String type, char[] password)
             throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException {
         try (InputStream in = Files.newInputStream(path)) {
             KeyStore store = KeyStore.getInstance(type);
@@ -108,7 +108,7 @@ public static X509Certificate[] readX509Certificates(List<Path> certPaths) throw
         return certificates.toArray(new X509Certificate[0]);
     }
 
-    static List<Certificate> readCertificates(InputStream input) throws CertificateException, IOException {
+    public static List<Certificate> readCertificates(InputStream input) throws CertificateException, IOException {
         CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
         Collection<Certificate> certificates = (Collection<Certificate>) certFactory.generateCertificates(input);
         return new ArrayList<>(certificates);
@@ -140,7 +140,7 @@ public static Map<Certificate, Key> readPkcs12KeyPairs(Path path, char[] passwor
     /**
      * Creates a {@link KeyStore} from a PEM encoded certificate and key file
      */
-    static KeyStore getKeyStoreFromPEM(Path certificatePath, Path keyPath, char[] keyPassword)
+    public static KeyStore getKeyStoreFromPEM(Path certificatePath, Path keyPath, char[] keyPassword)
             throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException {
         final PrivateKey key = PemUtils.readPrivateKey(keyPath, () -> keyPassword);
         final Certificate[] certificates = readCertificates(Collections.singletonList(certificatePath));
@@ -168,7 +168,7 @@ private static KeyStore getKeyStore(Certificate[] certificateChain, PrivateKey p
     /**
      * Returns a {@link X509ExtendedKeyManager} that is built from the provided keystore
      */
-    static X509ExtendedKeyManager keyManager(KeyStore keyStore, char[] password, String algorithm)
+    public static X509ExtendedKeyManager keyManager(KeyStore keyStore, char[] password, String algorithm)
             throws NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException {
         KeyManagerFactory kmf = KeyManagerFactory.getInstance(algorithm);
         kmf.init(keyStore, password);
@@ -271,7 +271,7 @@ public static X509ExtendedTrustManager trustManager(String trustStorePath, Strin
     /**
      * Creates a {@link X509ExtendedTrustManager} based on the trust material in the provided {@link KeyStore}
      */
-    static X509ExtendedTrustManager trustManager(KeyStore keyStore, String algorithm)
+    public static X509ExtendedTrustManager trustManager(KeyStore keyStore, String algorithm)
             throws NoSuchAlgorithmException, KeyStoreException {
         TrustManagerFactory tmf = TrustManagerFactory.getInstance(algorithm);
         tmf.init(keyStore);

x-pack/plugin/security/build.gradle

@@ -22,8 +22,8 @@ dependencies {
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 
     compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
+    compileOnly 'org.bouncycastle:bcprov-jdk15on:1.59'
+    compileOnly 'org.bouncycastle:bcpkix-jdk15on:1.59'
 
     // the following are all SAML dependencies - might as well download the whole internet
     compile "org.opensaml:opensaml-core:3.3.0"
@@ -79,7 +79,6 @@ sourceSets.test.resources {
     srcDir '../core/src/test/resources'
 }
 dependencyLicenses {
-    mapping from: /bc.*/, to: 'bouncycastle'
     mapping from: /java-support|opensaml-.*/, to: 'shibboleth'
     mapping from: /http.*/, to: 'httpclient'
 }

x-pack/plugin/security/cli/build.gradle

@@ -0,0 +1,20 @@
+apply plugin: 'elasticsearch.build'
+
+archivesBaseName = 'elasticsearch-security-cli'
+
+dependencies {
+    compileOnly "org.elasticsearch:elasticsearch:${version}"
+    compileOnly xpackProject('plugin:core')
+    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
+    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
+    testImplementation 'com.google.jimfs:jimfs:1.1'
+    testCompile "junit:junit:${versions.junit}"
+    testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+    testCompile 'org.elasticsearch:securemock:1.2'
+    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+}
+
+dependencyLicenses {
+    mapping from: /bc.*/, to: 'bouncycastle'
+}

x-pack/plugin/core/licenses/bcpkix-jdk15on-1.59.jar.sha1 renamed to x-pack/plugin/security/cli/licenses/bcpkix-jdk15on-1.59.jar.sha1

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ssl;
+package org.elasticsearch.xpack.security.cli;
 
 import org.bouncycastle.asn1.ASN1Encodable;
 import org.bouncycastle.asn1.ASN1ObjectIdentifier;
@@ -78,7 +78,7 @@ private CertGenUtils() {
      * Generates a CA certificate
      */
     public static X509Certificate generateCACertificate(X500Principal x500Principal, KeyPair keyPair, int days)
-            throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
+        throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
         return generateSignedCertificate(x500Principal, null, keyPair, null, null, true, days, null);
     }
 
@@ -100,7 +100,7 @@ public static X509Certificate generateCACertificate(X500Principal x500Principal,
      */
     public static X509Certificate generateSignedCertificate(X500Principal principal, GeneralNames subjectAltNames, KeyPair keyPair,
                                                             X509Certificate caCert, PrivateKey caPrivKey, int days)
-            throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
+        throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
         return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, false, days, null);
     }
 
@@ -125,7 +125,7 @@ public static X509Certificate generateSignedCertificate(X500Principal principal,
     public static X509Certificate generateSignedCertificate(X500Principal principal, GeneralNames subjectAltNames, KeyPair keyPair,
                                                             X509Certificate caCert, PrivateKey caPrivKey,
                                                             int days, String signatureAlgorithm)
-            throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
+        throws OperatorCreationException, CertificateException, CertIOException, NoSuchAlgorithmException {
         return generateSignedCertificate(principal, subjectAltNames, keyPair, caCert, caPrivKey, false, days, signatureAlgorithm);
     }
 
@@ -150,7 +150,7 @@ public static X509Certificate generateSignedCertificate(X500Principal principal,
     private static X509Certificate generateSignedCertificate(X500Principal principal, GeneralNames subjectAltNames, KeyPair keyPair,
                                                              X509Certificate caCert, PrivateKey caPrivKey, boolean isCa,
                                                              int days, String signatureAlgorithm)
-            throws NoSuchAlgorithmException, CertificateException, CertIOException, OperatorCreationException {
+        throws NoSuchAlgorithmException, CertificateException, CertIOException, OperatorCreationException {
         Objects.requireNonNull(keyPair, "Key-Pair must not be null");
         final DateTime notBefore = new DateTime(DateTimeZone.UTC);
         if (days < 1) {
@@ -175,8 +175,8 @@ private static X509Certificate generateSignedCertificate(X500Principal principal
         }
 
         JcaX509v3CertificateBuilder builder =
-                new JcaX509v3CertificateBuilder(issuer, serial,
-                        new Time(notBefore.toDate(), Locale.ROOT), new Time(notAfter.toDate(), Locale.ROOT), subject, keyPair.getPublic());
+            new JcaX509v3CertificateBuilder(issuer, serial,
+                new Time(notBefore.toDate(), Locale.ROOT), new Time(notAfter.toDate(), Locale.ROOT), subject, keyPair.getPublic());
 
         builder.addExtension(Extension.subjectKeyIdentifier, false, extUtils.createSubjectKeyIdentifier(keyPair.getPublic()));
         builder.addExtension(Extension.authorityKeyIdentifier, false, authorityKeyIdentifier);
@@ -187,8 +187,8 @@ private static X509Certificate generateSignedCertificate(X500Principal principal
 
         PrivateKey signingKey = caPrivKey != null ? caPrivKey : keyPair.getPrivate();
         ContentSigner signer = new JcaContentSignerBuilder(
-                (Strings.isNullOrEmpty(signatureAlgorithm)) ? getDefaultSignatureAlgorithm(signingKey) : signatureAlgorithm)
-                .setProvider(CertGenUtils.BC_PROV).build(signingKey);
+            (Strings.isNullOrEmpty(signatureAlgorithm)) ? getDefaultSignatureAlgorithm(signingKey) : signatureAlgorithm)
+            .setProvider(CertGenUtils.BC_PROV).build(signingKey);
         X509CertificateHolder certificateHolder = builder.build(signer);
         return new JcaX509CertificateConverter().getCertificate(certificateHolder);
     }
@@ -214,7 +214,7 @@ private static String getDefaultSignatureAlgorithm(PrivateKey key) {
                 break;
             default:
                 throw new IllegalArgumentException("Unsupported algorithm : " + key.getAlgorithm()
-                        + " for signature, allowed values for private key algorithm are [RSA, DSA, EC]");
+                    + " for signature, allowed values for private key algorithm are [RSA, DSA, EC]");
         }
         return signatureAlgorithm;
     }
@@ -229,7 +229,7 @@ private static String getDefaultSignatureAlgorithm(PrivateKey key) {
      * @return a certificate signing request
      */
     static PKCS10CertificationRequest generateCSR(KeyPair keyPair, X500Principal principal, GeneralNames sanList)
-            throws IOException, OperatorCreationException {
+        throws IOException, OperatorCreationException {
         Objects.requireNonNull(keyPair, "Key-Pair must not be null");
         Objects.requireNonNull(keyPair.getPublic(), "Public-Key must not be null");
         Objects.requireNonNull(principal, "Principal must not be null");