Enroll additional nodes to cluster (#77292)

This introduces a new CLI tool `elasticsearch-enroll-node`.
It takes an Enrollment Token as a parameter and using the
information in it, it attempts to

- Communicate with an existing node of the cluster
- Receive necessary key/certificate material
- Persist said material and configuration

This tool needs to be run before the first time the current
node starts and if it doesn't have any explicit security
related configuration already defined.

Author: jkakavas

docs/changelog/77292.yaml

@@ -0,0 +1,5 @@
+pr: 77292
+summary: Enroll additional nodes to cluster
+area: "Security"
+type: enhancement
+issues: []

libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/PemUtils.java

@@ -207,7 +207,18 @@ private static PrivateKey parsePKCS8(BufferedReader bReader) throws IOException,
         if (null == line || PKCS8_FOOTER.equals(line.trim()) == false) {
             throw new IOException("Malformed PEM file, PEM footer is invalid or missing");
         }
-        byte[] keyBytes = Base64.getDecoder().decode(sb.toString());
+        return parsePKCS8PemString(sb.toString());
+    }
+
+    /**
+     * Creates a {@link PrivateKey} from a String that contains the PEM encoded representation of a plaintext private key encoded in PKCS8
+     * @param pemString the PEM encoded representation of a plaintext private key encoded in PKCS8
+     * @return {@link PrivateKey}
+     * @throws IOException if the algorithm identifier can not be parsed from DER
+     * @throws GeneralSecurityException if the private key can't be generated from the {@link PKCS8EncodedKeySpec}
+     */
+    public static PrivateKey parsePKCS8PemString(String pemString) throws IOException, GeneralSecurityException{
+        byte[] keyBytes = Base64.getDecoder().decode(pemString);
         String keyAlgo = getKeyAlgorithmIdentifier(keyBytes);
         KeyFactory keyFactory = KeyFactory.getInstance(keyAlgo);
         return keyFactory.generatePrivate(new PKCS8EncodedKeySpec(keyBytes));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/CommandLineHttpClient.java renamed to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/CommandLineHttpClient.java

@@ -4,9 +4,11 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.security.tool;
+package org.elasticsearch.xpack.core.security;
 
+import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.core.CharArrays;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Strings;
@@ -23,9 +25,12 @@
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.ssl.SSLService;
-import org.elasticsearch.xpack.security.tool.HttpResponse.HttpResponseBuilder;
+import org.elasticsearch.xpack.core.security.HttpResponse.HttpResponseBuilder;
 
 import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -34,9 +39,16 @@
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.CharBuffer;
 import java.nio.charset.StandardCharsets;
 import java.security.AccessController;
-import java.security.PrivilegedAction;
+import java.security.MessageDigest;
+import java.security.PrivilegedExceptionAction;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -60,9 +72,16 @@ public class CommandLineHttpClient {
     private static final int READ_TIMEOUT = 35 * 1000;
 
     private final Environment env;
+    private final String pinnedCaCertFingerprint;
 
     public CommandLineHttpClient(Environment env) {
         this.env = env;
+        this.pinnedCaCertFingerprint = null;
+    }
+
+    public CommandLineHttpClient(Environment env, String pinnedCaCertFingerprint) {
+        this.env = env;
+        this.pinnedCaCertFingerprint = pinnedCaCertFingerprint;
     }
 
     /**
@@ -79,22 +98,55 @@ public CommandLineHttpClient(Environment env) {
      *            handler of the response Input Stream.
      * @return HTTP protocol response code.
      */
-    @SuppressForbidden(reason = "We call connect in doPrivileged and provide SocketPermission")
     public HttpResponse execute(String method, URL url, String user, SecureString password,
                                 CheckedSupplier<String, Exception> requestBodySupplier,
                                 CheckedFunction<InputStream, HttpResponseBuilder, Exception> responseHandler) throws Exception {
+
+        final String authorizationHeader = UsernamePasswordToken.basicAuthHeaderValue(user, password);
+        return execute(method, url, authorizationHeader, requestBodySupplier, responseHandler);
+    }
+
+    /**
+     * General purpose HTTP(S) call with JSON Content-Type and Authorization Header.
+     * SSL settings are read from the settings file, if any.
+     *
+     * @param apiKey
+     *            API key value to be used in the Authorization header
+     * @param requestBodySupplier
+     *            supplier for the JSON string body of the request.
+     * @param responseHandler
+     *            handler of the response Input Stream.
+     * @return HTTP protocol response code.
+     */
+    public HttpResponse execute(String method, URL url, SecureString apiKey,
+        CheckedSupplier<String, Exception> requestBodySupplier,
+        CheckedFunction<InputStream, HttpResponseBuilder, Exception> responseHandler) throws Exception {
+        final String authorizationHeaderValue = apiKeyHeaderValue(apiKey);
+        return execute(method, url, authorizationHeaderValue, requestBodySupplier, responseHandler);
+    }
+
+    @SuppressForbidden(reason = "We call connect in doPrivileged and provide SocketPermission")
+    private HttpResponse execute(String method, URL url, String authorizationHeader,
+        CheckedSupplier<String, Exception> requestBodySupplier,
+        CheckedFunction<InputStream, HttpResponseBuilder, Exception> responseHandler) throws Exception {
         final HttpURLConnection conn;
         // If using SSL, need a custom service because it's likely a self-signed certificate
         if ("https".equalsIgnoreCase(url.getProtocol())) {
             final SSLService sslService = new SSLService(env);
             final HttpsURLConnection httpsConn = (HttpsURLConnection) url.openConnection();
-            AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
-                final SslConfiguration sslConfiguration = sslService.getHttpTransportSSLConfiguration();
-                // Requires permission java.lang.RuntimePermission "setFactory";
-                httpsConn.setSSLSocketFactory(sslService.sslSocketFactory(sslConfiguration));
-                final boolean isHostnameVerificationEnabled = sslConfiguration.getVerificationMode().isHostnameVerificationEnabled();
-                if (isHostnameVerificationEnabled == false) {
-                    httpsConn.setHostnameVerifier((hostname, session) -> true);
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                if (pinnedCaCertFingerprint != null) {
+                    final SSLContext sslContext = SSLContext.getInstance("TLS");
+                    sslContext.init(null, new TrustManager[] { fingerprintTrustingTrustManager(pinnedCaCertFingerprint) }, null);
+                    httpsConn.setSSLSocketFactory(sslContext.getSocketFactory());
+                } else {
+                    final SslConfiguration sslConfiguration = sslService.getHttpTransportSSLConfiguration();
+                    // Requires permission java.lang.RuntimePermission "setFactory";
+                    httpsConn.setSSLSocketFactory(sslService.sslSocketFactory(sslConfiguration));
+                    final boolean isHostnameVerificationEnabled = sslConfiguration.getVerificationMode().isHostnameVerificationEnabled();
+                    if (isHostnameVerificationEnabled == false) {
+                        httpsConn.setHostnameVerifier((hostname, session) -> true);
+                    }
                 }
                 return null;
             });
@@ -105,8 +157,7 @@ public HttpResponse execute(String method, URL url, String user, SecureString pa
         conn.setRequestMethod(method);
         conn.setReadTimeout(READ_TIMEOUT);
         // Add basic-auth header
-        String token = UsernamePasswordToken.basicAuthHeaderValue(user, password);
-        conn.setRequestProperty("Authorization", token);
+        conn.setRequestProperty("Authorization", authorizationHeader);
         conn.setRequestProperty("Content-Type", XContentType.JSON.mediaType());
         String bodyString = requestBodySupplier.get();
         conn.setDoOutput(bodyString != null); // set true if we are sending a body
@@ -253,4 +304,48 @@ public static HttpResponse.HttpResponseBuilder responseBuilder(InputStream is) t
     public static URL createURL(URL url, String path, String query) throws MalformedURLException, URISyntaxException {
         return new URL(url, (url.toURI().getPath() + path).replaceAll("/+", "/") + query);
     }
+
+    public static String apiKeyHeaderValue(SecureString apiKey) {
+        CharBuffer chars = CharBuffer.allocate(apiKey.length());
+        byte[] charBytes = null;
+        try {
+            chars.put(apiKey.getChars());
+            charBytes = CharArrays.toUtf8Bytes(chars.array());
+
+            //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence?
+            String apiKeyToken = Base64.getEncoder().encodeToString(charBytes);
+            return "ApiKey " + apiKeyToken;
+        } finally {
+            Arrays.fill(chars.array(), (char) 0);
+            if (charBytes != null) {
+                Arrays.fill(charBytes, (byte) 0);
+            }
+        }
+    }
+
+    /**
+     * Returns a TrustManager to be used in a client SSLContext, which trusts all certificates that are signed
+     * by a specific CA certificate ( identified by its SHA256 fingerprint, {@code pinnedCaCertFingerPrint} )
+     */
+    private TrustManager fingerprintTrustingTrustManager(String pinnedCaCertFingerprint) {
+        final TrustManager trustManager = new X509TrustManager() {
+            public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+            }
+
+            public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                final Certificate caCertFromChain = chain[1];
+                MessageDigest sha256 = MessageDigests.sha256();
+                sha256.update(caCertFromChain.getEncoded());
+                if (MessageDigests.toHexString(sha256.digest()).equals(pinnedCaCertFingerprint) == false ) {
+                    throw new CertificateException();
+                }
+            }
+
+            @Override public X509Certificate[] getAcceptedIssuers() {
+                return new X509Certificate[0];
+            }
+        };
+
+        return trustManager;
+    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/EnrollmentToken.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ParseField;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class EnrollmentToken {
+    private final String apiKey;
+    private final String fingerprint;
+    private final String version;
+    private final List<String > boundAddress;
+
+    public String getApiKey() { return apiKey; }
+    public String getFingerprint() { return fingerprint; }
+    public String getVersion() { return version; }
+    public List<String> getBoundAddress() { return boundAddress; }
+
+    private static final ParseField API_KEY = new ParseField("key");
+    private static final ParseField FINGERPRINT = new ParseField("fgr");
+    private static final ParseField VERSION = new ParseField("ver");
+    private static final ParseField ADDRESS = new ParseField("adr");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<EnrollmentToken, Void> PARSER = new ConstructingObjectParser<>("enrollment_token", false,
+        a -> new EnrollmentToken((String) a[0], (String) a[1], (String) a[2], (List<String>) a[3]));
+
+    static {
+        PARSER.declareString(constructorArg(), API_KEY);
+        PARSER.declareString(constructorArg(), FINGERPRINT);
+        PARSER.declareString(constructorArg(), VERSION);
+        PARSER.declareStringArray(constructorArg(), ADDRESS);
+    }
+    /**
+     * Create an EnrollmentToken
+     *
+     * @param apiKey         API Key credential in the form apiKeyId:ApiKeySecret to be used for enroll calls
+     * @param fingerprint    hex encoded SHA256 fingerprint of the HTTP CA cert
+     * @param version        node version number
+     * @param boundAddress   IP Addresses and port numbers for the interfaces where the Elasticsearch node is listening on
+     */
+    public EnrollmentToken(String apiKey, String fingerprint, String version, List<String> boundAddress) {
+        this.apiKey = Objects.requireNonNull(apiKey);
+        this.fingerprint = Objects.requireNonNull(fingerprint);
+        this.version = Objects.requireNonNull(version);
+        this.boundAddress = Objects.requireNonNull(boundAddress);
+    }
+
+    public String getRaw() throws Exception {
+        final XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        builder.field("ver", version);
+        builder.startArray("adr");
+        for (String bound_address : boundAddress) {
+            builder.value(bound_address);
+        }
+        builder.endArray();
+        builder.field("fgr", fingerprint);
+        builder.field("key", apiKey);
+        builder.endObject();
+        return Strings.toString(builder);
+    }
+
+    public String getEncoded() throws Exception {
+        final String jsonString = getRaw();
+        return Base64.getUrlEncoder().encodeToString(jsonString.getBytes(StandardCharsets.UTF_8));
+    }
+
+    /**
+     * Decodes and parses an enrollment token from its serialized form (created with {@link EnrollmentToken#getEncoded()}
+     * @param encoded The Base64 encoded JSON representation of the enrollment token
+     * @return the parsed EnrollmentToken
+     * @throws IOException when failing to decode the serialized token
+     */
+    public static EnrollmentToken decodeFromString(String encoded) throws IOException {
+        if (Strings.isNullOrEmpty(encoded)) {
+            throw new IOException("Cannot decode enrollment token from an empty string");
+        }
+        final XContentParser jsonParser = JsonXContent.jsonXContent.createParser(
+            NamedXContentRegistry.EMPTY,
+            DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+            Base64.getDecoder().decode(encoded)
+        );
+        return EnrollmentToken.PARSER.parse(jsonParser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EnrollmentToken that = (EnrollmentToken) o;
+        return apiKey.equals(that.apiKey) && fingerprint.equals(that.fingerprint) && version.equals(that.version) && boundAddress.equals(
+            that.boundAddress);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(apiKey, fingerprint, version, boundAddress);
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/tool/HttpResponse.java renamed to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/HttpResponse.java

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.security.tool;
+package org.elasticsearch.xpack.core.security;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.xcontent.XContentHelper;

x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy

@@ -1,4 +1,6 @@
 grant {
+  // CommandLineHttpClient
+  permission java.lang.RuntimePermission "setFactory";
   // bouncy castle
   permission java.security.SecurityPermission "putProviderProperty.BC";