Repository uses passwords again

Author: albertzaharovits

x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/AESKeyUtils.java

@@ -9,15 +9,23 @@
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
 import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
 import javax.crypto.spec.SecretKeySpec;
 import java.nio.charset.StandardCharsets;
-import java.security.GeneralSecurityException;
 import java.security.InvalidKeyException;
 import java.security.Key;
 import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
 import java.util.Base64;
 
 public final class AESKeyUtils {
+    public static final int KEY_LENGTH_IN_BYTES = 32; // 256-bit AES key
+    public static final int WRAPPED_KEY_LENGTH_IN_BYTES = KEY_LENGTH_IN_BYTES + 8; // https://www.ietf.org/rfc/rfc3394.txt section 2.2
+    // parameter for the KDF function, it's a funny and unusual iter count larger than 60k
+    private static final int KDF_ITER = 61616;
+    // the KDF algorithm that generate the symmetric key given the password
+    private static final String KDF_ALGO = "PBKDF2WithHmacSHA512";
     // The Id of any AES SecretKey is the AES-Wrap-ciphertext of this fixed 32 byte wide array.
     // Key wrapping encryption is deterministic (same plaintext generates the same ciphertext)
     // and the probability that two different keys map the same plaintext to the same ciphertext is very small
@@ -43,6 +51,9 @@ public static SecretKey unwrap(SecretKey wrappingKey, byte[] keyToUnwrap) throws
         if (false == "AES".equals(wrappingKey.getAlgorithm())) {
             throw new IllegalArgumentException("wrappingKey argument is not an AES Key");
         }
+        if (keyToUnwrap.length != WRAPPED_KEY_LENGTH_IN_BYTES) {
+            throw new IllegalArgumentException("keyToUnwrap invalid length [" + keyToUnwrap.length + "]");
+        }
         Cipher c = Cipher.getInstance("AESWrap");
         c.init(Cipher.UNWRAP_MODE, wrappingKey);
         Key unwrappedKey = c.unwrap(keyToUnwrap, "AES", Cipher.SECRET_KEY);
@@ -59,8 +70,18 @@ public static SecretKey unwrap(SecretKey wrappingKey, byte[] keyToUnwrap) throws
      * Moreover, the ciphertext reveals no information on the key, and the probability of collision of ciphertexts given different
      * keys is statistically negligible.
      */
-    public static String computeId(SecretKey secretAESKey) throws GeneralSecurityException {
+    public static String computeId(SecretKey secretAESKey) throws IllegalBlockSizeException, InvalidKeyException,
+            NoSuchAlgorithmException, NoSuchPaddingException {
         byte[] ciphertextOfKnownPlaintext = wrap(secretAESKey, new SecretKeySpec(KEY_ID_PLAINTEXT, "AES"));
         return new String(Base64.getUrlEncoder().withoutPadding().encode(ciphertextOfKnownPlaintext), StandardCharsets.UTF_8);
     }
+
+    public static SecretKey generatePasswordBasedKey(char[] password, byte[] salt) throws NoSuchAlgorithmException,
+            InvalidKeySpecException {
+        PBEKeySpec keySpec = new PBEKeySpec(password, salt, KDF_ITER, KEY_LENGTH_IN_BYTES * Byte.SIZE);
+        SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(KDF_ALGO);
+        SecretKey secretKey = keyFactory.generateSecret(keySpec);
+        SecretKeySpec secret = new SecretKeySpec(secretKey.getEncoded(), "AES");
+        return secret;
+    }
 }

x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
@@ -24,11 +25,6 @@
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 import org.elasticsearch.xpack.core.XPackPlugin;
 
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -39,12 +35,11 @@
 public class EncryptedRepositoryPlugin extends Plugin implements RepositoryPlugin {
     static final Logger logger = LogManager.getLogger(EncryptedRepositoryPlugin.class);
     static final String REPOSITORY_TYPE_NAME = "encrypted";
-    static final Setting.AffixSetting<InputStream> KEY_ENCRYPTION_KEY_SETTING = Setting.affixKeySetting("repository.encrypted.",
-            "key", key -> SecureSetting.secureFile(key, null));
+    static final List<String> SUPPORTED_ENCRYPTED_TYPE_NAMES = Arrays.asList("fs", "gcs", "azure", "s3");
+    static final Setting.AffixSetting<SecureString> ENCRYPTION_PASSWORD_SETTING = Setting.affixKeySetting("repository.encrypted.",
+            "password", key -> SecureSetting.secureString(key, null));
     static final Setting<String> DELEGATE_TYPE_SETTING = Setting.simpleString("delegate_type", "");
-    static final Setting<String> KEK_NAME_SETTING = Setting.simpleString("key_name", "");
-    static final String KEK_CIPHER_ALGO = "AES";
-    static final int KEK_LENGTH_IN_BYTES = 32; // 256-bit AES symmetric key
+    static final Setting<String> PASSWORD_NAME_SETTING = Setting.simpleString("password_name", "");
 
     // "protected" because it is overloaded for tests
     protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
@@ -59,37 +54,22 @@ public EncryptedRepositoryPlugin() {
 
     @Override
     public List<Setting<?>> getSettings() {
-        return List.of(KEY_ENCRYPTION_KEY_SETTING);
+        return List.of(ENCRYPTION_PASSWORD_SETTING);
     }
 
     @Override
     public Map<String, Repository.Factory> getRepositories(Environment env,
                                                            NamedXContentRegistry registry,
                                                            ClusterService clusterService) {
-        // store all KEKs from the keystore in memory (because the keystore is not readable later on)
-        final Map<String, SecretKey> repositoryKEKMapBuilder = new HashMap<>();
-        for (String KEKName : KEY_ENCRYPTION_KEY_SETTING.getNamespaces(env.settings())) {
-            final Setting<InputStream> KEKSetting = KEY_ENCRYPTION_KEY_SETTING.getConcreteSettingForNamespace(KEKName);
-            final SecretKey KEK;
-            byte[] encodedKEKBytes = null;
-            try (InputStream KEKInputStream = KEKSetting.get(env.settings())) {
-                encodedKEKBytes = KEKInputStream.readAllBytes();
-                if (encodedKEKBytes.length != KEK_LENGTH_IN_BYTES) {
-                    throw new IllegalArgumentException("Expected a 32 bytes (256 bit) wide AES key, but key ["
-                            + KEKSetting.getKey() + "] is [" + encodedKEKBytes.length + "] bytes wide");
-                }
-                KEK = new SecretKeySpec(encodedKEKBytes, 0, KEK_LENGTH_IN_BYTES, KEK_CIPHER_ALGO);
-            } catch (IOException e) {
-                throw new UncheckedIOException("Exception while reading [" + KEKName + "] from the node keystore", e);
-            } finally {
-                if (encodedKEKBytes != null) {
-                    Arrays.fill(encodedKEKBytes, (byte) 0);
-                }
-            }
-            logger.debug(() -> new ParameterizedMessage("Loaded repository AES key [" + KEKName + "] from the node keystore"));
-            repositoryKEKMapBuilder.put(KEKName, KEK);
+        // load all the passwords from the keystore in memory because the keystore is not readable when the repository is created
+        final Map<String, char[]> repositoryPasswordsMapBuilder = new HashMap<>();
+        for (String passwordName : ENCRYPTION_PASSWORD_SETTING.getNamespaces(env.settings())) {
+            Setting<SecureString> passwordSetting = ENCRYPTION_PASSWORD_SETTING.getConcreteSettingForNamespace(passwordName);
+            SecureString encryptionPassword = passwordSetting.get(env.settings());
+            repositoryPasswordsMapBuilder.put(passwordName, encryptionPassword.getChars());
+            logger.debug(() -> new ParameterizedMessage("Loaded repository password [" + passwordName + "] from the node keystore"));
         }
-        final Map<String, SecretKey> repositoryKEKMap = Map.copyOf(repositoryKEKMapBuilder);
+        final Map<String, char[]> repositoryPasswordsMap = Map.copyOf(repositoryPasswordsMapBuilder);
 
         return Collections.singletonMap(REPOSITORY_TYPE_NAME, new Repository.Factory() {
 
@@ -109,23 +89,23 @@ public Repository create(RepositoryMetaData metaData, Function<String, Repositor
                             DELEGATE_TYPE_SETTING.getKey() + "] must not be equal to [" + REPOSITORY_TYPE_NAME + "]");
                 }
                 final Repository.Factory factory = typeLookup.apply(delegateType);
-                if (null == factory) {
-                    throw new IllegalArgumentException("Unrecognized delegate repository type [" + DELEGATE_TYPE_SETTING.getKey() + "]");
+                if (null == factory || false == SUPPORTED_ENCRYPTED_TYPE_NAMES.contains(delegateType)) {
+                    throw new IllegalArgumentException("Unsupported delegate repository type [" + DELEGATE_TYPE_SETTING.getKey() + "]");
                 }
-                final String repositoryKEKName = KEK_NAME_SETTING.get(metaData.settings());
-                if (Strings.hasLength(repositoryKEKName) == false) {
-                    throw new IllegalArgumentException("Repository setting [" + KEK_NAME_SETTING.getKey() + "] must be set");
+                final String repositoryPasswordName = PASSWORD_NAME_SETTING.get(metaData.settings());
+                if (Strings.hasLength(repositoryPasswordName) == false) {
+                    throw new IllegalArgumentException("Repository setting [" + PASSWORD_NAME_SETTING.getKey() + "] must be set");
                 }
-                final SecretKey repositoryKEK = repositoryKEKMap.get(repositoryKEKName);
-                if (repositoryKEK == null) {
+                final char[] repositoryPassword = repositoryPasswordsMap.get(repositoryPasswordName);
+                if (repositoryPassword == null) {
                     throw new IllegalArgumentException("Secure setting [" +
-                            KEY_ENCRYPTION_KEY_SETTING.getConcreteSettingForNamespace(repositoryKEKName).getKey() + "] must be set");
+                            ENCRYPTION_PASSWORD_SETTING.getConcreteSettingForNamespace(repositoryPasswordName).getKey() + "] must be set");
                 }
                 final Repository delegatedRepository = factory.create(new RepositoryMetaData(metaData.name(), delegateType,
                         metaData.settings()));
                 if (false == (delegatedRepository instanceof BlobStoreRepository)
                         || delegatedRepository instanceof EncryptedRepository) {
-                    throw new IllegalArgumentException("Unsupported delegate type [" + DELEGATE_TYPE_SETTING.getKey() + "]");
+                    throw new IllegalArgumentException("Unsupported delegate repository type [" + DELEGATE_TYPE_SETTING.getKey() + "]");
                 }
                 if (false == getLicenseState().isEncryptedSnapshotAllowed()) {
                     logger.warn("Encrypted snapshots are not allowed for the currently installed license." +
@@ -134,7 +114,7 @@ public Repository create(RepositoryMetaData metaData, Function<String, Repositor
                             LicenseUtils.newComplianceException("encrypted snapshots"));
                 }
                 return new EncryptedRepository(metaData, registry, clusterService, (BlobStoreRepository) delegatedRepository,
-                        () -> getLicenseState(), repositoryKEK);
+                        () -> getLicenseState(), repositoryPassword);
             }
         });
     }

x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.repositories.azure.AzureBlobStoreRepositoryTests;
 import org.junit.BeforeClass;
 
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -52,11 +51,8 @@ protected Settings nodeSettings(int nodeOrdinal) {
     protected MockSecureSettings nodeSecureSettings() {
         MockSecureSettings secureSettings = new MockSecureSettings();
         for (String repositoryName : repositoryNames) {
-            byte[] repositoryNameBytes = repositoryName.getBytes(StandardCharsets.UTF_8);
-            byte[] repositoryKEK = new byte[32];
-            System.arraycopy(repositoryNameBytes, 0, repositoryKEK, 0, repositoryNameBytes.length);
-            secureSettings.setFile(EncryptedRepositoryPlugin.KEY_ENCRYPTION_KEY_SETTING.
-                    getConcreteSettingForNamespace(repositoryName).getKey(), repositoryKEK);
+            secureSettings.setString(EncryptedRepositoryPlugin.ENCRYPTION_PASSWORD_SETTING.
+                    getConcreteSettingForNamespace(repositoryName).getKey(), repositoryName);
         }
         return secureSettings;
     }
@@ -81,7 +77,7 @@ protected Settings repositorySettings(String repositoryName) {
         return Settings.builder()
                 .put(super.repositorySettings())
                 .put(EncryptedRepositoryPlugin.DELEGATE_TYPE_SETTING.getKey(), "azure")
-                .put(EncryptedRepositoryPlugin.KEK_NAME_SETTING.getKey(), repositoryName)
+                .put(EncryptedRepositoryPlugin.PASSWORD_NAME_SETTING.getKey(), repositoryName)
                 .build();
     }