Remove keystore v1 and v2 formats (#87893)

The keystore format has been changed a few times since it was first
introduced. Part of Elasticsearch startup automatically upgrades the
format. Since Elasticsearch has fixed bounds of supported versions for
upgrades, there are also fixed bounds on the keystore formats we might
need to read.

The v3 keystore format was introduced in Elasticsearch 6.3.0. Since
current Elasticsearch master branch is 8.x, and 8.x only supports
offline upgrades from 7.x, it is therefore impossible to need to read
v1 or v2 formats. This commit removes support for those formats.

Author: rjernst

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java

@@ -34,12 +34,10 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.GeneralSecurityException;
-import java.security.KeyStore;
 import java.security.MessageDigest;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Base64;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
@@ -375,98 +373,6 @@ public void testIllegalSettingName() throws Exception {
         assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
     }
 
-    public void testBackcompatV1() throws Exception {
-        assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
-        Path configDir = env.configFile();
-        try (
-            Directory directory = newFSDirectory(configDir);
-            IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
-        ) {
-            CodecUtil.writeHeader(output, "elasticsearch.keystore", 1);
-            output.writeByte((byte) 0); // hasPassword = false
-            output.writeString("PKCS12");
-            output.writeString("PBE");
-
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
-            KeyStore keystore = KeyStore.getInstance("PKCS12");
-            keystore.load(null, null);
-            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
-            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
-            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
-            keystore.store(keystoreBytesStream, new char[0]);
-            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
-            output.writeInt(keystoreBytes.length);
-            output.writeBytes(keystoreBytes, keystoreBytes.length);
-            CodecUtil.writeFooter(output);
-        }
-
-        KeyStoreWrapper keystore = KeyStoreWrapper.load(configDir);
-        keystore.decrypt(new char[0]);
-        SecureString testValue = keystore.getString("string_setting");
-        assertThat(testValue.toString(), equalTo("stringSecretValue"));
-    }
-
-    public void testBackcompatV2() throws Exception {
-        assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
-        Path configDir = env.configFile();
-        byte[] fileBytes = new byte[20];
-        random().nextBytes(fileBytes);
-        try (
-            Directory directory = newFSDirectory(configDir);
-            IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
-        ) {
-            CodecUtil.writeHeader(output, "elasticsearch.keystore", KeyStoreWrapper.V2_VERSION);
-            output.writeByte((byte) 0); // hasPassword = false
-            output.writeString("PKCS12");
-            output.writeString("PBE"); // string algo
-            output.writeString("PBE"); // file algo
-
-            output.writeVInt(2); // num settings
-            output.writeString("string_setting");
-            output.writeString("STRING");
-            output.writeString("file_setting");
-            output.writeString("FILE");
-
-            SecretKeyFactory secretFactory = SecretKeyFactory.getInstance("PBE");
-            KeyStore keystore = KeyStore.getInstance("PKCS12");
-            keystore.load(null, null);
-            SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec("stringSecretValue".toCharArray()));
-            KeyStore.ProtectionParameter protectionParameter = new KeyStore.PasswordProtection(new char[0]);
-            keystore.setEntry("string_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            byte[] base64Bytes = Base64.getEncoder().encode(fileBytes);
-            char[] chars = new char[base64Bytes.length];
-            for (int i = 0; i < chars.length; ++i) {
-                chars[i] = (char) base64Bytes[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
-            }
-            secretKey = secretFactory.generateSecret(new PBEKeySpec(chars));
-            keystore.setEntry("file_setting", new KeyStore.SecretKeyEntry(secretKey), protectionParameter);
-
-            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
-            keystore.store(keystoreBytesStream, new char[0]);
-            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
-            output.writeInt(keystoreBytes.length);
-            output.writeBytes(keystoreBytes, keystoreBytes.length);
-            CodecUtil.writeFooter(output);
-        }
-
-        KeyStoreWrapper keystore = KeyStoreWrapper.load(configDir);
-        keystore.decrypt(new char[0]);
-        SecureString testValue = keystore.getString("string_setting");
-        assertThat(testValue.toString(), equalTo("stringSecretValue"));
-
-        try (InputStream fileInput = keystore.getFile("file_setting")) {
-            byte[] readBytes = new byte[20];
-            assertEquals(20, fileInput.read(readBytes));
-            for (int i = 0; i < fileBytes.length; ++i) {
-                assertThat("byte " + i, readBytes[i], equalTo(fileBytes[i]));
-            }
-            assertEquals(-1, fileInput.read());
-        }
-    }
-
     public void testBackcompatV4() throws Exception {
         assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
         Path configDir = env.configFile();

server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java

@@ -43,13 +43,9 @@
 import java.nio.file.attribute.PosixFileAttributeView;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.security.GeneralSecurityException;
-import java.security.KeyStore;
 import java.security.SecureRandom;
 import java.util.Arrays;
-import java.util.Base64;
-import java.util.Enumeration;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
@@ -109,7 +105,7 @@ private static class Entry {
     public static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
 
     /** The oldest metadata format version that can be read. */
-    private static final int MIN_FORMAT_VERSION = 1;
+    private static final int MIN_FORMAT_VERSION = 3;
     /** Legacy versions of the metadata written before the keystore data. */
     public static final int V2_VERSION = 2;
     public static final int V3_VERSION = 3;
@@ -268,59 +264,15 @@ public static KeyStoreWrapper load(Path configDir) throws IOException {
                 throw new IllegalStateException("hasPassword boolean is corrupt: " + String.format(Locale.ROOT, "%02x", hasPasswordByte));
             }
 
-            if (formatVersion <= V2_VERSION) {
-                String type = input.readString();
-                if (type.equals("PKCS12") == false) {
-                    throw new IllegalStateException("Corrupted legacy keystore string encryption algorithm");
-                }
-
-                final String stringKeyAlgo = input.readString();
-                if (stringKeyAlgo.equals("PBE") == false) {
-                    throw new IllegalStateException("Corrupted legacy keystore string encryption algorithm");
-                }
-                if (formatVersion == V2_VERSION) {
-                    final String fileKeyAlgo = input.readString();
-                    if (fileKeyAlgo.equals("PBE") == false) {
-                        throw new IllegalStateException("Corrupted legacy keystore file encryption algorithm");
-                    }
-                }
-            }
-
             final byte[] dataBytes;
-            if (formatVersion == V2_VERSION) {
-                // For v2 we had a map of strings containing the types for each setting. In v3 this map is now
-                // part of the encrypted bytes. Unfortunately we cannot seek backwards with checksum input, so
-                // we cannot just read the map and find out how long it is. So instead we read the map and
-                // store it back using java's builtin DataOutput in a byte array, along with the actual keystore bytes
-                Map<String, String> settingTypes = input.readMapOfStrings();
-                ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-                try (DataOutputStream output = new DataOutputStream(bytes)) {
-                    output.writeInt(settingTypes.size());
-                    for (Map.Entry<String, String> entry : settingTypes.entrySet()) {
-                        output.writeUTF(entry.getKey());
-                        output.writeUTF(entry.getValue());
-                    }
-                    final int keystoreLen;
-                    if (formatVersion < LE_VERSION) {
-                        keystoreLen = Integer.reverseBytes(input.readInt());
-                    } else {
-                        keystoreLen = input.readInt();
-                    }
-                    byte[] keystoreBytes = new byte[keystoreLen];
-                    input.readBytes(keystoreBytes, 0, keystoreLen);
-                    output.write(keystoreBytes);
-                }
-                dataBytes = bytes.toByteArray();
+            int dataBytesLen;
+            if (formatVersion < LE_VERSION) {
+                dataBytesLen = Integer.reverseBytes(input.readInt());
             } else {
-                int dataBytesLen;
-                if (formatVersion < LE_VERSION) {
-                    dataBytesLen = Integer.reverseBytes(input.readInt());
-                } else {
-                    dataBytesLen = input.readInt();
-                }
-                dataBytes = new byte[dataBytesLen];
-                input.readBytes(dataBytes, 0, dataBytesLen);
+                dataBytesLen = input.readInt();
             }
+            dataBytes = new byte[dataBytesLen];
+            input.readBytes(dataBytes, 0, dataBytesLen);
 
             CodecUtil.checkFooter(input);
             return new KeyStoreWrapper(formatVersion, hasPassword, dataBytes);
@@ -378,13 +330,6 @@ public void decrypt(char[] password) throws GeneralSecurityException, IOExceptio
         if (entries.get() != null) {
             throw new IllegalStateException("Keystore has already been decrypted");
         }
-        if (formatVersion <= V2_VERSION) {
-            decryptLegacyEntries();
-            if (password.length != 0) {
-                throw new IllegalArgumentException("Keystore format does not accept non-empty passwords");
-            }
-            return;
-        }
 
         final byte[] salt;
         final byte[] iv;
@@ -462,79 +407,6 @@ private byte[] encrypt(char[] password, byte[] salt, byte[] iv) throws GeneralSe
         return bytes.toByteArray();
     }
 
-    private void decryptLegacyEntries() throws GeneralSecurityException, IOException {
-        // v1 and v2 keystores never had passwords actually used, so we always use an empty password
-        KeyStore keystore = KeyStore.getInstance("PKCS12");
-        Map<String, EntryType> settingTypes = new HashMap<>();
-        ByteArrayInputStream inputBytes = new ByteArrayInputStream(dataBytes);
-        try (DataInputStream input = new DataInputStream(inputBytes)) {
-            // first read the setting types map
-            if (formatVersion == V2_VERSION) {
-                int numSettings = input.readInt();
-                for (int i = 0; i < numSettings; ++i) {
-                    String key = input.readUTF();
-                    String value = input.readUTF();
-                    settingTypes.put(key, EntryType.valueOf(value));
-                }
-            }
-            // then read the actual keystore
-            keystore.load(input, "".toCharArray());
-        }
-
-        // verify the settings metadata matches the keystore entries
-        Enumeration<String> aliases = keystore.aliases();
-        if (formatVersion == MIN_FORMAT_VERSION) {
-            while (aliases.hasMoreElements()) {
-                settingTypes.put(aliases.nextElement(), EntryType.STRING);
-            }
-        } else {
-            // verify integrity: keys in keystore match what the metadata thinks exist
-            Set<String> expectedSettings = new HashSet<>(settingTypes.keySet());
-            while (aliases.hasMoreElements()) {
-                String settingName = aliases.nextElement();
-                if (expectedSettings.remove(settingName) == false) {
-                    throw new SecurityException("Keystore has been corrupted or tampered with");
-                }
-            }
-            if (expectedSettings.isEmpty() == false) {
-                throw new SecurityException("Keystore has been corrupted or tampered with");
-            }
-        }
-
-        // fill in the entries now that we know all the types to expect
-        this.entries.set(new HashMap<>());
-        SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBE");
-        KeyStore.PasswordProtection password = new KeyStore.PasswordProtection("".toCharArray());
-
-        for (Map.Entry<String, EntryType> settingEntry : settingTypes.entrySet()) {
-            String setting = settingEntry.getKey();
-            EntryType settingType = settingEntry.getValue();
-            KeyStore.SecretKeyEntry keystoreEntry = (KeyStore.SecretKeyEntry) keystore.getEntry(setting, password);
-            PBEKeySpec keySpec = (PBEKeySpec) keyFactory.getKeySpec(keystoreEntry.getSecretKey(), PBEKeySpec.class);
-            char[] chars = keySpec.getPassword();
-            keySpec.clearPassword();
-
-            final byte[] bytes;
-            if (settingType == EntryType.STRING) {
-                ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(CharBuffer.wrap(chars));
-                bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
-                Arrays.fill(byteBuffer.array(), (byte) 0);
-            } else {
-                assert settingType == EntryType.FILE;
-                // The PBE keyspec gives us chars, we convert to bytes
-                byte[] tmpBytes = new byte[chars.length];
-                for (int i = 0; i < tmpBytes.length; ++i) {
-                    tmpBytes[i] = (byte) chars[i]; // PBE only stores the lower 8 bits, so this narrowing is ok
-                }
-                bytes = Base64.getDecoder().decode(tmpBytes);
-                Arrays.fill(tmpBytes, (byte) 0);
-            }
-            Arrays.fill(chars, '\0');
-
-            entries.get().put(setting, new Entry(bytes));
-        }
-    }
-
     /** Write the keystore to the given config directory. */
     public synchronized void save(Path configDir, char[] password) throws Exception {
         save(configDir, password, true);