Skip to content

Commit 8ae9090

Browse files
albertzaharovitsalbertzaharovits
committed
Packet-based enc/dec cipher streams (#49896)
This adds a new bare snapshot repository project which contains the classes implementing encryption (and decryption) input stream decorators that support mark and reset. Relates #48221 , #46170
1 parent a24bc6e commit 8ae9090

17 files changed

+4669
-0
lines changed

‎x-pack/plugin/repository-encrypted/build.gradle

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
evaluationDependsOn(xpackModule('core'))
2+
3+
apply plugin: 'elasticsearch.esplugin'
4+
esplugin {
5+
name 'repository-encrypted'
6+
description 'Elasticsearch Expanded Pack Plugin - client-side encrypted repositories.'
7+
classname 'org.elasticsearch.repositories.encrypted.EncryptedRepositoryPlugin'
8+
extendedPlugins = ['x-pack-core']
9+
}
10+
11+
integTest.enabled = false

‎x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/BufferOnMarkInputStream.java

+546
Large diffs are not rendered by default.

‎x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/ChainingInputStream.java

+375
Large diffs are not rendered by default.

‎x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/CountingInputStream.java

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.repositories.encrypted;
7+
8+
import java.io.IOException;
9+
import java.io.InputStream;
10+
import java.util.Objects;
11+
12+
/**
13+
* A {@code CountingInputStream} wraps another input stream and counts the number of bytes
14+
* that have been read or skipped.
15+
* <p>
16+
* This input stream does no buffering on its own and only supports {@code mark} and
17+
* {@code reset} if the underlying wrapped stream supports it.
18+
* <p>
19+
* If the stream supports {@code mark} and {@code reset} the byte count is also reset to the
20+
* value that it had on the last {@code mark} call, thereby not counting the same bytes twice.
21+
* <p>
22+
* If the {@code closeSource} constructor argument is {@code true}, closing this
23+
* stream will also close the wrapped input stream. Apart from closing the wrapped
24+
* stream in this case, the {@code close} method does nothing else.
25+
*/
26+
public final class CountingInputStream extends InputStream {
27+
28+
private final InputStream source;
29+
private final boolean closeSource;
30+
long count; // package-protected for tests
31+
long mark; // package-protected for tests
32+
boolean closed; // package-protected for tests
33+
34+
/**
35+
* Wraps another input stream, counting the number of bytes read.
36+
*
37+
* @param source the input stream to be wrapped
38+
* @param closeSource {@code true} if closing this stream will also close the wrapped stream
39+
*/
40+
public CountingInputStream(InputStream source, boolean closeSource) {
41+
this.source = Objects.requireNonNull(source);
42+
this.closeSource = closeSource;
43+
this.count = 0L;
44+
this.mark = -1L;
45+
this.closed = false;
46+
}
47+
48+
/** Returns the number of bytes read. */
49+
public long getCount() {
50+
return count;
51+
}
52+
53+
@Override
54+
public int read() throws IOException {
55+
int result = source.read();
56+
if (result != -1) {
57+
count++;
58+
}
59+
return result;
60+
}
61+
62+
@Override
63+
public int read(byte[] b, int off, int len) throws IOException {
64+
int result = source.read(b, off, len);
65+
if (result != -1) {
66+
count += result;
67+
}
68+
return result;
69+
}
70+
71+
@Override
72+
public long skip(long n) throws IOException {
73+
long result = source.skip(n);
74+
count += result;
75+
return result;
76+
}
77+
78+
@Override
79+
public int available() throws IOException {
80+
return source.available();
81+
}
82+
83+
@Override
84+
public boolean markSupported() {
85+
return source.markSupported();
86+
}
87+
88+
@Override
89+
public synchronized void mark(int readlimit) {
90+
source.mark(readlimit);
91+
mark = count;
92+
}
93+
94+
@Override
95+
public synchronized void reset() throws IOException {
96+
if (false == source.markSupported()) {
97+
throw new IOException("Mark not supported");
98+
}
99+
if (mark == -1L) {
100+
throw new IOException("Mark not set");
101+
}
102+
count = mark;
103+
source.reset();
104+
}
105+
106+
@Override
107+
public void close() throws IOException {
108+
if (false == closed) {
109+
closed = true;
110+
if (closeSource) {
111+
source.close();
112+
}
113+
}
114+
}
115+
}

‎x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/DecryptionPacketsInputStream.java

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.repositories.encrypted;
7+
8+
import javax.crypto.BadPaddingException;
9+
import javax.crypto.Cipher;
10+
import javax.crypto.IllegalBlockSizeException;
11+
import javax.crypto.NoSuchPaddingException;
12+
import javax.crypto.SecretKey;
13+
import javax.crypto.ShortBufferException;
14+
import javax.crypto.spec.GCMParameterSpec;
15+
import java.io.ByteArrayInputStream;
16+
import java.io.IOException;
17+
import java.io.InputStream;
18+
import java.nio.ByteBuffer;
19+
import java.nio.ByteOrder;
20+
import java.security.InvalidAlgorithmParameterException;
21+
import java.security.InvalidKeyException;
22+
import java.security.NoSuchAlgorithmException;
23+
import java.util.Objects;
24+
25+
import static org.elasticsearch.repositories.encrypted.EncryptedRepository.GCM_IV_LENGTH_IN_BYTES;
26+
import static org.elasticsearch.repositories.encrypted.EncryptedRepository.GCM_TAG_LENGTH_IN_BYTES;
27+
28+
/**
29+
* A {@code DecryptionPacketsInputStream} wraps an encrypted input stream and decrypts
30+
* its contents. This is designed (and tested) to decrypt only the encryption format that
31+
* {@link EncryptionPacketsInputStream} generates. No decrypted bytes are returned before
32+
* they are authenticated.
33+
* <p>
34+
* The same parameters, namely {@code secretKey}, {@code nonce} and {@code packetLength},
35+
* that have been used during encryption must also be used for decryption, otherwise
36+
* decryption will fail.
37+
* <p>
38+
* This implementation buffers the encrypted packet in memory. The maximum packet size it can
39+
* accommodate is {@link EncryptedRepository#MAX_PACKET_LENGTH_IN_BYTES}.
40+
* <p>
41+
* This implementation does not support {@code mark} and {@code reset}.
42+
* <p>
43+
* The {@code close} call will close the decryption input stream and any subsequent {@code read},
44+
* {@code skip}, {@code available} and {@code reset} calls will throw {@code IOException}s.
45+
* <p>
46+
* This is NOT thread-safe, multiple threads sharing a single instance must synchronize access.
47+
*
48+
* @see EncryptionPacketsInputStream
49+
*/
50+
public final class DecryptionPacketsInputStream extends ChainingInputStream {
51+
52+
private final InputStream source;
53+
private final SecretKey secretKey;
54+
private final int nonce;
55+
private final int packetLength;
56+
private final byte[] packetBuffer;
57+
58+
private boolean hasNext;
59+
private long counter;
60+
61+
/**
62+
* Computes and returns the length of the plaintext given the {@code ciphertextLength} and the {@code packetLength}
63+
* used during encryption.
64+
* Each ciphertext packet is prepended by the Initilization Vector and has the Authentication Tag appended.
65+
* Decryption is 1:1, and the ciphertext is not padded, but stripping away the IV and the AT amounts to a shorter
66+
* plaintext compared to the ciphertext.
67+
*
68+
* @see EncryptionPacketsInputStream#getEncryptionLength(long, int)
69+
*/
70+
public static long getDecryptionLength(long ciphertextLength, int packetLength) {
71+
long encryptedPacketLength = packetLength + GCM_TAG_LENGTH_IN_BYTES + GCM_IV_LENGTH_IN_BYTES;
72+
long completePackets = ciphertextLength / encryptedPacketLength;
73+
long decryptedSize = completePackets * packetLength;
74+
if (ciphertextLength % encryptedPacketLength != 0) {
75+
decryptedSize += (ciphertextLength % encryptedPacketLength) - GCM_IV_LENGTH_IN_BYTES - GCM_TAG_LENGTH_IN_BYTES;
76+
}
77+
return decryptedSize;
78+
}
79+
80+
public DecryptionPacketsInputStream(InputStream source, SecretKey secretKey, int nonce, int packetLength) {
81+
this.source = Objects.requireNonNull(source);
82+
this.secretKey = Objects.requireNonNull(secretKey);
83+
this.nonce = nonce;
84+
if (packetLength <= 0 || packetLength >= EncryptedRepository.MAX_PACKET_LENGTH_IN_BYTES) {
85+
throw new IllegalArgumentException("Invalid packet length [" + packetLength + "]");
86+
}
87+
this.packetLength = packetLength;
88+
this.packetBuffer = new byte[packetLength + GCM_TAG_LENGTH_IN_BYTES];
89+
this.hasNext = true;
90+
this.counter = EncryptedRepository.PACKET_START_COUNTER;
91+
}
92+
93+
@Override
94+
InputStream nextComponent(InputStream currentComponentIn) throws IOException {
95+
if (currentComponentIn != null && currentComponentIn.read() != -1) {
96+
throw new IllegalStateException("Stream for previous packet has not been fully processed");
97+
}
98+
if (false == hasNext) {
99+
return null;
100+
}
101+
PrefixInputStream packetInputStream = new PrefixInputStream(source,
102+
packetLength + GCM_IV_LENGTH_IN_BYTES + GCM_TAG_LENGTH_IN_BYTES,
103+
false);
104+
int currentPacketLength = decrypt(packetInputStream);
105+
// only the last packet is shorter, so this must be the last packet
106+
if (currentPacketLength != packetLength) {
107+
hasNext = false;
108+
}
109+
return new ByteArrayInputStream(packetBuffer, 0, currentPacketLength);
110+
}
111+
112+
@Override
113+
public boolean markSupported() {
114+
return false;
115+
}
116+
117+
@Override
118+
public void mark(int readlimit) {
119+
}
120+
121+
@Override
122+
public void reset() throws IOException {
123+
throw new IOException("Mark/reset not supported");
124+
}
125+
126+
private int decrypt(PrefixInputStream packetInputStream) throws IOException {
127+
// read only the IV prefix into the packet buffer
128+
int ivLength = packetInputStream.readNBytes(packetBuffer, 0, GCM_IV_LENGTH_IN_BYTES);
129+
if (ivLength != GCM_IV_LENGTH_IN_BYTES) {
130+
throw new IOException("Packet heading IV error. Unexpected length [" + ivLength + "].");
131+
}
132+
// extract the nonce and the counter from the packet IV
133+
ByteBuffer ivBuffer = ByteBuffer.wrap(packetBuffer, 0, GCM_IV_LENGTH_IN_BYTES).order(ByteOrder.LITTLE_ENDIAN);
134+
int packetIvNonce = ivBuffer.getInt(0);
135+
long packetIvCounter = ivBuffer.getLong(Integer.BYTES);
136+
if (packetIvNonce != nonce) {
137+
throw new IOException("Packet nonce mismatch. Expecting [" + nonce + "], but got [" + packetIvNonce + "].");
138+
}
139+
if (packetIvCounter != counter) {
140+
throw new IOException("Packet counter mismatch. Expecting [" + counter + "], but got [" + packetIvCounter + "].");
141+
}
142+
// counter increment for the subsequent packet
143+
counter++;
144+
// counter wrap around
145+
if (counter == EncryptedRepository.PACKET_START_COUNTER) {
146+
throw new IOException("Maximum packet count limit exceeded");
147+
}
148+
// cipher used to decrypt only the current packetInputStream
149+
Cipher packetCipher = getPacketDecryptionCipher(packetBuffer);
150+
// read the rest of the packet, reusing the packetBuffer
151+
int packetLength = packetInputStream.readNBytes(packetBuffer, 0, packetBuffer.length);
152+
if (packetLength < GCM_TAG_LENGTH_IN_BYTES) {
153+
throw new IOException("Encrypted packet is too short");
154+
}
155+
try {
156+
// in-place decryption of the whole packet and return decrypted length
157+
return packetCipher.doFinal(packetBuffer, 0, packetLength, packetBuffer);
158+
} catch (ShortBufferException | IllegalBlockSizeException | BadPaddingException e) {
159+
throw new IOException("Exception during packet decryption", e);
160+
}
161+
}
162+
163+
private Cipher getPacketDecryptionCipher(byte[] packet) throws IOException {
164+
GCMParameterSpec gcmParameterSpec = new GCMParameterSpec(GCM_TAG_LENGTH_IN_BYTES * Byte.SIZE, packet, 0, GCM_IV_LENGTH_IN_BYTES);
165+
try {
166+
Cipher packetCipher = Cipher.getInstance(EncryptedRepository.GCM_ENCRYPTION_SCHEME);
167+
packetCipher.init(Cipher.DECRYPT_MODE, secretKey, gcmParameterSpec);
168+
return packetCipher;
169+
} catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) {
170+
throw new IOException("Exception during packet cipher initialisation", e);
171+
}
172+
}
173+
}

‎x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepository.java

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.repositories.encrypted;
8+
9+
public class EncryptedRepository {
10+
static final int GCM_TAG_LENGTH_IN_BYTES = 16;
11+
static final int GCM_IV_LENGTH_IN_BYTES = 12;
12+
static final int AES_BLOCK_SIZE_IN_BYTES = 128;
13+
static final String GCM_ENCRYPTION_SCHEME = "AES/GCM/NoPadding";
14+
static final long PACKET_START_COUNTER = Long.MIN_VALUE;
15+
static final int MAX_PACKET_LENGTH_IN_BYTES = 1 << 30;
16+
}

‎x-pack/plugin/repository-encrypted/src/main/java/org/elasticsearch/repositories/encrypted/EncryptedRepositoryPlugin.java

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.repositories.encrypted;
8+
9+
import org.elasticsearch.common.settings.Setting;
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.plugins.Plugin;
12+
import org.elasticsearch.plugins.ReloadablePlugin;
13+
import org.elasticsearch.plugins.RepositoryPlugin;
14+
15+
import java.util.List;
16+
17+
public final class EncryptedRepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin {
18+
19+
public EncryptedRepositoryPlugin(final Settings settings) {
20+
}
21+
22+
@Override
23+
public List<Setting<?>> getSettings() {
24+
return List.of();
25+
}
26+
27+
@Override
28+
public void reload(Settings settings) {
29+
// Secure settings should be readable inside this method.
30+
}
31+
}

0 commit comments

Comments
 (0)