Skip to content

Commit b296936

Browse files
talevytalevy
committed
Introduce a Hashing Processor (#31087)
It is useful to have a processor similar to logstash-filter-fingerprint in Elasticsearch. A processor that leverages a variety of hashing algorithms to create cryptographically-secure one-way hashes of values in documents. This processor introduces a pbkdf2hmac hashing scheme to fields in documents for indexing
1 parent e057f1a commit b296936

File tree

6 files changed

+525
-1
lines changed

6 files changed

+525
-1
lines changed

‎x-pack/plugin/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ integTestCluster {
152152
keystoreSetting 'bootstrap.password', 'x-pack-test-password'
153153
keystoreSetting 'xpack.security.authc.token.passphrase', 'x-pack-token-service-password'
154154
keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
155+
keystoreSetting 'xpack.security.ingest.hash.processor.key', 'hmackey'
155156
distribution = 'zip' // this is important since we use the reindex module in ML
156157

157158
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@
176176
import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
177177
import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
178178
import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
179+
import org.elasticsearch.xpack.security.ingest.HashProcessor;
179180
import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
180181
import org.elasticsearch.xpack.security.rest.SecurityRestFilter;
181182
import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction;
@@ -600,6 +601,8 @@ public static List<Setting<?>> getSettings(boolean transportClientMode, List<Sec
600601
settingsList.add(Setting.listSetting(SecurityField.setting("hide_settings"), Collections.emptyList(), Function.identity(),
601602
Property.NodeScope, Property.Filtered));
602603
settingsList.add(INDICES_ADMIN_FILTERED_FIELDS_SETTING);
604+
// ingest processor settings
605+
settingsList.add(HashProcessor.HMAC_KEY_SETTING);
603606

604607
return settingsList;
605608
}
@@ -744,7 +747,10 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
744747

745748
@Override
746749
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
747-
return Collections.singletonMap(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
750+
Map<String, Processor.Factory> processors = new HashMap<>();
751+
processors.put(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
752+
processors.put(HashProcessor.TYPE, new HashProcessor.Factory(parameters.env.settings()));
753+
return processors;
748754
}
749755

750756
/**

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/HashProcessor.java

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.security.ingest;
7+
8+
import org.elasticsearch.ElasticsearchException;
9+
import org.elasticsearch.common.Nullable;
10+
import org.elasticsearch.common.Strings;
11+
import org.elasticsearch.common.collect.Tuple;
12+
import org.elasticsearch.common.settings.SecureSetting;
13+
import org.elasticsearch.common.settings.SecureString;
14+
import org.elasticsearch.common.settings.Setting;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.ingest.AbstractProcessor;
17+
import org.elasticsearch.ingest.ConfigurationUtils;
18+
import org.elasticsearch.ingest.IngestDocument;
19+
import org.elasticsearch.ingest.Processor;
20+
import org.elasticsearch.xpack.core.security.SecurityField;
21+
22+
import javax.crypto.Mac;
23+
import javax.crypto.SecretKeyFactory;
24+
import javax.crypto.spec.PBEKeySpec;
25+
import javax.crypto.spec.SecretKeySpec;
26+
import java.nio.charset.StandardCharsets;
27+
import java.security.InvalidKeyException;
28+
import java.security.NoSuchAlgorithmException;
29+
import java.security.spec.InvalidKeySpecException;
30+
import java.util.Arrays;
31+
import java.util.Base64;
32+
import java.util.HashMap;
33+
import java.util.List;
34+
import java.util.Locale;
35+
import java.util.Map;
36+
import java.util.Objects;
37+
import java.util.stream.Collectors;
38+
39+
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
40+
41+
/**
42+
* A processor that hashes the contents of a field (or fields) using various hashing algorithms
43+
*/
44+
public final class HashProcessor extends AbstractProcessor {
45+
public static final String TYPE = "hash";
46+
public static final Setting.AffixSetting<SecureString> HMAC_KEY_SETTING = SecureSetting
47+
.affixKeySetting(SecurityField.setting("ingest." + TYPE) + ".", "key",
48+
(key) -> SecureSetting.secureString(key, null));
49+
50+
private final List<String> fields;
51+
private final String targetField;
52+
private final Method method;
53+
private final Mac mac;
54+
private final byte[] salt;
55+
private final boolean ignoreMissing;
56+
57+
HashProcessor(String tag, List<String> fields, String targetField, byte[] salt, Method method, @Nullable Mac mac,
58+
boolean ignoreMissing) {
59+
super(tag);
60+
this.fields = fields;
61+
this.targetField = targetField;
62+
this.method = method;
63+
this.mac = mac;
64+
this.salt = salt;
65+
this.ignoreMissing = ignoreMissing;
66+
}
67+
68+
List<String> getFields() {
69+
return fields;
70+
}
71+
72+
String getTargetField() {
73+
return targetField;
74+
}
75+
76+
byte[] getSalt() {
77+
return salt;
78+
}
79+
80+
@Override
81+
public void execute(IngestDocument document) {
82+
Map<String, String> hashedFieldValues = fields.stream().map(f -> {
83+
String value = document.getFieldValue(f, String.class, ignoreMissing);
84+
if (value == null && ignoreMissing) {
85+
return new Tuple<String, String>(null, null);
86+
}
87+
try {
88+
return new Tuple<>(f, method.hash(mac, salt, value));
89+
} catch (Exception e) {
90+
throw new IllegalArgumentException("field[" + f + "] could not be hashed", e);
91+
}
92+
}).filter(tuple -> Objects.nonNull(tuple.v1())).collect(Collectors.toMap(Tuple::v1, Tuple::v2));
93+
if (fields.size() == 1) {
94+
document.setFieldValue(targetField, hashedFieldValues.values().iterator().next());
95+
} else {
96+
document.setFieldValue(targetField, hashedFieldValues);
97+
}
98+
}
99+
100+
@Override
101+
public String getType() {
102+
return TYPE;
103+
}
104+
105+
public static final class Factory implements Processor.Factory {
106+
107+
private final Settings settings;
108+
private final Map<String, SecureString> secureKeys;
109+
110+
public Factory(Settings settings) {
111+
this.settings = settings;
112+
this.secureKeys = new HashMap<>();
113+
HMAC_KEY_SETTING.getAllConcreteSettings(settings).forEach(k -> {
114+
secureKeys.put(k.getKey(), k.get(settings));
115+
});
116+
}
117+
118+
private static Mac createMac(Method method, SecureString password, byte[] salt, int iterations) {
119+
try {
120+
SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2With" + method.getAlgorithm());
121+
PBEKeySpec keySpec = new PBEKeySpec(password.getChars(), salt, iterations, 128);
122+
byte[] pbkdf2 = secretKeyFactory.generateSecret(keySpec).getEncoded();
123+
Mac mac = Mac.getInstance(method.getAlgorithm());
124+
mac.init(new SecretKeySpec(pbkdf2, method.getAlgorithm()));
125+
return mac;
126+
} catch (NoSuchAlgorithmException | InvalidKeySpecException | InvalidKeyException e) {
127+
throw new IllegalArgumentException("invalid settings", e);
128+
}
129+
}
130+
131+
@Override
132+
public HashProcessor create(Map<String, Processor.Factory> registry, String processorTag, Map<String, Object> config) {
133+
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
134+
List<String> fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields");
135+
if (fields.isEmpty()) {
136+
throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields", "must specify at least one field");
137+
} else if (fields.stream().anyMatch(Strings::isNullOrEmpty)) {
138+
throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "fields",
139+
"a field-name entry is either empty or null");
140+
}
141+
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
142+
String keySettingName = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "key_setting");
143+
SecureString key = secureKeys.get(keySettingName);
144+
if (key == null) {
145+
throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, "key_setting",
146+
"key [" + keySettingName + "] must match [xpack.security.ingest.hash.*.key]. It is not set");
147+
}
148+
String saltString = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "salt");
149+
byte[] salt = saltString.getBytes(StandardCharsets.UTF_8);
150+
String methodProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "method", "SHA256");
151+
Method method = Method.fromString(processorTag, "method", methodProperty);
152+
int iterations = ConfigurationUtils.readIntProperty(TYPE, processorTag, config, "iterations", 5);
153+
Mac mac = createMac(method, key, salt, iterations);
154+
return new HashProcessor(processorTag, fields, targetField, salt, method, mac, ignoreMissing);
155+
}
156+
}
157+
158+
enum Method {
159+
SHA1("HmacSHA1"),
160+
SHA256("HmacSHA256"),
161+
SHA384("HmacSHA384"),
162+
SHA512("HmacSHA512");
163+
164+
private final String algorithm;
165+
166+
Method(String algorithm) {
167+
this.algorithm = algorithm;
168+
}
169+
170+
public String getAlgorithm() {
171+
return algorithm;
172+
}
173+
174+
@Override
175+
public String toString() {
176+
return name().toLowerCase(Locale.ROOT);
177+
}
178+
179+
public String hash(Mac mac, byte[] salt, String input) {
180+
try {
181+
byte[] encrypted = mac.doFinal(input.getBytes(StandardCharsets.UTF_8));
182+
byte[] messageWithSalt = new byte[salt.length + encrypted.length];
183+
System.arraycopy(salt, 0, messageWithSalt, 0, salt.length);
184+
System.arraycopy(encrypted, 0, messageWithSalt, salt.length, encrypted.length);
185+
return Base64.getEncoder().encodeToString(messageWithSalt);
186+
} catch (IllegalStateException e) {
187+
throw new ElasticsearchException("error hashing data", e);
188+
}
189+
}
190+
191+
public static Method fromString(String processorTag, String propertyName, String type) {
192+
try {
193+
return Method.valueOf(type.toUpperCase(Locale.ROOT));
194+
} catch(IllegalArgumentException e) {
195+
throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type +
196+
"] not supported, cannot convert field. Valid hash methods: " + Arrays.toString(Method.values()));
197+
}
198+
}
199+
}
200+
}

‎x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/HashProcessorFactoryTests.java

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.security.ingest;
7+
8+
import org.elasticsearch.ElasticsearchException;
9+
import org.elasticsearch.common.settings.MockSecureSettings;
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.test.ESTestCase;
12+
13+
import java.nio.charset.StandardCharsets;
14+
import java.util.Collections;
15+
import java.util.HashMap;
16+
import java.util.Map;
17+
18+
import static org.hamcrest.Matchers.equalTo;
19+
20+
public class HashProcessorFactoryTests extends ESTestCase {
21+
22+
public void testProcessor() {
23+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
24+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
25+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
26+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
27+
Map<String, Object> config = new HashMap<>();
28+
config.put("fields", Collections.singletonList("_field"));
29+
config.put("target_field", "_target");
30+
config.put("salt", "_salt");
31+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
32+
for (HashProcessor.Method method : HashProcessor.Method.values()) {
33+
config.put("method", method.toString());
34+
HashProcessor processor = factory.create(null, "_tag", new HashMap<>(config));
35+
assertThat(processor.getFields(), equalTo(Collections.singletonList("_field")));
36+
assertThat(processor.getTargetField(), equalTo("_target"));
37+
assertArrayEquals(processor.getSalt(), "_salt".getBytes(StandardCharsets.UTF_8));
38+
}
39+
}
40+
41+
public void testProcessorNoFields() {
42+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
43+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
44+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
45+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
46+
Map<String, Object> config = new HashMap<>();
47+
config.put("target_field", "_target");
48+
config.put("salt", "_salt");
49+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
50+
config.put("method", HashProcessor.Method.SHA1.toString());
51+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
52+
() -> factory.create(null, "_tag", config));
53+
assertThat(e.getMessage(), equalTo("[fields] required property is missing"));
54+
}
55+
56+
public void testProcessorNoTargetField() {
57+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
58+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
59+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
60+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
61+
Map<String, Object> config = new HashMap<>();
62+
config.put("fields", Collections.singletonList("_field"));
63+
config.put("salt", "_salt");
64+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
65+
config.put("method", HashProcessor.Method.SHA1.toString());
66+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
67+
() -> factory.create(null, "_tag", config));
68+
assertThat(e.getMessage(), equalTo("[target_field] required property is missing"));
69+
}
70+
71+
public void testProcessorFieldsIsEmpty() {
72+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
73+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
74+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
75+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
76+
Map<String, Object> config = new HashMap<>();
77+
config.put("fields", Collections.singletonList(randomBoolean() ? "" : null));
78+
config.put("salt", "_salt");
79+
config.put("target_field", "_target");
80+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
81+
config.put("method", HashProcessor.Method.SHA1.toString());
82+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
83+
() -> factory.create(null, "_tag", config));
84+
assertThat(e.getMessage(), equalTo("[fields] a field-name entry is either empty or null"));
85+
}
86+
87+
public void testProcessorMissingSalt() {
88+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
89+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
90+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
91+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
92+
Map<String, Object> config = new HashMap<>();
93+
config.put("fields", Collections.singletonList("_field"));
94+
config.put("target_field", "_target");
95+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
96+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
97+
() -> factory.create(null, "_tag", config));
98+
assertThat(e.getMessage(), equalTo("[salt] required property is missing"));
99+
}
100+
101+
public void testProcessorInvalidMethod() {
102+
MockSecureSettings mockSecureSettings = new MockSecureSettings();
103+
mockSecureSettings.setString("xpack.security.ingest.hash.processor.key", "my_key");
104+
Settings settings = Settings.builder().setSecureSettings(mockSecureSettings).build();
105+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
106+
Map<String, Object> config = new HashMap<>();
107+
config.put("fields", Collections.singletonList("_field"));
108+
config.put("salt", "_salt");
109+
config.put("target_field", "_target");
110+
config.put("key_setting", "xpack.security.ingest.hash.processor.key");
111+
config.put("method", "invalid");
112+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
113+
() -> factory.create(null, "_tag", config));
114+
assertThat(e.getMessage(), equalTo("[method] type [invalid] not supported, cannot convert field. " +
115+
"Valid hash methods: [sha1, sha256, sha384, sha512]"));
116+
}
117+
118+
public void testProcessorInvalidOrMissingKeySetting() {
119+
Settings settings = Settings.builder().setSecureSettings(new MockSecureSettings()).build();
120+
HashProcessor.Factory factory = new HashProcessor.Factory(settings);
121+
Map<String, Object> config = new HashMap<>();
122+
config.put("fields", Collections.singletonList("_field"));
123+
config.put("salt", "_salt");
124+
config.put("target_field", "_target");
125+
config.put("key_setting", "invalid");
126+
config.put("method", HashProcessor.Method.SHA1.toString());
127+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
128+
() -> factory.create(null, "_tag", new HashMap<>(config)));
129+
assertThat(e.getMessage(),
130+
equalTo("[key_setting] key [invalid] must match [xpack.security.ingest.hash.*.key]. It is not set"));
131+
config.remove("key_setting");
132+
ElasticsearchException ex = expectThrows(ElasticsearchException.class,
133+
() -> factory.create(null, "_tag", config));
134+
assertThat(ex.getMessage(), equalTo("[key_setting] required property is missing"));
135+
}
136+
}

0 commit comments

Comments
 (0)