Security: cache users in PKI realm (elastic/x-pack-elasticsearch#4428)

The PKI realm has never been a caching realm as the need had not
presented itself until now. The PKI realm relies on role mappings to
map the DN from a certificate to roles so that the users have the
appropriate access permissions. Without caching, this role mapping will
happen on every request. For file based role mappings, this is not an
issue as the mappings are based on equality checks for the DN.

However, the design of the API based role mappings allows for more
complex matches. These matches are implemented using automata, which
are built on every request that needs role mappings. Building automata
is an expensive operation and in combination with the PKI realm's lack
of caching leads to a significant performance impact.

The change in this commit makes the PkiRealm a caching realm using the
same pattern as other caching realms. The cache provided by
elasticsearch core is used to map the fingerprint of a certificate to
the user that was resolved from this certificate. The semantics of
modifications to this cache during iteration requires that we use a
read-write lock to protect access. There can be multiple concurrent
modifications and retrievals but iteration must be protected from any
attempts to modify the cache.

Additionally, some PKI tests were converted to single node tests as
part of this change. One test only used a single node and the other did
not require multiple nodes.

relates elastic/x-pack-elasticsearch#4406

Original commit: elastic/x-pack-elasticsearch@214772e

Author: jaymode

docs/en/security/authentication/user-cache.asciidoc

@@ -6,7 +6,8 @@ remote authentication service or hitting the disk for every incoming request.
 You can configure characteristics of the user cache with the `cache.ttl`,
 `cache.max_users`, and `cache.hash_algo` realm settings.
 
-NOTE: PKI realms do not use the user cache.
+NOTE: PKI realms do not cache user credentials but do cache the resolved user
+object to avoid unnecessarily needing to perform role mapping on each request.
 
 The cached user credentials are hashed in memory. By default, {security} uses a
 salted `sha-256` hash algorithm. You can use a different hashing algorithm by

docs/en/settings/security-settings.asciidoc

@@ -640,6 +640,15 @@ Specifies the {xpack-ref}/security-files.html[location] of the
 {xpack-ref}/mapping-roles.html[YAML role  mapping configuration file].
 Defaults to `CONFIG_DIR/x-pack/role_mapping.yml`.
 
+`cache.ttl`::
+Specifies the time-to-live for cached user entries. Use the
+standard Elasticsearch {ref}/common-options.html#time-units[time units]).
+Defaults to `20m`.
+
+`cache.max_users`::
+Specifies the maximum number of user entries that the cache can contain.
+Defaults to `100000`.
+
 [[ref-saml-settings]]
 [float]
 ===== SAML Realm Settings

plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/pki/PkiRealmSettings.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.security.authc.pki;
 
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.CompositeRoleMapperSettings;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 
@@ -18,6 +19,11 @@ public final class PkiRealmSettings {
     public static final String DEFAULT_USERNAME_PATTERN = "CN=(.*?)(?:,|$)";
     public static final Setting<Pattern> USERNAME_PATTERN_SETTING = new Setting<>("username_pattern", DEFAULT_USERNAME_PATTERN,
             s -> Pattern.compile(s, Pattern.CASE_INSENSITIVE), Setting.Property.NodeScope);
+    private static final TimeValue DEFAULT_TTL = TimeValue.timeValueMinutes(20);
+    public static final Setting<TimeValue> CACHE_TTL_SETTING = Setting.timeSetting("cache.ttl", DEFAULT_TTL, Setting.Property.NodeScope);
+    private static final int DEFAULT_MAX_USERS = 100_000; //100k users
+    public static final Setting<Integer> CACHE_MAX_USERS_SETTING = Setting.intSetting("cache.max_users", DEFAULT_MAX_USERS,
+            Setting.Property.NodeScope);
     public static final SSLConfigurationSettings SSL_SETTINGS = SSLConfigurationSettings.withoutPrefix();
 
     private PkiRealmSettings() {}
@@ -28,6 +34,8 @@ private PkiRealmSettings() {}
     public static Set<Setting<?>> getSettings() {
         Set<Setting<?>> settings = new HashSet<>();
         settings.add(USERNAME_PATTERN_SETTING);
+        settings.add(CACHE_TTL_SETTING);
+        settings.add(CACHE_MAX_USERS_SETTING);
 
         settings.add(SSL_SETTINGS.truststorePath);
         settings.add(SSL_SETTINGS.truststorePassword);

plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/BytesKey.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.StringHelper;
+
+import java.util.Arrays;
+
+/**
+ * Simple wrapper around bytes so that it can be used as a cache key. The hashCode is computed
+ * once upon creation and cached.
+ */
+public class BytesKey {
+
+    final byte[] bytes;
+    private final int hashCode;
+
+    public BytesKey(byte[] bytes) {
+        this.bytes = bytes;
+        this.hashCode = StringHelper.murmurhash3_x86_32(bytes, 0, bytes.length, StringHelper.GOOD_FAST_HASH_SEED);
+    }
+
+    @Override
+    public int hashCode() {
+        return hashCode;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == null) {
+            return false;
+        }
+        if (other instanceof BytesKey == false) {
+            return false;
+        }
+
+        BytesKey otherBytes = (BytesKey) other;
+        return Arrays.equals(otherBytes.bytes, bytes);
+    }
+
+    @Override
+    public String toString() {
+        return new BytesRef(bytes).toString();
+    }
+}

plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java

@@ -1165,44 +1165,6 @@ public void onFailure(Exception e) {
         }
     }
 
-    /**
-     * Simple wrapper around bytes so that it can be used as a cache key. The hashCode is computed
-     * once upon creation and cached.
-     */
-    static class BytesKey {
-
-        final byte[] bytes;
-        private final int hashCode;
-
-        BytesKey(byte[] bytes) {
-            this.bytes = bytes;
-            this.hashCode = StringHelper.murmurhash3_x86_32(bytes, 0, bytes.length, StringHelper.GOOD_FAST_HASH_SEED);
-        }
-
-        @Override
-        public int hashCode() {
-            return hashCode;
-        }
-
-        @Override
-        public boolean equals(Object other) {
-            if (other == null) {
-                return false;
-            }
-            if (other instanceof BytesKey == false) {
-                return false;
-            }
-
-            BytesKey otherBytes = (BytesKey) other;
-            return Arrays.equals(otherBytes.bytes, bytes);
-        }
-
-        @Override
-        public String toString() {
-            return new BytesRef(bytes).toString();
-        }
-    }
-
     /**
      * Creates a new key unless present that is newer than the current active key and returns the corresponding metadata. Note:
      * this method doesn't modify the metadata used in this token service. See {@link #refreshMetaData(TokenMetaData)}

plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java

@@ -11,8 +11,12 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ReleasableLock;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -25,32 +29,52 @@
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.ssl.CertUtils;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
+import org.elasticsearch.xpack.security.authc.BytesKey;
+import org.elasticsearch.xpack.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.CompositeRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 
 import javax.net.ssl.X509TrustManager;
 
+import java.security.MessageDigest;
 import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-public class PkiRealm extends Realm {
+public class PkiRealm extends Realm implements CachingRealm {
 
     public static final String PKI_CERT_HEADER_NAME = "__SECURITY_CLIENT_CERTIFICATE";
 
     // For client based cert validation, the auth type must be specified but UNKNOWN is an acceptable value
     private static final String AUTH_TYPE = "UNKNOWN";
 
+    // the lock is used in an odd manner; when iterating over the cache we cannot have modifiers other than deletes using
+    // the iterator but when not iterating we can modify the cache without external locking. When making normal modifications to the cache
+    // the read lock is obtained so that we can allow concurrent modifications; however when we need to iterate over the keys or values of
+    // the cache the write lock must obtained to prevent any modifications
+    private final ReleasableLock readLock;
+    private final ReleasableLock writeLock;
+
+    {
+        final ReadWriteLock iterationLock = new ReentrantReadWriteLock();
+        readLock = new ReleasableLock(iterationLock.readLock());
+        writeLock = new ReleasableLock(iterationLock.writeLock());
+    }
+
     private final X509TrustManager trustManager;
     private final Pattern principalPattern;
     private final UserRoleMapper roleMapper;
-
+    private final Cache<BytesKey, User> cache;
 
     public PkiRealm(RealmConfig config, ResourceWatcherService watcherService, NativeRoleMappingStore nativeRoleMappingStore) {
         this(config, new CompositeRoleMapper(PkiRealmSettings.TYPE, config, watcherService, nativeRoleMappingStore));
@@ -62,6 +86,10 @@ public PkiRealm(RealmConfig config, ResourceWatcherService watcherService, Nativ
         this.trustManager = trustManagers(config);
         this.principalPattern = PkiRealmSettings.USERNAME_PATTERN_SETTING.get(config.settings());
         this.roleMapper = roleMapper;
+        this.cache = CacheBuilder.<BytesKey, User>builder()
+                .setExpireAfterWrite(PkiRealmSettings.CACHE_TTL_SETTING.get(config.settings()))
+                .setMaximumWeight(PkiRealmSettings.CACHE_MAX_USERS_SETTING.get(config.settings()))
+                .build();
     }
 
     @Override
@@ -77,18 +105,28 @@ public X509AuthenticationToken token(ThreadContext context) {
     @Override
     public void authenticate(AuthenticationToken authToken, ActionListener<AuthenticationResult> listener) {
         X509AuthenticationToken token = (X509AuthenticationToken)authToken;
-        if (isCertificateChainTrusted(trustManager, token, logger) == false) {
-            listener.onResponse(AuthenticationResult.unsuccessful("Certificate for " + token.dn() + " is not trusted", null));
-        } else {
-            final Map<String, Object> metadata = Collections.singletonMap("pki_dn", token.dn());
-            final UserRoleMapper.UserData user = new UserRoleMapper.UserData(token.principal(),
-                    token.dn(), Collections.emptySet(), metadata, this.config);
-            roleMapper.resolveRoles(user, ActionListener.wrap(
-                    roles -> listener.onResponse(AuthenticationResult.success(
-                            new User(token.principal(), roles.toArray(new String[roles.size()]), null, null, metadata, true)
-                    )),
-                    listener::onFailure
-            ));
+        try {
+            final BytesKey fingerprint = computeFingerprint(token.credentials()[0]);
+            User user = cache.get(fingerprint);
+            if (user != null) {
+                listener.onResponse(AuthenticationResult.success(user));
+            } else if (isCertificateChainTrusted(trustManager, token, logger) == false) {
+                listener.onResponse(AuthenticationResult.unsuccessful("Certificate for " + token.dn() + " is not trusted", null));
+            } else {
+                final Map<String, Object> metadata = Collections.singletonMap("pki_dn", token.dn());
+                final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(token.principal(),
+                        token.dn(), Collections.emptySet(), metadata, this.config);
+                roleMapper.resolveRoles(userData, ActionListener.wrap(roles -> {
+                    final User computedUser =
+                            new User(token.principal(), roles.toArray(new String[roles.size()]), null, null, metadata, true);
+                    try (ReleasableLock ignored = readLock.acquire()) {
+                        cache.put(fingerprint, computedUser);
+                    }
+                    listener.onResponse(AuthenticationResult.success(computedUser));
+                }, listener::onFailure));
+            }
+        } catch (CertificateEncodingException e) {
+            listener.onResponse(AuthenticationResult.unsuccessful("Certificate for " + token.dn() + " has encoding issues", e));
         }
     }
 
@@ -196,4 +234,29 @@ private static X509TrustManager trustManagersFromCAs(Settings settings, Environm
         }
     }
 
+    @Override
+    public void expire(String username) {
+        try (ReleasableLock ignored = writeLock.acquire()) {
+            Iterator<User> userIterator = cache.values().iterator();
+            while (userIterator.hasNext()) {
+                if (userIterator.next().principal().equals(username)) {
+                    userIterator.remove();
+                    // do not break since there is no guarantee username is unique in this realm
+                }
+            }
+        }
+    }
+
+    @Override
+    public void expireAll() {
+        try (ReleasableLock ignored = readLock.acquire()) {
+            cache.invalidateAll();
+        }
+    }
+
+    private static BytesKey computeFingerprint(X509Certificate certificate) throws CertificateEncodingException {
+        MessageDigest digest = MessageDigests.sha256();
+        digest.update(certificate.getEncoded());
+        return new BytesKey(digest.digest());
+    }
 }

plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -167,6 +168,12 @@ protected Settings nodeSettings() {
         return builder.build();
     }
 
+    protected Settings transportClientSettings() {
+        return Settings.builder()
+                .put(customSecuritySettingsSource.transportClientSettings())
+                .build();
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return customSecuritySettingsSource.nodePlugins();
@@ -271,22 +278,31 @@ protected RestClient getRestClient() {
         return getRestClient(client());
     }
 
+    protected RestClient createRestClient(RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback, String protocol) {
+        return createRestClient(client(), httpClientConfigCallback, protocol);
+    }
+
     private static synchronized RestClient getRestClient(Client client) {
         if (restClient == null) {
-            restClient = createRestClient(client);
+            restClient = createRestClient(client, null, "http");
         }
         return restClient;
     }
 
-    private static RestClient createRestClient(Client client) {
+    private static RestClient createRestClient(Client client, RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback,
+                                               String protocol) {
         NodesInfoResponse nodesInfoResponse = client.admin().cluster().prepareNodesInfo().get();
         assertFalse(nodesInfoResponse.hasFailures());
         assertEquals(nodesInfoResponse.getNodes().size(), 1);
         NodeInfo node = nodesInfoResponse.getNodes().get(0);
         assertNotNull(node.getHttp());
         TransportAddress publishAddress = node.getHttp().address().publishAddress();
         InetSocketAddress address = publishAddress.address();
-        final HttpHost host = new HttpHost(NetworkAddress.format(address.getAddress()), address.getPort(), "http");
-        return RestClient.builder(host).build();
+        final HttpHost host = new HttpHost(NetworkAddress.format(address.getAddress()), address.getPort(), protocol);
+        RestClientBuilder builder = RestClient.builder(host);
+        if (httpClientConfigCallback != null) {
+            builder.setHttpClientConfigCallback(httpClientConfigCallback);
+        }
+        return builder.build();
     }
 }

plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java

@@ -51,7 +51,6 @@
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.security.SecurityLifecycleService;
-import org.elasticsearch.xpack.security.authc.TokenService.BytesKey;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;