Introduce Application Privileges with support for Kibana RBAC (#32309)

This commit introduces "Application Privileges" to the X-Pack security
model.

Application Privileges are managed within Elasticsearch, and can be
tested with the _has_privileges API, but do not grant access to any
actions or resources within Elasticsearch. Their purpose is to allow
applications outside of Elasticsearch to represent and store their own
privileges model within Elasticsearch roles.

Access to manage application privileges is handled in a new way that
grants permission to specific application names only. This lays the
foundation for more OLS on cluster privileges, which is implemented by
allowing a cluster permission to inspect not just the action being
executed, but also the request to which the action is applied.
To support this, a "conditional cluster privilege" is introduced, which
is like the existing cluster privilege, except that it has a Predicate
over the request as well as over the action name.

Specifically, this adds
- GET/PUT/DELETE actions for defining application level privileges
- application privileges in role definitions
- application privileges in the has_privileges API
- changes to the cluster permission class to support checking of request
  objects
- a new "global" element on role definition to provide cluster object
  level security (only for manage application privileges)
- changes to `kibana_user`, `kibana_dashboard_only_user` and
  `kibana_system` roles to use and manage application privileges

Closes #29820
Closes #31559

Author: tvernum

server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

@@ -59,6 +59,7 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
@@ -931,8 +932,23 @@ public <T extends Streamable> List<T> readStreamableList(Supplier<T> constructor
      * Reads a list of objects
      */
     public <T> List<T> readList(Writeable.Reader<T> reader) throws IOException {
+        return readCollection(reader, ArrayList::new);
+    }
+
+    /**
+     * Reads a set of objects
+     */
+    public <T> Set<T> readSet(Writeable.Reader<T> reader) throws IOException {
+        return readCollection(reader, HashSet::new);
+    }
+
+    /**
+     * Reads a collection of objects
+     */
+    private <T, C extends Collection<? super T>> C readCollection(Writeable.Reader<T> reader,
+                                                                  IntFunction<C> constructor) throws IOException {
         int count = readArraySize();
-        List<T> builder = new ArrayList<>(count);
+        C builder = constructor.apply(count);
         for (int i=0; i<count; i++) {
             builder.add(reader.read(this));
         }

server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -55,6 +55,7 @@
 import java.nio.file.NoSuchFileException;
 import java.nio.file.NotDirectoryException;
 import java.time.ZonedDateTime;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumMap;
@@ -995,6 +996,16 @@ public void writeList(List<? extends Writeable> list) throws IOException {
         }
     }
 
+    /**
+     * Writes a collection of generic objects via a {@link Writer}
+     */
+    public <T> void writeCollection(Collection<T> collection, Writer<T> writer) throws IOException {
+        writeVInt(collection.size());
+        for (T val: collection) {
+            writer.write(this, val);
+        }
+    }
+
     /**
      * Writes a list of strings
      */

server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java

@@ -31,6 +31,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -42,6 +43,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.iterableWithSize;
 
 public class StreamTests extends ESTestCase {
 
@@ -65,7 +67,7 @@ public void testBooleanSerialization() throws IOException {
         final Set<Byte> set = IntStream.range(Byte.MIN_VALUE, Byte.MAX_VALUE).mapToObj(v -> (byte) v).collect(Collectors.toSet());
         set.remove((byte) 0);
         set.remove((byte) 1);
-        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final byte[] corruptBytes = new byte[]{randomFrom(set)};
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readBoolean());
         final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
@@ -100,7 +102,7 @@ public void testOptionalBooleanSerialization() throws IOException {
         set.remove((byte) 0);
         set.remove((byte) 1);
         set.remove((byte) 2);
-        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final byte[] corruptBytes = new byte[]{randomFrom(set)};
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readOptionalBoolean());
         final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
@@ -119,22 +121,22 @@ public void testRandomVLongSerialization() throws IOException {
 
     public void testSpecificVLongSerialization() throws IOException {
         List<Tuple<Long, byte[]>> values =
-                Arrays.asList(
-                        new Tuple<>(0L, new byte[]{0}),
-                        new Tuple<>(-1L, new byte[]{1}),
-                        new Tuple<>(1L, new byte[]{2}),
-                        new Tuple<>(-2L, new byte[]{3}),
-                        new Tuple<>(2L, new byte[]{4}),
-                        new Tuple<>(Long.MIN_VALUE, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1, 1}),
-                        new Tuple<>(Long.MAX_VALUE, new byte[]{-2, -1, -1, -1, -1, -1, -1, -1, -1, 1})
-
-                );
+            Arrays.asList(
+                new Tuple<>(0L, new byte[]{0}),
+                new Tuple<>(-1L, new byte[]{1}),
+                new Tuple<>(1L, new byte[]{2}),
+                new Tuple<>(-2L, new byte[]{3}),
+                new Tuple<>(2L, new byte[]{4}),
+                new Tuple<>(Long.MIN_VALUE, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1, 1}),
+                new Tuple<>(Long.MAX_VALUE, new byte[]{-2, -1, -1, -1, -1, -1, -1, -1, -1, 1})
+
+            );
         for (Tuple<Long, byte[]> value : values) {
             BytesStreamOutput out = new BytesStreamOutput();
             out.writeZLong(value.v1());
             assertArrayEquals(Long.toString(value.v1()), value.v2(), BytesReference.toBytes(out.bytes()));
             BytesReference bytes = new BytesArray(value.v2());
-            assertEquals(Arrays.toString(value.v2()), (long)value.v1(), bytes.streamInput().readZLong());
+            assertEquals(Arrays.toString(value.v2()), (long) value.v1(), bytes.streamInput().readZLong());
         }
     }
 
@@ -158,7 +160,7 @@ public void testLinkedHashMap() throws IOException {
         }
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeGenericValue(write);
-        LinkedHashMap<String, Integer> read = (LinkedHashMap<String, Integer>)out.bytes().streamInput().readGenericValue();
+        LinkedHashMap<String, Integer> read = (LinkedHashMap<String, Integer>) out.bytes().streamInput().readGenericValue();
         assertEquals(size, read.size());
         int index = 0;
         for (Map.Entry<String, Integer> entry : read.entrySet()) {
@@ -172,7 +174,8 @@ public void testFilterStreamInputDelegatesAvailable() throws IOException {
         final int length = randomIntBetween(1, 1024);
         StreamInput delegate = StreamInput.wrap(new byte[length]);
 
-        FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {};
+        FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {
+        };
         assertEquals(filterInputStream.available(), length);
 
         // read some bytes
@@ -201,7 +204,7 @@ public void testReadArraySize() throws IOException {
         }
         stream.writeByteArray(array);
         InputStreamStreamInput streamInput = new InputStreamStreamInput(StreamInput.wrap(BytesReference.toBytes(stream.bytes())), array
-            .length-1);
+            .length - 1);
         expectThrows(EOFException.class, streamInput::readByteArray);
         streamInput = new InputStreamStreamInput(StreamInput.wrap(BytesReference.toBytes(stream.bytes())), BytesReference.toBytes(stream
             .bytes()).length);
@@ -230,6 +233,21 @@ public void testWritableArrays() throws IOException {
         assertThat(targetArray, equalTo(sourceArray));
     }
 
+    public void testSetOfLongs() throws IOException {
+        final int size = randomIntBetween(0, 6);
+        final Set<Long> sourceSet = new HashSet<>(size);
+        for (int i = 0; i < size; i++) {
+            sourceSet.add(randomLongBetween(i * 1000, (i + 1) * 1000 - 1));
+        }
+        assertThat(sourceSet, iterableWithSize(size));
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.writeCollection(sourceSet, StreamOutput::writeLong);
+
+        final Set<Long> targetSet = out.bytes().streamInput().readSet(StreamInput::readLong);
+        assertThat(targetSet, equalTo(sourceSet));
+    }
+
     static final class WriteableString implements Writeable {
         final String string;
 

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -149,6 +149,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
+import java.util.function.IntFunction;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -717,6 +718,20 @@ public static String[] generateRandomStringArray(int maxArraySize, int stringSiz
         return generateRandomStringArray(maxArraySize, stringSize, allowNull, true);
     }
 
+    public static <T> T[] randomArray(int maxArraySize, IntFunction<T[]> arrayConstructor, Supplier<T> valueConstructor) {
+        return randomArray(0, maxArraySize, arrayConstructor, valueConstructor);
+    }
+
+    public static <T> T[] randomArray(int minArraySize, int maxArraySize, IntFunction<T[]> arrayConstructor, Supplier<T> valueConstructor) {
+        final int size = randomIntBetween(minArraySize, maxArraySize);
+        final T[] array = arrayConstructor.apply(size);
+        for (int i = 0; i < array.length; i++) {
+            array[i] = valueConstructor.get();
+        }
+        return array;
+    }
+
+
     private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m", "micros", "nanos"};
 
     public static String randomTimeValue(int lower, int upper, String... suffixes) {

x-pack/docs/en/rest-api/security/privileges.asciidoc

@@ -84,7 +84,8 @@ The following example output indicates which privileges the "rdeniro" user has:
       "read" : true,
       "write" : false
     }
-  }
+  },
+  "application" : {}
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"rdeniro"/"$body.username"/]

x-pack/docs/en/rest-api/security/roles.asciidoc

@@ -140,6 +140,7 @@ role. If the role is not defined in the `native` realm, the request 404s.
       },
       "query" : "{\"match\": {\"title\": \"foo\"}}"
     } ],
+    "applications" : [ ],
     "run_as" : [ "other_user" ],
     "metadata" : {
       "version" : 1

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -133,6 +133,8 @@
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExceptExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction;
@@ -342,6 +344,11 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, TokenMetaData.TYPE, TokenMetaData::readDiffFrom),
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SECURITY, SecurityFeatureSetUsage::new),
+                // security : conditional privileges
+                new NamedWriteableRegistry.Entry(ConditionalClusterPrivilege.class,
+                    ConditionalClusterPrivileges.ManageApplicationPrivileges.WRITEABLE_NAME,
+                    ConditionalClusterPrivileges.ManageApplicationPrivileges::createFrom),
+                // security : role-mappings
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AllExpression.NAME, AllExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ApplicationPrivilegesRequest.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import java.util.Collection;
+
+/**
+ * Interface implemented by all Requests that manage application privileges
+ */
+public interface ApplicationPrivilegesRequest {
+
+    Collection<String> getApplicationNames();
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.Action;
+
+/**
+ * Action for deleting application privileges.
+ */
+public final class DeletePrivilegesAction extends Action<DeletePrivilegesResponse> {
+
+    public static final DeletePrivilegesAction INSTANCE = new DeletePrivilegesAction();
+    public static final String NAME = "cluster:admin/xpack/security/privilege/delete";
+
+    private DeletePrivilegesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeletePrivilegesResponse newResponse() {
+        return new DeletePrivilegesResponse();
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequest.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request to delete an application privilege.
+ */
+public final class DeletePrivilegesRequest extends ActionRequest
+    implements ApplicationPrivilegesRequest, WriteRequest<DeletePrivilegesRequest> {
+
+    private String application;
+    private String[] privileges;
+    private RefreshPolicy refreshPolicy = RefreshPolicy.IMMEDIATE;
+
+    public DeletePrivilegesRequest() {
+        this(null, Strings.EMPTY_ARRAY);
+    }
+
+    public DeletePrivilegesRequest(String application, String[] privileges) {
+        this.application = application;
+        this.privileges = privileges;
+    }
+
+    @Override
+    public DeletePrivilegesRequest setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = refreshPolicy;
+        return this;
+    }
+
+    @Override
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(application)) {
+            validationException = addValidationError("application name is missing", validationException);
+        }
+        if (privileges == null || privileges.length == 0 || Arrays.stream(privileges).allMatch(Strings::isNullOrEmpty)) {
+            validationException = addValidationError("privileges are missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void application(String application) {
+        this.application = application;
+    }
+
+    public String application() {
+        return application;
+    }
+
+    @Override
+    public Collection<String> getApplicationNames() {
+        return Collections.singleton(application);
+    }
+
+    public String[] privileges() {
+        return this.privileges;
+    }
+
+    public void privileges(String[] privileges) {
+        this.privileges = privileges;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        application = in.readString();
+        privileges = in.readStringArray();
+        refreshPolicy = RefreshPolicy.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(application);
+        out.writeStringArray(privileges);
+        refreshPolicy.writeTo(out);
+    }
+
+}