Service Accounts - features required for Fleet integration (#71514)

* Service Accounts - Initial bootstrap plumbing for essential classes (#70391)

This PR is the initial effort to add essential classes for service accounts to
lay down the foundation of future works.  The classes are wired in places, but
not yet been used. Also intentionally left out the actual credential store
implementation. It is a good first commit which does not bring in too many
changes.

* Service Accounts - New CLI tool for managing file tokens (#70454)

This is the second PR for service accounts. It adds a new CLI tool
elasticsearch-service-tokens to manage file tokens. The file tokens are stored
in the service_tokens file under the config directory. Out of the planned create,
remove and list sub-commands, this PR only implements the create function since
it is the most important one. The other two sub-commands will be handled in
separate PRs.

* Service Accounts - Authentication with file tokens (#70543)

This the 3rd PR for service accounts. It adds support for authentication with
file tokens. It also adds a cache for performance so that expensive pbkdf2
hashing does not have to be performed on every request. Adding a cache comes
with its own housekeeping work around invalidation. This PR ensures that cache
gets invalidated when underlying token file is changed. It does not implement
APIs for active invalidation. It will be handled in a separate PR after the API
token is in place.

* [Test] Service Account - fix test assumption

* [Test] Service Accounts - handle token names with leading hyphen (#70983)

The CLI tool needs an option terminator (--) for another option names that
begin with a hyphen. Otherwise it errors out with message of "not recognized
option". The service account token name can begin with a hyphen. Hence we need
to use -- when it is the case. An example of equivalent command line is
./bin/elasticsearch-service-tokens create elastic/fleet -- -lead-with-hyphen.

* Service Accounts - Fleet integration (#70724)

This PR implements rest of the pieces needed for Fleet integration, including:

* Get service account role descriptor for authorization
* API for creating service account token and storing in the security index
* API for list tokens for a service account
* New named privilege for manage service account
* Mandate HTTP TLS for both service account auth and service account related
  APIs
* Tests for API key related operations using service account

* [Test] Service Accounts - Remove colon from invalid token name generator (#71099)

The colon character is interpreted as the separate between token name and token
secret. So if a token name contains a colon, it is in theory invalid. But the
parser takes only the part before the colon as the token name and thus consider
it as a valid token name. Subsequent authentication will still fail. But for
tests, this generates a different exception and fails the expectation. This PR
removes the colon char from being used to generate invalid token names for
simplicity.

* Fix for 7.x quirks

Author: ywangd

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -459,6 +459,8 @@ public void test94ElasticsearchNodeExecuteCliNotEsHomeWorkDir() throws Exception
             assertThat(result.stdout, containsString("Sets the passwords for reserved users"));
             result = sh.run(bin.usersTool + " -h");
             assertThat(result.stdout, containsString("Manages elasticsearch file users"));
+            result = sh.run(bin.serviceTokensTool + " -h");
+            assertThat(result.stdout, containsString("Manages elasticsearch service account file-tokens"));
         };
 
         Platforms.onLinux(action);

qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java

@@ -199,6 +199,7 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist
             "elasticsearch-sql-cli",
             "elasticsearch-syskeygen",
             "elasticsearch-users",
+            "elasticsearch-service-tokens",
             "x-pack-env",
             "x-pack-security-env",
             "x-pack-watcher-env"

qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java

@@ -472,6 +472,7 @@ private static void verifyDefaultInstallation(Installation es) {
             "elasticsearch-sql-cli",
             "elasticsearch-syskeygen",
             "elasticsearch-users",
+            "elasticsearch-service-tokens",
             "x-pack-env",
             "x-pack-security-env",
             "x-pack-watcher-env"

qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -189,5 +189,6 @@ public class Executables {
         public final Executable sqlCli = new Executable("elasticsearch-sql-cli");
         public final Executable syskeygenTool = new Executable("elasticsearch-syskeygen");
         public final Executable usersTool = new Executable("elasticsearch-users");
+        public final Executable serviceTokensTool = new Executable("elasticsearch-service-tokens");
     }
 }

qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -227,6 +227,7 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist
             "elasticsearch-sql-cli",
             "elasticsearch-syskeygen",
             "elasticsearch-users",
+            "elasticsearch-service-tokens",
             "x-pack-env",
             "x-pack-security-env",
             "x-pack-watcher-env"

x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc

@@ -1,3 +1,4 @@
+
 [role="xpack"]
 [[security-api-get-builtin-privileges]]
 === Get builtin privileges API
@@ -83,6 +84,7 @@ A successful call returns an object with "cluster" and "index" fields.
     "manage_rollup",
     "manage_saml",
     "manage_security",
+    "manage_service_account",
     "manage_slm",
     "manage_token",
     "manage_transform",

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -202,6 +202,26 @@ private XPackSettings() {
     public static final Setting<Boolean> DIAGNOSE_TRUST_EXCEPTIONS_SETTING = Setting.boolSetting(
         "xpack.security.ssl.diagnose.trust", true, Setting.Property.NodeScope);
 
+    // TODO: This setting of hashing algorithm can share code with the one for password when pbkdf2_stretch is the default for both
+    public static final Setting<String> SERVICE_TOKEN_HASHING_ALGORITHM = new Setting<>(
+        new Setting.SimpleKey("xpack.security.authc.service_token_hashing.algorithm"),
+        (s) -> "PBKDF2_STRETCH",
+        Function.identity(),
+        v -> {
+            if (Hasher.getAvailableAlgoStoredHash().contains(v.toLowerCase(Locale.ROOT)) == false) {
+                throw new IllegalArgumentException("Invalid algorithm: " + v + ". Valid values for password hashing are " +
+                    Hasher.getAvailableAlgoStoredHash().toString());
+            } else if (v.regionMatches(true, 0, "pbkdf2", 0, "pbkdf2".length())) {
+                try {
+                    SecretKeyFactory.getInstance("PBKDF2withHMACSHA512");
+                } catch (NoSuchAlgorithmException e) {
+                    throw new IllegalArgumentException(
+                        "Support for PBKDF2WithHMACSHA512 must be available in order to use any of the " +
+                            "PBKDF2 algorithms for the [xpack.security.authc.service_token_hashing.algorithm] setting.", e);
+                }
+            }
+        }, Property.NodeScope);
+
     public static final List<String> DEFAULT_SUPPORTED_PROTOCOLS;
 
     static {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/CreateServiceAccountTokenAction.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.service;
+
+import org.elasticsearch.action.ActionType;
+
+public class CreateServiceAccountTokenAction extends ActionType<CreateServiceAccountTokenResponse> {
+
+    public static final String NAME = "cluster:admin/xpack/security/service_account/token/create";
+    public static final CreateServiceAccountTokenAction INSTANCE = new CreateServiceAccountTokenAction();
+
+    private CreateServiceAccountTokenAction() {
+        super(NAME, CreateServiceAccountTokenResponse::new);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/CreateServiceAccountTokenRequest.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.service;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class CreateServiceAccountTokenRequest extends ActionRequest {
+
+    private final String namespace;
+    private final String serviceName;
+    private final String tokenName;
+    private WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.WAIT_UNTIL;
+
+    public CreateServiceAccountTokenRequest(String namespace, String serviceName, String tokenName) {
+        this.namespace = namespace;
+        this.serviceName = serviceName;
+        this.tokenName = tokenName;
+    }
+
+    public CreateServiceAccountTokenRequest(StreamInput in) throws IOException {
+        super(in);
+        this.namespace = in.readString();
+        this.serviceName = in.readString();
+        this.tokenName = in.readString();
+        this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public String getServiceName() {
+        return serviceName;
+    }
+
+    public String getTokenName() {
+        return tokenName;
+    }
+
+    public WriteRequest.RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        CreateServiceAccountTokenRequest that = (CreateServiceAccountTokenRequest) o;
+        return Objects.equals(namespace, that.namespace) && Objects.equals(serviceName, that.serviceName)
+            && Objects.equals(tokenName, that.tokenName) && refreshPolicy == that.refreshPolicy;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(namespace, serviceName, tokenName, refreshPolicy);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(namespace);
+        out.writeString(serviceName);
+        out.writeString(tokenName);
+        refreshPolicy.writeTo(out);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(namespace)) {
+            validationException = addValidationError("service account namespace is required", validationException);
+        }
+
+        if (Strings.isNullOrEmpty(serviceName)) {
+            validationException = addValidationError("service account service-name is required", validationException);
+        }
+
+        if (Strings.isNullOrEmpty(tokenName)) {
+            validationException = addValidationError("service account token name is required", validationException);
+        } else {
+            if (tokenName.length() > 256) {
+                validationException = addValidationError(
+                    "service account token name may not be more than 256 characters long", validationException);
+            }
+            if (tokenName.equals(tokenName.trim()) == false) {
+                validationException = addValidationError(
+                    "service account token name may not begin or end with whitespace", validationException);
+            }
+            if (tokenName.startsWith("_")) {
+                validationException = addValidationError(
+                    "service account token name may not begin with an underscore", validationException);
+            }
+        }
+        return validationException;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/CreateServiceAccountTokenResponse.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.service;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class CreateServiceAccountTokenResponse extends ActionResponse implements ToXContentObject {
+
+    @Nullable
+    private final String name;
+    @Nullable
+    private final SecureString value;
+
+    private CreateServiceAccountTokenResponse(boolean created, String name, SecureString value) {
+        this.name = name;
+        this.value = value;
+    }
+
+    public CreateServiceAccountTokenResponse(StreamInput in) throws IOException {
+        super(in);
+        this.name = in.readOptionalString();
+        this.value = in.readOptionalSecureString();
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public SecureString getValue() {
+        return value;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+            .field("created", true)
+            .field("token")
+            .startObject()
+            .field("name", name)
+            .field("value", value.toString())
+            .endObject()
+            .endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(name);
+        out.writeOptionalSecureString(value);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        CreateServiceAccountTokenResponse that = (CreateServiceAccountTokenResponse) o;
+        return Objects.equals(name, that.name) && Objects.equals(value, that.value);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, value);
+    }
+
+    public static CreateServiceAccountTokenResponse created(String name, SecureString value) {
+        return new CreateServiceAccountTokenResponse(true, name, value);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountTokensAction.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.action.service;
+
+import org.elasticsearch.action.ActionType;
+
+public class GetServiceAccountTokensAction extends ActionType<GetServiceAccountTokensResponse> {
+
+    public static final String NAME = "cluster:admin/xpack/security/service_account/token/get";
+    public static final GetServiceAccountTokensAction INSTANCE = new GetServiceAccountTokensAction();
+
+    public GetServiceAccountTokensAction() {
+        super(NAME, GetServiceAccountTokensResponse::new);
+    }
+}