System indices treated as restricted indices (#74212)

System indices should be treated as a special set of indices and not be
accessible by all users. The existing security codebase has the notion
of restricted indices, which are currently a subset of system indices.

This change unifies the two concepts by making system indices the set
of restricted indices. This means that going forward, consumers of
system indices will need access to restricted indices.

Our intention is that this will be handled internally as much as
possible.  For example, restricted index access can be used implicitly 
by setting a valid origin on a request or using a system index plugin.
In these cases, the security module will apply internally defined
privileges when necessary.

The main impact of this change for developers is that system index
deletions will require superuser privileges, so we have to make sure
we are using an admin role for test cleanup.

Closes #69298

Co-authored-by: William Brafford <[email protected]>
Co-authored-by: Albert Zaharovits <[email protected]>

Author: 3 people

client/rest-high-level/qa/ssl-enabled/src/javaRestTest/java/org/elasticsearch/client/EnrollmentIT.java

@@ -59,6 +59,17 @@ protected Settings restClientSettings() {
             .build();
     }
 
+    /**
+     * Cleanup for these tests requires the admin client to have access to the
+     * truststore, so we use the same settings that we're using for the rest
+     * client in this test class.
+     * @return Settings for the admin client.
+     */
+    @Override
+    protected Settings restAdminSettings() {
+        return restClientSettings();
+    }
+
     public void testEnrollNode() throws Exception {
         final NodeEnrollmentResponse nodeEnrollmentResponse =
             execute(highLevelClient().security()::enrollNode, highLevelClient().security()::enrollNodeAsync, RequestOptions.DEFAULT);

client/rest-high-level/qa/ssl-enabled/src/javaRestTest/java/org/elasticsearch/client/documentation/EnrollmentDocumentationIT.java

@@ -52,6 +52,17 @@ protected Settings restClientSettings() {
             .build();
     }
 
+    /**
+     * Cleanup for these tests requires the admin client to have access to the
+     * truststore, so we use the same settings that we're using for the rest
+     * client in this test class.
+     * @return Settings for the admin client.
+     */
+    @Override
+    protected Settings restAdminSettings() {
+        return restClientSettings();
+    }
+
     public void testNodeEnrollment() throws Exception {
         RestHighLevelClient client = highLevelClient();
 

client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.client.cluster.RemoteInfoRequest;
 import org.elasticsearch.client.cluster.RemoteInfoResponse;
 import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -66,6 +67,7 @@ public abstract class ESRestHighLevelClientTestCase extends ESRestTestCase {
     protected static final String CONFLICT_PIPELINE_ID = "conflict_pipeline";
 
     private static RestHighLevelClient restHighLevelClient;
+    private static RestHighLevelClient adminRestHighLevelClient;
     private static boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false"));
 
     @Before
@@ -74,18 +76,35 @@ public void initHighLevelClient() throws IOException {
         if (restHighLevelClient == null) {
             restHighLevelClient = new HighLevelClient(client());
         }
+        if (adminRestHighLevelClient == null) {
+            adminRestHighLevelClient = new HighLevelClient(adminClient());
+        }
     }
 
     @AfterClass
     public static void cleanupClient() throws IOException {
         IOUtils.close(restHighLevelClient);
+        IOUtils.close(adminRestHighLevelClient);
         restHighLevelClient = null;
+        adminRestHighLevelClient = null;
     }
 
     protected static RestHighLevelClient highLevelClient() {
         return restHighLevelClient;
     }
 
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("admin_user", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    protected static RestHighLevelClient adminHighLevelClient() {
+        return adminRestHighLevelClient;
+    }
+
     /**
      * Executes the provided request using either the sync method or its async variant, both provided as functions
      */

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java

@@ -274,7 +274,7 @@ private void addModelSnapshotIndexRequests(BulkRequest bulkRequest) throws IOExc
 
     @After
     public void deleteJob() throws IOException {
-        new MlTestStateCleaner(logger, highLevelClient()).clearMlMetadata();
+        new MlTestStateCleaner(logger, adminHighLevelClient()).clearMlMetadata();
     }
 
     public void testGetModelSnapshots() throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.client.feature.ResetFeaturesRequest;
+import org.elasticsearch.client.feature.ResetFeaturesResponse;
 import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest;
 
 import java.io.IOException;
@@ -36,7 +37,15 @@ public MlTestStateCleaner(Logger logger, RestHighLevelClient client) {
     public void clearMlMetadata() throws IOException {
         deleteAllTrainedModelIngestPipelines();
         // This resets all features, not just ML, but they should have been getting reset between tests anyway so it shouldn't matter
-        client.features().resetFeatures(new ResetFeaturesRequest(), RequestOptions.DEFAULT);
+        ResetFeaturesResponse response = client.features().resetFeatures(new ResetFeaturesRequest(), RequestOptions.DEFAULT);
+        if (response.getFeatureResetStatuses().stream().anyMatch(status -> "FAILURE".equals(status.getStatus()))) {
+            logger.warn("Not all feature states could be reset while clearing ML Metadata:");
+            for (ResetFeaturesResponse.ResetFeatureStateStatus status : response.getFeatureResetStatuses()) {
+                if (status.getStatus().equals("FAILURE")) {
+                    logger.warn("Feature {} failed with response: {}", status.getFeatureName(), status.getException());
+                }
+            }
+        }
     }
 
     @SuppressWarnings("unchecked")

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.apache.lucene.util.automaton.Automaton;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.IndicesRequest;
@@ -786,6 +787,10 @@ public Predicate<String> getSystemIndexAccessPredicate() {
         return systemIndexAccessLevelPredicate;
     }
 
+    public Automaton getSystemNameAutomaton() {
+        return systemIndices.getSystemNameAutomaton();
+    }
+
     public Predicate<String> getNetNewSystemIndexPredicate() {
         return systemIndices::isNetNewSystemIndex;
     }

server/src/main/java/org/elasticsearch/indices/SystemIndices.java

@@ -42,7 +42,6 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -66,10 +65,12 @@ public class SystemIndices {
         TASKS_FEATURE_NAME, new Feature(TASKS_FEATURE_NAME, "Manages task results", List.of(TASKS_DESCRIPTOR))
     );
 
-    private final CharacterRunAutomaton systemIndexAutomaton;
-    private final CharacterRunAutomaton systemDataStreamIndicesAutomaton;
+    private final Automaton systemNameAutomaton;
     private final CharacterRunAutomaton netNewSystemIndexAutomaton;
-    private final Predicate<String> systemDataStreamAutomaton;
+    private final CharacterRunAutomaton systemNameRunAutomaton;
+    private final CharacterRunAutomaton systemIndexRunAutomaton;
+    private final CharacterRunAutomaton systemDataStreamIndicesRunAutomaton;
+    private final Predicate<String> systemDataStreamPredicate;
     private final Map<String, Feature> featureDescriptors;
     private final Map<String, CharacterRunAutomaton> productToSystemIndicesMatcher;
     private final ExecutorSelector executorSelector;
@@ -83,12 +84,19 @@ public SystemIndices(Map<String, Feature> pluginAndModulesDescriptors) {
         featureDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors);
         checkForOverlappingPatterns(featureDescriptors);
         checkForDuplicateAliases(this.getSystemIndexDescriptors());
-        this.systemIndexAutomaton = buildIndexCharacterRunAutomaton(featureDescriptors);
+        Automaton systemIndexAutomata = buildIndexAutomaton(featureDescriptors);
+        this.systemIndexRunAutomaton = new CharacterRunAutomaton(systemIndexAutomata);
+        Automaton systemDataStreamIndicesAutomata = buildDataStreamBackingIndicesAutomaton(featureDescriptors);
+        this.systemDataStreamIndicesRunAutomaton = new CharacterRunAutomaton(systemDataStreamIndicesAutomata);
+        this.systemDataStreamPredicate = buildDataStreamNamePredicate(featureDescriptors);
         this.netNewSystemIndexAutomaton = buildNetNewIndexCharacterRunAutomaton(featureDescriptors);
-        this.systemDataStreamIndicesAutomaton = buildDataStreamBackingIndicesAutomaton(featureDescriptors);
-        this.systemDataStreamAutomaton = buildDataStreamNamePredicate(featureDescriptors);
         this.productToSystemIndicesMatcher = getProductToSystemIndicesMap(featureDescriptors);
         this.executorSelector = new ExecutorSelector(this);
+        this.systemNameAutomaton = MinimizationOperations.minimize(
+            Operations.union(List.of(systemIndexAutomata, systemDataStreamIndicesAutomata, buildDataStreamAutomaton(featureDescriptors))),
+            Integer.MAX_VALUE
+        );
+        this.systemNameRunAutomaton = new CharacterRunAutomaton(systemNameAutomaton);
     }
 
     private static void checkForDuplicateAliases(Collection<SystemIndexDescriptor> descriptors) {
@@ -150,7 +158,7 @@ private static Map<String, CharacterRunAutomaton> getProductToSystemIndicesMap(M
      * is checked against index names, aliases, data stream names, and the names of indices that back a system data stream.
      */
     public boolean isSystemName(String name) {
-        return isSystemIndex(name) || isSystemDataStream(name) || isSystemIndexBackingDataStream(name);
+        return systemNameRunAutomaton.run(name);
     }
 
     /**
@@ -169,22 +177,30 @@ public boolean isSystemIndex(Index index) {
      * @return true if the index name matches a pattern from a {@link SystemIndexDescriptor}
      */
     public boolean isSystemIndex(String indexName) {
-        return systemIndexAutomaton.run(indexName);
+        return systemIndexRunAutomaton.run(indexName);
     }
 
     /**
      * Determines whether the provided name matches that of a system data stream that has been defined by a
      * {@link SystemDataStreamDescriptor}
      */
     public boolean isSystemDataStream(String name) {
-        return systemDataStreamAutomaton.test(name);
+        return systemDataStreamPredicate.test(name);
     }
 
     /**
      * Determines whether the provided name matches that of an index that backs a system data stream.
      */
     public boolean isSystemIndexBackingDataStream(String name) {
-        return systemDataStreamIndicesAutomaton.run(name);
+        return systemDataStreamIndicesRunAutomaton.run(name);
+    }
+
+    /**
+     * @return An {@link Automaton} that tests whether strings are names of system indices, aliases, or
+     * data streams.
+     */
+    public Automaton getSystemNameAutomaton() {
+        return systemNameAutomaton;
     }
 
     public boolean isNetNewSystemIndex(String indexName) {
@@ -293,11 +309,11 @@ public Map<String, Feature> getFeatures() {
         return featureDescriptors;
     }
 
-    private static CharacterRunAutomaton buildIndexCharacterRunAutomaton(Map<String, Feature> descriptors) {
+    private static Automaton buildIndexAutomaton(Map<String, Feature> descriptors) {
         Optional<Automaton> automaton = descriptors.values().stream()
             .map(SystemIndices::featureToIndexAutomaton)
             .reduce(Operations::union);
-        return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE));
+        return MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE);
     }
 
     private static CharacterRunAutomaton buildNetNewIndexCharacterRunAutomaton(Map<String, Feature> featureDescriptors) {
@@ -317,19 +333,26 @@ private static Automaton featureToIndexAutomaton(Feature feature) {
         return systemIndexAutomaton.orElse(EMPTY);
     }
 
-    private static Predicate<String> buildDataStreamNamePredicate(Map<String, Feature> descriptors) {
-        Set<String> systemDataStreamNames = descriptors.values().stream()
+    private static Automaton buildDataStreamAutomaton(Map<String, Feature> descriptors) {
+        Optional<Automaton> automaton = descriptors.values().stream()
             .flatMap(feature -> feature.getDataStreamDescriptors().stream())
             .map(SystemDataStreamDescriptor::getDataStreamName)
-            .collect(Collectors.toUnmodifiableSet());
-        return systemDataStreamNames::contains;
+            .map(dsName -> SystemIndexDescriptor.buildAutomaton(dsName, null))
+            .reduce(Operations::union);
+
+        return automaton.isPresent() ? MinimizationOperations.minimize(automaton.get(), Integer.MAX_VALUE) : EMPTY;
+    }
+
+    private static Predicate<String> buildDataStreamNamePredicate(Map<String, Feature> descriptors) {
+        CharacterRunAutomaton characterRunAutomaton = new CharacterRunAutomaton(buildDataStreamAutomaton(descriptors));
+        return characterRunAutomaton::run;
     }
 
-    private static CharacterRunAutomaton buildDataStreamBackingIndicesAutomaton(Map<String, Feature> descriptors) {
+    private static Automaton buildDataStreamBackingIndicesAutomaton(Map<String, Feature> descriptors) {
         Optional<Automaton> automaton = descriptors.values().stream()
             .map(SystemIndices::featureToDataStreamBackingIndicesAutomaton)
             .reduce(Operations::union);
-        return new CharacterRunAutomaton(automaton.orElse(EMPTY));
+        return MinimizationOperations.minimize(automaton.orElse(EMPTY), Integer.MAX_VALUE);
     }
 
     private static Automaton featureToDataStreamBackingIndicesAutomaton(Feature feature) {
@@ -343,7 +366,7 @@ private static Automaton featureToDataStreamBackingIndicesAutomaton(Feature feat
     }
 
     public SystemDataStreamDescriptor validateDataStreamAccess(String dataStreamName, ThreadContext threadContext) {
-        if (systemDataStreamAutomaton.test(dataStreamName)) {
+        if (systemDataStreamPredicate.test(dataStreamName)) {
             SystemDataStreamDescriptor dataStreamDescriptor = featureDescriptors.values().stream()
                 .flatMap(feature -> feature.getDataStreamDescriptors().stream())
                 .filter(descriptor -> descriptor.getDataStreamName().equals(dataStreamName))
@@ -405,6 +428,7 @@ IllegalArgumentException dataStreamAccessException(@Nullable String product, Str
     /**
      * Determines what level of system index access should be allowed in the current context.
      *
+     * @param threadContext the current thread context that has headers associated with the current request
      * @return {@link SystemIndexAccessLevel#ALL} if unrestricted system index access should be allowed,
      * {@link SystemIndexAccessLevel#RESTRICTED} if a subset of system index access should be allowed, or
      * {@link SystemIndexAccessLevel#NONE} if no system index access should be allowed.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java

@@ -142,7 +142,7 @@ private static XContentBuilder mappings() {
 
     public static SystemIndexDescriptor getSystemIndexDescriptor() {
         return SystemIndexDescriptor.builder()
-            .setIndexPattern(XPackPlugin.ASYNC_RESULTS_INDEX)
+            .setIndexPattern(XPackPlugin.ASYNC_RESULTS_INDEX + "*")
             .setDescription("Async search results")
             .setPrimaryIndex(XPackPlugin.ASYNC_RESULTS_INDEX)
             .setMappings(mappings())