All system indices are hidden indices

modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java

@@ -14,6 +14,7 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -23,6 +24,7 @@
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class KibanaSystemIndexIT extends ESRestTestCase {
@@ -218,6 +220,26 @@ public void testUpdateIndexSettings() throws IOException {
         assertThat(response.getStatusLine().getStatusCode(), is(200));
     }
 
+    public void testCannotCreateVisibleSystemIndex() {
+        Request request = request("PUT", "/" + indexName);
+        request.setJsonEntity("{\"settings\": {\"index.hidden\":\"false\"}}");
+        ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        assertThat(exception.getResponse().getStatusLine().getStatusCode(), is(400));
+        assertThat(exception.getMessage(), containsString("System indices must have index.hidden set to true."));
+    }
+
+    public void testCannotSetVisible() throws IOException {
+        Request putIndexRequest = request("PUT", "/" + indexName);
+        Response response = client().performRequest(putIndexRequest);
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+
+        Request putSettingsRequest = request("PUT", "/" + indexName + "/_settings");
+        putSettingsRequest.setJsonEntity("{ \"index.hidden\" : false }");
+        ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(putSettingsRequest));
+        assertThat(exception.getResponse().getStatusLine().getStatusCode(), is(400));
+        assertThat(exception.getMessage(), containsString("System indices must have index.hidden set to true."));
+    }
+
     public void testGetIndex() throws IOException {
         Request request = request("PUT", "/" + indexName);
         Response response = client().performRequest(request);

modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java

@@ -227,15 +227,17 @@ public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) thr
         String indexName = Optional.ofNullable(descriptor.getPrimaryIndex()).orElse(descriptor.getIndexPattern().replace("*", "old"));
         CreateIndexRequestBuilder createRequest = prepareCreate(indexName);
         createRequest.setWaitForActiveShards(ActiveShardCount.ALL);
-        if (descriptor.getSettings() != null) {
-            createRequest.setSettings(Settings.builder().put("index.version.created", Version.CURRENT).build());
-        } else {
+        if (SystemIndexDescriptor.DEFAULT_SETTINGS.equals(descriptor.getSettings())) {
+            // unmanaged
             createRequest.setSettings(
                 createSimpleSettings(
                     Version.V_7_0_0,
                     descriptor.isInternal() ? INTERNAL_UNMANAGED_FLAG_VALUE : EXTERNAL_UNMANAGED_FLAG_VALUE
                 )
             );
+        } else {
+            // managed
+            createRequest.setSettings(Settings.builder().put("index.version.created", Version.CURRENT).build());
         }
         if (descriptor.getMappings() == null) {
             createRequest.setMapping(createSimpleMapping(false, descriptor.isInternal()));

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml

@@ -126,7 +126,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
 
   - do:
       count:

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml

@@ -155,7 +155,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -270,7 +271,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml

@@ -164,7 +164,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -282,7 +283,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml

@@ -147,7 +147,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
@@ -256,7 +257,8 @@
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"
-      indices.refresh: {}
+      indices.refresh:
+        expand_wildcards: "open,hidden"
   - do:
       warnings:
         - "this request accesses system indices: [.tasks], but in a future major version, direct access to system indices will be prevented by default"

qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -45,6 +46,7 @@
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.test.rest.ESRestTestCase.entityAsMap;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
@@ -88,6 +90,20 @@ public void testSystemIndexAccessBlockedByDefault() throws Exception {
         // And with a total wildcard
         assertDeprecationWarningOnAccess(randomFrom("*", "_all"), SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
 
+        // If we're not expanding wildcards, we don't get anything
+        {
+            Request searchRequest = new Request("GET", "/" + randomFrom("*", "_all") + randomFrom("/_count", "/_search"));
+            searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
+            searchRequest.addParameter("allow_no_indices", "false");
+
+            try {
+                getRestClient().performRequest(searchRequest);
+            } catch (ResponseException e) {
+                assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
+                assertThat(e.getMessage(), containsString("no such index"));
+            }
+        }
+
         // Try to index a doc directly
         {
             String expectedWarning = "this request accesses system indices: [" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "], but in a " +
@@ -107,6 +123,7 @@ private void assertDeprecationWarningOnAccess(String queryPattern, String warnin
         searchRequest.setJsonEntity("{\"query\": {\"match\":  {\"some_field\":  \"some_value\"}}}");
         // Disallow no indices to cause an exception if this resolves to zero indices, so that we're sure it resolved the index
         searchRequest.addParameter("allow_no_indices", "false");
+        searchRequest.addParameter("expand_wildcards", "open,hidden");
         searchRequest.setOptions(expectWarnings(expectedWarning));
 
         Response response = getRestClient().performRequest(searchRequest);

server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java

@@ -159,12 +159,12 @@ private Map<String, List<String>> checkForSystemIndexViolations(Index[] concrete
     }
 
     /**
-     * Checks that the request isn't trying to add the "hidden" setting to a system
+     * Checks that the request isn't trying to remove the "hidden" setting on a system
      * index
      *
      * @param concreteIndices the indices being updated
      * @param request the update request
-     * @return a list of system indexes that this request would set to hidden
+     * @return a list of system indexes that this request would make visible
      */
     private List<String> checkForHidingSystemIndex(Index[] concreteIndices, UpdateSettingsRequest request) {
         // Requests that a cluster generates itself are permitted to have a difference in settings

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -111,6 +111,7 @@ public DataStream(String name, TimestampField timeStampField, List<Index> indice
         this.indices = Collections.unmodifiableList(indices);
         this.generation = generation;
         this.metadata = metadata;
+        assert system == false || hidden; // system indices must be hidden
         this.hidden = hidden;
         this.replicated = replicated;
         this.timeProvider = timeProvider;

server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java

@@ -220,6 +220,7 @@ public Alias(AliasMetadata aliasMetadata, List<IndexMetadata> indices) {
                     Strings.collectionToCommaDelimitedString(writeIndicesStrings) + "]");
             }
 
+            // TODO[wrb]: how do we handle system alias visibility?
             this.isHidden = aliasMetadata.isHidden() == null ? false : aliasMetadata.isHidden();
             this.isSystem = indices.stream().allMatch(IndexMetadata::isSystem);
             dataStreamAlias = false;
@@ -362,7 +363,7 @@ public DataStream getParentDataStream() {
 
         @Override
         public boolean isHidden() {
-            return dataStream.isHidden();
+            return isSystem() || dataStream.isHidden();
         }
 
         @Override

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -479,6 +479,7 @@ private IndexMetadata(
         this.rolloverInfos = rolloverInfos;
         this.isSystem = isSystem;
         assert isHidden == INDEX_HIDDEN_SETTING.get(settings);
+        assert isSystem == false || isHidden; // system indices must be hidden indices
         this.isHidden = isHidden;
         this.timestampRange = timestampRange;
         this.priority = priority;
@@ -1356,6 +1357,10 @@ public IndexMetadata build() {
                 tierPreference = null;
             }
 
+            if (isSystem) {
+                settings = Settings.builder().put(settings).put(INDEX_HIDDEN_SETTING.getKey(), true).build();
+            }
+
             return new IndexMetadata(
                     new Index(index, uuid),
                     version,

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java

@@ -216,7 +216,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn
         DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName);
         List<Index> dsBackingIndices = backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList());
         dsBackingIndices.add(writeIndex.getIndex());
-        boolean hidden = isSystem ? false : template.getDataStreamTemplate().isHidden();
+        boolean hidden = isSystem || template.getDataStreamTemplate().isHidden();
         DataStream newDataStream = new DataStream(
             dataStreamName,
             timestampField,

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -97,6 +97,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
@@ -356,6 +357,13 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd
                 return applyCreateIndexRequestForSystemDataStream(currentState, request, silent, metadataTransformer);
             }
 
+            if (systemIndices.isSystemIndex(request.index())
+                && request.settings().hasValue(SETTING_INDEX_HIDDEN)
+                && "false".equals(request.settings().get(SETTING_INDEX_HIDDEN))) {
+                    throw new IllegalArgumentException("System indices must have " + IndexMetadata.SETTING_INDEX_HIDDEN +
+                        " set to true.");
+                }
+
             // Hidden indices apply templates slightly differently (ignoring wildcard '*'
             // templates), so we need to check to see if the request is creating a hidden index
             // prior to resolving which templates it matches
@@ -575,7 +583,7 @@ private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterSta
         final List<Map<String, Object>> mappings =
             collectSystemV2Mappings(template, componentTemplates, xContentRegistry, request.index());
 
-        final Settings aggregatedIndexSettings = aggregateIndexSettings(
+        final Settings aggregatedIndexSettings = Settings.builder().put(aggregateIndexSettings(
             currentState,
             request,
             resolveSettings(template, componentTemplates),
@@ -585,7 +593,9 @@ private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterSta
             shardLimitValidator,
             indexSettingProviders,
             this.enforceDefaultTierPreference
-        );
+        ))
+            .put(SETTING_INDEX_HIDDEN, true)
+            .build();
         final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
         final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.ShardLimitValidator;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -39,6 +40,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN;
 import static org.elasticsearch.index.IndexSettings.same;
 
 /**
@@ -55,16 +57,18 @@ public class MetadataUpdateSettingsService {
     private final IndicesService indicesService;
     private final ShardLimitValidator shardLimitValidator;
     private final ThreadPool threadPool;
+    private final SystemIndices systemIndices;
 
     public MetadataUpdateSettingsService(ClusterService clusterService, AllocationService allocationService,
                                          IndexScopedSettings indexScopedSettings, IndicesService indicesService,
-                                         ShardLimitValidator shardLimitValidator, ThreadPool threadPool) {
+                                         ShardLimitValidator shardLimitValidator, ThreadPool threadPool, SystemIndices systemIndices) {
         this.clusterService = clusterService;
         this.threadPool = threadPool;
         this.allocationService = allocationService;
         this.indexScopedSettings = indexScopedSettings;
         this.indicesService = indicesService;
         this.shardLimitValidator = shardLimitValidator;
+        this.systemIndices = systemIndices;
     }
 
     public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request,
@@ -110,6 +114,7 @@ public ClusterState execute(ClusterState currentState) {
                 // on an open index
                 Set<Index> openIndices = new HashSet<>();
                 Set<Index> closeIndices = new HashSet<>();
+                Set<Index> systemIndicesInRequest = new HashSet<>();
                 final String[] actualIndices = new String[request.indices().length];
                 for (int i = 0; i < request.indices().length; i++) {
                     Index index = request.indices()[i];
@@ -120,6 +125,16 @@ public ClusterState execute(ClusterState currentState) {
                     } else {
                         closeIndices.add(index);
                     }
+                    if (systemIndices.isSystemIndex(index)) {
+                        systemIndicesInRequest.add(index);
+                    }
+                }
+
+                if (systemIndicesInRequest.size() > 0
+                    && normalizedSettings.hasValue(SETTING_INDEX_HIDDEN)
+                    && "false".equals(normalizedSettings.get(SETTING_INDEX_HIDDEN))) {
+                    throw new IllegalArgumentException("System indices must have " + IndexMetadata.SETTING_INDEX_HIDDEN +
+                        " set to true.");
                 }
 
                 if (skippedSettings.isEmpty() == false && openIndices.isEmpty() == false) {

server/src/main/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeService.java

@@ -90,10 +90,11 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         builder.system(cursor.value.isSystem() == false);
                         updated = true;
                     }
-                    if (isSystem && cursor.value.getSettings().getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false)) {
+                    boolean isHidden = cursor.value.getSettings().getAsBoolean(IndexMetadata.SETTING_INDEX_HIDDEN, false);
+                    if (isSystem && isHidden == false) {
                         builder.settings(Settings.builder()
                             .put(cursor.value.getSettings())
-                            .put(IndexMetadata.SETTING_INDEX_HIDDEN, false));
+                            .put(IndexMetadata.SETTING_INDEX_HIDDEN, true));
                         updated = true;
                     }
                     if (updated) {