From 6a596af934d296f9cc65ead1c3d91ff8cb357cf9 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Thu, 12 Dec 2019 22:54:12 -0700
Subject: [PATCH 01/26] Basic working version of hidden indices

---
 .../cluster/health/ClusterHealthRequest.java  |   2 +-
 .../action/support/IndicesOptions.java        | 131 +++++++++++-------
 .../cluster/metadata/IndexMetaData.java       |   7 +
 .../metadata/IndexNameExpressionResolver.java |  75 ++++++----
 .../cluster/metadata/MetaData.java            |  66 +++++++--
 .../common/settings/IndexScopedSettings.java  |   1 +
 .../action/support/IndicesOptionsTests.java   |  25 ----
 .../elasticsearch/index/HiddenIndexIT.java    |  64 +++++++++
 .../org/elasticsearch/test/TestCluster.java   |   5 +-
 9 files changed, 259 insertions(+), 117 deletions(-)
 create mode 100644 server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java

diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
index 9bea7b68caf0b..bbe24d7c0443f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
@@ -39,7 +39,7 @@
 public class ClusterHealthRequest extends MasterNodeReadRequest<ClusterHealthRequest> implements IndicesRequest.Replaceable {
 
     private String[] indices;
-    private IndicesOptions indicesOptions = IndicesOptions.lenientExpand();
+    private IndicesOptions indicesOptions = IndicesOptions.lenientExpandHidden();
     private TimeValue timeout = new TimeValue(30, TimeUnit.SECONDS);
     private ClusterHealthStatus waitForStatus;
     private boolean waitForNoRelocatingShards = false;
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index 25e7a25335d66..cfc4585194de7 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.action.support;
 
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -45,7 +46,8 @@ public class IndicesOptions implements ToXContentFragment {
 
     public enum WildcardStates {
         OPEN,
-        CLOSED;
+        CLOSED,
+        HIDDEN;
 
         public static final EnumSet<WildcardStates> NONE = EnumSet.noneOf(WildcardStates.class);
 
@@ -56,18 +58,24 @@ public static EnumSet<WildcardStates> parseParameter(Object value, EnumSet<Wildc
 
             Set<WildcardStates> states = new HashSet<>();
             String[] wildcards = nodeStringArrayValue(value);
+            // TODO why do we let patterns like "none,all" or "open,none,closed" get used. The location of 'none' in the array changes the
+            // meaning of the resulting value
             for (String wildcard : wildcards) {
-                if ("open".equals(wildcard)) {
-                    states.add(OPEN);
-                } else if ("closed".equals(wildcard)) {
-                    states.add(CLOSED);
-                } else if ("none".equals(wildcard)) {
-                    states.clear();
-                } else if ("all".equals(wildcard)) {
-                    states.add(OPEN);
-                    states.add(CLOSED);
-                } else {
-                    throw new IllegalArgumentException("No valid expand wildcard value [" + wildcard + "]");
+                switch (wildcard) {
+                    case "open":
+                        states.add(OPEN);
+                        break;
+                    case "closed":
+                        states.add(CLOSED);
+                        break;
+                    case "none":
+                        states.clear();
+                        break;
+                    case "all":
+                        states.addAll(EnumSet.allOf(WildcardStates.class));
+                        break;
+                    default:
+                        throw new IllegalArgumentException("No valid expand wildcard value [" + wildcard + "]");
                 }
             }
 
@@ -91,9 +99,15 @@ public enum Option {
     public static final IndicesOptions LENIENT_EXPAND_OPEN =
         new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
             EnumSet.of(WildcardStates.OPEN));
+    public static final IndicesOptions LENIENT_EXPAND_OPEN_HIDDEN =
+        new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
+            EnumSet.of(WildcardStates.OPEN, WildcardStates.HIDDEN));
     public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED =
         new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
             EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED));
+    public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN =
+        new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE),
+            EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED, WildcardStates.HIDDEN));
     public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED =
         new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED));
     public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED =
@@ -118,42 +132,6 @@ private IndicesOptions(Collection<Option> options, Collection<WildcardStates> ex
             expandWildcards.isEmpty() ? WildcardStates.NONE : EnumSet.copyOf(expandWildcards));
     }
 
-    // Package visible for testing
-    static IndicesOptions fromByte(final byte id) {
-        // IGNORE_UNAVAILABLE = 1;
-        // ALLOW_NO_INDICES = 2;
-        // EXPAND_WILDCARDS_OPEN = 4;
-        // EXPAND_WILDCARDS_CLOSED = 8;
-        // FORBID_ALIASES_TO_MULTIPLE_INDICES = 16;
-        // FORBID_CLOSED_INDICES = 32;
-        // IGNORE_ALIASES = 64;
-
-        Set<Option> opts = new HashSet<>();
-        Set<WildcardStates> wildcards = new HashSet<>();
-        if ((id & 1) != 0) {
-            opts.add(Option.IGNORE_UNAVAILABLE);
-        }
-        if ((id & 2) != 0) {
-            opts.add(Option.ALLOW_NO_INDICES);
-        }
-        if ((id & 4) != 0) {
-            wildcards.add(WildcardStates.OPEN);
-        }
-        if ((id & 8) != 0) {
-            wildcards.add(WildcardStates.CLOSED);
-        }
-        if ((id & 16) != 0) {
-            opts.add(Option.FORBID_ALIASES_TO_MULTIPLE_INDICES);
-        }
-        if ((id & 32) != 0) {
-            opts.add(Option.FORBID_CLOSED_INDICES);
-        }
-        if ((id & 64) != 0) {
-            opts.add(Option.IGNORE_ALIASES);
-        }
-        return new IndicesOptions(opts, wildcards);
-    }
-
     /**
      * @return Whether specified concrete indices should be ignored when unavailable (missing or closed)
      */
@@ -185,6 +163,13 @@ public boolean expandWildcardsClosed() {
         return expandWildcards.contains(WildcardStates.CLOSED);
     }
 
+    /**
+     * @return Whether wildcard expressions should get expanded to hidden indices
+     */
+    public boolean expandWildcardsHidden() {
+        return expandWildcards.contains(WildcardStates.HIDDEN);
+    }
+
     /**
      * @return Whether execution on closed indices is allowed.
      */
@@ -218,7 +203,17 @@ public boolean ignoreThrottled() {
 
     public void writeIndicesOptions(StreamOutput out) throws IOException {
         out.writeEnumSet(options);
-        out.writeEnumSet(expandWildcards);
+        if (out.getVersion().before(Version.V_8_0_0) && expandWildcards.contains(WildcardStates.HIDDEN)) {
+            Set<WildcardStates> states = new HashSet<>();
+            for (WildcardStates state : expandWildcards) {
+                if (state != WildcardStates.HIDDEN) {
+                    states.add(state);
+                }
+            }
+            out.writeEnumSet(EnumSet.copyOf(states));
+        } else {
+            out.writeEnumSet(expandWildcards);
+        }
     }
 
     public static IndicesOptions readIndicesOptions(StreamInput in) throws IOException {
@@ -227,18 +222,27 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
 
     public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
                                              boolean expandToClosedIndices) {
-        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, true, false, false, false);
+        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, false, true, false, false, false);
     }
 
     public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
                                              boolean expandToClosedIndices, IndicesOptions defaultOptions) {
         return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices,
-            defaultOptions.allowAliasesToMultipleIndices(), defaultOptions.forbidClosedIndices(), defaultOptions.ignoreAliases(),
-            defaultOptions.ignoreThrottled());
+            defaultOptions.expandWildcardsHidden(), defaultOptions.allowAliasesToMultipleIndices(),
+            defaultOptions.forbidClosedIndices(), defaultOptions.ignoreAliases(), defaultOptions.ignoreThrottled());
     }
 
     public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
-            boolean expandToClosedIndices, boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices, boolean ignoreAliases,
+                                             boolean expandToClosedIndices, boolean allowAliasesToMultipleIndices,
+                                             boolean forbidClosedIndices, boolean ignoreAliases,
+                                             boolean ignoreThrottled) {
+        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, false,
+            allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases, ignoreThrottled);
+    }
+
+    public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
+                                             boolean expandToClosedIndices, boolean expandToHiddenIndices,
+                                             boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices, boolean ignoreAliases,
                                              boolean ignoreThrottled) {
         final Set<Option> opts = new HashSet<>();
         final Set<WildcardStates> wildcards = new HashSet<>();
@@ -255,6 +259,9 @@ public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allo
         if (expandToClosedIndices) {
             wildcards.add(WildcardStates.CLOSED);
         }
+        if (expandToHiddenIndices) {
+            wildcards.add(WildcardStates.HIDDEN);
+        }
         if (allowAliasesToMultipleIndices == false) {
             opts.add(Option.FORBID_ALIASES_TO_MULTIPLE_INDICES);
         }
@@ -314,6 +321,7 @@ public static IndicesOptions fromParameters(Object wildcardsString, Object ignor
                 nodeBooleanValue(allowNoIndicesString, "allow_no_indices", defaultSettings.allowNoIndices()),
                 wildcards.contains(WildcardStates.OPEN),
                 wildcards.contains(WildcardStates.CLOSED),
+                wildcards.contains(WildcardStates.HIDDEN),
                 defaultSettings.allowAliasesToMultipleIndices(),
                 defaultSettings.forbidClosedIndices(),
                 defaultSettings.ignoreAliases(),
@@ -384,6 +392,14 @@ public static IndicesOptions lenientExpandOpen() {
         return LENIENT_EXPAND_OPEN;
     }
 
+    /**
+     * @return indices options that ignores unavailable indices, expands wildcards to open and hidden indices, and
+     *         allows that no indices are resolved from wildcard expressions (not returning an error).
+     */
+    public static IndicesOptions lenientExpandOpenHidden() {
+        return LENIENT_EXPAND_OPEN_HIDDEN;
+    }
+
     /**
      * @return indices options that ignores unavailable indices,  expands wildcards to both open and closed
      * indices and allows that no indices are resolved from wildcard expressions (not returning an error).
@@ -392,6 +408,14 @@ public static IndicesOptions lenientExpand() {
         return LENIENT_EXPAND_OPEN_CLOSED;
     }
 
+    /**
+     * @return indices options that ignores unavailable indices,  expands wildcards to all open and closed
+     * indices and allows that no indices are resolved from wildcard expressions (not returning an error).
+     */
+    public static IndicesOptions lenientExpandHidden() {
+        return LENIENT_EXPAND_OPEN_CLOSED_HIDDEN;
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (obj == null) {
@@ -419,6 +443,7 @@ public String toString() {
                 ", allow_no_indices=" + allowNoIndices() +
                 ", expand_wildcards_open=" + expandWildcardsOpen() +
                 ", expand_wildcards_closed=" + expandWildcardsClosed() +
+                ", expand_wildcards_hidden=" + expandWildcardsHidden() +
                 ", allow_aliases_to_multiple_indices=" + allowAliasesToMultipleIndices() +
                 ", forbid_closed_indices=" + forbidClosedIndices() +
                 ", ignore_aliases=" + ignoreAliases() +
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index f8e1b48c6dd83..5d5faacc10a2a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -256,6 +256,13 @@ public Iterator<Setting<?>> settings() {
                       Setting.Property.Dynamic,
                       Setting.Property.IndexScope);
 
+    /**
+     * Whether the index is considered hidden or not. A hidden index will not be resolved in
+     * normal wildcard searches unless explicitly allowed
+     */
+    public static final Setting<Boolean> INDEX_HIDDEN_SETTING =
+        Setting.boolSetting("index.hidden", false, Property.IndexScope, Property.Final);
+
     /**
      * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading
      */
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 93e8b84936133..6c4ebf371f76a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -144,8 +144,11 @@ String[] concreteIndexNames(Context context, String... indexExpressions) {
     }
 
     Index[] concreteIndices(Context context, String... indexExpressions) {
+        final List<String> expressions;
         if (indexExpressions == null || indexExpressions.length == 0) {
-            indexExpressions = new String[]{MetaData.ALL};
+            expressions = List.of(MetaData.ALL);
+        } else {
+            expressions = List.of(indexExpressions);
         }
         MetaData metaData = context.getState().metaData();
         IndicesOptions options = context.getOptions();
@@ -153,17 +156,17 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         // If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices
         // option. At some point we should change this, because there shouldn't be a reason why whether a single index
         // or multiple indices are specified yield different behaviour.
-        final boolean failNoIndices = indexExpressions.length == 1 ? !options.allowNoIndices() : !options.ignoreUnavailable();
-        List<String> expressions = Arrays.asList(indexExpressions);
+        final boolean failNoIndices = expressions.size() == 1 ? !options.allowNoIndices() : !options.ignoreUnavailable();
+        List<String> resolvedExpressions = expressions;
         for (ExpressionResolver expressionResolver : expressionResolvers) {
-            expressions = expressionResolver.resolve(context, expressions);
+            resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
         }
-        
-        if (expressions.isEmpty()) {
+
+        if (resolvedExpressions.isEmpty()) {
             if (!options.allowNoIndices()) {
                 IndexNotFoundException infe;
-                if (indexExpressions.length == 1) {
-                    if (indexExpressions[0].equals(MetaData.ALL)) {
+                if (expressions.size() == 1) {
+                    if (MetaData.ALL.equals(expressions.get(0))) {
                         infe = new IndexNotFoundException("no indices exist", (String)null);
                     } else {
                         infe = new IndexNotFoundException((String)null);
@@ -179,7 +182,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         }
 
         final Set<Index> concreteIndices = new HashSet<>(expressions.size());
-        for (String expression : expressions) {
+        for (String expression : resolvedExpressions) {
             AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression);
             if (aliasOrIndex == null ) {
                 if (failNoIndices) {
@@ -187,7 +190,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                     if (expression.equals(MetaData.ALL)) {
                         infe = new IndexNotFoundException("no indices exist", expression);
                     } else {
-                        infe = new IndexNotFoundException(expression);  
+                        infe = new IndexNotFoundException(expression);
                     }
                     infe.setResources("index_expression", expression);
                     throw infe;
@@ -345,7 +348,7 @@ public String resolveDateMathExpression(String dateExpression) {
      */
     public Set<String> resolveExpressions(ClusterState state, String... expressions) {
         Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false);
-        List<String> resolvedExpressions = Arrays.asList(expressions);
+        List<String> resolvedExpressions = List.of(expressions);
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
         }
@@ -427,7 +430,7 @@ public String[] indexAliases(ClusterState state, String index, Predicate<AliasMe
         if (aliases == null) {
             return null;
         }
-        return aliases.toArray(new String[aliases.size()]);
+        return aliases.toArray(Strings.EMPTY_ARRAY);
     }
 
     /**
@@ -670,7 +673,8 @@ static final class WildcardExpressionResolver implements ExpressionResolver {
         public List<String> resolve(Context context, List<String> expressions) {
             IndicesOptions options = context.getOptions();
             MetaData metaData = context.getState().metaData();
-            if (options.expandWildcardsClosed() == false && options.expandWildcardsOpen() == false) {
+            if (options.expandWildcardsClosed() == false && options.expandWildcardsOpen() == false &&
+                options.expandWildcardsHidden() == false) {
                 return expressions;
             }
 
@@ -678,26 +682,24 @@ public List<String> resolve(Context context, List<String> expressions) {
                 return resolveEmptyOrTrivialWildcard(options, metaData);
             }
 
-            // TODO: Fix API to work with sets rather than lists since we need to convert to sets
-            // internally anyway.
-            Set<String> result = innerResolve(context, expressions, options, metaData);
+            final Set<String> result = innerResolve(context, expressions, options, metaData);
 
             if (result == null) {
                 return expressions;
             }
             if (result.isEmpty() && !options.allowNoIndices()) {
                 IndexNotFoundException infe = new IndexNotFoundException((String)null);
-                infe.setResources("index_or_alias", expressions.toArray(new String[0]));
+                infe.setResources("index_or_alias", expressions.toArray(Strings.EMPTY_ARRAY));
                 throw infe;
             }
-            return new ArrayList<>(result);
+            return List.copyOf(result);
         }
 
         private Set<String> innerResolve(Context context, List<String> expressions, IndicesOptions options, MetaData metaData) {
             Set<String> result = null;
             boolean wildcardSeen = false;
             for (int i = 0; i < expressions.size(); i++) {
-                String expression = expressions.get(i);
+                String expression = expressions.get(0);
                 if (Strings.isEmpty(expression)) {
                     throw indexNotFoundException(expression);
                 }
@@ -739,7 +741,7 @@ private Set<String> innerResolve(Context context, List<String> expressions, Indi
 
                 final IndexMetaData.State excludeState = excludeState(options);
                 final Map<String, AliasOrIndex> matches = matches(context, metaData, expression);
-                Set<String> expand = expand(context, excludeState, matches);
+                Set<String> expand = expand(context, excludeState, matches, options.expandWildcardsHidden());
                 if (add) {
                     result.addAll(expand);
                 } else {
@@ -834,7 +836,8 @@ private static Map<String, AliasOrIndex> otherWildcard(Context context, MetaData
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         }
 
-        private static Set<String> expand(Context context, IndexMetaData.State excludeState, Map<String, AliasOrIndex> matches) {
+        private static Set<String> expand(Context context, IndexMetaData.State excludeState, Map<String, AliasOrIndex> matches,
+                                          boolean includeHidden) {
             Set<String> expand = new HashSet<>();
             for (Map.Entry<String, AliasOrIndex> entry : matches.entrySet()) {
                 AliasOrIndex aliasOrIndex = entry.getValue();
@@ -843,7 +846,9 @@ private static Set<String> expand(Context context, IndexMetaData.State excludeSt
                 } else {
                     for (IndexMetaData meta : aliasOrIndex.getIndices()) {
                         if (excludeState == null || meta.getState() != excludeState) {
-                            expand.add(meta.getIndex().getName());
+                            if (includeHidden || IndexMetaData.INDEX_HIDDEN_SETTING.get(meta.getSettings()) == false) {
+                                expand.add(meta.getIndex().getName());
+                            }
                         }
                     }
                 }
@@ -852,19 +857,31 @@ private static Set<String> expand(Context context, IndexMetaData.State excludeSt
         }
 
         private boolean isEmptyOrTrivialWildcard(List<String> expressions) {
-            return expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0)) ||
-                Regex.isMatchAllPattern(expressions.get(0))));
+            if (expressions.isEmpty()) {
+                return true;
+            } else if (expressions.size() == 1) {
+                final String expression = expressions.get(0);
+                return MetaData.ALL.equals(expression) || Regex.isMatchAllPattern(expression);
+            } else {
+                return false;
+            }
         }
 
         private static List<String> resolveEmptyOrTrivialWildcard(IndicesOptions options, MetaData metaData) {
-            if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) {
-                return Arrays.asList(metaData.getConcreteAllIndices());
+            if (options.expandWildcardsOpen() && options.expandWildcardsClosed() && options.expandWildcardsHidden()) {
+                return List.of(metaData.getConcreteAllIndices());
+            } else if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) {
+                return List.of(metaData.getConcreteVisibleIndices());
+            } else if (options.expandWildcardsOpen() && options.expandWildcardsHidden()) {
+                return List.of(metaData.getConcreteAllOpenIndices());
             } else if (options.expandWildcardsOpen()) {
-                return Arrays.asList(metaData.getConcreteAllOpenIndices());
+                return List.of(metaData.getConcreteVisibleOpenIndices());
+            } else if (options.expandWildcardsClosed() && options.expandWildcardsHidden()) {
+                return List.of(metaData.getConcreteAllClosedIndices());
             } else if (options.expandWildcardsClosed()) {
-                return Arrays.asList(metaData.getConcreteAllClosedIndices());
+                return List.of(metaData.getConcreteVisibleClosedIndices());
             } else {
-                return Collections.emptyList();
+                return List.of();
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index 482d57bed54b5..02dca032a11c2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -177,15 +177,19 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment {
     private final int totalOpenIndexShards;
 
     private final String[] allIndices;
+    private final String[] visibleIndices;
     private final String[] allOpenIndices;
+    private final String[] visibleOpenIndices;
     private final String[] allClosedIndices;
+    private final String[] visibleClosedIndices;
 
     private final SortedMap<String, AliasOrIndex> aliasAndIndexLookup;
 
     MetaData(String clusterUUID, boolean clusterUUIDCommitted, long version, CoordinationMetaData coordinationMetaData,
              Settings transientSettings, Settings persistentSettings, DiffableStringMap hashesOfConsistentSettings,
              ImmutableOpenMap<String, IndexMetaData> indices, ImmutableOpenMap<String, IndexTemplateMetaData> templates,
-             ImmutableOpenMap<String, Custom> customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices,
+             ImmutableOpenMap<String, Custom> customs, String[] allIndices, String[] visibleIndices, String[] allOpenIndices,
+             String[] visibleOpenIndices, String[] allClosedIndices, String[] visibleClosedIndices,
              SortedMap<String, AliasOrIndex> aliasAndIndexLookup) {
         this.clusterUUID = clusterUUID;
         this.clusterUUIDCommitted = clusterUUIDCommitted;
@@ -210,8 +214,11 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment {
         this.totalOpenIndexShards = totalOpenIndexShards;
 
         this.allIndices = allIndices;
+        this.visibleIndices = visibleIndices;
         this.allOpenIndices = allOpenIndices;
+        this.visibleOpenIndices = visibleOpenIndices;
         this.allClosedIndices = allClosedIndices;
+        this.visibleClosedIndices = visibleClosedIndices;
         this.aliasAndIndexLookup = aliasAndIndexLookup;
     }
 
@@ -479,14 +486,41 @@ public String[] getConcreteAllIndices() {
         return allIndices;
     }
 
+    /**
+     * Returns all the concrete indices that are not hidden.
+     */
+    public String[] getConcreteVisibleIndices() {
+        return visibleIndices;
+    }
+
+    /**
+     * Returns all of the concrete indices that are open.
+     */
     public String[] getConcreteAllOpenIndices() {
         return allOpenIndices;
     }
 
+    /**
+     * Returns all of the concrete indices that are open and not hidden.
+     */
+    public String[] getConcreteVisibleOpenIndices() {
+        return visibleOpenIndices;
+    }
+
+    /**
+     * Returns all of the concrete indices that are closed.
+     */
     public String[] getConcreteAllClosedIndices() {
         return allClosedIndices;
     }
 
+    /**
+     * Returns all of the concrete indices that are closed and not hidden.
+     */
+    public String[] getConcreteVisibleClosedIndices() {
+        return visibleClosedIndices;
+    }
+
     /**
      * Returns indexing routing for the given <code>aliasOrIndex</code>. Resolves routing from the alias metadata used
      * in the write index.
@@ -1147,18 +1181,31 @@ public MetaData build() {
             // 2) The aliasAndIndexLookup can be updated instead of rebuilding it all the time.
 
             final Set<String> allIndices = new HashSet<>(indices.size());
+            final List<String> visibleIndices = new ArrayList<>();
             final List<String> allOpenIndices = new ArrayList<>();
+            final List<String> visibleOpenIndices = new ArrayList<>();
             final List<String> allClosedIndices = new ArrayList<>();
+            final List<String> visibleClosedIndices = new ArrayList<>();
             final Set<String> duplicateAliasesIndices = new HashSet<>();
             for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
                 final IndexMetaData indexMetaData = cursor.value;
                 final String name = indexMetaData.getIndex().getName();
                 boolean added = allIndices.add(name);
                 assert added : "double index named [" + name + "]";
+                final boolean visible = IndexMetaData.INDEX_HIDDEN_SETTING.get(indexMetaData.getSettings()) == false;
+                if (visible) {
+                    visibleIndices.add(name);
+                }
                 if (indexMetaData.getState() == IndexMetaData.State.OPEN) {
-                    allOpenIndices.add(indexMetaData.getIndex().getName());
+                    allOpenIndices.add(name);
+                    if (visible) {
+                        visibleOpenIndices.add(name);
+                    }
                 } else if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                    allClosedIndices.add(indexMetaData.getIndex().getName());
+                    allClosedIndices.add(name);
+                    if (visible) {
+                        visibleClosedIndices.add(name);
+                    }
                 }
                 indexMetaData.getAliases().keysIt().forEachRemaining(duplicateAliasesIndices::add);
             }
@@ -1186,13 +1233,16 @@ public MetaData build() {
             // TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all indices.
             // When doing an operation across all indices, most of the time is spent on actually going to all shards and
             // do the required operations, the bottleneck isn't resolving expressions into concrete indices.
-            String[] allIndicesArray = allIndices.toArray(new String[allIndices.size()]);
-            String[] allOpenIndicesArray = allOpenIndices.toArray(new String[allOpenIndices.size()]);
-            String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]);
+            String[] allIndicesArray = allIndices.toArray(Strings.EMPTY_ARRAY);
+            String[] visibleIndicesArray = visibleIndices.toArray(Strings.EMPTY_ARRAY);
+            String[] allOpenIndicesArray = allOpenIndices.toArray(Strings.EMPTY_ARRAY);
+            String[] visibleOpenIndicesArray = visibleOpenIndices.toArray(Strings.EMPTY_ARRAY);
+            String[] allClosedIndicesArray = allClosedIndices.toArray(Strings.EMPTY_ARRAY);
+            String[] visibleClosedIndicesArray = visibleClosedIndices.toArray(Strings.EMPTY_ARRAY);
 
             return new MetaData(clusterUUID, clusterUUIDCommitted, version, coordinationMetaData, transientSettings, persistentSettings,
-                    hashesOfConsistentSettings, indices.build(), templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray,
-                    allClosedIndicesArray, aliasAndIndexLookup);
+                hashesOfConsistentSettings, indices.build(), templates.build(), customs.build(), allIndicesArray, visibleIndicesArray,
+                allOpenIndicesArray, visibleOpenIndicesArray, allClosedIndicesArray, visibleClosedIndicesArray, aliasAndIndexLookup);
         }
 
         private SortedMap<String, AliasOrIndex> buildAliasAndIndexLookup() {
diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index f83012f2db32b..89ef64126c190 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -76,6 +76,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
             IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING,
             IndexMetaData.INDEX_PRIORITY_SETTING,
             IndexMetaData.INDEX_DATA_PATH_SETTING,
+            IndexMetaData.INDEX_HIDDEN_SETTING,
             IndexMetaData.INDEX_FORMAT_SETTING,
             SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING,
             SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING,
diff --git a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index 10edb0b30d6cc..f28873e155102 100644
--- a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -160,31 +160,6 @@ public void testFromParameters() {
         assertEquals(defaultOptions.ignoreAliases(), updatedOptions.ignoreAliases());
     }
 
-    public void testSimpleByteBWC() {
-        Map<Byte, IndicesOptions> old = new HashMap<>();
-        // These correspond to each individual option (bit) in the old byte-based IndicesOptions
-        old.put((byte) 0, IndicesOptions.fromOptions(false, false, false, false, true, false, false, false));
-        old.put((byte) 1, IndicesOptions.fromOptions(true, false, false, false, true, false, false, false));
-        old.put((byte) 2, IndicesOptions.fromOptions(false, true, false, false, true, false, false, false));
-        old.put((byte) 4, IndicesOptions.fromOptions(false, false, true, false, true, false, false, false));
-        old.put((byte) 8, IndicesOptions.fromOptions(false, false, false, true, true, false, false, false));
-        old.put((byte) 16, IndicesOptions.fromOptions(false, false, false, false, false, false, false, false));
-        old.put((byte) 32, IndicesOptions.fromOptions(false, false, false, false, true, true, false, false));
-        old.put((byte) 64, IndicesOptions.fromOptions(false, false, false, false, true, false, true, false));
-        // Test a few multi-selected options
-        old.put((byte) 13, IndicesOptions.fromOptions(true, false, true, true, true, false, false, false));
-        old.put((byte) 19, IndicesOptions.fromOptions(true, true, false, false, false, false, false, false));
-        old.put((byte) 24, IndicesOptions.fromOptions(false, false, false, true, false, false, false, false));
-        old.put((byte) 123, IndicesOptions.fromOptions(true, true, false, true, false, true, true, false));
-
-        for (Map.Entry<Byte, IndicesOptions> entry : old.entrySet()) {
-            IndicesOptions indicesOptions2 = IndicesOptions.fromByte(entry.getKey());
-            logger.info("--> 1 {}", entry.getValue().toString());
-            logger.info("--> 2 {}", indicesOptions2.toString());
-            assertThat("IndicesOptions for byte " + entry.getKey() + " differ for conversion",indicesOptions2, equalTo(entry.getValue()));
-        }
-    }
-
     public void testEqualityAndHashCode() {
         IndicesOptions indicesOptions = IndicesOptions.fromOptions(
             randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
diff --git a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
new file mode 100644
index 0000000000000..e8872b97b9454
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Arrays;
+
+public class HiddenIndexIT extends ESIntegTestCase {
+
+    public void testHiddenIndexSearch() {
+        CreateIndexResponse createResponse = client().admin().indices().prepareCreate("hidden-index")
+            .setSettings(Settings.builder().put("index.hidden", true).build())
+            .setWaitForActiveShards(ActiveShardCount.ALL)
+            .get();
+        assertTrue(createResponse.isAcknowledged());
+        assertTrue(createResponse.isShardsAcknowledged());
+
+        // default not visible
+        client().prepareIndex("hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
+        SearchResponse searchResponse =
+            client().prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()).get();
+        boolean matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex()));
+        assertFalse(matchedHidden);
+
+        // direct access allowed
+        searchResponse = client().prepareSearch("hidden-index").setSize(1000).setQuery(QueryBuilders.matchAllQuery()).get();
+        matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex()));
+        assertTrue(matchedHidden);
+
+        // with indices option to include hidden
+        searchResponse = client().prepareSearch(randomFrom("*", "_all", "h*", "*index"))
+            .setSize(1000)
+            .setQuery(QueryBuilders.matchAllQuery())
+            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN)
+            .get();
+        matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex()));
+        assertTrue(matchedHidden);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
index a469088bcb6c3..b11e4b0c35e3e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java
@@ -25,6 +25,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@@ -134,7 +135,9 @@ public void wipeIndices(String... indices) {
         assert indices != null && indices.length > 0;
         if (size() > 0) {
             try {
-                assertAcked(client().admin().indices().prepareDelete(indices));
+                // include wiping hidden indices!
+                assertAcked(client().admin().indices().prepareDelete(indices)
+                    .setIndicesOptions(IndicesOptions.fromOptions(false, true, true, true, true, false, false, true, false)));
             } catch (IndexNotFoundException e) {
                 // ignore
             } catch (IllegalArgumentException e) {

From e1c4d8972c0de909d14b79e8c6477ab47f598858 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Fri, 20 Dec 2019 14:58:44 -0700
Subject: [PATCH 02/26] fix oops and add back todo

---
 .../cluster/metadata/IndexNameExpressionResolver.java         | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 6c4ebf371f76a..488b39b3399f6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -682,6 +682,8 @@ public List<String> resolve(Context context, List<String> expressions) {
                 return resolveEmptyOrTrivialWildcard(options, metaData);
             }
 
+            // TODO: Fix API to work with sets rather than lists since we need to convert to sets
+            // internally anyway
             final Set<String> result = innerResolve(context, expressions, options, metaData);
 
             if (result == null) {
@@ -699,7 +701,7 @@ private Set<String> innerResolve(Context context, List<String> expressions, Indi
             Set<String> result = null;
             boolean wildcardSeen = false;
             for (int i = 0; i < expressions.size(); i++) {
-                String expression = expressions.get(0);
+                String expression = expressions.get(i);
                 if (Strings.isEmpty(expression)) {
                     throw indexNotFoundException(expression);
                 }

From b6d2a212c011b08a95872c14b0386bde6e4b9b78 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Fri, 20 Dec 2019 15:03:10 -0700
Subject: [PATCH 03/26] fix INFE expressions

---
 .../cluster/metadata/IndexNameExpressionResolver.java           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 488b39b3399f6..855915b71cdfe 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -174,7 +174,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                 } else {
                     infe = new IndexNotFoundException((String)null);
                 }
-                infe.setResources("index_expression", indexExpressions);
+                infe.setResources("index_expression", expressions.toArray(Strings.EMPTY_ARRAY));
                 throw infe;
             } else {
                 return Index.EMPTY_ARRAY;

From a891e4cbb6450da3aaedb32f6bac4a8d443d0d03 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Fri, 20 Dec 2019 15:20:20 -0700
Subject: [PATCH 04/26] fix ml test

---
 .../xpack/ml/job/persistence/JobResultsProviderTests.java        | 1 +
 1 file changed, 1 insertion(+)

diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
index 866fc3337c5a7..969ecbaa75c81 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
@@ -160,6 +160,7 @@ public void testCreateJobWithExistingIndex() {
 
         ImmutableOpenMap<String, AliasMetaData> aliases = ImmutableOpenMap.of();
         when(indexMetaData.getAliases()).thenReturn(aliases);
+        when(indexMetaData.getSettings()).thenReturn(Settings.EMPTY);
 
         ImmutableOpenMap<String, IndexMetaData> indexMap = ImmutableOpenMap.<String, IndexMetaData>builder()
                 .fPut(AnomalyDetectorsIndex.jobResultsAliasedName("foo"), indexMetaData).build();

From a3b48561ee757c61d5153b8903c842d9a88f4801 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Mon, 6 Jan 2020 08:34:59 -0700
Subject: [PATCH 05/26] simplify serialization

---
 .../elasticsearch/action/support/IndicesOptions.java   | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index cfc4585194de7..066f68d95f448 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -204,13 +204,9 @@ public boolean ignoreThrottled() {
     public void writeIndicesOptions(StreamOutput out) throws IOException {
         out.writeEnumSet(options);
         if (out.getVersion().before(Version.V_8_0_0) && expandWildcards.contains(WildcardStates.HIDDEN)) {
-            Set<WildcardStates> states = new HashSet<>();
-            for (WildcardStates state : expandWildcards) {
-                if (state != WildcardStates.HIDDEN) {
-                    states.add(state);
-                }
-            }
-            out.writeEnumSet(EnumSet.copyOf(states));
+            final EnumSet<WildcardStates> states = EnumSet.copyOf(expandWildcards);
+            states.remove(WildcardStates.HIDDEN);
+            out.writeEnumSet(states);
         } else {
             out.writeEnumSet(expandWildcards);
         }

From d957215ba5141dd52bcf7274b1a06915c23169a9 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Mon, 6 Jan 2020 12:45:35 -0700
Subject: [PATCH 06/26] handle hidden in wildcard state parsing

---
 .../java/org/elasticsearch/action/support/IndicesOptions.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index 066f68d95f448..a232263dc840b 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -68,6 +68,9 @@ public static EnumSet<WildcardStates> parseParameter(Object value, EnumSet<Wildc
                     case "closed":
                         states.add(CLOSED);
                         break;
+                    case "hidden":
+                        states.add(HIDDEN);
+                        break;
                     case "none":
                         states.clear();
                         break;

From 873763b81a00a83f01f4f5802b98ba246d648af5 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Mon, 6 Jan 2020 13:22:35 -0700
Subject: [PATCH 07/26] fix multisearchrequest writing of expand wildcards

---
 .../action/search/MultiSearchRequest.java             | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index e6e11e5eeb5c4..a48ab3f8a6b16 100644
--- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -304,12 +304,21 @@ public static void writeSearchRequestParams(SearchRequest request, XContentBuild
             xContentBuilder.field("index", request.indices());
         }
         if (request.indicesOptions() != null && request.indicesOptions() != SearchRequest.DEFAULT_INDICES_OPTIONS) {
-            if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsClosed()) {
+            if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsClosed() &&
+                request.indicesOptions().expandWildcardsHidden()) {
                 xContentBuilder.field("expand_wildcards", "all");
+            } else if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsClosed()) {
+                xContentBuilder.field("expand_wildcards", "open,closed");
+            } else if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsHidden()) {
+                xContentBuilder.field("expand_wildcards", "open,hidden");
             } else if (request.indicesOptions().expandWildcardsOpen()) {
                 xContentBuilder.field("expand_wildcards", "open");
+            } else if (request.indicesOptions().expandWildcardsClosed() && request.indicesOptions().expandWildcardsHidden()) {
+                xContentBuilder.field("expand_wildcards", "closed,hidden");
             } else if (request.indicesOptions().expandWildcardsClosed()) {
                 xContentBuilder.field("expand_wildcards", "closed");
+            } else if (request.indicesOptions().expandWildcardsHidden()) {
+                xContentBuilder.field("expand_wildcards", "hidden");
             } else {
                 xContentBuilder.field("expand_wildcards", "none");
             }

From b5ea6af4704d58dd010bac44a5095e49c314d013 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Tue, 7 Jan 2020 20:55:01 -0700
Subject: [PATCH 08/26] global index templates do not apply to hidden indices

---
 .../rollover/TransportRolloverAction.java     | 12 ++--
 .../action/bulk/TransportBulkAction.java      |  2 +-
 .../metadata/MetaDataCreateIndexService.java  |  4 +-
 .../MetaDataIndexTemplateService.java         | 60 +++++++++++++++++--
 .../TransportRolloverActionTests.java         |  2 +-
 .../MetaDataIndexTemplateServiceTests.java    |  6 +-
 .../elasticsearch/index/HiddenIndexIT.java    | 60 +++++++++++++++++++
 7 files changed, 131 insertions(+), 15 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 9c1c9d71d6708..e71500409977d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -43,7 +43,6 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
-import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
@@ -63,6 +62,8 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.*;
+
 /**
  * Main class to swap the index pointed to by an alias, given some conditions
  */
@@ -122,7 +123,9 @@ protected void masterOperation(Task task, final RolloverRequest rolloverRequest,
             : generateRolloverIndexName(sourceProvidedName, indexNameExpressionResolver);
         final String rolloverIndexName = indexNameExpressionResolver.resolveDateMathExpression(unresolvedName);
         MetaDataCreateIndexService.validateIndexName(rolloverIndexName, state); // will fail if the index already exists
-        checkNoDuplicatedAliasInIndexTemplate(metaData, rolloverIndexName, rolloverRequest.getAlias());
+        final Boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.exists(rolloverRequest.getCreateIndexRequest().settings()) ?
+            IndexMetaData.INDEX_HIDDEN_SETTING.get(rolloverRequest.getCreateIndexRequest().settings()) : null;
+        checkNoDuplicatedAliasInIndexTemplate(metaData, rolloverIndexName, rolloverRequest.getAlias(), isHidden);
         IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getAlias())
             .clear()
             .indicesOptions(IndicesOptions.fromOptions(true, false, true, true))
@@ -291,8 +294,9 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Stri
      * the rollover alias will point to multiple indices. This causes indexing requests to be rejected.
      * To avoid this, we make sure that there is no duplicated alias in index templates before creating a new index.
      */
-    static void checkNoDuplicatedAliasInIndexTemplate(MetaData metaData, String rolloverIndexName, String rolloverRequestAlias) {
-        final List<IndexTemplateMetaData> matchedTemplates = MetaDataIndexTemplateService.findTemplates(metaData, rolloverIndexName);
+    static void checkNoDuplicatedAliasInIndexTemplate(MetaData metaData, String rolloverIndexName, String rolloverRequestAlias,
+                                                      @Nullable Boolean isHidden) {
+        final List<IndexTemplateMetaData> matchedTemplates = findTemplates(metaData, rolloverIndexName, isHidden);
         for (IndexTemplateMetaData template : matchedTemplates) {
             if (template.aliases().containsKey(rolloverRequestAlias)) {
                 throw new IllegalArgumentException(String.format(Locale.ROOT,
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 6a8912cc1dc82..b0a88c512a75e 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -307,7 +307,7 @@ static boolean resolvePipelines(final DocWriteRequest<?> originalRequest, final
                 }
             } else if (indexRequest.index() != null) {
                 // the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
-                List<IndexTemplateMetaData> templates = MetaDataIndexTemplateService.findTemplates(metaData, indexRequest.index());
+                List<IndexTemplateMetaData> templates = MetaDataIndexTemplateService.findTemplates(metaData, indexRequest.index(), null);
                 assert (templates != null);
                 // order of templates are highest order first
                 for (final IndexTemplateMetaData template : templates) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 1ed7fb04407f1..bb517c2f899b1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -278,8 +278,10 @@ public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateInd
 
         // we only find a template when its an API call (a new index)
         // find templates, highest order are better matching
+        final Boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.exists(request.settings()) ?
+            IndexMetaData.INDEX_HIDDEN_SETTING.get(request.settings()) : null;
         final List<IndexTemplateMetaData> templates = sourceMetaData == null ?
-            Collections.unmodifiableList(MetaDataIndexTemplateService.findTemplates(currentState.metaData(), request.index())) :
+            Collections.unmodifiableList(MetaDataIndexTemplateService.findTemplates(currentState.metaData(), request.index(), isHidden)) :
             List.of();
 
         final Map<String, Object> mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), templates, xContentRegistry));
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 31a612341d76a..d5e9d88533d1d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -55,6 +56,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
@@ -197,18 +199,59 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
 
     /**
      * Finds index templates whose index pattern matched with the given index name.
-     * The result is sorted by {@link IndexTemplateMetaData#order} descending.
+     * The result is sorted by {@link IndexTemplateMetaData#order()} descending.
      */
-    public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, String indexName) {
+    public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, String indexName, @Nullable Boolean isHidden) {
         final List<IndexTemplateMetaData> matchedTemplates = new ArrayList<>();
         for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
             final IndexTemplateMetaData template = cursor.value;
-            final boolean matched = template.patterns().stream().anyMatch(pattern -> Regex.simpleMatch(pattern, indexName));
-            if (matched) {
-                matchedTemplates.add(template);
+            if (isHidden == null || isHidden == Boolean.FALSE) {
+                final boolean matched = template.patterns().stream()
+                    .anyMatch(pattern -> Regex.simpleMatch(pattern, indexName));
+                if (matched) {
+                    matchedTemplates.add(template);
+                }
+            } else {
+                assert isHidden == Boolean.TRUE;
+
+                boolean skip = false;
+                boolean matched = false;
+                for (String pattern : template.patterns()) {
+                    if (Regex.isMatchAllPattern(pattern)) {
+                        skip = true;
+                        break;
+                    }
+                    matched = matched || Regex.simpleMatch(pattern, indexName);
+                }
+
+                if (skip == false && matched) {
+                    matchedTemplates.add(template);
+                }
             }
         }
         CollectionUtil.timSort(matchedTemplates, Comparator.comparingInt(IndexTemplateMetaData::order).reversed());
+
+        // this is complex but if the index is not hidden in the create request but hidden is the result of template application,
+        // then we need to exclude global templates
+        if (isHidden == null) {
+            final Optional<IndexTemplateMetaData> templateWithHiddenSetting = matchedTemplates.stream()
+                .filter(template -> IndexMetaData.INDEX_HIDDEN_SETTING.exists(template.settings())).findFirst();
+            if (templateWithHiddenSetting.isPresent()) {
+                final boolean templatedIsHidden = IndexMetaData.INDEX_HIDDEN_SETTING.get(templateWithHiddenSetting.get().settings());
+                if (templatedIsHidden) {
+                    // remove the global templates
+                    matchedTemplates.removeIf(current -> current.patterns().stream().anyMatch(Regex::isMatchAllPattern));
+                }
+                // validate that hidden didn't change
+                final Optional<IndexTemplateMetaData> templateWithHiddenSettingPostRemoval = matchedTemplates.stream()
+                    .filter(template -> IndexMetaData.INDEX_HIDDEN_SETTING.exists(template.settings())).findFirst();
+                if (templateWithHiddenSettingPostRemoval.isEmpty() ||
+                    templateWithHiddenSetting.get() != templateWithHiddenSettingPostRemoval.get()) {
+                    throw new IllegalStateException("A global index template [" + templateWithHiddenSetting.get().name() +
+                        "] defined the index hidden setting, which is not allowed");
+                }
+            }
+        }
         return matchedTemplates;
     }
 
@@ -307,6 +350,13 @@ private void validate(PutRequest request) {
         }
         List<String> indexSettingsValidation = metaDataCreateIndexService.getIndexSettingsValidationErrors(request.settings, true);
         validationErrors.addAll(indexSettingsValidation);
+
+        if (request.indexPatterns.stream().anyMatch(Regex::isMatchAllPattern)) {
+            if (IndexMetaData.INDEX_HIDDEN_SETTING.exists(request.settings)) {
+                validationErrors.add("global templates may not specify the setting " + IndexMetaData.INDEX_HIDDEN_SETTING.getKey());
+            }
+        }
+
         if (!validationErrors.isEmpty()) {
             ValidationException validationException = new ValidationException();
             validationException.addValidationErrors(validationErrors);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 54454cd9f7ecb..10aa5d6925be3 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -359,7 +359,7 @@ public void testRejectDuplicateAlias() {
         String indexName = randomFrom("foo-123", "bar-xyz");
         String aliasName = randomFrom("foo-write", "bar-write");
         final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-            () -> TransportRolloverAction.checkNoDuplicatedAliasInIndexTemplate(metaData, indexName, aliasName));
+            () -> TransportRolloverAction.checkNoDuplicatedAliasInIndexTemplate(metaData, indexName, aliasName, randomBoolean()));
         assertThat(ex.getMessage(), containsString("index template [test-template]"));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index 5652f20e8b390..3bcf3ff226cc8 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -144,11 +144,11 @@ public void testFindTemplates() throws Exception {
         putTemplateDetail(new PutRequest("test", "foo-2").patterns(Arrays.asList("foo-*")).order(2));
         putTemplateDetail(new PutRequest("test", "bar").patterns(Arrays.asList("bar-*")).order(between(0, 100)));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "foo-1234").stream()
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "foo-1234", randomBoolean()).stream()
             .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("foo-2", "foo-1"));
-        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "bar-xyz").stream()
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "bar-xyz", randomBoolean()).stream()
             .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("bar"));
-        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "baz"), empty());
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "baz", randomBoolean()), empty());
     }
 
     private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
diff --git a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
index e8872b97b9454..542f9af389281 100644
--- a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -20,15 +20,27 @@
 package org.elasticsearch.index;
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.InvalidIndexTemplateException;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 
 public class HiddenIndexIT extends ESIntegTestCase {
 
@@ -61,4 +73,52 @@ public void testHiddenIndexSearch() {
         matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex()));
         assertTrue(matchedHidden);
     }
+
+    public void testGlobalTemplatesDoNotApply() {
+        assertAcked(client().admin().indices().preparePutTemplate("a_global_template").setPatterns(List.of("*"))
+            .addMapping("_doc", "foo", "type=text").get());
+        assertAcked(client().admin().indices().preparePutTemplate("not_global_template").setPatterns(List.of("a*"))
+            .addMapping("_doc", "bar", "type=text").get());
+        assertAcked(client().admin().indices().preparePutTemplate("specific_template").setPatterns(List.of("a_hidden_index"))
+            .addMapping("_doc", "baz", "type=text").get());
+        assertAcked(client().admin().indices().preparePutTemplate("unused_template").setPatterns(List.of("not_used"))
+            .addMapping("_doc", "foobar", "type=text").get());
+
+        assertAcked(client().admin().indices().prepareCreate("a_hidden_index")
+            .setSettings(Settings.builder().put("index.hidden", true).build()).get());
+
+        GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings("a_hidden_index").get();
+        assertThat(mappingsResponse.mappings().size(), is(1));
+        MappingMetaData mappingMetaData = mappingsResponse.mappings().get("a_hidden_index");
+        assertNotNull(mappingMetaData);
+        Map<String, Object> propertiesMap = (Map<String, Object>) mappingMetaData.getSourceAsMap().get("properties");
+        assertNotNull(propertiesMap);
+        assertThat(propertiesMap.size(), is(2));
+        Map<String, Object> barMap = (Map<String, Object>) propertiesMap.get("bar");
+        assertNotNull(barMap);
+        assertThat(barMap.get("type"), is("text"));
+        Map<String, Object> bazMap = (Map<String, Object>) propertiesMap.get("baz");
+        assertNotNull(bazMap);
+        assertThat(bazMap.get("type"), is("text"));
+    }
+
+    public void testGlobalTemplateCannotMakeIndexHidden() {
+        InvalidIndexTemplateException invalidIndexTemplateException = expectThrows(InvalidIndexTemplateException.class,
+            () -> client().admin().indices().preparePutTemplate("a_global_template")
+                .setPatterns(List.of("*"))
+                .setSettings(Settings.builder().put("index.hidden", randomBoolean()).build())
+                .get());
+        assertThat(invalidIndexTemplateException.getMessage(), containsString("global templates may not specify the setting index.hidden"));
+    }
+
+    public void testNonGlobalTemplateCanMakeIndexHidden() {
+        assertAcked(client().admin().indices().preparePutTemplate("a_global_template")
+            .setPatterns(List.of("my_hidden_pattern*"))
+            .addMapping("_doc", "foo", "type=text")
+            .setSettings(Settings.builder().put("index.hidden", true).build())
+            .get());
+        assertAcked(client().admin().indices().prepareCreate("my_hidden_pattern1").get());
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("my_hidden_pattern1").get();
+        assertThat(getSettingsResponse.getSetting("my_hidden_pattern1", "index.hidden"), is("true"));
+    }
 }

From 919161a332a09ccd6115d18e3eb76a2ebf81e0ec Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Tue, 7 Jan 2020 21:32:17 -0700
Subject: [PATCH 09/26] fix checkstyle

---
 .../action/admin/indices/rollover/TransportRolloverAction.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index e71500409977d..5f6ee87e87fd8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -62,7 +62,7 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.*;
+import static org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.findTemplates;
 
 /**
  * Main class to swap the index pointed to by an alias, given some conditions

From fc7f1b9672e35c556a6df3e6d4e728fa6cf0e72b Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Tue, 7 Jan 2020 21:32:37 -0700
Subject: [PATCH 10/26] integrate with security

---
 .../authz/IndicesAndAliasesResolver.java      | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index 7245bf7356611..8381316dc6e09 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData.State;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -120,10 +121,10 @@ ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, MetaData
                     : "indices are: " + Arrays.toString(indicesRequest.indices()); // Arrays.toString() can handle null values - all good
             resolvedIndicesBuilder.addLocal(getPutMappingIndexOrAlias((PutMappingRequest) indicesRequest, authorizedIndices, metaData));
         } else if (indicesRequest instanceof IndicesRequest.Replaceable) {
-            IndicesRequest.Replaceable replaceable = (IndicesRequest.Replaceable) indicesRequest;
-            final boolean replaceWildcards = indicesRequest.indicesOptions().expandWildcardsOpen()
-                    || indicesRequest.indicesOptions().expandWildcardsClosed();
-            IndicesOptions indicesOptions = indicesRequest.indicesOptions();
+            final IndicesRequest.Replaceable replaceable = (IndicesRequest.Replaceable) indicesRequest;
+            final IndicesOptions indicesOptions = indicesRequest.indicesOptions();
+            final boolean replaceWildcards = indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed() ||
+                indicesOptions.expandWildcardsHidden();
 
             // check for all and return list of authorized indices
             if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
@@ -422,10 +423,20 @@ private static boolean isIndexVisible(String index, IndicesOptions indicesOption
         }
         assert aliasOrIndex.getIndices().size() == 1 : "concrete index must point to a single index";
         IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
-        if (indexMetaData.getState() == IndexMetaData.State.CLOSE && (indicesOptions.expandWildcardsClosed() || dateMathExpression)) {
+        final boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.get(indexMetaData.getSettings());
+        if (isHidden && indicesOptions.expandWildcardsHidden() == false) {
+            return false;
+        }
+
+        // the index is not hidden and since it is a date math expression, we consider it visible regardless of open/closed
+        if (dateMathExpression) {
+            assert State.values().length == 2 : "a new IndexMetaData.State value may need to be handled!";
+            return true;
+        }
+        if (indexMetaData.getState() == IndexMetaData.State.CLOSE && indicesOptions.expandWildcardsClosed()) {
             return true;
         }
-        if (indexMetaData.getState() == IndexMetaData.State.OPEN && (indicesOptions.expandWildcardsOpen() || dateMathExpression)) {
+        if (indexMetaData.getState() == IndexMetaData.State.OPEN && indicesOptions.expandWildcardsOpen()) {
             return true;
         }
         return false;

From 6fdf2e6152faed8e95822d0f7306727486b1cd2a Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Wed, 8 Jan 2020 08:31:55 -0700
Subject: [PATCH 11/26] fix checkstyle

---
 server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
index 542f9af389281..98206e093d866 100644
--- a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
@@ -39,7 +38,6 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
 public class HiddenIndexIT extends ESIntegTestCase {

From 3bd2592c73f770ffcade4027ceef29f35379e3f9 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Wed, 8 Jan 2020 09:04:30 -0700
Subject: [PATCH 12/26] cleanup

---
 .../metadata/IndexNameExpressionResolver.java | 57 ++++++++-----------
 .../elasticsearch/index/HiddenIndexIT.java    | 10 +---
 2 files changed, 27 insertions(+), 40 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 855915b71cdfe..9897c425a3737 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -144,11 +144,8 @@ String[] concreteIndexNames(Context context, String... indexExpressions) {
     }
 
     Index[] concreteIndices(Context context, String... indexExpressions) {
-        final List<String> expressions;
         if (indexExpressions == null || indexExpressions.length == 0) {
-            expressions = List.of(MetaData.ALL);
-        } else {
-            expressions = List.of(indexExpressions);
+            indexExpressions = new String[]{MetaData.ALL};
         }
         MetaData metaData = context.getState().metaData();
         IndicesOptions options = context.getOptions();
@@ -156,17 +153,17 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         // If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices
         // option. At some point we should change this, because there shouldn't be a reason why whether a single index
         // or multiple indices are specified yield different behaviour.
-        final boolean failNoIndices = expressions.size() == 1 ? !options.allowNoIndices() : !options.ignoreUnavailable();
-        List<String> resolvedExpressions = expressions;
+        final boolean failNoIndices = indexExpressions.length == 1 ? !options.allowNoIndices() : !options.ignoreUnavailable();
+        List<String> expressions = Arrays.asList(indexExpressions);
         for (ExpressionResolver expressionResolver : expressionResolvers) {
-            resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
+            expressions = expressionResolver.resolve(context, expressions);
         }
 
-        if (resolvedExpressions.isEmpty()) {
+        if (expressions.isEmpty()) {
             if (!options.allowNoIndices()) {
                 IndexNotFoundException infe;
-                if (expressions.size() == 1) {
-                    if (MetaData.ALL.equals(expressions.get(0))) {
+                if (indexExpressions.length == 1) {
+                    if (indexExpressions[0].equals(MetaData.ALL)) {
                         infe = new IndexNotFoundException("no indices exist", (String)null);
                     } else {
                         infe = new IndexNotFoundException((String)null);
@@ -174,7 +171,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                 } else {
                     infe = new IndexNotFoundException((String)null);
                 }
-                infe.setResources("index_expression", expressions.toArray(Strings.EMPTY_ARRAY));
+                infe.setResources("index_expression", indexExpressions);
                 throw infe;
             } else {
                 return Index.EMPTY_ARRAY;
@@ -182,7 +179,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         }
 
         final Set<Index> concreteIndices = new HashSet<>(expressions.size());
-        for (String expression : resolvedExpressions) {
+        for (String expression : expressions) {
             AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression);
             if (aliasOrIndex == null ) {
                 if (failNoIndices) {
@@ -348,7 +345,7 @@ public String resolveDateMathExpression(String dateExpression) {
      */
     public Set<String> resolveExpressions(ClusterState state, String... expressions) {
         Context context = new Context(state, IndicesOptions.lenientExpandOpen(), true, false);
-        List<String> resolvedExpressions = List.of(expressions);
+        List<String> resolvedExpressions = Arrays.asList(expressions);
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions);
         }
@@ -430,7 +427,7 @@ public String[] indexAliases(ClusterState state, String index, Predicate<AliasMe
         if (aliases == null) {
             return null;
         }
-        return aliases.toArray(Strings.EMPTY_ARRAY);
+        return aliases.toArray(new String[aliases.size()]);
     }
 
     /**
@@ -683,18 +680,18 @@ public List<String> resolve(Context context, List<String> expressions) {
             }
 
             // TODO: Fix API to work with sets rather than lists since we need to convert to sets
-            // internally anyway
-            final Set<String> result = innerResolve(context, expressions, options, metaData);
+            // internally anyway.
+            Set<String> result = innerResolve(context, expressions, options, metaData);
 
             if (result == null) {
                 return expressions;
             }
             if (result.isEmpty() && !options.allowNoIndices()) {
                 IndexNotFoundException infe = new IndexNotFoundException((String)null);
-                infe.setResources("index_or_alias", expressions.toArray(Strings.EMPTY_ARRAY));
+                infe.setResources("index_or_alias", expressions.toArray(new String[0]));
                 throw infe;
             }
-            return List.copyOf(result);
+            return new ArrayList<>(result);
         }
 
         private Set<String> innerResolve(Context context, List<String> expressions, IndicesOptions options, MetaData metaData) {
@@ -859,31 +856,25 @@ private static Set<String> expand(Context context, IndexMetaData.State excludeSt
         }
 
         private boolean isEmptyOrTrivialWildcard(List<String> expressions) {
-            if (expressions.isEmpty()) {
-                return true;
-            } else if (expressions.size() == 1) {
-                final String expression = expressions.get(0);
-                return MetaData.ALL.equals(expression) || Regex.isMatchAllPattern(expression);
-            } else {
-                return false;
-            }
+            return expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0)) ||
+                Regex.isMatchAllPattern(expressions.get(0))));
         }
 
         private static List<String> resolveEmptyOrTrivialWildcard(IndicesOptions options, MetaData metaData) {
             if (options.expandWildcardsOpen() && options.expandWildcardsClosed() && options.expandWildcardsHidden()) {
-                return List.of(metaData.getConcreteAllIndices());
+                return Arrays.asList(metaData.getConcreteAllIndices());
             } else if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) {
-                return List.of(metaData.getConcreteVisibleIndices());
+                return Arrays.asList(metaData.getConcreteVisibleIndices());
             } else if (options.expandWildcardsOpen() && options.expandWildcardsHidden()) {
-                return List.of(metaData.getConcreteAllOpenIndices());
+                return Arrays.asList(metaData.getConcreteAllOpenIndices());
             } else if (options.expandWildcardsOpen()) {
-                return List.of(metaData.getConcreteVisibleOpenIndices());
+                return Arrays.asList(metaData.getConcreteVisibleOpenIndices());
             } else if (options.expandWildcardsClosed() && options.expandWildcardsHidden()) {
-                return List.of(metaData.getConcreteAllClosedIndices());
+                return Arrays.asList(metaData.getConcreteAllClosedIndices());
             } else if (options.expandWildcardsClosed()) {
-                return List.of(metaData.getConcreteVisibleClosedIndices());
+                return Arrays.asList(metaData.getConcreteVisibleClosedIndices());
             } else {
-                return List.of();
+                return Collections.emptyList();
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
index 98206e093d866..361c6fc5b435d 100644
--- a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -19,11 +19,9 @@
 
 package org.elasticsearch.index;
 
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -43,12 +41,10 @@
 public class HiddenIndexIT extends ESIntegTestCase {
 
     public void testHiddenIndexSearch() {
-        CreateIndexResponse createResponse = client().admin().indices().prepareCreate("hidden-index")
+        assertAcked(client().admin().indices().prepareCreate("hidden-index")
             .setSettings(Settings.builder().put("index.hidden", true).build())
-            .setWaitForActiveShards(ActiveShardCount.ALL)
-            .get();
-        assertTrue(createResponse.isAcknowledged());
-        assertTrue(createResponse.isShardsAcknowledged());
+            .get());
+        ensureGreen("hidden-index");
 
         // default not visible
         client().prepareIndex("hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();

From 55307268309ac04ce2c6b25a022f8c05ba2e4f2a Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Wed, 8 Jan 2020 10:20:07 -0700
Subject: [PATCH 13/26] all copies cannot be active if there are not enough
 nodes

---
 server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
index 361c6fc5b435d..5d4702545d477 100644
--- a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -44,7 +44,6 @@ public void testHiddenIndexSearch() {
         assertAcked(client().admin().indices().prepareCreate("hidden-index")
             .setSettings(Settings.builder().put("index.hidden", true).build())
             .get());
-        ensureGreen("hidden-index");
 
         // default not visible
         client().prepareIndex("hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();

From f6b3f8516e7a2b0dcf8b568b57c29c9fcc890a18 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Wed, 8 Jan 2020 13:35:32 -0700
Subject: [PATCH 14/26] use enumset

---
 .../org/elasticsearch/action/support/IndicesOptions.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index a232263dc840b..e5d19571755c9 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -56,7 +56,7 @@ public static EnumSet<WildcardStates> parseParameter(Object value, EnumSet<Wildc
                 return defaultStates;
             }
 
-            Set<WildcardStates> states = new HashSet<>();
+            EnumSet<WildcardStates> states = EnumSet.noneOf(WildcardStates.class);
             String[] wildcards = nodeStringArrayValue(value);
             // TODO why do we let patterns like "none,all" or "open,none,closed" get used. The location of 'none' in the array changes the
             // meaning of the resulting value
@@ -75,14 +75,14 @@ public static EnumSet<WildcardStates> parseParameter(Object value, EnumSet<Wildc
                         states.clear();
                         break;
                     case "all":
-                        states.addAll(EnumSet.allOf(WildcardStates.class));
+                        states = EnumSet.allOf(WildcardStates.class);
                         break;
                     default:
                         throw new IllegalArgumentException("No valid expand wildcard value [" + wildcard + "]");
                 }
             }
 
-            return states.isEmpty() ? NONE : EnumSet.copyOf(states);
+            return states;
         }
     }
 

From 0057e3884310e7b2690c178ba6464ee94c3b2cdc Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Thu, 9 Jan 2020 21:27:42 -0700
Subject: [PATCH 15/26] add more unit tests

---
 .../action/support/IndicesOptions.java        |  13 ++-
 .../metadata/IndexNameExpressionResolver.java |   5 +-
 .../health/ClusterHealthRequestTests.java     |   5 +
 .../TransportRolloverActionTests.java         |  18 +++
 .../search/MultiSearchRequestTests.java       |  36 ++++++
 .../action/support/IndicesOptionsTests.java   | 106 +++++++++++-------
 6 files changed, 137 insertions(+), 46 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index e5d19571755c9..f58cd90d35a7c 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -30,10 +30,8 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
-import java.util.HashSet;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
@@ -216,7 +214,12 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
     }
 
     public static IndicesOptions readIndicesOptions(StreamInput in) throws IOException {
-        return new IndicesOptions(in.readEnumSet(Option.class), in.readEnumSet(WildcardStates.class));
+        EnumSet<Option> options = in.readEnumSet(Option.class);
+        EnumSet<WildcardStates> states = in.readEnumSet(WildcardStates.class);
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            states.add(WildcardStates.HIDDEN);
+        }
+        return new IndicesOptions(options, states);
     }
 
     public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
@@ -243,8 +246,8 @@ public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allo
                                              boolean expandToClosedIndices, boolean expandToHiddenIndices,
                                              boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices, boolean ignoreAliases,
                                              boolean ignoreThrottled) {
-        final Set<Option> opts = new HashSet<>();
-        final Set<WildcardStates> wildcards = new HashSet<>();
+        final EnumSet<Option> opts = EnumSet.noneOf(Option.class);
+        final EnumSet<WildcardStates> wildcards = EnumSet.noneOf(WildcardStates.class);
 
         if (ignoreUnavailable) {
             opts.add(Option.IGNORE_UNAVAILABLE);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 9897c425a3737..80a25e366c093 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -670,8 +670,9 @@ static final class WildcardExpressionResolver implements ExpressionResolver {
         public List<String> resolve(Context context, List<String> expressions) {
             IndicesOptions options = context.getOptions();
             MetaData metaData = context.getState().metaData();
-            if (options.expandWildcardsClosed() == false && options.expandWildcardsOpen() == false &&
-                options.expandWildcardsHidden() == false) {
+            // only check open/closed since if we do not expand to open or closed it doesn't make sense to
+            // expand to hidden
+            if (options.expandWildcardsClosed() == false && options.expandWildcardsOpen() == false) {
                 return expressions;
             }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java
index a901c8c9bc2e1..40be77001bb71 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java
@@ -56,6 +56,11 @@ public void testSerialize() throws Exception {
         assertThat(cloneRequest.indicesOptions(), equalTo(originalRequest.indicesOptions()));
     }
 
+    public void testRequestReturnsHiddenIndicesByDefault() {
+        final ClusterHealthRequest defaultRequest = new ClusterHealthRequest();
+        assertTrue(defaultRequest.indicesOptions().expandWildcardsHidden());
+    }
+
     public void testBwcSerialization() throws Exception {
         for (int runs = 0; runs < randomIntBetween(5, 20); runs++) {
             // Generate a random cluster health request in version < 7.2.0 and serializes it
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 10aa5d6925be3..c4afcf57e599a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -79,6 +79,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -363,6 +364,23 @@ public void testRejectDuplicateAlias() {
         assertThat(ex.getMessage(), containsString("index template [test-template]"));
     }
 
+    public void testHiddenAffectsResolvedTemplates() {
+        final IndexTemplateMetaData template = IndexTemplateMetaData.builder("test-template")
+            .patterns(Collections.singletonList("*"))
+            .putAlias(AliasMetaData.builder("foo-write")).putAlias(AliasMetaData.builder("bar-write").writeIndex(randomBoolean()))
+            .build();
+        final MetaData metaData = MetaData.builder().put(createMetaData(randomAlphaOfLengthBetween(5, 7)), false).put(template).build();
+        String indexName = randomFrom("foo-123", "bar-xyz");
+        String aliasName = randomFrom("foo-write", "bar-write");
+
+        // hidden shouldn't throw
+        TransportRolloverAction.checkNoDuplicatedAliasInIndexTemplate(metaData, indexName, aliasName, Boolean.TRUE);
+        // not hidden will throw
+        final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () ->
+            TransportRolloverAction.checkNoDuplicatedAliasInIndexTemplate(metaData, indexName, aliasName, randomFrom(Boolean.FALSE, null)));
+        assertThat(ex.getMessage(), containsString("index template [test-template]"));
+    }
+
     public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPrimariesFromWriteIndex() {
         final TransportService mockTransportService = mock(TransportService.class);
         final ClusterService mockClusterService = mock(ClusterService.class);
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 35f60546bb023..11e586a6724a9 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -25,9 +25,13 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.RestRequest;
@@ -43,6 +47,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
@@ -256,6 +261,37 @@ public void testMultiLineSerialization() throws IOException {
         }
     }
 
+    public void testWritingExpandWildcards() throws IOException {
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, true, true, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "all");
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, true, false, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "open,closed");
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "open,hidden");
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, false, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "open");
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, true, true, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "closed,hidden");
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, true, false, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "closed");
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, false, true, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "hidden");
+        assertExpandWildcardsValue(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), false, false, false, randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean()), "none");
+    }
+
+    private void assertExpandWildcardsValue(IndicesOptions options, String expectedValue) throws IOException {
+        SearchRequest request = new SearchRequest();
+        request.indicesOptions(options);
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            MultiSearchRequest.writeSearchRequestParams(request, builder);
+            Map<String, Object> map =
+                XContentHelper.convertToMap(XContentType.JSON.xContent(), BytesReference.bytes(builder).streamInput(), false);
+            final String value = (String) map.get("expand_wildcards");
+            assertEquals(expectedValue, value);
+        }
+    }
+
     public void testEqualsAndHashcode() {
         checkEqualsAndHashCode(createMultiSearchRequest(), MultiSearchRequestTests::copyRequest, MultiSearchRequestTests::mutate);
     }
diff --git a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index f28873e155102..b5e60966e51fe 100644
--- a/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -45,6 +46,7 @@
 
 import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class IndicesOptionsTests extends ESTestCase {
 
@@ -54,7 +56,7 @@ public void testSerialization() throws Exception {
             Version version = randomVersionBetween(random(), Version.V_7_0_0, null);
             IndicesOptions indicesOptions = IndicesOptions.fromOptions(
                 randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
-                randomBoolean());
+                randomBoolean(), randomBoolean());
 
             BytesStreamOutput output = new BytesStreamOutput();
             output.setVersion(version);
@@ -68,6 +70,11 @@ public void testSerialization() throws Exception {
             assertThat(indicesOptions2.allowNoIndices(), equalTo(indicesOptions.allowNoIndices()));
             assertThat(indicesOptions2.expandWildcardsOpen(), equalTo(indicesOptions.expandWildcardsOpen()));
             assertThat(indicesOptions2.expandWildcardsClosed(), equalTo(indicesOptions.expandWildcardsClosed()));
+            if (version.before(Version.V_8_0_0)) {
+                assertThat(indicesOptions2.expandWildcardsHidden(), is(true));
+            } else {
+                assertThat(indicesOptions2.expandWildcardsHidden(), equalTo(indicesOptions.expandWildcardsHidden()));
+            }
 
             assertThat(indicesOptions2.forbidClosedIndices(), equalTo(indicesOptions.forbidClosedIndices()));
             assertThat(indicesOptions2.allowAliasesToMultipleIndices(), equalTo(indicesOptions.allowAliasesToMultipleIndices()));
@@ -77,27 +84,30 @@ public void testSerialization() throws Exception {
     }
 
     public void testFromOptions() {
-        boolean ignoreUnavailable = randomBoolean();
-        boolean allowNoIndices = randomBoolean();
-        boolean expandToOpenIndices = randomBoolean();
-        boolean expandToClosedIndices = randomBoolean();
-        boolean allowAliasesToMultipleIndices = randomBoolean();
-        boolean forbidClosedIndices = randomBoolean();
-        boolean ignoreAliases = randomBoolean();
-        boolean ingoreThrottled = randomBoolean();
-
-        IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices,expandToOpenIndices,
-                expandToClosedIndices, allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases, ingoreThrottled);
+        final boolean ignoreUnavailable = randomBoolean();
+        final boolean allowNoIndices = randomBoolean();
+        final boolean expandToOpenIndices = randomBoolean();
+        final boolean expandToClosedIndices = randomBoolean();
+        final boolean expandToHiddenIndices = randomBoolean();
+        final boolean allowAliasesToMultipleIndices = randomBoolean();
+        final boolean forbidClosedIndices = randomBoolean();
+        final boolean ignoreAliases = randomBoolean();
+        final boolean ignoreThrottled = randomBoolean();
+
+        IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices,
+            expandToClosedIndices, expandToHiddenIndices, allowAliasesToMultipleIndices, forbidClosedIndices, ignoreAliases,
+            ignoreThrottled);
 
         assertThat(indicesOptions.ignoreUnavailable(), equalTo(ignoreUnavailable));
         assertThat(indicesOptions.allowNoIndices(), equalTo(allowNoIndices));
         assertThat(indicesOptions.expandWildcardsOpen(), equalTo(expandToOpenIndices));
         assertThat(indicesOptions.expandWildcardsClosed(), equalTo(expandToClosedIndices));
+        assertThat(indicesOptions.expandWildcardsHidden(), equalTo(expandToHiddenIndices));
         assertThat(indicesOptions.allowAliasesToMultipleIndices(), equalTo(allowAliasesToMultipleIndices));
         assertThat(indicesOptions.allowAliasesToMultipleIndices(), equalTo(allowAliasesToMultipleIndices));
         assertThat(indicesOptions.forbidClosedIndices(), equalTo(forbidClosedIndices));
         assertEquals(ignoreAliases, indicesOptions.ignoreAliases());
-        assertEquals(ingoreThrottled, indicesOptions.ignoreThrottled());
+        assertEquals(ignoreThrottled, indicesOptions.ignoreThrottled());
     }
 
     public void testFromOptionsWithDefaultOptions() {
@@ -106,53 +116,64 @@ public void testFromOptionsWithDefaultOptions() {
         boolean expandToOpenIndices = randomBoolean();
         boolean expandToClosedIndices = randomBoolean();
 
-        IndicesOptions defaultOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(),
+        IndicesOptions defaultOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
                 randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
 
-        IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices,expandToOpenIndices,
+        IndicesOptions indicesOptions = IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices,
                 expandToClosedIndices, defaultOptions);
 
         assertEquals(ignoreUnavailable, indicesOptions.ignoreUnavailable());
         assertEquals(allowNoIndices, indicesOptions.allowNoIndices());
         assertEquals(expandToOpenIndices, indicesOptions.expandWildcardsOpen());
         assertEquals(expandToClosedIndices, indicesOptions.expandWildcardsClosed());
+        assertEquals(defaultOptions.expandWildcardsHidden(), indicesOptions.expandWildcardsHidden());
         assertEquals(defaultOptions.allowAliasesToMultipleIndices(), indicesOptions.allowAliasesToMultipleIndices());
         assertEquals(defaultOptions.forbidClosedIndices(), indicesOptions.forbidClosedIndices());
         assertEquals(defaultOptions.ignoreAliases(), indicesOptions.ignoreAliases());
     }
 
     public void testFromParameters() {
-        boolean expandWildcardsOpen = randomBoolean();
-        boolean expandWildcardsClosed = randomBoolean();
-        String expandWildcardsString;
-        if (expandWildcardsOpen && expandWildcardsClosed) {
+        final boolean expandWildcardsOpen = randomBoolean();
+        final boolean expandWildcardsClosed = randomBoolean();
+        final boolean expandWildcardsHidden = randomBoolean();
+        final String expandWildcardsString;
+        if (expandWildcardsOpen && expandWildcardsClosed && expandWildcardsHidden) {
             if (randomBoolean()) {
-                expandWildcardsString = "open,closed";
+                expandWildcardsString = "open,closed,hidden";
             } else {
                 expandWildcardsString = "all";
             }
-        } else if (expandWildcardsOpen) {
-            expandWildcardsString = "open";
-        } else if (expandWildcardsClosed) {
-            expandWildcardsString = "closed";
         } else {
-            expandWildcardsString = "none";
+            List<String> values = new ArrayList<>();
+            if (expandWildcardsOpen) {
+                values.add("open");
+            }
+            if (expandWildcardsClosed) {
+                values.add("closed");
+            }
+            if (expandWildcardsHidden) {
+                values.add("hidden");
+            }
+            if (values.isEmpty() && randomBoolean()) {
+                values.add("none");
+            }
+            expandWildcardsString = String.join(",", values);
         }
         boolean ignoreUnavailable = randomBoolean();
         String ignoreUnavailableString = Boolean.toString(ignoreUnavailable);
         boolean ignoreThrottled = randomBoolean();
-        String ignoreThrottledString = Boolean.toString(ignoreThrottled);
         boolean allowNoIndices = randomBoolean();
         String allowNoIndicesString = Boolean.toString(allowNoIndices);
 
         IndicesOptions defaultOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
-                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
 
         IndicesOptions updatedOptions = IndicesOptions.fromParameters(expandWildcardsString, ignoreUnavailableString,
                 allowNoIndicesString, ignoreThrottled, defaultOptions);
 
         assertEquals(expandWildcardsOpen, updatedOptions.expandWildcardsOpen());
         assertEquals(expandWildcardsClosed, updatedOptions.expandWildcardsClosed());
+        assertEquals(expandWildcardsHidden, updatedOptions.expandWildcardsHidden());
         assertEquals(ignoreUnavailable, updatedOptions.ignoreUnavailable());
         assertEquals(allowNoIndices, updatedOptions.allowNoIndices());
         assertEquals(defaultOptions.allowAliasesToMultipleIndices(), updatedOptions.allowAliasesToMultipleIndices());
@@ -161,20 +182,20 @@ public void testFromParameters() {
     }
 
     public void testEqualityAndHashCode() {
-        IndicesOptions indicesOptions = IndicesOptions.fromOptions(
-            randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
-            randomBoolean());
+        IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+            randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
 
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(indicesOptions, opts -> {
             return IndicesOptions.fromOptions(opts.ignoreUnavailable(), opts.allowNoIndices(), opts.expandWildcardsOpen(),
-                opts.expandWildcardsClosed(), opts.allowAliasesToMultipleIndices(), opts.forbidClosedIndices(), opts.ignoreAliases(),
-                opts.ignoreThrottled());
+                opts.expandWildcardsClosed(), opts.expandWildcardsHidden(), opts.allowAliasesToMultipleIndices(),
+                opts.forbidClosedIndices(), opts.ignoreAliases(), opts.ignoreThrottled());
         }, opts -> {
             boolean mutated = false;
             boolean ignoreUnavailable = opts.ignoreUnavailable();
             boolean allowNoIndices = opts.allowNoIndices();
             boolean expandOpen = opts.expandWildcardsOpen();
             boolean expandClosed = opts.expandWildcardsClosed();
+            boolean expandHidden = opts.expandWildcardsHidden();
             boolean allowAliasesToMulti = opts.allowAliasesToMultipleIndices();
             boolean forbidClosed = opts.forbidClosedIndices();
             boolean ignoreAliases = opts.ignoreAliases();
@@ -196,6 +217,10 @@ public void testEqualityAndHashCode() {
                     expandClosed = !expandClosed;
                     mutated = true;
                 }
+                if (randomBoolean()) {
+                    expandHidden = !expandHidden;
+                    mutated = true;
+                }
                 if (randomBoolean()) {
                     allowAliasesToMulti = !allowAliasesToMulti;
                     mutated = true;
@@ -213,7 +238,7 @@ public void testEqualityAndHashCode() {
                     mutated = true;
                 }
             }
-            return IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed,
+            return IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandOpen, expandClosed, expandHidden,
                 allowAliasesToMulti, forbidClosed, ignoreAliases, ignoreThrottled);
         });
     }
@@ -221,7 +246,7 @@ public void testEqualityAndHashCode() {
     public void testFromMap() {
         IndicesOptions defaults = IndicesOptions.strictExpandOpen();
         Collection<String> wildcardStates = randomBoolean() ?
-                null : randomSubsetOf(Arrays.asList("open", "closed"));
+                null : randomSubsetOf(Arrays.asList("open", "closed", "hidden"));
         Boolean ignoreUnavailable = randomBoolean() ? null : randomBoolean();
         Boolean allowNoIndices = randomBoolean() ? null : randomBoolean();
         Boolean ignoreThrottled = randomBoolean() ? null : randomBoolean();
@@ -247,13 +272,15 @@ public void testFromMap() {
         IndicesOptions fromMap = IndicesOptions.fromMap(settings, defaults);
 
         boolean open = wildcardStates != null ? wildcardStates.contains("open") : defaults.expandWildcardsOpen();
-        assertEquals(fromMap.expandWildcardsOpen(), open);
+        assertEquals(open, fromMap.expandWildcardsOpen());
         boolean closed = wildcardStates != null ? wildcardStates.contains("closed") : defaults.expandWildcardsClosed();
-        assertEquals(fromMap.expandWildcardsClosed(), closed);
+        assertEquals(closed, fromMap.expandWildcardsClosed());
+        boolean hidden = wildcardStates != null ? wildcardStates.contains("hidden") : defaults.expandWildcardsHidden();
+        assertEquals(hidden, fromMap.expandWildcardsHidden());
 
-        assertEquals(fromMap.ignoreUnavailable(), ignoreUnavailable == null ? defaults.ignoreUnavailable() : ignoreUnavailable);
-        assertEquals(fromMap.allowNoIndices(), allowNoIndices == null ? defaults.allowNoIndices() : allowNoIndices);
-        assertEquals(fromMap.ignoreThrottled(), ignoreThrottled == null ? defaults.ignoreThrottled() : ignoreThrottled);
+        assertEquals(ignoreUnavailable == null ? defaults.ignoreUnavailable() : ignoreUnavailable, fromMap.ignoreUnavailable());
+        assertEquals(allowNoIndices == null ? defaults.allowNoIndices() : allowNoIndices, fromMap.allowNoIndices());
+        assertEquals(ignoreThrottled == null ? defaults.ignoreThrottled() : ignoreThrottled, fromMap.ignoreThrottled());
     }
 
     public void testToXContent() throws IOException {
@@ -284,6 +311,7 @@ public void testToXContent() throws IOException {
         } else {
             assertFalse(((List<?>) map.get("expand_wildcards")).contains("closed"));
         }
+        assertEquals(wildcardStates.contains(WildcardStates.HIDDEN), ((List<?>) map.get("expand_wildcards")).contains("hidden"));
         assertEquals(map.get("ignore_unavailable"), options.contains(Option.IGNORE_UNAVAILABLE));
         assertEquals(map.get("allow_no_indices"), options.contains(Option.ALLOW_NO_INDICES));
         assertEquals(map.get("ignore_throttled"), options.contains(Option.IGNORE_THROTTLED));

From 61ffc1f2b1a0065e3853684cc7cd33fc19e29040 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Fri, 10 Jan 2020 08:49:32 -0700
Subject: [PATCH 16/26] fix test compile

---
 .../java/org/elasticsearch/index/HiddenIndexIT.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
index 5d4702545d477..1209a18785d49 100644
--- a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -69,13 +69,13 @@ public void testHiddenIndexSearch() {
 
     public void testGlobalTemplatesDoNotApply() {
         assertAcked(client().admin().indices().preparePutTemplate("a_global_template").setPatterns(List.of("*"))
-            .addMapping("_doc", "foo", "type=text").get());
+            .setMapping("foo", "type=text").get());
         assertAcked(client().admin().indices().preparePutTemplate("not_global_template").setPatterns(List.of("a*"))
-            .addMapping("_doc", "bar", "type=text").get());
+            .setMapping("bar", "type=text").get());
         assertAcked(client().admin().indices().preparePutTemplate("specific_template").setPatterns(List.of("a_hidden_index"))
-            .addMapping("_doc", "baz", "type=text").get());
+            .setMapping("baz", "type=text").get());
         assertAcked(client().admin().indices().preparePutTemplate("unused_template").setPatterns(List.of("not_used"))
-            .addMapping("_doc", "foobar", "type=text").get());
+            .setMapping("foobar", "type=text").get());
 
         assertAcked(client().admin().indices().prepareCreate("a_hidden_index")
             .setSettings(Settings.builder().put("index.hidden", true).build()).get());
@@ -107,7 +107,7 @@ public void testGlobalTemplateCannotMakeIndexHidden() {
     public void testNonGlobalTemplateCanMakeIndexHidden() {
         assertAcked(client().admin().indices().preparePutTemplate("a_global_template")
             .setPatterns(List.of("my_hidden_pattern*"))
-            .addMapping("_doc", "foo", "type=text")
+            .setMapping("foo", "type=text")
             .setSettings(Settings.builder().put("index.hidden", true).build())
             .get());
         assertAcked(client().admin().indices().prepareCreate("my_hidden_pattern1").get());

From 984d9ddcbed41331aee9e152faf33b860657b220 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Fri, 10 Jan 2020 09:47:11 -0700
Subject: [PATCH 17/26] more tests and test fixes

---
 .../action/support/IndicesOptions.java        |  8 ++-
 .../action/OriginalIndicesTests.java          | 15 ++++--
 .../ClusterSearchShardsRequestTests.java      | 11 ++--
 .../indices/close/CloseIndexRequestTests.java | 51 ++++++-------------
 .../IndexNameExpressionResolverTests.java     | 30 ++++++-----
 5 files changed, 61 insertions(+), 54 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index f58cd90d35a7c..4b7ec594e978e 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -224,7 +224,13 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
 
     public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
                                              boolean expandToClosedIndices) {
-        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, false, true, false, false, false);
+        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, false);
+    }
+
+    public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
+                                             boolean expandToClosedIndices, boolean expandToHiddenIndices) {
+        return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, expandToHiddenIndices, true,
+            false, false, false);
     }
 
     public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices,
diff --git a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
index 177c0870803d7..39f453d448fd7 100644
--- a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
+++ b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -26,7 +27,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class OriginalIndicesTests extends ESTestCase {
@@ -41,7 +42,7 @@ public void testOriginalIndicesSerialization() throws IOException {
             OriginalIndices originalIndices = randomOriginalIndices();
 
             BytesStreamOutput out = new BytesStreamOutput();
-            out.setVersion(randomVersion(random()));
+            out.setVersion(randomCompatibleVersion(random(), Version.CURRENT));
             OriginalIndices.writeOriginalIndices(originalIndices, out);
 
             StreamInput in = out.bytes().streamInput();
@@ -49,7 +50,15 @@ public void testOriginalIndicesSerialization() throws IOException {
             OriginalIndices originalIndices2 = OriginalIndices.readOriginalIndices(in);
 
             assertThat(originalIndices2.indices(), equalTo(originalIndices.indices()));
-            assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions()));
+            // indices options are not equivalent when sent to an older version and re-read due
+            // to the addition of hidden indices as expand to hidden indices is always true when
+            // read from a prior version
+            // TODO update version on backport!
+            if (out.getVersion().onOrAfter(Version.V_8_0_0) || originalIndices.indicesOptions().expandWildcardsHidden()) {
+                assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions()));
+            } else if (originalIndices.indicesOptions().expandWildcardsHidden()) {
+                assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions()));
+            }
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
index 600920267fcb3..ee943518ac1c2 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java
@@ -40,7 +40,7 @@ public void testSerialization() throws Exception {
         }
         if (randomBoolean()) {
             request.indicesOptions(
-                    IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+                    IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
         }
         if (randomBoolean()) {
             request.preference(randomAlphaOfLengthBetween(3, 10));
@@ -54,7 +54,7 @@ public void testSerialization() throws Exception {
             request.routing(routings);
         }
 
-        Version version = VersionUtils.randomIndexCompatibleVersion(random());
+        Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.setVersion(version);
             request.writeTo(out);
@@ -62,7 +62,12 @@ public void testSerialization() throws Exception {
                 in.setVersion(version);
                 ClusterSearchShardsRequest deserialized = new ClusterSearchShardsRequest(in);
                 assertArrayEquals(request.indices(), deserialized.indices());
-                assertEquals(request.indicesOptions(), deserialized.indicesOptions());
+                // indices options are not equivalent when sent to an older version and re-read due
+                // to the addition of hidden indices as expand to hidden indices is always true when
+                // read from a prior version
+                if (version.onOrAfter(Version.V_8_0_0) || request.indicesOptions().expandWildcardsHidden()) { // TODO change on backport!
+                    assertEquals(request.indicesOptions(), deserialized.indicesOptions());
+                }
                 assertEquals(request.routing(), deserialized.routing());
                 assertEquals(request.preference(), deserialized.preference());
             }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
index f6c500fe3f22b..54f9da362c1aa 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
@@ -49,52 +49,33 @@ public void testSerialization() throws Exception {
     }
 
     public void testBwcSerialization() throws Exception {
-        {
-            final CloseIndexRequest request = randomRequest();
-            try (BytesStreamOutput out = new BytesStreamOutput()) {
-                out.setVersion(VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
-                    VersionUtils.getPreviousVersion(Version.V_7_2_0)));
-                request.writeTo(out);
+        final CloseIndexRequest request = randomRequest();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
+            request.writeTo(out);
 
-                try (StreamInput in = out.bytes().streamInput()) {
-                    assertEquals(request.getParentTask(), TaskId.readFromStream(in));
-                    assertEquals(request.masterNodeTimeout(), in.readTimeValue());
-                    assertEquals(request.timeout(), in.readTimeValue());
-                    assertArrayEquals(request.indices(), in.readStringArray());
+            try (StreamInput in = out.bytes().streamInput()) {
+                assertEquals(request.getParentTask(), TaskId.readFromStream(in));
+                assertEquals(request.masterNodeTimeout(), in.readTimeValue());
+                assertEquals(request.timeout(), in.readTimeValue());
+                assertArrayEquals(request.indices(), in.readStringArray());
+                // indices options are not equivalent when sent to an older version and re-read due
+                // to the addition of hidden indices as expand to hidden indices is always true when
+                // read from a prior version
+                // TODO update version on backport!
+                if (out.getVersion().onOrAfter(Version.V_8_0_0) || request.indicesOptions().expandWildcardsHidden()) {
                     assertEquals(request.indicesOptions(), IndicesOptions.readIndicesOptions(in));
                 }
             }
         }
-        {
-            final CloseIndexRequest sample = randomRequest();
-            try (BytesStreamOutput out = new BytesStreamOutput()) {
-                sample.getParentTask().writeTo(out);
-                out.writeTimeValue(sample.masterNodeTimeout());
-                out.writeTimeValue(sample.timeout());
-                out.writeStringArray(sample.indices());
-                sample.indicesOptions().writeIndicesOptions(out);
-
-                final CloseIndexRequest deserializedRequest;
-                try (StreamInput in = out.bytes().streamInput()) {
-                    in.setVersion(VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
-                        VersionUtils.getPreviousVersion(Version.V_7_2_0)));
-                    deserializedRequest = new CloseIndexRequest(in);
-                }
-                assertEquals(sample.getParentTask(), deserializedRequest.getParentTask());
-                assertEquals(sample.masterNodeTimeout(), deserializedRequest.masterNodeTimeout());
-                assertEquals(sample.timeout(), deserializedRequest.timeout());
-                assertArrayEquals(sample.indices(), deserializedRequest.indices());
-                assertEquals(sample.indicesOptions(), deserializedRequest.indicesOptions());
-                assertEquals(ActiveShardCount.NONE, deserializedRequest.waitForActiveShards());
-            }
-        }
     }
 
     private CloseIndexRequest randomRequest() {
         CloseIndexRequest request = new CloseIndexRequest();
         request.indices(generateRandomStringArray(10, 5, false, false));
         if (randomBoolean()) {
-            request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+            request.indicesOptions(
+                IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
         }
         if (randomBoolean()) {
             request.timeout(randomPositiveTimeValue());
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 371ce0be6aa87..7b0e3390c2095 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -272,11 +272,12 @@ public void testIndexOptionsWildcardExpansion() {
         MetaData.Builder mdBuilder = MetaData.builder()
                 .put(indexBuilder("foo").state(IndexMetaData.State.CLOSE))
                 .put(indexBuilder("bar"))
-                .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("barbaz")));
+                .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("barbaz")))
+                .put(indexBuilder("hidden", Settings.builder().put("index.hidden", true).build()));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
 
         // Only closed
-        IndicesOptions options = IndicesOptions.fromOptions(false, true, false, true);
+        IndicesOptions options = IndicesOptions.fromOptions(false, true, false, true, false);
         IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
         String[] results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(1, results.length);
@@ -292,7 +293,7 @@ public void testIndexOptionsWildcardExpansion() {
         assertEquals("bar", results[0]);
 
         // Only open
-        options = IndicesOptions.fromOptions(false, true, true, false);
+        options = IndicesOptions.fromOptions(false, true, true, false, false);
         context = new IndexNameExpressionResolver.Context(state, options);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(2, results.length);
@@ -307,7 +308,7 @@ public void testIndexOptionsWildcardExpansion() {
         assertEquals("bar", results[0]);
 
         // Open and closed
-        options = IndicesOptions.fromOptions(false, true, true, true);
+        options = IndicesOptions.fromOptions(false, true, true, true, false);
         context = new IndexNameExpressionResolver.Context(state, options);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
         assertEquals(3, results.length);
@@ -332,7 +333,7 @@ public void testIndexOptionsWildcardExpansion() {
         results = indexNameExpressionResolver.concreteIndexNames(context, "-*");
         assertEquals(0, results.length);
 
-        options = IndicesOptions.fromOptions(false, false, true, true);
+        options = IndicesOptions.fromOptions(false, false, true, true, true);
         IndexNameExpressionResolver.Context context2 = new IndexNameExpressionResolver.Context(state, options);
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndexNames(context2, "-*"));
@@ -539,12 +540,17 @@ public void testIndexOptionsEmptyCluster() {
     }
 
     private static IndexMetaData.Builder indexBuilder(String index) {
-        return IndexMetaData.builder(index).settings(settings());
+        return indexBuilder(index, Settings.EMPTY);
     }
 
-    private static Settings.Builder settings() {
+    private static IndexMetaData.Builder indexBuilder(String index, Settings additionalSettings) {
+        return IndexMetaData.builder(index).settings(settings(additionalSettings));
+    }
+
+    private static Settings.Builder settings(Settings additionalSettings) {
         return settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(additionalSettings);
     }
 
     public void testConcreteIndicesIgnoreIndicesOneMissingIndex() {
@@ -1397,13 +1403,13 @@ public void testInvalidIndex() {
 
     public void testIgnoreThrottled() {
         MetaData.Builder mdBuilder = MetaData.builder()
-            .put(indexBuilder("test-index").state(State.OPEN)
-                .settings(settings().put(IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), true))
+            .put(indexBuilder("test-index", Settings.builder().put(IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), true).build())
+                .state(State.OPEN)
                 .putAlias(AliasMetaData.builder("test-alias")))
             .put(indexBuilder("index").state(State.OPEN)
                 .putAlias(AliasMetaData.builder("test-alias2")))
-            .put(indexBuilder("index-closed").state(State.CLOSE)
-                .settings(settings().put(IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), true))
+            .put(indexBuilder("index-closed", Settings.builder().put(IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), true).build())
+                .state(State.CLOSE)
                 .putAlias(AliasMetaData.builder("test-alias-closed")));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
         {

From 14c6f722f570a50a8d0b6515b825874094422239 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Fri, 10 Jan 2020 14:46:57 -0700
Subject: [PATCH 18/26] more more tests

---
 .../MetaDataIndexTemplateService.java         |  13 ++-
 .../MetaDataIndexTemplateServiceTests.java    |  69 ++++++++++--
 .../IndexNameExpressionResolverTests.java     | 106 +++++++++++++++++-
 .../authz/IndicesAndAliasesResolver.java      |   3 +-
 4 files changed, 173 insertions(+), 18 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index d5e9d88533d1d..81d6a177c0922 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -198,8 +198,17 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
     }
 
     /**
-     * Finds index templates whose index pattern matched with the given index name.
-     * The result is sorted by {@link IndexTemplateMetaData#order()} descending.
+     * Finds index templates whose index pattern matched with the given index name. In the case of
+     * hidden indices, a template with a match all pattern or global template will not be returned.
+     *
+     * @param metaData The {@link MetaData} containing all of the {@link IndexTemplateMetaData} values
+     * @param indexName The name of the index that templates are being found for
+     * @param isHidden Whether or not the index is know to be hidden. May be {@code null} if the index
+     *                 being hidden has not been explicitly requested. When {@code null} if the result
+     *                 of template application results in a hidden index, then global templates will
+     *                 not be returned
+     * @return a list of templates sorted by {@link IndexTemplateMetaData#order()} descending.
+     *
      */
     public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, String indexName, @Nullable Boolean isHidden) {
         final List<IndexTemplateMetaData> matchedTemplates = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index 3bcf3ff226cc8..31add95f55401 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -48,17 +48,20 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.is;
 
 public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
     public void testIndexTemplateInvalidNumberOfShards() {
         PutRequest request = new PutRequest("test", "test_shards");
-        request.patterns(Collections.singletonList("test_shards*"));
+        request.patterns(singletonList("test_shards*"));
 
         request.settings(builder()
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "0")
@@ -76,7 +79,7 @@ public void testIndexTemplateInvalidNumberOfShards() {
 
     public void testIndexTemplateValidationAccumulatesValidationErrors() {
         PutRequest request = new PutRequest("test", "putTemplate shards");
-        request.patterns(Collections.singletonList("_test_shards*"));
+        request.patterns(singletonList("_test_shards*"));
         request.settings(builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "0").build());
 
         List<Throwable> throwables = putTemplate(xContentRegistry(), request);
@@ -101,7 +104,7 @@ public void testIndexTemplateWithAliasNameEqualToTemplatePattern() {
 
     public void testIndexTemplateWithValidateMapping() throws Exception {
         PutRequest request = new PutRequest("api", "validate_template");
-        request.patterns(Collections.singletonList("te*"));
+        request.patterns(singletonList("te*"));
         request.putMapping("type1", Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
                         .startObject("properties").startObject("field2").field("type", "text").field("analyzer", "custom_1").endObject()
                         .endObject().endObject().endObject()));
@@ -114,7 +117,7 @@ public void testIndexTemplateWithValidateMapping() throws Exception {
 
     public void testBrokenMapping() throws Exception {
         PutRequest request = new PutRequest("api", "broken_mapping");
-        request.patterns(Collections.singletonList("te*"));
+        request.patterns(singletonList("te*"));
         request.putMapping("type1", "abcde");
 
         List<Throwable> errors = putTemplateDetail(request);
@@ -126,7 +129,7 @@ public void testBrokenMapping() throws Exception {
     public void testAliasInvalidFilterInvalidJson() throws Exception {
         //invalid json: put index template fails
         PutRequest request = new PutRequest("api", "blank_mapping");
-        request.patterns(Collections.singletonList("te*"));
+        request.patterns(singletonList("te*"));
         request.putMapping("type1", "{}");
         Set<Alias> aliases = new HashSet<>();
         aliases.add(new Alias("invalid_alias").filter("abcde"));
@@ -140,9 +143,9 @@ public void testAliasInvalidFilterInvalidJson() throws Exception {
 
     public void testFindTemplates() throws Exception {
         client().admin().indices().prepareDeleteTemplate("*").get(); // Delete all existing templates
-        putTemplateDetail(new PutRequest("test", "foo-1").patterns(Arrays.asList("foo-*")).order(1));
-        putTemplateDetail(new PutRequest("test", "foo-2").patterns(Arrays.asList("foo-*")).order(2));
-        putTemplateDetail(new PutRequest("test", "bar").patterns(Arrays.asList("bar-*")).order(between(0, 100)));
+        putTemplateDetail(new PutRequest("test", "foo-1").patterns(singletonList("foo-*")).order(1));
+        putTemplateDetail(new PutRequest("test", "foo-2").patterns(singletonList("foo-*")).order(2));
+        putTemplateDetail(new PutRequest("test", "bar").patterns(singletonList("bar-*")).order(between(0, 100)));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "foo-1234", randomBoolean()).stream()
             .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("foo-2", "foo-1"));
@@ -151,6 +154,54 @@ public void testFindTemplates() throws Exception {
         assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "baz", randomBoolean()), empty());
     }
 
+    public void testFindTemplatesWithHiddenIndices() throws Exception {
+        client().admin().indices().prepareDeleteTemplate("*").get(); // Delete all existing templates
+        putTemplateDetail(new PutRequest("testFindTemplatesWithHiddenIndices", "foo-1").patterns(singletonList("foo-*")).order(1));
+        putTemplateDetail(new PutRequest("testFindTemplatesWithHiddenIndices", "foo-2").patterns(singletonList("foo-*")).order(2));
+        putTemplateDetail(
+            new PutRequest("testFindTemplatesWithHiddenIndices", "bar").patterns(singletonList("bar-*")).order(between(0, 100)));
+        putTemplateDetail(new PutRequest("testFindTemplatesWithHiddenIndices", "global").patterns(singletonList("*")));
+        putTemplateDetail(new PutRequest("testFindTemplatesWithHiddenIndices", "sneaky-hidden")
+            .patterns(singletonList("sneaky*")).settings(Settings.builder().put("index.hidden", true).build()));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+
+        // hidden
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "foo-1234", true).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("foo-2", "foo-1"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "bar-xyz", true).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("bar"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "baz", true), empty());
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "sneaky1", true).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("sneaky-hidden"));
+
+        // not hidden
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "foo-1234", false).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("foo-2", "foo-1", "global"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "bar-xyz", false).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("bar", "global"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "baz", false).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("global"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "sneaky1", false).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), containsInAnyOrder("global", "sneaky-hidden"));
+
+        // unknown
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "foo-1234", null).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("foo-2", "foo-1", "global"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "bar-xyz", null).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("bar", "global"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "baz", null).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("global"));
+        assertThat(MetaDataIndexTemplateService.findTemplates(state.metaData(), "sneaky1", null).stream()
+            .map(IndexTemplateMetaData::name).collect(Collectors.toList()), contains("sneaky-hidden"));
+    }
+
+    public void testPutGlobalTemplateWithIndexHiddenSetting() throws Exception {
+        List<Throwable> errors = putTemplateDetail(new PutRequest("testPutGlobalTemplateWithIndexHiddenSetting", "sneaky-hidden")
+            .patterns(singletonList("*")).settings(Settings.builder().put("index.hidden", true).build()));
+        assertThat(errors.size(), is(1));
+        assertThat(errors.get(0).getMessage(), containsString("global templates may not specify the setting index.hidden"));
+    }
+
     private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
                 Settings.EMPTY,
@@ -192,7 +243,7 @@ private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
                 null,
                 null,
                 new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
-                null,
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
                 null,
                 xContentRegistry(),
                 true);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 7b0e3390c2095..81d81014c9b77 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -273,7 +273,8 @@ public void testIndexOptionsWildcardExpansion() {
                 .put(indexBuilder("foo").state(IndexMetaData.State.CLOSE))
                 .put(indexBuilder("bar"))
                 .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("barbaz")))
-                .put(indexBuilder("hidden", Settings.builder().put("index.hidden", true).build()));
+                .put(indexBuilder("hidden", Settings.builder().put("index.hidden", true).build()))
+                .put(indexBuilder("hidden-closed", Settings.builder().put("index.hidden", true).build()).state(IndexMetaData.State.CLOSE));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
 
         // Only closed
@@ -333,6 +334,101 @@ public void testIndexOptionsWildcardExpansion() {
         results = indexNameExpressionResolver.concreteIndexNames(context, "-*");
         assertEquals(0, results.length);
 
+        // open closed and hidden
+        options = IndicesOptions.fromOptions(false, true, true, true, true);
+        context = new IndexNameExpressionResolver.Context(state, options);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
+        assertEquals(5, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "foo", "hidden", "hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
+        assertEquals(2, results.length);
+        assertThat(results, arrayContainingInAnyOrder("foobar", "foo"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "bar");
+        assertEquals(1, results.length);
+        assertEquals("bar", results[0]);
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "*", "-foo*");
+        assertEquals(3, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "hidden", "hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "*", "-foo", "-foobar");
+        assertEquals(3, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "hidden", "hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "*", "-foo", "-foobar", "-hidden*");
+        assertEquals(1, results.length);
+        assertEquals("bar", results[0]);
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "hidden*");
+        assertEquals(2, results.length);
+        assertThat(results, arrayContainingInAnyOrder("hidden", "hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "hidden");
+        assertEquals(1, results.length);
+        assertThat(results, arrayContainingInAnyOrder("hidden"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "hidden-closed");
+        assertEquals(1, results.length);
+        assertThat(results, arrayContainingInAnyOrder("hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "-*");
+        assertEquals(0, results.length);
+
+        // open and hidden
+        options = IndicesOptions.fromOptions(false, true, true, false, true);
+        context = new IndexNameExpressionResolver.Context(state, options);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
+        assertEquals(3, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "hidden"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
+        assertEquals(1, results.length);
+        assertEquals("foobar", results[0]);
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "bar");
+        assertEquals(1, results.length);
+        assertEquals("bar", results[0]);
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "h*");
+        assertEquals(1, results.length);
+        assertEquals("hidden", results[0]);
+
+        // closed and hidden
+        options = IndicesOptions.fromOptions(false, true, false, true, true);
+        context = new IndexNameExpressionResolver.Context(state, options);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
+        assertEquals(2, results.length);
+        assertThat(results, arrayContainingInAnyOrder("foo", "hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
+        assertEquals(1, results.length);
+        assertEquals("foo", results[0]);
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "bar");
+        assertEquals(1, results.length);
+        assertEquals("bar", results[0]);
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "h*");
+        assertEquals(1, results.length);
+        assertEquals("hidden-closed", results[0]);
+
+        // only hidden
+        options = IndicesOptions.fromOptions(false, true, false, false, true);
+        context = new IndexNameExpressionResolver.Context(state, options);
+        results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
+        assertThat(results, emptyArray());
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "h*");
+        assertThat(results, emptyArray());
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "hidden");
+        assertThat(results, arrayContainingInAnyOrder("hidden"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "hidden-closed");
+        assertThat(results, arrayContainingInAnyOrder("hidden-closed"));
+
         options = IndicesOptions.fromOptions(false, false, true, true, true);
         IndexNameExpressionResolver.Context context2 = new IndexNameExpressionResolver.Context(state, options);
         IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
@@ -350,7 +446,7 @@ public void testIndexOptionsNoExpandWildcards() {
 
         //ignore unavailable and allow no indices
         {
-            IndicesOptions noExpandLenient = IndicesOptions.fromOptions(true, true, false, false);
+            IndicesOptions noExpandLenient = IndicesOptions.fromOptions(true, true, false, false, randomBoolean());
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandLenient);
             String[] results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
             assertThat(results, emptyArray());
@@ -372,7 +468,7 @@ public void testIndexOptionsNoExpandWildcards() {
 
         //ignore unavailable but don't allow no indices
         {
-            IndicesOptions noExpandDisallowEmpty = IndicesOptions.fromOptions(true, false, false, false);
+            IndicesOptions noExpandDisallowEmpty = IndicesOptions.fromOptions(true, false, false, false, randomBoolean());
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandDisallowEmpty);
 
             {
@@ -397,7 +493,7 @@ public void testIndexOptionsNoExpandWildcards() {
 
         //error on unavailable but allow no indices
         {
-            IndicesOptions noExpandErrorUnavailable = IndicesOptions.fromOptions(false, true, false, false);
+            IndicesOptions noExpandErrorUnavailable = IndicesOptions.fromOptions(false, true, false, false, randomBoolean());
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandErrorUnavailable);
             {
                 String[] results = indexNameExpressionResolver.concreteIndexNames(context, "baz*");
@@ -423,7 +519,7 @@ public void testIndexOptionsNoExpandWildcards() {
 
         //error on both unavailable and no indices
         {
-            IndicesOptions noExpandStrict = IndicesOptions.fromOptions(false, false, false, false);
+            IndicesOptions noExpandStrict = IndicesOptions.fromOptions(false, false, false, false, randomBoolean());
             IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, noExpandStrict);
             IndexNotFoundException infe = expectThrows(IndexNotFoundException.class,
                     () -> indexNameExpressionResolver.concreteIndexNames(context, "baz*"));
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index 8381316dc6e09..cfb32d871f471 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -123,8 +123,7 @@ ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, MetaData
         } else if (indicesRequest instanceof IndicesRequest.Replaceable) {
             final IndicesRequest.Replaceable replaceable = (IndicesRequest.Replaceable) indicesRequest;
             final IndicesOptions indicesOptions = indicesRequest.indicesOptions();
-            final boolean replaceWildcards = indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed() ||
-                indicesOptions.expandWildcardsHidden();
+            final boolean replaceWildcards = indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed();
 
             // check for all and return list of authorized indices
             if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {

From 0ccb7f2ff8725958ee0b4cc7e94d7e69dfdb1794 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Mon, 13 Jan 2020 10:19:06 -0700
Subject: [PATCH 19/26] update test

---
 .../action/admin/indices/close/CloseIndexRequestTests.java       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
index 54f9da362c1aa..a18e602bdf718 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
@@ -55,6 +55,7 @@ public void testBwcSerialization() throws Exception {
             request.writeTo(out);
 
             try (StreamInput in = out.bytes().streamInput()) {
+                in.setVersion(out.getVersion());
                 assertEquals(request.getParentTask(), TaskId.readFromStream(in));
                 assertEquals(request.masterNodeTimeout(), in.readTimeValue());
                 assertEquals(request.timeout(), in.readTimeValue());

From d5d7e2271ecf76de3a4eb7e082216adab5c8fb99 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Mon, 13 Jan 2020 10:46:19 -0700
Subject: [PATCH 20/26] add security test

---
 .../authz/IndicesAndAliasesResolverTests.java | 50 ++++++++++++++++---
 1 file changed, 43 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index e79f6bea966b2..88cbed2f8315d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData.State;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -123,13 +124,13 @@ public void setup() {
                         .putAlias(AliasMetaData.builder("foounauthorized")).settings(settings))
                 .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("foofoobar"))
                         .putAlias(AliasMetaData.builder("foobarfoo")).settings(settings))
-                .put(indexBuilder("closed").state(IndexMetaData.State.CLOSE)
+                .put(indexBuilder("closed").state(State.CLOSE)
                         .putAlias(AliasMetaData.builder("foofoobar")).settings(settings))
-                .put(indexBuilder("foofoo-closed").state(IndexMetaData.State.CLOSE).settings(settings))
-                .put(indexBuilder("foobar-closed").state(IndexMetaData.State.CLOSE).settings(settings))
+                .put(indexBuilder("foofoo-closed").state(State.CLOSE).settings(settings))
+                .put(indexBuilder("foobar-closed").state(State.CLOSE).settings(settings))
                 .put(indexBuilder("foofoo").putAlias(AliasMetaData.builder("barbaz")).settings(settings))
                 .put(indexBuilder("bar").settings(settings))
-                .put(indexBuilder("bar-closed").state(IndexMetaData.State.CLOSE).settings(settings))
+                .put(indexBuilder("bar-closed").state(State.CLOSE).settings(settings))
                 .put(indexBuilder("bar2").settings(settings))
                 .put(indexBuilder(indexNameExpressionResolver.resolveDateMathExpression("<datetime-{now/M}>")).settings(settings))
                 .put(indexBuilder("-index10").settings(settings))
@@ -139,6 +140,9 @@ public void setup() {
                 .put(indexBuilder("logs-00001").putAlias(AliasMetaData.builder("logs-alias").writeIndex(false)).settings(settings))
                 .put(indexBuilder("logs-00002").putAlias(AliasMetaData.builder("logs-alias").writeIndex(false)).settings(settings))
                 .put(indexBuilder("logs-00003").putAlias(AliasMetaData.builder("logs-alias").writeIndex(true)).settings(settings))
+                .put(indexBuilder("hidden-open").settings(Settings.builder().put(settings).put("index.hidden", true).build()))
+                .put(indexBuilder("hidden-closed").state(State.CLOSE)
+                    .settings(Settings.builder().put(settings).put("index.hidden", true).build()))
                 .put(indexBuilder(securityIndexName).settings(settings)).build();
 
         if (withAlias) {
@@ -150,7 +154,8 @@ public void setup() {
         userDashIndices = new User("dash", "dash");
         userNoIndices = new User("test", "test");
         rolesStore = mock(CompositeRolesStore.class);
-        String[] authorizedIndices = new String[] { "bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "missing", "foofoo-closed"};
+        String[] authorizedIndices = new String[] { "bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "missing", "foofoo-closed",
+            "hidden-open", "hidden-closed"};
         String[] dashIndices = new String[]{"-index10", "-index11", "-index20", "-index21"};
         roleMap = new HashMap<>();
         roleMap.put("role", new RoleDescriptor("role", null,
@@ -1368,8 +1373,39 @@ public void testWhenAliasToMultipleIndicesAndUserIsAuthorizedUsingAliasReturnsIn
         assertEquals(message, index, putMappingIndexOrAlias);
     }
 
-    // TODO with the removal of DeleteByQuery is there another way to test resolving a write action?
-
+    public void testHiddenIndicesResolution() {
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, true, true));
+        List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
+        assertThat(resolvedIndices.getLocal(),
+            containsInAnyOrder("bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open", "hidden-closed"));
+        assertThat(resolvedIndices.getRemote(), emptyIterable());
+
+        // open + hidden
+        searchRequest = new SearchRequest();
+        searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true));
+        authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
+        assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open"));
+        assertThat(resolvedIndices.getRemote(), emptyIterable());
+
+        // closed + hidden, ignore aliases
+        searchRequest = new SearchRequest();
+        searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, false, true, true, true, false, true, false));
+        authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
+        assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar-closed", "foofoo-closed", "hidden-closed"));
+        assertThat(resolvedIndices.getRemote(), emptyIterable());
+
+        // allow no indices, do not expand to open or closed, expand hidden, ignore aliases
+        searchRequest = new SearchRequest();
+        searchRequest.indicesOptions(IndicesOptions.fromOptions(false, true, false, false, false, true, false, true, false));
+        authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
+        assertThat(resolvedIndices.getLocal(), contains("-*"));
+        assertThat(resolvedIndices.getRemote(), emptyIterable());
+    }
 
     private List<String> buildAuthorizedIndices(User user, String action) {
         PlainActionFuture<Role> rolesListener = new PlainActionFuture<>();

From 1ccce789891fd7fdf41793fe86170bf55721e256 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Mon, 13 Jan 2020 13:13:09 -0700
Subject: [PATCH 21/26] update docs

---
 docs/reference/index-modules.asciidoc         |  7 +++
 docs/reference/rest-api/common-parms.asciidoc | 55 ++++++++++---------
 2 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index a6cc8019c96fb..69a45555e9401 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -79,6 +79,13 @@ indices.
     Indicates whether <<query-filter-context, cached filters>> are pre-loaded for
     nested queries. Possible values are `true` (default) and `false`.
 
+`index.hidden`::
+
+    Indicates whether the index should be hidden by default. Hidden indices are not
+    returned by default when using a wildcard expression. This behavior is controlled
+    per request through the use of the `expand_wildcards` parameter. Possible values are
+    `true` and `false` (default).
+
 [float]
 [[dynamic-index-settings]]
 === Dynamic index settings
diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index 36b9791ebfdf0..1e8208003df4f 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -53,7 +53,7 @@ Specifies what to do when the request:
 --
 * Contains wildcard expressions and there are no {transforms} that match.
 * Contains the `_all` string or no identifiers and there are no matches.
-* Contains wildcard expressions and there are only partial matches. 
+* Contains wildcard expressions and there are only partial matches.
 
 The default value is `true`, which returns an empty `transforms` array when
 there are no matches and the subset of results when there are partial matches.
@@ -69,7 +69,7 @@ Specifies what to do when the request:
 --
 * Contains wildcard expressions and there are no {transforms} that match.
 * Contains the `_all` string or no identifiers and there are no matches.
-* Contains wildcard expressions and there are only partial matches. 
+* Contains wildcard expressions and there are only partial matches.
 
 The default value is `true`, which returns a successful acknowledgement message
 when there are no matches. When there are only partial matches, the API stops
@@ -90,7 +90,7 @@ end::analyzer[]
 
 tag::analyze_wildcard[]
 `analyze_wildcard`::
-(Optional, boolean) If `true`, wildcard and prefix queries are 
+(Optional, boolean) If `true`, wildcard and prefix queries are
 analyzed. Defaults to `false`.
 end::analyze_wildcard[]
 
@@ -118,7 +118,7 @@ end::completion-fields[]
 
 tag::default_operator[]
 `default_operator`::
-(Optional, string) The default operator for query string query: AND or OR. 
+(Optional, string) The default operator for query string query: AND or OR.
 Defaults to `OR`.
 end::default_operator[]
 
@@ -144,7 +144,7 @@ end::detailed[]
 
 tag::df[]
 `df`::
-(Optional, string) Field to use as default where no field prefix is 
+(Optional, string) Field to use as default where no field prefix is
 given in the query string.
 end::df[]
 
@@ -181,6 +181,9 @@ Expand only to open indices.
 `closed`::
 Expand only to closed indices.
 
+`hidden`::
+Expansion of wildcards will include hidden indices.
+
 `none`::
 Wildcard expressions are not accepted.
 --
@@ -188,7 +191,7 @@ end::expand-wildcards[]
 
 tag::field_statistics[]
 `field_statistics`::
-(Optional, boolean) If `true`, the response includes the document count, sum of document frequencies, 
+(Optional, boolean) If `true`, the response includes the document count, sum of document frequencies,
 and sum of total term frequencies.
 Defaults to `true`.
 end::field_statistics[]
@@ -297,9 +300,9 @@ end::help[]
 
 tag::bulk-id[]
 `_id`::
-(Optional, string) 
+(Optional, string)
 The document ID.
-If no ID is specified, a document ID is automatically generated. 
+If no ID is specified, a document ID is automatically generated.
 end::bulk-id[]
 
 tag::if_primary_term[]
@@ -316,7 +319,7 @@ end::if_seq_no[]
 
 tag::ignore_throttled[]
 `ignore_throttled`::
-(Optional, boolean) If `true`, concrete, expanded or aliased indices are 
+(Optional, boolean) If `true`, concrete, expanded or aliased indices are
 ignored when throttled.
 end::ignore_throttled[]
 
@@ -362,7 +365,7 @@ end::index[]
 tag::bulk-index[]
 `_index`::
 (Optional, string)
-The name of the target index. 
+The name of the target index.
 Required if not specified as a path parameter.
 end::bulk-index[]
 
@@ -447,7 +450,7 @@ end::index-template[]
 
 tag::lenient[]
 `lenient`::
-(Optional, boolean) If `true`, format-based query failures (such as 
+(Optional, boolean) If `true`, format-based query failures (such as
 providing text to a numeric field) will be ignored. Defaults to `false`.
 end::lenient[]
 
@@ -618,13 +621,13 @@ end::search-q[]
 
 tag::query[]
 `query`::
-(Optional, <<query-dsl,query object>>) Defines the search definition using the 
+(Optional, <<query-dsl,query object>>) Defines the search definition using the
 <<query-dsl,Query DSL>>.
 end::query[]
 
 tag::realtime[]
 `realtime`::
-(Optional, boolean) If `true`, the request is real-time as opposed to near-real-time. 
+(Optional, boolean) If `true`, the request is real-time as opposed to near-real-time.
 Defaults to `true`. See <<realtime>>.
 end::realtime[]
 
@@ -638,7 +641,7 @@ end::refresh[]
 
 tag::request_cache[]
 `request_cache`::
-(Optional, boolean) If `true`, the request cache is used for this request. 
+(Optional, boolean) If `true`, the request cache is used for this request.
 Defaults to the index-level setting.
 end::request_cache[]
 
@@ -668,14 +671,14 @@ end::cat-s[]
 
 tag::scroll[]
 `scroll`::
-(Optional, <<time-units, time units>>) Specifies how long a consistent view of 
+(Optional, <<time-units, time units>>) Specifies how long a consistent view of
 the index should be maintained for scrolled search.
 end::scroll[]
 
 tag::scroll_size[]
 `scroll_size`::
-(Optional, integer) Size of the scroll request that powers the operation. 
-Defaults to 100. 
+(Optional, integer) Size of the scroll request that powers the operation.
+Defaults to 100.
 end::scroll_size[]
 
 tag::search_timeout[]
@@ -731,7 +734,7 @@ end::size-transforms[]
 
 tag::slices[]
 `slices`::
-(Optional, integer) The number of slices this task should be divided into. 
+(Optional, integer) The number of slices this task should be divided into.
 Defaults to 1 meaning the task isn't sliced into subtasks.
 end::slices[]
 
@@ -742,24 +745,24 @@ end::sort[]
 
 tag::source[]
 `_source`::
-(Optional, string) True or false to return the `_source` field or not, or a 
+(Optional, string) True or false to return the `_source` field or not, or a
 list of fields to return.
 end::source[]
 
 tag::source_excludes[]
 `_source_excludes`::
-(Optional, string) A list of fields to exclude from the returned `_source` 
+(Optional, string) A list of fields to exclude from the returned `_source`
 field.
 end::source_excludes[]
 
 tag::source_includes[]
 `_source_includes`::
-(Optional, string) A list of fields to extract and return from the `_source` 
+(Optional, string) A list of fields to extract and return from the `_source`
 field.
 end::source_includes[]
 
 tag::source-transforms[]
-The source of the data for the {transform}. 
+The source of the data for the {transform}.
 end::source-transforms[]
 
 tag::source-index-transforms[]
@@ -822,13 +825,13 @@ end::task-id[]
 
 tag::term_statistics[]
 `term_statistics`::
-(Optional, boolean) If `true`, the response includes term frequency and document frequency. 
+(Optional, boolean) If `true`, the response includes term frequency and document frequency.
 Defaults to `false`.
 end::term_statistics[]
 
 tag::terminate_after[]
 `terminate_after`::
-(Optional, integer) The maximum number of documents to collect for each shard, 
+(Optional, integer) The maximum number of documents to collect for each shard,
 upon reaching which the query execution will terminate early.
 end::terminate_after[]
 
@@ -870,7 +873,7 @@ end::transform-id-wildcard[]
 
 tag::cat-v[]
 `v`::
-(Optional, boolean) If `true`, the response includes column headings. 
+(Optional, boolean) If `true`, the response includes column headings.
 Defaults to `false`.
 end::cat-v[]
 
@@ -911,6 +914,6 @@ end::wait_for_active_shards[]
 
 tag::wait_for_completion[]
 `wait_for_completion`::
-(Optional, boolean) If `true`, the request blocks until the operation is complete. 
+(Optional, boolean) If `true`, the request blocks until the operation is complete.
 Defaults to `true`.
 end::wait_for_completion[]

From 4f1098f58834a35f16c99de943d64dcb5691255b Mon Sep 17 00:00:00 2001
From: Jay Modi <jaymode@users.noreply.github.com>
Date: Wed, 15 Jan 2020 08:12:11 -0700
Subject: [PATCH 22/26] Apply suggestions from code review

Co-Authored-By: Gordon Brown <gordon.brown@elastic.co>
---
 .../cluster/metadata/MetaDataIndexTemplateService.java        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index a8cd04bcfe494..cd1394868b84f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -203,7 +203,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
      *
      * @param metaData The {@link MetaData} containing all of the {@link IndexTemplateMetaData} values
      * @param indexName The name of the index that templates are being found for
-     * @param isHidden Whether or not the index is know to be hidden. May be {@code null} if the index
+     * @param isHidden Whether or not the index is known to be hidden. May be {@code null} if the index
      *                 being hidden has not been explicitly requested. When {@code null} if the result
      *                 of template application results in a hidden index, then global templates will
      *                 not be returned
@@ -240,7 +240,7 @@ public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, Strin
         }
         CollectionUtil.timSort(matchedTemplates, Comparator.comparingInt(IndexTemplateMetaData::order).reversed());
 
-        // this is complex but if the index is not hidden in the create request but hidden is the result of template application,
+        // this is complex but if the index is not hidden in the create request but is hidden as the result of template application,
         // then we need to exclude global templates
         if (isHidden == null) {
             final Optional<IndexTemplateMetaData> templateWithHiddenSetting = matchedTemplates.stream()

From 27cee0500dcafee3e6e9e6ea9d0f245b43b18167 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Wed, 15 Jan 2020 08:20:18 -0700
Subject: [PATCH 23/26] clean up find templates

---
 .../MetaDataIndexTemplateService.java         | 21 +++++++------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index cd1394868b84f..55e03a604ec55 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -58,6 +58,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
 
@@ -211,30 +212,22 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
      *
      */
     public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, String indexName, @Nullable Boolean isHidden) {
+        final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, indexName);
         final List<IndexTemplateMetaData> matchedTemplates = new ArrayList<>();
         for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
             final IndexTemplateMetaData template = cursor.value;
             if (isHidden == null || isHidden == Boolean.FALSE) {
-                final boolean matched = template.patterns().stream()
-                    .anyMatch(pattern -> Regex.simpleMatch(pattern, indexName));
+                final boolean matched = template.patterns().stream().anyMatch(patternMatchPredicate);
                 if (matched) {
                     matchedTemplates.add(template);
                 }
             } else {
                 assert isHidden == Boolean.TRUE;
-
-                boolean skip = false;
-                boolean matched = false;
-                for (String pattern : template.patterns()) {
-                    if (Regex.isMatchAllPattern(pattern)) {
-                        skip = true;
-                        break;
+                final boolean isNotMatchAllTemplate = template.patterns().stream().noneMatch(Regex::isMatchAllPattern);
+                if (isNotMatchAllTemplate) {
+                    if (template.patterns().stream().anyMatch(patternMatchPredicate)) {
+                        matchedTemplates.add(template);
                     }
-                    matched = matched || Regex.simpleMatch(pattern, indexName);
-                }
-
-                if (skip == false && matched) {
-                    matchedTemplates.add(template);
                 }
             }
         }

From 94afea41abd0179f22f2731453a546e16c81c674 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Wed, 15 Jan 2020 08:44:58 -0700
Subject: [PATCH 24/26] refactor writing expand_wildcards

---
 .../action/search/MultiSearchRequest.java     | 20 ++----------------
 .../action/support/IndicesOptions.java        | 21 ++++++++++++++++++-
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index a48ab3f8a6b16..2fba92f5bab26 100644
--- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.IndicesOptions.WildcardStates;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -304,24 +305,7 @@ public static void writeSearchRequestParams(SearchRequest request, XContentBuild
             xContentBuilder.field("index", request.indices());
         }
         if (request.indicesOptions() != null && request.indicesOptions() != SearchRequest.DEFAULT_INDICES_OPTIONS) {
-            if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsClosed() &&
-                request.indicesOptions().expandWildcardsHidden()) {
-                xContentBuilder.field("expand_wildcards", "all");
-            } else if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsClosed()) {
-                xContentBuilder.field("expand_wildcards", "open,closed");
-            } else if (request.indicesOptions().expandWildcardsOpen() && request.indicesOptions().expandWildcardsHidden()) {
-                xContentBuilder.field("expand_wildcards", "open,hidden");
-            } else if (request.indicesOptions().expandWildcardsOpen()) {
-                xContentBuilder.field("expand_wildcards", "open");
-            } else if (request.indicesOptions().expandWildcardsClosed() && request.indicesOptions().expandWildcardsHidden()) {
-                xContentBuilder.field("expand_wildcards", "closed,hidden");
-            } else if (request.indicesOptions().expandWildcardsClosed()) {
-                xContentBuilder.field("expand_wildcards", "closed");
-            } else if (request.indicesOptions().expandWildcardsHidden()) {
-                xContentBuilder.field("expand_wildcards", "hidden");
-            } else {
-                xContentBuilder.field("expand_wildcards", "none");
-            }
+            WildcardStates.toXContent(request.indicesOptions().getExpandWildcards(), xContentBuilder);
             xContentBuilder.field("ignore_unavailable", request.indicesOptions().ignoreUnavailable());
             xContentBuilder.field("allow_no_indices", request.indicesOptions().allowNoIndices());
         }
diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index 4b7ec594e978e..42c97b25b950e 100644
--- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -32,6 +32,7 @@
 import java.util.EnumSet;
 import java.util.Locale;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
@@ -82,6 +83,18 @@ public static EnumSet<WildcardStates> parseParameter(Object value, EnumSet<Wildc
 
             return states;
         }
+
+        public static XContentBuilder toXContent(EnumSet<WildcardStates> states, XContentBuilder builder) throws IOException {
+            if (states.isEmpty()) {
+                builder.field("expand_wildcards", "none");
+            } else if (states.containsAll(EnumSet.allOf(WildcardStates.class))) {
+                builder.field("expand_wildcards", "all");
+            } else {
+                builder.field("expand_wildcards",
+                    states.stream().map(state -> state.toString().toLowerCase(Locale.ROOT)).collect(Collectors.joining(",")));
+            }
+            return builder;
+        }
     }
 
     public enum Option {
@@ -195,13 +208,19 @@ public boolean ignoreAliases() {
     }
 
     /**
-     *
      * @return whether indices that are marked as throttled should be ignored
      */
     public boolean ignoreThrottled() {
         return options.contains(Option.IGNORE_THROTTLED);
     }
 
+    /**
+     * @return a copy of the {@link WildcardStates} that these indices options will expand to
+     */
+    public EnumSet<WildcardStates> getExpandWildcards() {
+        return EnumSet.copyOf(expandWildcards);
+    }
+
     public void writeIndicesOptions(StreamOutput out) throws IOException {
         out.writeEnumSet(options);
         if (out.getVersion().before(Version.V_8_0_0) && expandWildcards.contains(WildcardStates.HIDDEN)) {

From 5746bdd29acf1b5457232a35ef8e0d8e5c2a5c48 Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Thu, 16 Jan 2020 13:23:48 -0700
Subject: [PATCH 25/26] add implicit support for hidden index resolution based
 upon use of wildcard expression starting with .

---
 .../metadata/IndexNameExpressionResolver.java | 11 ++--
 .../IndexNameExpressionResolverTests.java     | 53 ++++++++++++++-----
 .../elasticsearch/index/HiddenIndexIT.java    | 16 +++++-
 .../authz/IndicesAndAliasesResolver.java      | 21 +++++---
 .../authz/IndicesAndAliasesResolverTests.java | 30 +++++++++--
 5 files changed, 102 insertions(+), 29 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 80a25e366c093..d68d69bf24e8c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -741,7 +741,7 @@ private Set<String> innerResolve(Context context, List<String> expressions, Indi
 
                 final IndexMetaData.State excludeState = excludeState(options);
                 final Map<String, AliasOrIndex> matches = matches(context, metaData, expression);
-                Set<String> expand = expand(context, excludeState, matches, options.expandWildcardsHidden());
+                Set<String> expand = expand(context, excludeState, matches, expression, options.expandWildcardsHidden());
                 if (add) {
                     result.addAll(expand);
                 } else {
@@ -837,7 +837,7 @@ private static Map<String, AliasOrIndex> otherWildcard(Context context, MetaData
         }
 
         private static Set<String> expand(Context context, IndexMetaData.State excludeState, Map<String, AliasOrIndex> matches,
-                                          boolean includeHidden) {
+                                          String expression, boolean includeHidden) {
             Set<String> expand = new HashSet<>();
             for (Map.Entry<String, AliasOrIndex> entry : matches.entrySet()) {
                 AliasOrIndex aliasOrIndex = entry.getValue();
@@ -846,7 +846,12 @@ private static Set<String> expand(Context context, IndexMetaData.State excludeSt
                 } else {
                     for (IndexMetaData meta : aliasOrIndex.getIndices()) {
                         if (excludeState == null || meta.getState() != excludeState) {
-                            if (includeHidden || IndexMetaData.INDEX_HIDDEN_SETTING.get(meta.getSettings()) == false) {
+                            if (includeHidden) {
+                                expand.add(meta.getIndex().getName());
+                            } else if (IndexMetaData.INDEX_HIDDEN_SETTING.get(meta.getSettings()) == false) {
+                                expand.add(meta.getIndex().getName());
+                            } else if (meta.getIndex().getName().startsWith(".") &&
+                                expression.startsWith(".") && Regex.isSimpleMatchPattern(expression)) {
                                 expand.add(meta.getIndex().getName());
                             }
                         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 81d81014c9b77..ab2177b51d684 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -274,6 +274,8 @@ public void testIndexOptionsWildcardExpansion() {
                 .put(indexBuilder("bar"))
                 .put(indexBuilder("foobar").putAlias(AliasMetaData.builder("barbaz")))
                 .put(indexBuilder("hidden", Settings.builder().put("index.hidden", true).build()))
+                .put(indexBuilder(".hidden", Settings.builder().put("index.hidden", true).build()))
+                .put(indexBuilder(".hidden-closed", Settings.builder().put("index.hidden", true).build()).state(IndexMetaData.State.CLOSE))
                 .put(indexBuilder("hidden-closed", Settings.builder().put("index.hidden", true).build()).state(IndexMetaData.State.CLOSE));
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
 
@@ -293,6 +295,15 @@ public void testIndexOptionsWildcardExpansion() {
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
+        // implicit hidden for dot indices based on wildcard starting with .
+        results = indexNameExpressionResolver.concreteIndexNames(context, ".*");
+        assertEquals(1, results.length);
+        assertThat(results, arrayContainingInAnyOrder(".hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, ".hidd*");
+        assertEquals(1, results.length);
+        assertThat(results, arrayContainingInAnyOrder(".hidden-closed"));
+
         // Only open
         options = IndicesOptions.fromOptions(false, true, true, false, false);
         context = new IndexNameExpressionResolver.Context(state, options);
@@ -308,6 +319,15 @@ public void testIndexOptionsWildcardExpansion() {
         assertEquals(1, results.length);
         assertEquals("bar", results[0]);
 
+        // implicit hidden for dot indices based on wildcard starting with .
+        results = indexNameExpressionResolver.concreteIndexNames(context, ".*");
+        assertEquals(1, results.length);
+        assertThat(results, arrayContainingInAnyOrder(".hidden"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, ".hidd*");
+        assertEquals(1, results.length);
+        assertThat(results, arrayContainingInAnyOrder(".hidden"));
+
         // Open and closed
         options = IndicesOptions.fromOptions(false, true, true, true, false);
         context = new IndexNameExpressionResolver.Context(state, options);
@@ -334,12 +354,21 @@ public void testIndexOptionsWildcardExpansion() {
         results = indexNameExpressionResolver.concreteIndexNames(context, "-*");
         assertEquals(0, results.length);
 
+        // implicit hidden for dot indices based on wildcard starting with .
+        results = indexNameExpressionResolver.concreteIndexNames(context, ".*");
+        assertEquals(2, results.length);
+        assertThat(results, arrayContainingInAnyOrder(".hidden", ".hidden-closed"));
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, ".hidd*");
+        assertEquals(2, results.length);
+        assertThat(results, arrayContainingInAnyOrder(".hidden", ".hidden-closed"));
+
         // open closed and hidden
         options = IndicesOptions.fromOptions(false, true, true, true, true);
         context = new IndexNameExpressionResolver.Context(state, options);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
-        assertEquals(5, results.length);
-        assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "foo", "hidden", "hidden-closed"));
+        assertEquals(7, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "foo", "hidden", "hidden-closed", ".hidden", ".hidden-closed"));
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertEquals(2, results.length);
@@ -350,16 +379,16 @@ public void testIndexOptionsWildcardExpansion() {
         assertEquals("bar", results[0]);
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "*", "-foo*");
-        assertEquals(3, results.length);
-        assertThat(results, arrayContainingInAnyOrder("bar", "hidden", "hidden-closed"));
+        assertEquals(5, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "hidden", "hidden-closed", ".hidden", ".hidden-closed"));
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "*", "-foo", "-foobar");
-        assertEquals(3, results.length);
-        assertThat(results, arrayContainingInAnyOrder("bar", "hidden", "hidden-closed"));
+        assertEquals(5, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "hidden", "hidden-closed", ".hidden", ".hidden-closed"));
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "*", "-foo", "-foobar", "-hidden*");
-        assertEquals(1, results.length);
-        assertEquals("bar", results[0]);
+        assertEquals(3, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", ".hidden", ".hidden-closed"));
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "hidden*");
         assertEquals(2, results.length);
@@ -380,8 +409,8 @@ public void testIndexOptionsWildcardExpansion() {
         options = IndicesOptions.fromOptions(false, true, true, false, true);
         context = new IndexNameExpressionResolver.Context(state, options);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
-        assertEquals(3, results.length);
-        assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "hidden"));
+        assertEquals(4, results.length);
+        assertThat(results, arrayContainingInAnyOrder("bar", "foobar", "hidden", ".hidden"));
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertEquals(1, results.length);
@@ -399,8 +428,8 @@ public void testIndexOptionsWildcardExpansion() {
         options = IndicesOptions.fromOptions(false, true, false, true, true);
         context = new IndexNameExpressionResolver.Context(state, options);
         results = indexNameExpressionResolver.concreteIndexNames(context, Strings.EMPTY_ARRAY);
-        assertEquals(2, results.length);
-        assertThat(results, arrayContainingInAnyOrder("foo", "hidden-closed"));
+        assertEquals(3, results.length);
+        assertThat(results, arrayContainingInAnyOrder("foo", "hidden-closed", ".hidden-closed"));
 
         results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
         assertEquals(1, results.length);
diff --git a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
index 1209a18785d49..ca4fa588b144a 100644
--- a/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/index/HiddenIndexIT.java
@@ -44,9 +44,9 @@ public void testHiddenIndexSearch() {
         assertAcked(client().admin().indices().prepareCreate("hidden-index")
             .setSettings(Settings.builder().put("index.hidden", true).build())
             .get());
-
-        // default not visible
         client().prepareIndex("hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
+
+        // default not visible to wildcard expansion
         SearchResponse searchResponse =
             client().prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()).get();
         boolean matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex()));
@@ -65,6 +65,18 @@ public void testHiddenIndexSearch() {
             .get();
         matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex()));
         assertTrue(matchedHidden);
+
+        // implicit based on use of pattern starting with . and a wildcard
+        assertAcked(client().admin().indices().prepareCreate(".hidden-index")
+            .setSettings(Settings.builder().put("index.hidden", true).build())
+            .get());
+        client().prepareIndex(".hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
+        searchResponse = client().prepareSearch(randomFrom(".*", ".hidden-*"))
+            .setSize(1000)
+            .setQuery(QueryBuilders.matchAllQuery())
+            .get();
+        matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> ".hidden-index".equals(hit.getIndex()));
+        assertTrue(matchedHidden);
     }
 
     public void testGlobalTemplatesDoNotApply() {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index cfb32d871f471..21af40707aec9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -129,7 +129,7 @@ ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, MetaData
             if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
                 if (replaceWildcards) {
                     for (String authorizedIndex : authorizedIndices) {
-                        if (isIndexVisible(authorizedIndex, indicesOptions, metaData)) {
+                        if (isIndexVisible("*", authorizedIndex, indicesOptions, metaData)) {
                             resolvedIndicesBuilder.addLocal(authorizedIndex);
                         }
                     }
@@ -356,7 +356,8 @@ private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indi
                 if (replaceWildcards && Regex.isSimpleMatchPattern(dateMathName)) {
                     // continue
                     aliasOrIndex = dateMathName;
-                } else if (authorizedIndices.contains(dateMathName) && isIndexVisible(dateMathName, indicesOptions, metaData, true)) {
+                } else if (authorizedIndices.contains(dateMathName) &&
+                    isIndexVisible(aliasOrIndex, dateMathName, indicesOptions, metaData, true)) {
                     if (minus) {
                         finalIndices.remove(dateMathName);
                     } else {
@@ -373,7 +374,8 @@ private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indi
                 wildcardSeen = true;
                 Set<String> resolvedIndices = new HashSet<>();
                 for (String authorizedIndex : authorizedIndices) {
-                    if (Regex.simpleMatch(aliasOrIndex, authorizedIndex) && isIndexVisible(authorizedIndex, indicesOptions, metaData)) {
+                    if (Regex.simpleMatch(aliasOrIndex, authorizedIndex) &&
+                        isIndexVisible(aliasOrIndex, authorizedIndex, indicesOptions, metaData)) {
                         resolvedIndices.add(authorizedIndex);
                     }
                 }
@@ -408,11 +410,12 @@ private List<String> replaceWildcardsWithAuthorizedIndices(Iterable<String> indi
         return finalIndices;
     }
 
-    private static boolean isIndexVisible(String index, IndicesOptions indicesOptions, MetaData metaData) {
-        return isIndexVisible(index, indicesOptions, metaData, false);
+    private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, MetaData metaData) {
+        return isIndexVisible(expression, index, indicesOptions, metaData, false);
     }
 
-    private static boolean isIndexVisible(String index, IndicesOptions indicesOptions, MetaData metaData, boolean dateMathExpression) {
+    private static boolean isIndexVisible(String expression, String index, IndicesOptions indicesOptions, MetaData metaData,
+                                          boolean dateMathExpression) {
         AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(index);
         if (aliasOrIndex.isAlias()) {
             //it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed.
@@ -423,7 +426,7 @@ private static boolean isIndexVisible(String index, IndicesOptions indicesOption
         assert aliasOrIndex.getIndices().size() == 1 : "concrete index must point to a single index";
         IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
         final boolean isHidden = IndexMetaData.INDEX_HIDDEN_SETTING.get(indexMetaData.getSettings());
-        if (isHidden && indicesOptions.expandWildcardsHidden() == false) {
+        if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {
             return false;
         }
 
@@ -441,6 +444,10 @@ private static boolean isIndexVisible(String index, IndicesOptions indicesOption
         return false;
     }
 
+    private static boolean isVisibleDueToImplicitHidden(String expression, String index) {
+        return index.startsWith(".") && expression.startsWith(".") && Regex.isSimpleMatchPattern(expression);
+    }
+
     private static List<String> indicesList(String[] list) {
         return (list == null) ? null : Arrays.asList(list);
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index 88cbed2f8315d..3f3334c3646f1 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -141,6 +141,9 @@ public void setup() {
                 .put(indexBuilder("logs-00002").putAlias(AliasMetaData.builder("logs-alias").writeIndex(false)).settings(settings))
                 .put(indexBuilder("logs-00003").putAlias(AliasMetaData.builder("logs-alias").writeIndex(true)).settings(settings))
                 .put(indexBuilder("hidden-open").settings(Settings.builder().put(settings).put("index.hidden", true).build()))
+                .put(indexBuilder(".hidden-open").settings(Settings.builder().put(settings).put("index.hidden", true).build()))
+                .put(indexBuilder(".hidden-closed").state(State.CLOSE)
+                    .settings(Settings.builder().put(settings).put("index.hidden", true).build()))
                 .put(indexBuilder("hidden-closed").state(State.CLOSE)
                     .settings(Settings.builder().put(settings).put("index.hidden", true).build()))
                 .put(indexBuilder(securityIndexName).settings(settings)).build();
@@ -155,7 +158,7 @@ public void setup() {
         userNoIndices = new User("test", "test");
         rolesStore = mock(CompositeRolesStore.class);
         String[] authorizedIndices = new String[] { "bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "missing", "foofoo-closed",
-            "hidden-open", "hidden-closed"};
+            "hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"};
         String[] dashIndices = new String[]{"-index10", "-index11", "-index20", "-index21"};
         roleMap = new HashMap<>();
         roleMap.put("role", new RoleDescriptor("role", null,
@@ -1378,8 +1381,8 @@ public void testHiddenIndicesResolution() {
         searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, true, true));
         List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
         ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
-        assertThat(resolvedIndices.getLocal(),
-            containsInAnyOrder("bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open", "hidden-closed"));
+        assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed",
+            "hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"));
         assertThat(resolvedIndices.getRemote(), emptyIterable());
 
         // open + hidden
@@ -1387,7 +1390,16 @@ public void testHiddenIndicesResolution() {
         searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true));
         authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
         resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
-        assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open"));
+        assertThat(resolvedIndices.getLocal(),
+            containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open", ".hidden-open"));
+        assertThat(resolvedIndices.getRemote(), emptyIterable());
+
+        // open + implicit hidden for . indices
+        searchRequest = new SearchRequest(randomFrom(".*", ".hid*"));
+        searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false));
+        authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
+        assertThat(resolvedIndices.getLocal(), containsInAnyOrder(".hidden-open"));
         assertThat(resolvedIndices.getRemote(), emptyIterable());
 
         // closed + hidden, ignore aliases
@@ -1395,7 +1407,15 @@ public void testHiddenIndicesResolution() {
         searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, false, true, true, true, false, true, false));
         authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
         resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
-        assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar-closed", "foofoo-closed", "hidden-closed"));
+        assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar-closed", "foofoo-closed", "hidden-closed", ".hidden-closed"));
+        assertThat(resolvedIndices.getRemote(), emptyIterable());
+
+        // closed + implicit hidden for . indices
+        searchRequest = new SearchRequest(randomFrom(".*", ".hid*"));
+        searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, false, true, false));
+        authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metaData, authorizedIndices);
+        assertThat(resolvedIndices.getLocal(), containsInAnyOrder(".hidden-closed"));
         assertThat(resolvedIndices.getRemote(), emptyIterable());
 
         // allow no indices, do not expand to open or closed, expand hidden, ignore aliases

From 7a6ac3adbde94b4bb0e894776895ceba3caa906a Mon Sep 17 00:00:00 2001
From: jaymode <jay@elastic.co>
Date: Fri, 17 Jan 2020 08:25:07 -0700
Subject: [PATCH 26/26] restore CloseIndexRequestTests and fix them

---
 .../indices/close/CloseIndexRequestTests.java | 71 +++++++++++++++----
 1 file changed, 58 insertions(+), 13 deletions(-)

diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
index a18e602bdf718..3b33791b9bfe8 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
@@ -49,23 +49,68 @@ public void testSerialization() throws Exception {
     }
 
     public void testBwcSerialization() throws Exception {
-        final CloseIndexRequest request = randomRequest();
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            out.setVersion(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
-            request.writeTo(out);
+        {
+            final CloseIndexRequest request = randomRequest();
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
+                request.writeTo(out);
 
-            try (StreamInput in = out.bytes().streamInput()) {
-                in.setVersion(out.getVersion());
-                assertEquals(request.getParentTask(), TaskId.readFromStream(in));
-                assertEquals(request.masterNodeTimeout(), in.readTimeValue());
-                assertEquals(request.timeout(), in.readTimeValue());
-                assertArrayEquals(request.indices(), in.readStringArray());
+                try (StreamInput in = out.bytes().streamInput()) {
+                    in.setVersion(out.getVersion());
+                    assertEquals(request.getParentTask(), TaskId.readFromStream(in));
+                    assertEquals(request.masterNodeTimeout(), in.readTimeValue());
+                    assertEquals(request.timeout(), in.readTimeValue());
+                    assertArrayEquals(request.indices(), in.readStringArray());
+                    final IndicesOptions indicesOptions = IndicesOptions.readIndicesOptions(in);
+                    // indices options are not equivalent when sent to an older version and re-read due
+                    // to the addition of hidden indices as expand to hidden indices is always true when
+                    // read from a prior version
+                    // TODO update version on backport!
+                    if (out.getVersion().onOrAfter(Version.V_8_0_0) || request.indicesOptions().expandWildcardsHidden()) {
+                        assertEquals(request.indicesOptions(), indicesOptions);
+                    }
+                    if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+                        assertEquals(request.waitForActiveShards(), ActiveShardCount.readFrom(in));
+                    } else {
+                        assertEquals(0, in.available());
+                    }
+                }
+            }
+        }
+        {
+            final CloseIndexRequest sample = randomRequest();
+            final Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                sample.getParentTask().writeTo(out);
+                out.writeTimeValue(sample.masterNodeTimeout());
+                out.writeTimeValue(sample.timeout());
+                out.writeStringArray(sample.indices());
+                sample.indicesOptions().writeIndicesOptions(out);
+                if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+                    sample.waitForActiveShards().writeTo(out);
+                }
+
+                final CloseIndexRequest deserializedRequest;
+                try (StreamInput in = out.bytes().streamInput()) {
+                    in.setVersion(version);
+                    deserializedRequest = new CloseIndexRequest(in);
+                }
+                assertEquals(sample.getParentTask(), deserializedRequest.getParentTask());
+                assertEquals(sample.masterNodeTimeout(), deserializedRequest.masterNodeTimeout());
+                assertEquals(sample.timeout(), deserializedRequest.timeout());
+                assertArrayEquals(sample.indices(), deserializedRequest.indices());
                 // indices options are not equivalent when sent to an older version and re-read due
                 // to the addition of hidden indices as expand to hidden indices is always true when
                 // read from a prior version
-                // TODO update version on backport!
-                if (out.getVersion().onOrAfter(Version.V_8_0_0) || request.indicesOptions().expandWildcardsHidden()) {
-                    assertEquals(request.indicesOptions(), IndicesOptions.readIndicesOptions(in));
+                // TODO change version on backport
+                if (out.getVersion().onOrAfter(Version.V_8_0_0) || sample.indicesOptions().expandWildcardsHidden()) {
+                    assertEquals(sample.indicesOptions(), deserializedRequest.indicesOptions());
+                }
+                if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+                    assertEquals(sample.waitForActiveShards(), deserializedRequest.waitForActiveShards());
+                } else {
+                    assertEquals(ActiveShardCount.NONE, deserializedRequest.waitForActiveShards());
                 }
             }
         }