Add a hard limit for index.number_of_shard (#20682)

this change adds a hard limit to `index.number_of_shard` that prevents
indices from being created that have more than 1024 shards. This is still
a huge limit and can only be changed via settings a system property.

Author: s1monw

core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -157,10 +157,25 @@ public static State fromString(String state) {
         }
     }
 
+    static {
+        final int maxNumShards = Integer.parseInt(System.getProperty("es.index.max_number_of_shards", "1024"));
+        if (maxNumShards < 1) {
+            throw new IllegalArgumentException("es.index.max_number_of_shards must be > 0");
+        }
+        MAX_NUMBER_OF_SHARDS = maxNumShards;
+    }
+    /* This is a safety limit that should only be exceeded in very rare and special cases. The assumption is that
+     * 99% of the users have less than 1024 shards per index. We also make it a hard check that requires restart of nodes
+     * if a cluster should allow to create more than 1024 shards per index. NOTE: this does not limit the number of shards per cluster.
+     * this also prevents creating stuff like a new index with millions of shards by accident which essentially kills the entire cluster
+     * with OOM on the spot.*/
+    private static final int MAX_NUMBER_OF_SHARDS;
+
     public static final String INDEX_SETTING_PREFIX = "index.";
     public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards";
     public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING =
-        Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 5, 1, Property.IndexScope);
+        Setting.intSetting(SETTING_NUMBER_OF_SHARDS, Math.min(5, MAX_NUMBER_OF_SHARDS), 1, MAX_NUMBER_OF_SHARDS,
+            Property.IndexScope);
     public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas";
     public static final Setting<Integer> INDEX_NUMBER_OF_REPLICAS_SETTING =
         Setting.intSetting(SETTING_NUMBER_OF_REPLICAS, 1, 0, Property.Dynamic, Property.IndexScope);

docs/reference/index-modules.asciidoc

@@ -38,7 +38,11 @@ specific index module:
 
     The number of primary shards that an index should have.  Defaults to 5.
     This setting can only be set at index creation time.  It cannot be
-    changed on a closed index.
+    changed on a closed index. Note: the number of shards are limited to `1024` per
+    index. This limitation is a safety limit to prevent accidental creation of indices
+    that can destabilize a cluster due to resource allocation. The limit can be modified
+    by specifying `export ES_JAVA_OPTS="-Des.index.max_number_of_shards=128"` system property on every node that is
+    part of the cluster.
 
 `index.shard.check_on_startup`::
 +

qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSystemPropertyTests.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+public class EvilSystemPropertyTests extends ESTestCase {
+
+    @SuppressForbidden(reason = "manipulates system properties for testing")
+    public void testMaxNumShards() {
+        int limit = randomIntBetween(1, 10);
+        System.setProperty("es.index.max_number_of_shards", Integer.toString(limit));
+        try {
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+                IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING
+                    .get(Settings.builder().put("index.number_of_shards", 11).build()));
+            assertEquals("Failed to parse value [11] for setting [index.number_of_shards] must be <= " + limit, exception.getMessage());
+        } finally {
+            System.clearProperty("es.index.max_number_of_shards");
+        }
+    }
+}