diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 4ab3b85e46ac3..353616c5a94fd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -157,10 +157,25 @@ public static State fromString(String state) { } } + static { + final int maxNumShards = Integer.parseInt(System.getProperty("es.index.max_number_of_shards", "1024")); + if (maxNumShards < 1) { + throw new IllegalArgumentException("es.index.max_number_of_shards must be > 0"); + } + MAX_NUMBER_OF_SHARDS = maxNumShards; + } + /* This is a safety limit that should only be exceeded in very rare and special cases. The assumption is that + * 99% of the users have less than 1024 shards per index. We also make it a hard check that requires restart of nodes + * if a cluster should allow to create more than 1024 shards per index. NOTE: this does not limit the number of shards per cluster. + * this also prevents creating stuff like a new index with millions of shards by accident which essentially kills the entire cluster + * with OOM on the spot.*/ + private static final int MAX_NUMBER_OF_SHARDS; + public static final String INDEX_SETTING_PREFIX = "index."; public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards"; public static final Setting INDEX_NUMBER_OF_SHARDS_SETTING = - Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 5, 1, Property.IndexScope); + Setting.intSetting(SETTING_NUMBER_OF_SHARDS, Math.min(5, MAX_NUMBER_OF_SHARDS), 1, MAX_NUMBER_OF_SHARDS, + Property.IndexScope); public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas"; public static final Setting INDEX_NUMBER_OF_REPLICAS_SETTING = Setting.intSetting(SETTING_NUMBER_OF_REPLICAS, 1, 0, Property.Dynamic, Property.IndexScope); diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index ff1a3c62c7fd6..28e9e6a114e75 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -38,7 +38,11 @@ specific index module: The number of primary shards that an index should have. Defaults to 5. This setting can only be set at index creation time. It cannot be - changed on a closed index. + changed on a closed index. Note: the number of shards are limited to `1024` per + index. This limitation is a safety limit to prevent accidental creation of indices + that can destabilize a cluster due to resource allocation. The limit can be modified + by specifying `export ES_JAVA_OPTS="-Des.index.max_number_of_shards=128"` system property on every node that is + part of the cluster. `index.shard.check_on_startup`:: + diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSystemPropertyTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSystemPropertyTests.java new file mode 100644 index 0000000000000..878803c007cfa --- /dev/null +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSystemPropertyTests.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.bootstrap; + +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +public class EvilSystemPropertyTests extends ESTestCase { + + @SuppressForbidden(reason = "manipulates system properties for testing") + public void testMaxNumShards() { + int limit = randomIntBetween(1, 10); + System.setProperty("es.index.max_number_of_shards", Integer.toString(limit)); + try { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> + IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING + .get(Settings.builder().put("index.number_of_shards", 11).build())); + assertEquals("Failed to parse value [11] for setting [index.number_of_shards] must be <= " + limit, exception.getMessage()); + } finally { + System.clearProperty("es.index.max_number_of_shards"); + } + } +}