Automatically prepare indices for splitting

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -264,7 +264,7 @@ public void testSearchWithMatrixStats() throws IOException {
         assertEquals(5, matrixStats.getFieldCount("num"));
         assertEquals(56d, matrixStats.getMean("num"), 0d);
         assertEquals(1830d, matrixStats.getVariance("num"), 0d);
-        assertEquals(0.09340198804973057, matrixStats.getSkewness("num"), 0d);
+        assertEquals(0.09340198804973046, matrixStats.getSkewness("num"), 0d);
         assertEquals(1.2741646510794589, matrixStats.getKurtosis("num"), 0d);
         assertEquals(5, matrixStats.getFieldCount("num2"));
         assertEquals(29d, matrixStats.getMean("num2"), 0d);

core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -1344,13 +1344,17 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet
         }
         int routingFactor = getRoutingFactor(numSourceShards, numTargetShards);
         // now we verify that the numRoutingShards is valid in the source index
-        int routingNumShards = sourceIndexMetadata.getRoutingNumShards();
+        // note: if the number of shards is 1 in the source index we can just assume it's correct since from 1 we can split into anything
+        // this is important to special case here since we use this to validate this in various places in the code but allow to split form
+        // 1 to N but we never modify the sourceIndexMetadata to accommodate for that
+        int routingNumShards = numSourceShards == 1 ? numTargetShards : sourceIndexMetadata.getRoutingNumShards();
         if (routingNumShards % numTargetShards != 0) {
             throw new IllegalStateException("the number of routing shards ["
                 + routingNumShards + "] must be a multiple of the target shards [" + numTargetShards + "]");
         }
         // this is just an additional assertion that ensures we are a factor of the routing num shards.
-        assert getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0;
+        assert sourceIndexMetadata.getNumberOfShards() == 1 // special case - we can split into anything from 1 shard
+            || getRoutingFactor(numTargetShards, routingNumShards) >= 0;
         return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor);
     }
 

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -379,16 +379,28 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                 indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName());
                 indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
                 final IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index());
-
+                final Settings idxSettings = indexSettingsBuilder.build();
+                int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings);
                 final int routingNumShards;
+                final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null);
                 if (recoverFromIndex == null) {
-                    Settings idxSettings = indexSettingsBuilder.build();
-                    routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings);
+                    if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) {
+                        routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings);
+                    } else {
+                        routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated);
+                    }
                 } else {
                     assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false
-                        : "index.number_of_routing_shards should be present on the target index on resize";
+                        : "index.number_of_routing_shards should not be present on the target index on resize";
                     final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex);
-                    routingNumShards = sourceMetaData.getRoutingNumShards();
+                    if (shouldAutoCalculateNumRoutingShards(numTargetShards, sourceMetaData.getNumberOfShards(),
+                        sourceMetaData.getRoutingNumShards())) {
+                        // in this case we have a source index with 1 shard and without an explicit split factor
+                        // or one that is valid in that case we can split into whatever and auto-generate a new factor.
+                        routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated);
+                    } else {
+                        routingNumShards = sourceMetaData.getRoutingNumShards();
+                    }
                 }
                 // remove the setting it's temporary and is only relevant once we create the index
                 indexSettingsBuilder.remove(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey());
@@ -717,4 +729,37 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set<String> ma
             .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), resizeSourceIndex.getName())
             .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID());
     }
+
+    /**
+     * Returns a default number of routing shards based on the number of shards of the index. The default number of routing shards will
+     * allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024
+     * the less default split operations are supported
+     */
+    public static int calculateNumRoutingShards(int numShards, Version indexVersionCreated) {
+        if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) {
+            // only select this automatically for indices that are created on or after 7.0
+            int base = 10; // logBase2(1024)
+            return numShards * 1 << Math.max(1, (base - (int) (Math.log(numShards) / Math.log(2))));
+        } else {
+            return numShards;
+        }
+    }
+
+
+    /**
+     * Returns <code>true</code> iff the we should calculate the number of routing shards for the split index based on
+     * the source index. This only applies to source indices that have only 1 shards. We don't want to recalculate the num routing
+     * shards on indices that already have a valid number of routing shards.
+     */
+    static boolean shouldAutoCalculateNumRoutingShards(int numTargetShards, int numSourceShards, int numSourceRoutingShards) {
+        if (numSourceShards == 1) {
+            if (numSourceRoutingShards < numTargetShards) {
+                // we have a source index that has less routing shards than our target shards -- we should reset
+                return true;
+            }
+            int factor = numSourceRoutingShards /  numTargetShards;
+            return factor * numTargetShards != numSourceRoutingShards || factor <= 1;
+        }
+        return false;
+    }
 }

core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java

@@ -317,6 +317,7 @@ public void testInvalidPartitionSize() {
                 response = prepareCreate("test_" + shards + "_" + partitionSize)
                     .setSettings(Settings.builder()
                         .put("index.number_of_shards", shards)
+                        .put("index.number_of_routing_shards", shards)
                         .put("index.routing_partition_size", partitionSize))
                     .execute().actionGet();
             } catch (IllegalStateException | IllegalArgumentException e) {

core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -66,7 +67,6 @@
 import java.util.List;
 import java.util.Set;
 import java.util.function.BiFunction;
-import java.util.function.IntFunction;
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -88,21 +88,31 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testCreateSplitIndexToN() throws IOException {
-        int[][] possibleShardSplits = new int[][] {{2,4,8}, {3, 6, 12}, {1, 2, 4}};
+        int[][] possibleShardSplits = new int[][]{{2, 4, 8}, {3, 6, 12}, {1, 2, 4}};
         int[] shardSplits = randomFrom(possibleShardSplits);
+        splitToN(shardSplits);
+    }
+
+    public void testSplitFromOneToN() {
+        splitToN(1, 5, 10);
+        client().admin().indices().prepareDelete("*").get();
+        int randomSplit = randomIntBetween(2, 6);
+        splitToN(1, randomSplit, randomSplit * 2);
+    }
+
+    private void splitToN(int... shardSplits) {
+        assertEquals(3, shardSplits.length);
         assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]);
         assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]);
         internalCluster().ensureAtLeastNumDataNodes(2);
         final boolean useRouting =  randomBoolean();
         final boolean useNested = randomBoolean();
         final boolean useMixedRouting = useRouting ? randomBoolean() : false;
         CreateIndexRequestBuilder createInitialIndex = prepareCreate("source");
-        final int routingShards = shardSplits[2] * randomIntBetween(1, 10);
-        Settings.Builder settings = Settings.builder().put(indexSettings())
-            .put("number_of_shards", shardSplits[0])
-            .put("index.number_of_routing_shards", routingShards);
+        Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0]);
         if (useRouting && useMixedRouting == false && randomBoolean()) {
-            settings.put("index.routing_partition_size", randomIntBetween(1, routingShards - 1));
+            settings.put("index.routing_partition_size", randomIntBetween(1,  MetaDataCreateIndexService.calculateNumRoutingShards
+                (shardSplits[0], Version.CURRENT) -1));
             if (useNested) {
                 createInitialIndex.addMapping("t1", "_routing", "required=true", "nested1", "type=nested");
             } else {
@@ -340,7 +350,6 @@ public void testCreateSplitIndex() {
         prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
             .put("number_of_shards", 1)
             .put("index.version.created", version)
-            .put("index.number_of_routing_shards",  2)
         ).get();
         final int docs = randomIntBetween(0, 128);
         for (int i = 0; i < docs; i++) {
@@ -443,7 +452,6 @@ public void testCreateSplitWithIndexSort() throws Exception {
                 Settings.builder()
                     .put(indexSettings())
                     .put("sort.field", "id")
-                    .put("index.number_of_routing_shards",  16)
                     .put("sort.order", "desc")
                     .put("number_of_shards", 2)
                     .put("number_of_replicas", 0)

core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

@@ -34,7 +34,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.ResourceAlreadyExistsException;
@@ -299,4 +298,39 @@ private void validateIndexName(String indexName, String errorMessage) {
                 .getDefault(Settings.EMPTY)).build()));
         assertThat(e.getMessage(), endsWith(errorMessage));
     }
+
+    public void testShouldAutoCalculateNumRoutingShards() {
+        assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(2, 1, 1));
+        assertFalse(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 20));
+        assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 5));
+        assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 16));
+        assertFalse(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(2, 1, 8));
+    }
+
+    public void testCalculateNumRoutingShards() {
+        assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT));
+        assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT));
+        assertEquals(1536, MetaDataCreateIndexService.calculateNumRoutingShards(3, Version.CURRENT));
+        assertEquals(1152, MetaDataCreateIndexService.calculateNumRoutingShards(9, Version.CURRENT));
+        assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512, Version.CURRENT));
+        assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024, Version.CURRENT));
+        assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048, Version.CURRENT));
+
+        Version latestV6 = VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1);
+        int numShards = randomIntBetween(1, 1000);
+        assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards, latestV6));
+        assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards,
+            VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), latestV6)));
+
+        for (int i = 0; i < 1000; i++) {
+            int randomNumShards = randomIntBetween(1, 10000);
+            int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards, Version.CURRENT);
+            double ratio = numRoutingShards / randomNumShards;
+            int intRatio = (int) ratio;
+            assertEquals(ratio, (double)(intRatio), 0.0d);
+            assertTrue(1 < ratio);
+            assertTrue(ratio <= 1024);
+            assertEquals(0, intRatio % 2);
+        }
+    }
 }

core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java

@@ -52,7 +52,6 @@
 
 public class OperationRoutingTests extends ESTestCase{
 
-
     public void testGenerateShardId() {
         int[][] possibleValues = new int[][] {
             {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1}

core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java

@@ -95,7 +95,8 @@ public void testQueryRewrite() throws Exception {
         Client client = client();
         assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
                 .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get());
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put("index.number_of_routing_shards", 5)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get());
         indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"),
                 client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"),
                 client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"),
@@ -362,7 +363,8 @@ public void testQueryRewriteDatesWithNow() throws Exception {
     public void testCanCache() throws Exception {
         Client client = client();
         Settings settings = Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
         assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date")
                 .setSettings(settings)
                 .get());

core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java

@@ -840,15 +840,17 @@ public void testPartitionedTemplate() throws Exception {
         // create an index with too few shards
         IllegalArgumentException eBadIndex = expectThrows(IllegalArgumentException.class,
                 () -> prepareCreate("test_bad", Settings.builder()
-            .put("index.number_of_shards", 5))
+            .put("index.number_of_shards", 5)
+            .put("index.number_of_routing_shards", 5))
             .get());
 
         assertThat(eBadIndex.getMessage(), containsString("partition size [6] should be a positive number "
                 + "less than the number of shards [5]"));
 
         // finally, create a valid index
         prepareCreate("test_good", Settings.builder()
-            .put("index.number_of_shards", 7))
+            .put("index.number_of_shards", 7)
+            .put("index.number_of_routing_shards", 7))
             .get();
 
         GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test_good").get();

core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java

@@ -24,6 +24,8 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -196,6 +198,13 @@ public void testAliasSearchRouting() throws Exception {
 
     }
 
+    @Override
+    public Settings indexSettings() {
+        Settings settings = super.indexSettings();
+        int numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(settings);
+        return Settings.builder().put(settings).put("index.number_of_routing_shards", numShards).build();
+    }
+
     public void testAliasSearchRoutingWithTwoIndices() throws Exception {
         createIndex("test-a");
         createIndex("test-b");

core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java

@@ -42,6 +42,7 @@ public void testVariousPartitionSizes() throws Exception {
                 client().admin().indices().prepareCreate(index)
                     .setSettings(Settings.builder()
                         .put("index.number_of_shards", shards)
+                        .put("index.number_of_routing_shards", shards)
                         .put("index.routing_partition_size", partitionSize))
                     .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON)
                     .execute().actionGet();
@@ -67,6 +68,7 @@ public void testShrinking() throws Exception {
         client().admin().indices().prepareCreate(index)
             .setSettings(Settings.builder()
                 .put("index.number_of_shards", currentShards)
+                .put("index.number_of_routing_shards", currentShards)
                 .put("index.number_of_replicas", numberOfReplicas())
                 .put("index.routing_partition_size", partitionSize))
             .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON)

core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java

@@ -258,7 +258,7 @@ public void testExplainWithRewriteValidateQuery() throws Exception {
     public void testExplainWithRewriteValidateQueryAllShards() throws Exception {
         client().admin().indices().prepareCreate("test")
             .addMapping("type1", "field", "type=text,analyzer=whitespace")
-            .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2)).get();
+            .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)).get();
         // We are relying on specific routing behaviors for the result to be right, so
         // we cannot randomize the number of shards or change ids here.
         client().prepareIndex("test", "type1", "1")

docs/build.gradle

@@ -194,6 +194,13 @@ buildRestTests.setups['sales'] = '''
 
 // Dummy bank account data used by getting-started.asciidoc
 buildRestTests.setups['bank'] = '''
+  - do:
+        indices.create:
+          index: bank
+          body:
+            settings:
+              number_of_shards: 5
+              number_of_routing_shards: 5
   - do:
         bulk:
           index: bank