From 005215e9776dbcb331411c252a200c1f308d39d8 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 16 Nov 2017 11:51:33 +0100 Subject: [PATCH 01/17] Automatically prepare indices for splitting Today we require users to prepare their indices for split operations. Yet, we can do this automatically when an index is created which would make the split feature a much more appealing option since it doesn't have any 3rd party prerequisites anymore. This change automatically sets the number of routinng shards such that an index is guaranteed to be able to split once into twice as many shards. The number of routing shards is scaled towards the default shard limit per index such that indices with a smaller amount of shards can be split more often than larger ones. For instance an index with 1 or 2 shards can be split 10x (until it approaches 1024 shards) while an index created with 128 shards can only be split 3x by a factor of 2. Please note this is just a default value and users can still prepare their indices with `index.number_of_routing_shards` for custom splitting. NOTE: this change has an impact on the document distribution since we are changing the hash space. Documents are still uniformly distributed across all shards but since we are artificually changing the number of buckets in the consistent hashign space document might be hashed into different shards compared to previous versions. This is a 7.0 only change. --- .../org/elasticsearch/client/SearchIT.java | 2 +- .../indices/shrink/TransportResizeAction.java | 4 +- .../cluster/metadata/IndexMetaData.java | 3 +- .../metadata/MetaDataCreateIndexService.java | 39 ++++++++++++++++--- .../admin/indices/create/CreateIndexIT.java | 1 + .../admin/indices/create/SplitIndexIT.java | 26 ++++++++----- .../MetaDataCreateIndexServiceTests.java | 21 ++++++++++ .../routing/OperationRoutingTests.java | 1 - .../indices/IndicesRequestCacheIT.java | 6 ++- .../template/SimpleIndexTemplateIT.java | 6 ++- .../elasticsearch/routing/AliasRoutingIT.java | 9 +++++ .../routing/PartitionedRoutingIT.java | 2 + .../validate/SimpleValidateQueryIT.java | 2 +- docs/build.gradle | 7 ++++ .../metrics/geocentroid-aggregation.asciidoc | 4 +- docs/reference/cat/segments.asciidoc | 4 +- .../how-to/recipes/stemming.asciidoc | 8 ++-- docs/reference/indices/split-index.asciidoc | 21 ++++++---- .../mapping/params/normalizer.asciidoc | 12 +++--- docs/reference/search/search-shards.asciidoc | 14 +++---- docs/reference/search/validate.asciidoc | 7 ++-- .../test/stats/30_single_value_field.yml | 1 + .../test/stats/40_multi_value_field.yml | 1 + .../test/lang_mustache/30_search_template.yml | 2 +- .../rest-api-spec/test/create/40_routing.yml | 1 + .../rest-api-spec/test/delete/40_parent.yml | 1 + .../rest-api-spec/test/delete/50_refresh.yml | 1 + .../rest-api-spec/test/exists/40_routing.yml | 1 + .../rest-api-spec/test/get/40_routing.yml | 1 + .../test/get_source/40_routing.yml | 1 + .../rest-api-spec/test/index/40_routing.yml | 1 + .../test/indices.split/10_basic.yml | 1 - .../test/indices.split/20_source_mapping.yml | 1 - .../rest-api-spec/test/mget/40_routing.yml | 1 + .../rest-api-spec/test/scroll/12_slices.yml | 4 ++ .../190_percentiles_hdr_metric.yml | 2 + .../rest-api-spec/test/update/40_routing.yml | 1 + 37 files changed, 161 insertions(+), 59 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index d73e746528dc6..06c8625d270ea 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -264,7 +264,7 @@ public void testSearchWithMatrixStats() throws IOException { assertEquals(5, matrixStats.getFieldCount("num")); assertEquals(56d, matrixStats.getMean("num"), 0d); assertEquals(1830d, matrixStats.getVariance("num"), 0d); - assertEquals(0.09340198804973057, matrixStats.getSkewness("num"), 0d); + assertEquals(0.09340198804973046, matrixStats.getSkewness("num"), 0d); assertEquals(1.2741646510794589, matrixStats.getKurtosis("num"), 0d); assertEquals(5, matrixStats.getFieldCount("num2")); assertEquals(29d, matrixStats.getMean("num2"), 0d); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 87dd9f9fa2d21..597e3e7e2089f 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -156,7 +156,9 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi } } } else { - Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); + if (metaData.getNumberOfShards() != 1) { // we can split into anything from 1 + Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); + } // we just execute this to ensure we get the right exceptions if the number of shards is wrong or less then etc. } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 3d14670e52771..da5c0c6caa1c9 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1344,7 +1344,8 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet } int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); // this is just an additional assertion that ensures we are a factor of the routing num shards. - assert getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0; + assert sourceIndexMetadata.getNumberOfShards() == 1 // special case - we can split into anything from 1 shard + || getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0; return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 64f2383fe7797..d2eb7340e0455 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; +import java.util.function.IntFunction; import java.util.function.Predicate; import java.util.stream.IntStream; @@ -379,16 +380,25 @@ public ClusterState execute(ClusterState currentState) throws Exception { indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName()); indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); final IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index()); - + final Settings idxSettings = indexSettingsBuilder.build(); + int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); final int routingNumShards; if (recoverFromIndex == null) { - Settings idxSettings = indexSettingsBuilder.build(); - routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); + if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { + routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); + } else { + routingNumShards = calculateNumRoutingShards(numTargetShards); + } } else { assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false - : "index.number_of_routing_shards should be present on the target index on resize"; + : "index.number_of_routing_shards should not be present on the target index on resize"; final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); - routingNumShards = sourceMetaData.getRoutingNumShards(); + if (sourceMetaData.getNumberOfShards() == 1 + && isInvalidSplitFactor(numTargetShards, sourceMetaData.getRoutingNumShards())) { + routingNumShards = calculateNumRoutingShards(numTargetShards); + } else { + routingNumShards = sourceMetaData.getRoutingNumShards(); + } } // remove the setting it's temporary and is only relevant once we create the index indexSettingsBuilder.remove(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); @@ -643,7 +653,9 @@ static void validateSplitIndex(ClusterState state, String sourceIndex, Set targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) { IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); - IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); + if (sourceMetaData.getNumberOfShards() != 1) { // we can split into anything if we only have 1 shard + IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); + } if (sourceMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) { // ensure we have a single type since this would make the splitting code considerably more complex // and a 5.x index would not be splittable unless it has been shrunk before so rather opt out of the complexity @@ -717,4 +729,19 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); } + + /** + * Returns a default number of routing shards based on the number of shards of the index. The default number of routing shards will + * allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024 + * the less default split operations are supported + */ + public static int calculateNumRoutingShards(int numShards) { + int base = 10; // logBase2(1024) + return numShards * 1 << Math.max(1, (base - (int) (Math.log(numShards) / Math.log(2)))); + } + + private static boolean isInvalidSplitFactor(int sourceNumberOfShards, int targetNumberOfShards) { + int factor = sourceNumberOfShards / targetNumberOfShards; + return factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1; + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 3c55d0df9c142..5385902bc3690 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -317,6 +317,7 @@ public void testInvalidPartitionSize() { response = prepareCreate("test_" + shards + "_" + partitionSize) .setSettings(Settings.builder() .put("index.number_of_shards", shards) + .put("index.number_of_routing_shards", shards) .put("index.routing_partition_size", partitionSize)) .execute().actionGet(); } catch (IllegalStateException | IllegalArgumentException e) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 995d86c3939ba..ca325b7bcb56d 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -39,6 +39,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.cluster.routing.ShardRouting; @@ -66,7 +67,6 @@ import java.util.List; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.IntFunction; import java.util.stream.IntStream; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -88,8 +88,20 @@ protected Collection> nodePlugins() { } public void testCreateSplitIndexToN() throws IOException { - int[][] possibleShardSplits = new int[][] {{2,4,8}, {3, 6, 12}, {1, 2, 4}}; + int[][] possibleShardSplits = new int[][]{{2, 4, 8}, {3, 6, 12}, {1, 2, 4}}; int[] shardSplits = randomFrom(possibleShardSplits); + splitToN(shardSplits); + } + + public void testSplitFromOneToN() { + splitToN(1, 5, 10); + client().admin().indices().prepareDelete("*").get(); + int randomSplit = randomIntBetween(2, 6); + splitToN(1, randomSplit, randomSplit * 2); + } + + private void splitToN(int... shardSplits) { + assertEquals(3, shardSplits.length); assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); internalCluster().ensureAtLeastNumDataNodes(2); @@ -97,12 +109,10 @@ public void testCreateSplitIndexToN() throws IOException { final boolean useNested = randomBoolean(); final boolean useMixedRouting = useRouting ? randomBoolean() : false; CreateIndexRequestBuilder createInitialIndex = prepareCreate("source"); - final int routingShards = shardSplits[2] * randomIntBetween(1, 10); - Settings.Builder settings = Settings.builder().put(indexSettings()) - .put("number_of_shards", shardSplits[0]) - .put("index.number_of_routing_shards", routingShards); + Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0]); if (useRouting && useMixedRouting == false && randomBoolean()) { - settings.put("index.routing_partition_size", randomIntBetween(1, routingShards - 1)); + settings.put("index.routing_partition_size", randomIntBetween(1, MetaDataCreateIndexService.calculateNumRoutingShards + (shardSplits[0]) -1)); if (useNested) { createInitialIndex.addMapping("t1", "_routing", "required=true", "nested1", "type=nested"); } else { @@ -340,7 +350,6 @@ public void testCreateSplitIndex() { prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) .put("number_of_shards", 1) .put("index.version.created", version) - .put("index.number_of_routing_shards", 2) ).get(); final int docs = randomIntBetween(0, 128); for (int i = 0; i < docs; i++) { @@ -443,7 +452,6 @@ public void testCreateSplitWithIndexSort() throws Exception { Settings.builder() .put(indexSettings()) .put("sort.field", "id") - .put("index.number_of_routing_shards", 16) .put("sort.order", "desc") .put("number_of_shards", 2) .put("number_of_replicas", 0) diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 39e4a18440931..9cbe4fb0f5730 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -297,4 +297,25 @@ private void validateIndexName(String indexName, String errorMessage) { .getDefault(Settings.EMPTY)).build())); assertThat(e.getMessage(), endsWith(errorMessage)); } + + public void testCalculateNumRoutingShards() { + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2)); + assertEquals(1536, MetaDataCreateIndexService.calculateNumRoutingShards(3)); + assertEquals(1152, MetaDataCreateIndexService.calculateNumRoutingShards(9)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512)); + assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024)); + assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048)); + + for (int i = 0; i < 1000; i++) { + int randomNumShards = randomIntBetween(1, 10000); + int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards); + double ratio = numRoutingShards / randomNumShards; + int intRatio = (int) ratio; + assertEquals(ratio, (double)(intRatio), 0.0d); + assertTrue(1 < ratio); + assertTrue(ratio <= 1024); + assertEquals(0, intRatio % 2); + } + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java index 1f8de1ca02fd7..dc3c4c8bc8e96 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java @@ -52,7 +52,6 @@ public class OperationRoutingTests extends ESTestCase{ - public void testGenerateShardId() { int[][] possibleValues = new int[][] { {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1} diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java index 1884361a47f69..70a633f02f4dc 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java @@ -95,7 +95,8 @@ public void testQueryRewrite() throws Exception { Client client = client(); assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get()); + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5).put("index.number_of_routing_shards", 5) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get()); indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"), client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"), client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"), @@ -362,7 +363,8 @@ public void testQueryRewriteDatesWithNow() throws Exception { public void testCanCache() throws Exception { Client client = client(); Settings settings = Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build(); + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build(); assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") .setSettings(settings) .get()); diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java index e81538b9057ba..9a09ba215fb71 100644 --- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java @@ -840,7 +840,8 @@ public void testPartitionedTemplate() throws Exception { // create an index with too few shards IllegalArgumentException eBadIndex = expectThrows(IllegalArgumentException.class, () -> prepareCreate("test_bad", Settings.builder() - .put("index.number_of_shards", 5)) + .put("index.number_of_shards", 5) + .put("index.number_of_routing_shards", 5)) .get()); assertThat(eBadIndex.getMessage(), containsString("partition size [6] should be a positive number " @@ -848,7 +849,8 @@ public void testPartitionedTemplate() throws Exception { // finally, create a valid index prepareCreate("test_good", Settings.builder() - .put("index.number_of_shards", 7)) + .put("index.number_of_shards", 7) + .put("index.number_of_routing_shards", 7)) .get(); GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test_good").get(); diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java index b99353812c887..14ec800c3a65d 100644 --- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -24,6 +24,8 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -196,6 +198,13 @@ public void testAliasSearchRouting() throws Exception { } + @Override + public Settings indexSettings() { + Settings settings = super.indexSettings(); + int numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(settings); + return Settings.builder().put(settings).put("index.number_of_routing_shards", numShards).build(); + } + public void testAliasSearchRoutingWithTwoIndices() throws Exception { createIndex("test-a"); createIndex("test-b"); diff --git a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 07a73a09f4ab4..74fbcc20a0278 100644 --- a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -42,6 +42,7 @@ public void testVariousPartitionSizes() throws Exception { client().admin().indices().prepareCreate(index) .setSettings(Settings.builder() .put("index.number_of_shards", shards) + .put("index.number_of_routing_shards", shards) .put("index.routing_partition_size", partitionSize)) .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON) .execute().actionGet(); @@ -67,6 +68,7 @@ public void testShrinking() throws Exception { client().admin().indices().prepareCreate(index) .setSettings(Settings.builder() .put("index.number_of_shards", currentShards) + .put("index.number_of_routing_shards", currentShards) .put("index.number_of_replicas", numberOfReplicas()) .put("index.routing_partition_size", partitionSize)) .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}", XContentType.JSON) diff --git a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java index a87f428fec51e..3c4666148d8f5 100644 --- a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java +++ b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java @@ -258,7 +258,7 @@ public void testExplainWithRewriteValidateQuery() throws Exception { public void testExplainWithRewriteValidateQueryAllShards() throws Exception { client().admin().indices().prepareCreate("test") .addMapping("type1", "field", "type=text,analyzer=whitespace") - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2)).get(); + .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)).get(); // We are relying on specific routing behaviors for the result to be right, so // we cannot randomize the number of shards or change ids here. client().prepareIndex("test", "type1", "1") diff --git a/docs/build.gradle b/docs/build.gradle index 4cb82b97152f8..e0d562416e5e6 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -193,6 +193,13 @@ buildRestTests.setups['sales'] = ''' // Dummy bank account data used by getting-started.asciidoc buildRestTests.setups['bank'] = ''' + - do: + indices.create: + index: bank + body: + settings: + number_of_shards: 5 + number_of_routing_shards: 5 - do: bulk: index: bank diff --git a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc index 7a355c7b12605..9aafcee56bc90 100644 --- a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc @@ -60,7 +60,7 @@ The response for the above aggregation: "aggregations": { "centroid": { "location": { - "lat": 51.00982963107526, + "lat": 51.00982963806018, "lon": 3.9662130922079086 }, "count": 6 @@ -114,7 +114,7 @@ The response for the above aggregation: "centroid": { "location": { "lat": 52.371655656024814, - "lon": 4.909563297405839 + "lon": 4.909563269466162 }, "count": 3 } diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc index 05d86758a8c74..2fa7e309bccf4 100644 --- a/docs/reference/cat/segments.asciidoc +++ b/docs/reference/cat/segments.asciidoc @@ -17,8 +17,8 @@ might look like: ["source","txt",subs="attributes,callouts"] -------------------------------------------------- index shard prirep ip segment generation docs.count docs.deleted size size.memory committed searchable version compound -test 3 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true -test1 3 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true +test 2 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true +test1 2 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true -------------------------------------------------- // TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ _cat] diff --git a/docs/reference/how-to/recipes/stemming.asciidoc b/docs/reference/how-to/recipes/stemming.asciidoc index 49d9f8bafa305..c8e0c2a109e60 100644 --- a/docs/reference/how-to/recipes/stemming.asciidoc +++ b/docs/reference/how-to/recipes/stemming.asciidoc @@ -90,19 +90,19 @@ GET index/_search { "_index": "index", "_type": "type", - "_id": "2", + "_id": "1", "_score": 0.2876821, "_source": { - "body": "A pair of skis" + "body": "Ski resort" } }, { "_index": "index", "_type": "type", - "_id": "1", + "_id": "2", "_score": 0.2876821, "_source": { - "body": "Ski resort" + "body": "A pair of skis" } } ] diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index 467c09baa2432..d9760659d478b 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -1,13 +1,19 @@ [[indices-split-index]] == Split Index -number_of_routing_shards - The split index API allows you to split an existing index into a new index with multiple of it's primary shards. Similarly to the <> where the number of primary shards in the shrunk index must be a factor of the source index. -The `_split` API requires the source index to be created with a specific number of routing shards -in order to be split in the future. (Note: this requirement might be remove in future releases) +Every index is by default created with an implicit `index.number_of_routing_shards` that specify +how often an index can be split. By default every index can at least be split once by a factor of 2. +The default number of routing shards is dependent on the number of primary shards in an index. +Indices with a small number of primary shards can be split more often compared to indices that +have number of shards closer to `1024`. Note: an index with a single shard can be split into +a new index with an arbitrary number of shards greater than 1. All properties of default number of +routing shards then apply to the newly created index. + +Optionally, an index can also be created with be created with a specific number of routing shards +in order to be split in the future. The number of routing shards specify the hashing space that is used internally to distribute documents across shards, in oder to have a consistent hashing that is compatible with the method elasticsearch uses today. @@ -37,15 +43,14 @@ Splitting works as follows: [float] === Preparing an index for splitting -Create an index with a routing shards factor: +Create a new index: [source,js] -------------------------------------------------- PUT my_source_index { "settings": { - "index.number_of_shards" : 1, - "index.number_of_routing_shards" : 2 <1> + "index.number_of_shards" : 1 } } ------------------------------------------------- @@ -128,7 +133,7 @@ POST my_source_index/_split/my_target_index } -------------------------------------------------- // CONSOLE -// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_routing_shards" : 5, "index.number_of_shards": "1"}}\n/] +// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_shards": "1"}}\n/] <1> The number of shards in the target index. This must be a factor of the number of shards in the source index. diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc index 44a27dab7589d..1b3d753e04125 100644 --- a/docs/reference/mapping/params/normalizer.asciidoc +++ b/docs/reference/mapping/params/normalizer.asciidoc @@ -80,24 +80,24 @@ both index and query time. }, "hits": { "total": 2, - "max_score": 0.2876821, + "max_score": 0.6931472, "hits": [ { "_index": "index", "_type": "type", - "_id": "2", - "_score": 0.2876821, + "_id": "1", + "_score": 0.6931472, "_source": { - "foo": "bar" + "foo": "BÀR" } }, { "_index": "index", "_type": "type", - "_id": "1", + "_id": "2", "_score": 0.2876821, "_source": { - "foo": "BÀR" + "foo": "bar" } } ] diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc index b20117bb75dbd..2acb2f8aa2b2e 100644 --- a/docs/reference/search/search-shards.asciidoc +++ b/docs/reference/search/search-shards.asciidoc @@ -120,9 +120,9 @@ This will yield the following result: "index": "twitter", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, - "shard": 0, + "shard": 1, "state": "STARTED", - "allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"}, + "allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"}, "relocating_node": null } ], @@ -131,9 +131,9 @@ This will yield the following result: "index": "twitter", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, - "shard": 1, + "shard": 3, "state": "STARTED", - "allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"}, + "allocation_id": {"id":"0TvkCyF7TAmM1wHP4a42-A"}, "relocating_node": null } ] @@ -141,9 +141,9 @@ This will yield the following result: } -------------------------------------------------- // TESTRESPONSE[s/"nodes": ...,/"nodes": $body.nodes,/] -// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.0.0.node/] -// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.0.0.allocation_id.id/] -// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.1.0.allocation_id.id/] +// TESTRESPONSE[s/JklnKbD7Tyqi9TP3_Q_tBg/$body.shards.1.0.node/] +// TESTRESPONSE[s/0TvkCyF7TAmM1wHP4a42-A/$body.shards.1.0.allocation_id.id/] +// TESTRESPONSE[s/fMju3hd1QHWmWrIgFnI4Ww/$body.shards.0.0.allocation_id.id/] This time the search will only be executed against two of the shards, because routing values have been specified. diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 7218f6e6b23c3..d330264fd58f0 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -239,19 +239,20 @@ Response: "index": "twitter", "shard": 2, "valid": true, - "explanation": "(user:kimchi)^0.8333333" + "explanation": "user:kimchy" }, { "index": "twitter", "shard": 3, "valid": true, - "explanation": "user:kimchy" + "explanation": "user:kimchy~2" + }, { "index": "twitter", "shard": 4, "valid": true, - "explanation": "user:kimchy~2" + "explanation": "(user:kimchi)^0.8333333" } ] } diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml index 782e5d5ebbc4a..fbc73974f8ac9 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml @@ -7,6 +7,7 @@ setup: body: settings: number_of_shards: 3 + number_of_routing_shards: 3 mappings: test: "properties": diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml index b4abf41a515e9..dff452d43cfd0 100644 --- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml +++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml @@ -7,6 +7,7 @@ setup: body: settings: number_of_shards: 3 + number_of_routing_shards: 3 mappings: test: "properties": diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml index d2608a48e733e..b3875982b2eb9 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml @@ -113,7 +113,7 @@ - match: { hits.total: 1 } - length: { hits.hits: 1 } - - match: { hits.hits.0._explanation.description: "weight(otherField:foo in 0) [PerFieldSimilarity], result of:" } + - match: { hits.hits.0._explanation.description: "weight(otherField:foo in 2) [PerFieldSimilarity], result of:" } - do: search_template: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml index 44447ebee8ce7..752489f722c9e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml index d9aa68704607f..82fc8a325d614 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/40_parent.yml @@ -6,6 +6,7 @@ body: settings: number_of_shards: 5 + number_of_routing_shards: 5 mappings: test: _parent: { type: "foo" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml index 589d2cf77a34d..f86c7250a37ca 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yml @@ -8,6 +8,7 @@ settings: refresh_interval: -1 number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: cluster.health: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml index 6b55a3bee37f7..25315628d7ece 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml index 94a40c0437a4d..276346cda4f98 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml index bd9a29184472a..b4eebfa8ab947 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml index 7b3c21df4e065..5b0cf94f4236b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 82881564a2043..50b5f3305ba4c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -11,7 +11,6 @@ settings: index.number_of_shards: 1 index.number_of_replicas: 0 - index.number_of_routing_shards: 2 - do: index: index: source diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml index 38f36c405a1cb..91f10578f8530 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -13,7 +13,6 @@ settings: number_of_shards: 1 number_of_replicas: 0 - index.number_of_routing_shards: 2 mappings: test: properties: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml index 7196412ebf3f6..d550dd26657c9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml index 4acc4d132327e..47bcbdb83c47c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yml @@ -3,6 +3,10 @@ setup: - do: indices.create: index: test_sliced_scroll + body: + settings: + number_of_shards: 5 + number_of_routing_shards: 5 - do: index: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml index 785e3be8d009c..a8b7f1c629906 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/190_percentiles_hdr_metric.yml @@ -5,6 +5,8 @@ setup: body: settings: number_of_replicas: 0 + number_of_shards: 5 + number_of_routing_shards: 5 mappings: doc: properties: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml index 097f49007e55f..977db506710c7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/40_routing.yml @@ -8,6 +8,7 @@ settings: index: number_of_shards: 5 + number_of_routing_shards: 5 number_of_replicas: 0 - do: From 40723798939e4bb93beff9d8a9a36179e5afaea7 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 20 Nov 2017 13:40:28 +0100 Subject: [PATCH 02/17] prevent automatic num routing shards for pre 7.0 indices --- .../metadata/MetaDataCreateIndexService.java | 19 ++++++++++----- .../admin/indices/create/SplitIndexIT.java | 2 +- .../MetaDataCreateIndexServiceTests.java | 23 +++++++++++-------- .../test/update_by_query/10_script.yml | 2 +- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index d2eb7340e0455..4b4cdf177a2db 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -92,7 +92,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; -import java.util.function.IntFunction; import java.util.function.Predicate; import java.util.stream.IntStream; @@ -383,11 +382,12 @@ public ClusterState execute(ClusterState currentState) throws Exception { final Settings idxSettings = indexSettingsBuilder.build(); int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); final int routingNumShards; + final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null); if (recoverFromIndex == null) { if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); } else { - routingNumShards = calculateNumRoutingShards(numTargetShards); + routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); } } else { assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false @@ -395,7 +395,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); if (sourceMetaData.getNumberOfShards() == 1 && isInvalidSplitFactor(numTargetShards, sourceMetaData.getRoutingNumShards())) { - routingNumShards = calculateNumRoutingShards(numTargetShards); + // in this case we have a source index with 1 shard and without an explicit split factor + // or one that is valid in that case we can split into whatever and auto-generate a new factor. + routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); } else { routingNumShards = sourceMetaData.getRoutingNumShards(); } @@ -735,9 +737,14 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma * allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024 * the less default split operations are supported */ - public static int calculateNumRoutingShards(int numShards) { - int base = 10; // logBase2(1024) - return numShards * 1 << Math.max(1, (base - (int) (Math.log(numShards) / Math.log(2)))); + public static int calculateNumRoutingShards(int numShards, Version indexVersionCreated) { + if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) { + // only select this automatically for indices that are created on or after 7.0 + int base = 10; // logBase2(1024) + return numShards * 1 << Math.max(1, (base - (int) (Math.log(numShards) / Math.log(2)))); + } else { + return numShards; + } } private static boolean isInvalidSplitFactor(int sourceNumberOfShards, int targetNumberOfShards) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index ca325b7bcb56d..28f96a4e500de 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -112,7 +112,7 @@ private void splitToN(int... shardSplits) { Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0]); if (useRouting && useMixedRouting == false && randomBoolean()) { settings.put("index.routing_partition_size", randomIntBetween(1, MetaDataCreateIndexService.calculateNumRoutingShards - (shardSplits[0]) -1)); + (shardSplits[0], Version.CURRENT) -1)); if (useNested) { createInitialIndex.addMapping("t1", "_routing", "required=true", "nested1", "type=nested"); } else { diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 9cbe4fb0f5730..2b68ef2fd1450 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.ResourceAlreadyExistsException; @@ -299,17 +298,23 @@ private void validateIndexName(String indexName, String errorMessage) { } public void testCalculateNumRoutingShards() { - assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1)); - assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2)); - assertEquals(1536, MetaDataCreateIndexService.calculateNumRoutingShards(3)); - assertEquals(1152, MetaDataCreateIndexService.calculateNumRoutingShards(9)); - assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512)); - assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024)); - assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT)); + assertEquals(1536, MetaDataCreateIndexService.calculateNumRoutingShards(3, Version.CURRENT)); + assertEquals(1152, MetaDataCreateIndexService.calculateNumRoutingShards(9, Version.CURRENT)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512, Version.CURRENT)); + assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024, Version.CURRENT)); + assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048, Version.CURRENT)); + + Version latestV6 = VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1); + int numShards = randomIntBetween(1, 1000); + assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards, latestV6)); + assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards, + VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), latestV6))); for (int i = 0; i < 1000; i++) { int randomNumShards = randomIntBetween(1, 10000); - int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards); + int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards, Version.CURRENT); double ratio = numRoutingShards / randomNumShards; int intRatio = (int) ratio; assertEquals(ratio, (double)(intRatio), 0.0d); diff --git a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml index ea9fa33e6a9cf..aee2b2fb05759 100644 --- a/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml +++ b/qa/smoke-test-reindex-with-all-modules/src/test/resources/rest-api-spec/test/update_by_query/10_script.yml @@ -337,6 +337,6 @@ body: script: lang: painless - source: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"} + source: if (ctx._source.user == "kimchy") {ctx.op = "index"} else {ctx.op = "junk"} - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' } From 6ab582e96f7c75cc4c337c7b3fc8b7b02a884180 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 20 Nov 2017 16:24:02 +0100 Subject: [PATCH 03/17] fix auto routing shards for 1 shard source indices --- .../indices/shrink/TransportResizeAction.java | 3 +- .../metadata/MetaDataCreateIndexService.java | 53 ++++++++++++------- .../MetaDataCreateIndexServiceTests.java | 8 +++ 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 597e3e7e2089f..7ccf391033090 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -156,7 +156,8 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi } } } else { - if (metaData.getNumberOfShards() != 1) { // we can split into anything from 1 + if (metaData.getNumberOfShards() != 1) { + // we can split into anything from 1 Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); } // we just execute this to ensure we get the right exceptions if the number of shards is wrong or less then etc. diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 4b4cdf177a2db..e2fd719f68cb6 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -383,24 +383,24 @@ public ClusterState execute(ClusterState currentState) throws Exception { int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); final int routingNumShards; final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null); - if (recoverFromIndex == null) { - if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { - routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); - } else { - routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); - } - } else { - assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false - : "index.number_of_routing_shards should not be present on the target index on resize"; - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); - if (sourceMetaData.getNumberOfShards() == 1 - && isInvalidSplitFactor(numTargetShards, sourceMetaData.getRoutingNumShards())) { - // in this case we have a source index with 1 shard and without an explicit split factor - // or one that is valid in that case we can split into whatever and auto-generate a new factor. - routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); + if (recoverFromIndex == null) { + if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { + routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); + } else { + routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); + } } else { - routingNumShards = sourceMetaData.getRoutingNumShards(); - } + assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false + : "index.number_of_routing_shards should not be present on the target index on resize"; + final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); + if (shouldAutoCalculateNumRoutingShards(numTargetShards, sourceMetaData.getNumberOfShards(), + sourceMetaData.getRoutingNumShards())) { + // in this case we have a source index with 1 shard and without an explicit split factor + // or one that is valid in that case we can split into whatever and auto-generate a new factor. + routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); + } else { + routingNumShards = sourceMetaData.getRoutingNumShards(); + } } // remove the setting it's temporary and is only relevant once we create the index indexSettingsBuilder.remove(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); @@ -747,8 +747,21 @@ public static int calculateNumRoutingShards(int numShards, Version indexVersionC } } - private static boolean isInvalidSplitFactor(int sourceNumberOfShards, int targetNumberOfShards) { - int factor = sourceNumberOfShards / targetNumberOfShards; - return factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1; + + /** + * Returns true iff the we should calculate the number of routing shards for the split index based on + * the source index. This only applies to source indices that have only 1 shards. We don't want to recalculate the num routing + * shards on indices that already have a valid number of routing shards. + */ + static boolean shouldAutoCalculateNumRoutingShards(int numTargetShards, int numSourceShards, int numSourceRoutingShards) { + if (numSourceShards == 1) { + if (numSourceRoutingShards < numTargetShards) { + // we have a source index that has less routing shards than our target shards -- we should reset + return true; + } + int factor = numSourceRoutingShards / numTargetShards; + return factor * numTargetShards != numSourceRoutingShards || factor <= 1; + } + return false; } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 2b68ef2fd1450..48a257c28047f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -297,6 +297,14 @@ private void validateIndexName(String indexName, String errorMessage) { assertThat(e.getMessage(), endsWith(errorMessage)); } + public void testShouldAutoCalculateNumRoutingShards() { + assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(2, 1, 1)); + assertFalse(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 20)); + assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 5)); + assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 16)); + assertFalse(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(2, 1, 8)); + } + public void testCalculateNumRoutingShards() { assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT)); assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT)); From 6d90ad8438867d42881438a32c2ee24ea1694077 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 20 Nov 2017 23:47:03 +0100 Subject: [PATCH 04/17] add additional verification for broken split setups --- .../indices/shrink/TransportResizeAction.java | 5 +-- .../cluster/metadata/IndexMetaData.java | 12 +++++- .../metadata/MetaDataCreateIndexService.java | 38 +++++++++---------- .../cluster/metadata/IndexMetaDataTests.java | 7 +++- .../MetaDataCreateIndexServiceTests.java | 25 ++++++++---- .../test/indices.split/10_basic.yml | 1 + .../test/indices.split/20_source_mapping.yml | 1 + 7 files changed, 55 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 7ccf391033090..87dd9f9fa2d21 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -156,10 +156,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi } } } else { - if (metaData.getNumberOfShards() != 1) { - // we can split into anything from 1 - Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); - } + Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); // we just execute this to ensure we get the right exceptions if the number of shards is wrong or less then etc. } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index da5c0c6caa1c9..d022293b302a3 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1343,9 +1343,19 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet + "] must be less that the number of target shards [" + numTargetShards + "]"); } int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + // now we verify that the numRoutingShards is valid in the source index + // note: if the number of shards is 1 in the source index we can just assume it's correct since from 1 we can split into anything + // this is important to special case here since we use this to validate this in various places in the code but allow to split form + // 1 to N but we never modify the sourceIndexMetadata to accommodate for that + int routingNumShards = numSourceShards == 1 ? numTargetShards : sourceIndexMetadata.getRoutingNumShards(); + int factor = routingNumShards / numTargetShards; + if (factor * numTargetShards != routingNumShards) { + throw new IllegalStateException("the number of routingShards [" + + routingNumShards + "] must be a multiple of the target shards [" + numTargetShards + "]"); + } // this is just an additional assertion that ensures we are a factor of the routing num shards. assert sourceIndexMetadata.getNumberOfShards() == 1 // special case - we can split into anything from 1 shard - || getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0; + || getRoutingFactor(numTargetShards, routingNumShards) >= 0; return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index e2fd719f68cb6..d5f31f72f503c 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -383,24 +383,24 @@ public ClusterState execute(ClusterState currentState) throws Exception { int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); final int routingNumShards; final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null); - if (recoverFromIndex == null) { - if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { - routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); - } else { - routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); - } + if (recoverFromIndex == null) { + if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { + routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); } else { - assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false - : "index.number_of_routing_shards should not be present on the target index on resize"; - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); - if (shouldAutoCalculateNumRoutingShards(numTargetShards, sourceMetaData.getNumberOfShards(), - sourceMetaData.getRoutingNumShards())) { - // in this case we have a source index with 1 shard and without an explicit split factor - // or one that is valid in that case we can split into whatever and auto-generate a new factor. - routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); - } else { - routingNumShards = sourceMetaData.getRoutingNumShards(); - } + routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); + } + } else { + assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false + : "index.number_of_routing_shards should not be present on the target index on resize"; + final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); + if (shouldAutoCalculateNumRoutingShards(numTargetShards, sourceMetaData.getNumberOfShards(), + sourceMetaData.getRoutingNumShards())) { + // in this case we have a source index with 1 shard and without an explicit split factor + // or one that is valid in that case we can split into whatever and auto-generate a new factor. + routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); + } else { + routingNumShards = sourceMetaData.getRoutingNumShards(); + } } // remove the setting it's temporary and is only relevant once we create the index indexSettingsBuilder.remove(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); @@ -655,9 +655,7 @@ static void validateSplitIndex(ClusterState state, String sourceIndex, Set targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) { IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); - if (sourceMetaData.getNumberOfShards() != 1) { // we can split into anything if we only have 1 shard - IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); - } + IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); if (sourceMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) { // ensure we have a single type since this would make the splitting code considerably more complex // and a 5.x index would not be splittable unless it has been shrunk before so rather opt out of the complexity diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index e83d1fa706cfd..b28a3994ca6f8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -118,6 +118,8 @@ public void testSelectShrinkShards() { } public void testSelectResizeShards() { + int numTargetShards = randomFrom(4, 6, 8, 12); + IndexMetaData split = IndexMetaData.builder("foo") .settings(Settings.builder() .put("index.version.created", 1) @@ -125,6 +127,7 @@ public void testSelectResizeShards() { .put("index.number_of_replicas", 0) .build()) .creationDate(randomLong()) + .setRoutingNumShards(numTargetShards * 2) .build(); IndexMetaData shrink = IndexMetaData.builder("foo") @@ -135,7 +138,6 @@ public void testSelectResizeShards() { .build()) .creationDate(randomLong()) .build(); - int numTargetShards = randomFrom(4, 6, 8, 12); int shard = randomIntBetween(0, numTargetShards-1); assertEquals(Collections.singleton(IndexMetaData.selectSplitShard(shard, split, numTargetShards)), IndexMetaData.selectRecoverFromShards(shard, split, numTargetShards)); @@ -173,6 +175,9 @@ public void testSelectSplitShard() { assertEquals("the number of source shards [2] must be a must be a factor of [3]", expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 3)).getMessage()); + + assertEquals("the number of routingShards [4] must be a multiple of the target shards [6]", + expectThrows(IllegalStateException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 6)).getMessage()); } public void testIndexFormat() { diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 48a257c28047f..3a229a839fe60 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -55,10 +55,13 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase { private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { + int numRoutingShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(settings) ? IndexMetaData + .INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(settings) : numShards; MetaData.Builder metaBuilder = MetaData.builder(); IndexMetaData indexMetaData = IndexMetaData.builder(name).settings(settings(Version.CURRENT) .put(settings)) - .numberOfShards(numShards).numberOfReplicas(numReplicas).build(); + .numberOfShards(numShards).numberOfReplicas(numReplicas) + .setRoutingNumShards(numRoutingShards).build(); metaBuilder.put(indexMetaData, false); MetaData metaData = metaBuilder.build(); RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); @@ -203,10 +206,19 @@ public void testValidateSplitIndex() { } ).getMessage()); - + assertEquals("mappings are not allowed when resizing indices, all mappings are copied from the source index", + expectThrows(IllegalArgumentException.class, () -> { + MetaDataCreateIndexService.validateSplitIndex(state, "source", Collections.singleton("foo"), + "target", targetSettings); + } + ).getMessage()); + int targetShards; + do { + targetShards = randomIntBetween(numShards+1, 100); + } while (isSplitable(numShards, targetShards) == false); ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, - Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) - .build(); + Settings.builder().put("index.blocks.write", true).put("index.number_of_routing_shards", targetShards).build())) + .nodes(DiscoveryNodes.builder().add(newNode("node1"))).build(); AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); @@ -217,10 +229,7 @@ public void testValidateSplitIndex() { routingTable = service.applyStartedShards(clusterState, routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); - int targetShards; - do { - targetShards = randomIntBetween(numShards+1, 100); - } while (isSplitable(numShards, targetShards) == false); + MetaDataCreateIndexService.validateSplitIndex(clusterState, "source", Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", targetShards).build()); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 50b5f3305ba4c..82881564a2043 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -11,6 +11,7 @@ settings: index.number_of_shards: 1 index.number_of_replicas: 0 + index.number_of_routing_shards: 2 - do: index: index: source diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml index 91f10578f8530..38f36c405a1cb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -13,6 +13,7 @@ settings: number_of_shards: 1 number_of_replicas: 0 + index.number_of_routing_shards: 2 mappings: test: properties: From e4da795840df649f598ffd4903ad09a6a969c1d7 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 21 Nov 2017 07:22:33 +0100 Subject: [PATCH 05/17] fix test: --- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 3a229a839fe60..e4de205b4556a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -55,8 +55,7 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase { private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { - int numRoutingShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(settings) ? IndexMetaData - .INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(settings) : numShards; + int numRoutingShards = settings.getAsInt(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey(), numShards); MetaData.Builder metaBuilder = MetaData.builder(); IndexMetaData indexMetaData = IndexMetaData.builder(name).settings(settings(Version.CURRENT) .put(settings)) From fb1396960237d2e8dc6f8aa85fe90aed7cee1d26 Mon Sep 17 00:00:00 2001 From: Clinton Gormley Date: Tue, 21 Nov 2017 14:04:27 +0100 Subject: [PATCH 06/17] Improved index-split docs --- docs/reference/indices/split-index.asciidoc | 70 +++++++++++---------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index d9760659d478b..4a7e81e4f4de8 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -1,29 +1,36 @@ [[indices-split-index]] == Split Index -The split index API allows you to split an existing index into a new index -with multiple of it's primary shards. Similarly to the <> -where the number of primary shards in the shrunk index must be a factor of the source index. -Every index is by default created with an implicit `index.number_of_routing_shards` that specify -how often an index can be split. By default every index can at least be split once by a factor of 2. -The default number of routing shards is dependent on the number of primary shards in an index. -Indices with a small number of primary shards can be split more often compared to indices that -have number of shards closer to `1024`. Note: an index with a single shard can be split into -a new index with an arbitrary number of shards greater than 1. All properties of default number of -routing shards then apply to the newly created index. - -Optionally, an index can also be created with be created with a specific number of routing shards -in order to be split in the future. -The number of routing shards specify the hashing space that is used internally to distribute documents -across shards, in oder to have a consistent hashing that is compatible with the method elasticsearch -uses today. -For example an index with `8` primary shards and a `index.number_of_routing_shards` of `32` -can be split into `16` and `32` primary shards. An index with `1` primary shard -and `index.number_of_routing_shards` of `64` can be split into `2`, `4`, `8`, `16`, `32` or `64`. -The same works for non power of two routing shards ie. an index with `1` primary shard and -`index.number_of_routing_shards` set to `15` can be split into `3` and `15` or alternatively`5` and `15`. -The number of shards in the split index must always be a factor of `index.number_of_routing_shards` -in the source index. Before splitting, a (primary) copy of every shard in the index must be active in the cluster. +The split index API allows you to split an existing index into a new index, +where each original primary shard is split into two or more primary shards in +the new index. + +The number of times the index can be split (and the number of shards that each +original shard can be split into) is determined by the +`index.number_of_routing_shards` setting. The number of routing shards +specifies the hashing space that is used internally to distribute documents +across shards with consistent hashing. For instance, a 5 shard index with +`number_of_routing_shards` set to `30` (`5 x 2 x 3`) could be split by a +factor of `2` or `3`. In other words, it could be split as follows: + +* `5` -> `10` -> `30` (split by 2, then by 3) +* `5` -> `15` -> `30` (split by 3, then by 2) +* `5` -> `30` (split by 6) + +While you can set the `index.number_of_routing_shards` setting explicitly at +index creation time, the default value depends upon the number of primary +shards in the original index. The default is designed to allow you to split +by factors of 2 up to a maximum of 1024 shards. However, the original number +of primary shards must taken into account. For instance, an index created +with 5 primary shards could be split into 10, 20, 40, 80, 160, 320, or a +maximum of 740 shards (with a single split action or multiple split actions). + +If the original index contains one primary shard (or a multi-shard index has +been <> down to a single primary shard), then the +index may by split into an arbitrary number of shards greater than 1. The +properties of the default number of routing shards will then apply to the +newly split index. + Splitting works as follows: @@ -35,7 +42,7 @@ Splitting works as follows: into the new index, which is a much more time consuming process.) * Once the low level files are created all documents will be `hashed` again to delete - documents that belong in a different shard. + documents that belong to a different shard. * Finally, it recovers the target index as though it were a closed index which had just been re-opened. @@ -49,16 +56,13 @@ Create a new index: -------------------------------------------------- PUT my_source_index { - "settings": { - "index.number_of_shards" : 1 - } + "settings": { + "index.number_of_shards" : 1 + } } ------------------------------------------------- // CONSOLE -<1> Allows to split the index into two shards or in other words, it allows - for a single split operation. - In order to split an index, the index must be marked as read-only, and have <> `green`. @@ -80,7 +84,7 @@ PUT /my_source_index/_settings changes like deleting the index. [float] -=== Spitting an index +=== Splitting an index To split `my_source_index` into a new index called `my_target_index`, issue the following request: @@ -107,7 +111,7 @@ Indices can only be split if they satisfy the following requirements: * the target index must not exist -* The index must have less primary shards than the target index. +* The source index must have fewer primary shards than the target index. * The number of primary shards in the target index must be a factor of the number of primary shards in the source index. @@ -167,4 +171,4 @@ replicas and may decide to relocate the primary shard to another node. Because the split operation creates a new index to split the shards to, the <> setting -on index creation applies to the split index action as well. +on index creation applies to the split index action as well. \ No newline at end of file From 27a6d3421ba1f8b78046558e16c0d4bc3ce06c6d Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 21 Nov 2017 17:44:13 +0100 Subject: [PATCH 07/17] apply review comments --- .../indices/shrink/TransportResizeAction.java | 7 +- .../cluster/metadata/IndexMetaData.java | 12 +- .../metadata/MetaDataCreateIndexService.java | 44 ++----- .../admin/indices/create/SplitIndexIT.java | 35 ++++-- .../shrink/TransportResizeActionTests.java | 66 ++++++++++- .../MetaDataCreateIndexServiceTests.java | 17 +-- .../test/indices.split/10_basic.yml | 111 +++++++++++++++++- 7 files changed, 225 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 87dd9f9fa2d21..c5a15be22a847 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -163,9 +163,14 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) { throw new IllegalArgumentException("cannot provide a routing partition size value when resizing an index"); + } if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(targetIndexSettings)) { - throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize"); + // if we have a source index with 1 shards it's legal to set this + final boolean splitFromSingleShards = resizeRequest.getResizeType() == ResizeType.SPLIT && metaData.getNumberOfShards() == 1; + if (splitFromSingleShards == false) { + throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize"); + } } String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index"; targetIndex.cause(cause); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index d9b9d78eb02ad..ac8e80a15c5df 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1333,16 +1333,21 @@ public int getRoutingFactor() { * @return a the source shard ID to split off from */ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { + int numSourceShards = sourceIndexMetadata.getNumberOfShards(); if (shardId >= numTargetShards) { throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId); } - int numSourceShards = sourceIndexMetadata.getNumberOfShards(); + assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); + final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); + } + + private static void assertSplitMetadata(int numSourceShards, int numTargetShards, IndexMetaData sourceIndexMetadata) { if (numSourceShards > numTargetShards) { throw new IllegalArgumentException("the number of source shards [" + numSourceShards - + "] must be less that the number of target shards [" + numTargetShards + "]"); + + "] must be less that the number of target shards [" + numTargetShards + "]"); } - int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); // now we verify that the numRoutingShards is valid in the source index // note: if the number of shards is 1 in the source index we can just assume it's correct since from 1 we can split into anything // this is important to special case here since we use this to validate this in various places in the code but allow to split form @@ -1355,7 +1360,6 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet // this is just an additional assertion that ensures we are a factor of the routing num shards. assert sourceIndexMetadata.getNumberOfShards() == 1 // special case - we can split into anything from 1 shard || getRoutingFactor(numTargetShards, routingNumShards) >= 0; - return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); } /** diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index d5f31f72f503c..5cf8ee065a883 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -383,7 +383,12 @@ public ClusterState execute(ClusterState currentState) throws Exception { int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); final int routingNumShards; final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null); - if (recoverFromIndex == null) { + final IndexMetaData sourceMetaData = recoverFromIndex == null ? null : + currentState.metaData().getIndexSafe(recoverFromIndex); + if (sourceMetaData == null || sourceMetaData.getNumberOfShards() == 1) { + // in this case we either have no index to recover from or + // we have a source index with 1 shard and without an explicit split factor + // or one that is valid in that case we can split into whatever and auto-generate a new factor. if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(idxSettings)) { routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); } else { @@ -392,15 +397,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { } else { assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false : "index.number_of_routing_shards should not be present on the target index on resize"; - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); - if (shouldAutoCalculateNumRoutingShards(numTargetShards, sourceMetaData.getNumberOfShards(), - sourceMetaData.getRoutingNumShards())) { - // in this case we have a source index with 1 shard and without an explicit split factor - // or one that is valid in that case we can split into whatever and auto-generate a new factor. - routingNumShards = calculateNumRoutingShards(numTargetShards, indexVersionCreated); - } else { - routingNumShards = sourceMetaData.getRoutingNumShards(); - } + routingNumShards = sourceMetaData.getRoutingNumShards(); } // remove the setting it's temporary and is only relevant once we create the index indexSettingsBuilder.remove(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); @@ -420,7 +417,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { * the maximum primary term on all the shards in the source index. This ensures that we have correct * document-level semantics regarding sequence numbers in the shrunken index. */ - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); final long primaryTerm = IntStream .range(0, sourceMetaData.getNumberOfShards()) @@ -737,29 +733,13 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma */ public static int calculateNumRoutingShards(int numShards, Version indexVersionCreated) { if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) { - // only select this automatically for indices that are created on or after 7.0 - int base = 10; // logBase2(1024) - return numShards * 1 << Math.max(1, (base - (int) (Math.log(numShards) / Math.log(2)))); + // only select this automatically for indices that are created on or after 7.0 this will prevent this new behaviour + // until we have a fully upgraded cluster see {@link IndexMetaDataE# + int base = 9; // logBase2(512) + final int minNumSplits = 1; + return numShards * 1 << Math.max(minNumSplits, (base - (int) (Math.log(numShards) / Math.log(2)))); } else { return numShards; } } - - - /** - * Returns true iff the we should calculate the number of routing shards for the split index based on - * the source index. This only applies to source indices that have only 1 shards. We don't want to recalculate the num routing - * shards on indices that already have a valid number of routing shards. - */ - static boolean shouldAutoCalculateNumRoutingShards(int numTargetShards, int numSourceShards, int numSourceRoutingShards) { - if (numSourceShards == 1) { - if (numSourceRoutingShards < numTargetShards) { - // we have a source index that has less routing shards than our target shards -- we should reset - return true; - } - int factor = numSourceRoutingShards / numTargetShards; - return factor * numTargetShards != numSourceRoutingShards || factor <= 1; - } - return false; - } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 28f96a4e500de..76235df91a98b 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -90,7 +90,7 @@ protected Collection> nodePlugins() { public void testCreateSplitIndexToN() throws IOException { int[][] possibleShardSplits = new int[][]{{2, 4, 8}, {3, 6, 12}, {1, 2, 4}}; int[] shardSplits = randomFrom(possibleShardSplits); - splitToN(shardSplits); + splitToN(shardSplits[0], shardSplits[1], shardSplits[2]); } public void testSplitFromOneToN() { @@ -100,19 +100,26 @@ public void testSplitFromOneToN() { splitToN(1, randomSplit, randomSplit * 2); } - private void splitToN(int... shardSplits) { - assertEquals(3, shardSplits.length); - assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); - assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); + private void splitToN(int sourceShards, int firstSplitShards, int secondSplitShards) { + + assertEquals(sourceShards, (sourceShards * firstSplitShards) / firstSplitShards); + assertEquals(firstSplitShards, (firstSplitShards * secondSplitShards) / secondSplitShards); internalCluster().ensureAtLeastNumDataNodes(2); final boolean useRouting = randomBoolean(); final boolean useNested = randomBoolean(); final boolean useMixedRouting = useRouting ? randomBoolean() : false; CreateIndexRequestBuilder createInitialIndex = prepareCreate("source"); - Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0]); + Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", sourceShards); + final int routingShards; + if (randomBoolean()) { + // randomly set the value manually + routingShards = secondSplitShards * randomIntBetween(1, 10); + settings.put("index.number_of_routing_shards", routingShards); + } else { + routingShards = MetaDataCreateIndexService.calculateNumRoutingShards(sourceShards, Version.CURRENT); + } if (useRouting && useMixedRouting == false && randomBoolean()) { - settings.put("index.routing_partition_size", randomIntBetween(1, MetaDataCreateIndexService.calculateNumRoutingShards - (shardSplits[0], Version.CURRENT) -1)); + settings.put("index.routing_partition_size", randomIntBetween(1, routingShards-1)); if (useNested) { createInitialIndex.addMapping("t1", "_routing", "required=true", "nested1", "type=nested"); } else { @@ -182,11 +189,15 @@ private void splitToN(int... shardSplits) { .setSettings(Settings.builder() .put("index.blocks.write", true)).get(); ensureGreen(); + Settings.Builder firstSplitSettingsBuilder = Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", firstSplitShards); + if (sourceShards == 1 && randomBoolean()) { // try to set it if we have a source index with 1 shard + firstSplitSettingsBuilder.put("index.number_of_routing_shards", routingShards); + } assertAcked(client().admin().indices().prepareResizeIndex("source", "first_split") .setResizeType(ResizeType.SPLIT) - .setSettings(Settings.builder() - .put("index.number_of_replicas", 0) - .put("index.number_of_shards", shardSplits[1]).build()).get()); + .setSettings(firstSplitSettingsBuilder.build()).get()); ensureGreen(); assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); @@ -214,7 +225,7 @@ private void splitToN(int... shardSplits) { .setResizeType(ResizeType.SPLIT) .setSettings(Settings.builder() .put("index.number_of_replicas", 0) - .put("index.number_of_shards", shardSplits[2]).build()).get()); + .put("index.number_of_shards", secondSplitShards).build()).get()); ensureGreen(); assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); // let it be allocated anywhere and bump replicas diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index b03b043f03e14..bd43182f00756 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -51,10 +51,13 @@ public class TransportResizeActionTests extends ESTestCase { private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { + return createClusterState(name, numShards, numReplicas, numShards, settings); + } + private ClusterState createClusterState(String name, int numShards, int numReplicas, int numRoutingShards, Settings settings) { MetaData.Builder metaBuilder = MetaData.builder(); IndexMetaData indexMetaData = IndexMetaData.builder(name).settings(settings(Version.CURRENT) .put(settings)) - .numberOfShards(numShards).numberOfReplicas(numReplicas).build(); + .numberOfShards(numShards).numberOfReplicas(numReplicas).setRoutingNumShards(numRoutingShards).build(); metaBuilder.put(indexMetaData, false); MetaData metaData = metaBuilder.build(); RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); @@ -108,6 +111,67 @@ public void testErrorCondition() { (i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)), "source", "target"); } + public void testPassNumRoutingShards() { + ClusterState clusterState = ClusterState.builder(createClusterState("source", 1, 0, + Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) + .build(); + AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + // now we start the shard + routingTable = service.applyStartedShards(clusterState, + routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + ResizeRequest resizeRequest = new ResizeRequest("target", "source"); + resizeRequest.setResizeType(ResizeType.SPLIT); + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder().put("index.number_of_shards", 2).build()); + TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target"); + + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder() + .put("index.number_of_routing_shards", randomIntBetween(2, 10)) + .put("index.number_of_shards", 2) + .build()); + TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target"); + } + + public void testPassNumRoutingShardsAndFail() { + int numShards = randomIntBetween(2, 100); + ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, numShards * 4, + Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) + .build(); + AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + // now we start the shard + routingTable = service.applyStartedShards(clusterState, + routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + ResizeRequest resizeRequest = new ResizeRequest("target", "source"); + resizeRequest.setResizeType(ResizeType.SPLIT); + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder().put("index.number_of_shards", numShards * 2).build()); + TransportResizeAction.prepareCreateIndexRequest(resizeRequest, clusterState, null, "source", "target"); + + resizeRequest.getTargetIndexRequest() + .settings(Settings.builder() + .put("index.number_of_shards", numShards * 2) + .put("index.number_of_routing_shards", numShards * 2).build()); + ClusterState finalState = clusterState; + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, + () -> TransportResizeAction.prepareCreateIndexRequest(resizeRequest, finalState, null, "source", "target")); + assertEquals("cannot provide index.number_of_routing_shards on resize", iae.getMessage()); + } + public void testShrinkIndexSettings() { String indexName = randomAlphaOfLength(10); // create one that won't fail diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 02de61a900a69..97fa4e960914f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -299,19 +299,11 @@ private void validateIndexName(String indexName, String errorMessage) { assertThat(e.getMessage(), endsWith(errorMessage)); } - public void testShouldAutoCalculateNumRoutingShards() { - assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(2, 1, 1)); - assertFalse(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 20)); - assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 5)); - assertTrue(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(10, 1, 16)); - assertFalse(MetaDataCreateIndexService.shouldAutoCalculateNumRoutingShards(2, 1, 8)); - } - public void testCalculateNumRoutingShards() { - assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT)); - assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT)); - assertEquals(1536, MetaDataCreateIndexService.calculateNumRoutingShards(3, Version.CURRENT)); - assertEquals(1152, MetaDataCreateIndexService.calculateNumRoutingShards(9, Version.CURRENT)); + assertEquals(512, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT)); + assertEquals(512, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT)); + assertEquals(768, MetaDataCreateIndexService.calculateNumRoutingShards(3, Version.CURRENT)); + assertEquals(576, MetaDataCreateIndexService.calculateNumRoutingShards(9, Version.CURRENT)); assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512, Version.CURRENT)); assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024, Version.CURRENT)); assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048, Version.CURRENT)); @@ -331,6 +323,7 @@ public void testCalculateNumRoutingShards() { assertTrue(1 < ratio); assertTrue(ratio <= 1024); assertEquals(0, intRatio % 2); + assertEquals("ration is not a power of two", intRatio, Integer.highestOneBit(intRatio)); } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 7f0b294a69d2b..923a6540b1398 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -6,9 +6,9 @@ setup: wait_for_active_shards: 1 body: settings: - index.number_of_shards: 1 + index.number_of_shards: 2 index.number_of_replicas: 0 - index.number_of_routing_shards: 2 + index.number_of_routing_shards: 4 - do: index: index: source @@ -59,7 +59,7 @@ setup: body: settings: index.number_of_replicas: 0 - index.number_of_shards: 2 + index.number_of_shards: 4 - do: cluster.health: @@ -100,6 +100,107 @@ setup: - match: { _id: "3" } - match: { _source: { foo: "hello world 3" } } + +--- +"Split from 1 to N": + - skip: + version: " - 6.99.99" + reason: Added in 7.0.0 + - do: + indices.create: + index: source_one_shard + wait_for_active_shards: 1 + body: + settings: + index.number_of_shards: 1 + index.number_of_replicas: 0 + - do: + index: + index: source_one_shard + type: doc + id: "1" + body: { "foo": "hello world" } + + - do: + index: + index: source_one_shard + type: doc + id: "2" + body: { "foo": "hello world 2" } + + - do: + index: + index: source_one_shard + type: doc + id: "3" + body: { "foo": "hello world 3" } + + # make it read-only + - do: + indices.put_settings: + index: source_one_shard + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source_one_shard + + # now we do the actual split from 1 to 5 + - do: + indices.split: + index: "source_one_shard" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 5 + + - do: + cluster.health: + wait_for_status: green + + - do: + get: + index: target + type: doc + id: "1" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "1" } + - match: { _source: { foo: "hello world" } } + + + - do: + get: + index: target + type: doc + id: "2" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "2" } + - match: { _source: { foo: "hello world 2" } } + + + - do: + get: + index: target + type: doc + id: "3" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "3" } + - match: { _source: { foo: "hello world 3" } } + + + --- "Create illegal split indices": - skip: @@ -117,8 +218,8 @@ setup: body: settings: index.number_of_replicas: 0 - index.number_of_shards: 2 - index.number_of_routing_shards: 4 + index.number_of_shards: 4 + index.number_of_routing_shards: 8 # try to do an illegal split with illegal number_of_shards - do: From 9262c9e7fd644376cf530468e0787cb39c7cb26e Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 21 Nov 2017 20:32:07 +0100 Subject: [PATCH 08/17] fix unittest --- .../java/org/elasticsearch/cluster/metadata/IndexMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index ac8e80a15c5df..b532a1ec841bb 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1338,8 +1338,8 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId); } - assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); final int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + assertSplitMetadata(numSourceShards, numTargetShards, sourceIndexMetadata); return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); } From 8482a60038f75a3fa9f44d7f62584945fbe2d37e Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 21 Nov 2017 20:50:33 +0100 Subject: [PATCH 09/17] stabelize SplitIndexIT --- .../admin/indices/create/SplitIndexIT.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 76235df91a98b..bfa6f5a2ccd09 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -110,16 +110,18 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha final boolean useMixedRouting = useRouting ? randomBoolean() : false; CreateIndexRequestBuilder createInitialIndex = prepareCreate("source"); Settings.Builder settings = Settings.builder().put(indexSettings()).put("number_of_shards", sourceShards); - final int routingShards; + final boolean useRoutingPartition; if (randomBoolean()) { // randomly set the value manually - routingShards = secondSplitShards * randomIntBetween(1, 10); + int routingShards = secondSplitShards * randomIntBetween(1, 10); settings.put("index.number_of_routing_shards", routingShards); + useRoutingPartition = false; } else { - routingShards = MetaDataCreateIndexService.calculateNumRoutingShards(sourceShards, Version.CURRENT); + useRoutingPartition = randomBoolean(); } - if (useRouting && useMixedRouting == false && randomBoolean()) { - settings.put("index.routing_partition_size", randomIntBetween(1, routingShards-1)); + if (useRouting && useMixedRouting == false && useRoutingPartition) { + settings.put("index.routing_partition_size", + randomIntBetween(1, MetaDataCreateIndexService.calculateNumRoutingShards(sourceShards, Version.CURRENT)-1)); if (useNested) { createInitialIndex.addMapping("t1", "_routing", "required=true", "nested1", "type=nested"); } else { @@ -192,8 +194,8 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha Settings.Builder firstSplitSettingsBuilder = Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", firstSplitShards); - if (sourceShards == 1 && randomBoolean()) { // try to set it if we have a source index with 1 shard - firstSplitSettingsBuilder.put("index.number_of_routing_shards", routingShards); + if (sourceShards == 1 && useRoutingPartition == false && randomBoolean()) { // try to set it if we have a source index with 1 shard + firstSplitSettingsBuilder.put("index.number_of_routing_shards", secondSplitShards); } assertAcked(client().admin().indices().prepareResizeIndex("source", "first_split") .setResizeType(ResizeType.SPLIT) From 6fe2bff4d10da1b33877fd65bb29d11022a63a05 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 21 Nov 2017 21:35:57 +0100 Subject: [PATCH 10/17] Fix routing test to actually be sane --- .../routing/SimpleRoutingIT.java | 116 +++++++++++------- 1 file changed, 70 insertions(+), 46 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java index 031bac145a340..d5f805d49a702 100644 --- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -36,8 +36,14 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -53,12 +59,27 @@ protected int minimumNumberOfShards() { return 2; } + public String findNonMatchingRoutingValue(String index, String id) { + OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); + ClusterState state = client().admin().cluster().prepareState().all().get().getState(); + int routing = -1; + ShardId idShard; + ShardId routingShard; + do { + idShard = operationRouting.shardId(state, index, id, null); + routingShard = operationRouting.shardId(state, index, id, Integer.toString(++routing)); + } while (idShard.getId() == routingShard.id()); + + return Integer.toString(routing); + } + public void testSimpleCrudRouting() throws Exception { createIndex("test"); ensureGreen(); - - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + String routingValue = findNonMatchingRoutingValue("test", "1"); + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { @@ -66,25 +87,25 @@ public void testSimpleCrudRouting() throws Exception { } logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> deleting with no routing, should not delete anything"); client().prepareDelete("test", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> deleting with routing, should delete"); - client().prepareDelete("test", "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + client().prepareDelete("test", "type1", "1").setRouting(routingValue).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(false)); } logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { @@ -92,16 +113,17 @@ public void testSimpleCrudRouting() throws Exception { } logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } } public void testSimpleSearchRouting() { createIndex("test"); ensureGreen(); + String routingValue = findNonMatchingRoutingValue("test", "1"); - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex("test", "type1", "1").setRouting(routingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { @@ -109,7 +131,7 @@ public void testSimpleSearchRouting() { } logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet("test", "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> search with no routing, should fine one"); @@ -125,12 +147,13 @@ public void testSimpleSearchRouting() { logger.info("--> search with correct routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> indexing with id [2], and routing [1]"); - client().prepareIndex("test", "type1", "2").setRouting("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + String secondRouting = findNonMatchingRoutingValue("test", "2"); + logger.info("--> indexing with id [2], and routing [{}]", secondRouting); + client().prepareIndex("test", "type1", "2").setRouting(secondRouting).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { @@ -138,28 +161,28 @@ public void testSimpleSearchRouting() { assertThat(client().prepareSearch().setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } - logger.info("--> search with 0 routing, should find one"); + logger.info("--> search with {} routing, should find one", routingValue); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> search with 1 routing, should find one"); + logger.info("--> search with {} routing, should find one", secondRouting); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); - assertThat(client().prepareSearch().setSize(0).setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setRouting(secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setSize(0).setRouting(secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> search with 0,1 routings , should find two"); + logger.info("--> search with {},{} routings , should find two", routingValue, secondRouting); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0", "1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setRouting(routingValue, secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } - logger.info("--> search with 0,1,0 routings , should find two"); + logger.info("--> search with {},{},{} routings , should find two", routingValue, secondRouting, routingValue); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); - assertThat(client().prepareSearch().setSize(0).setRouting("0", "1", "0").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setRouting(routingValue, secondRouting, routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRouting,routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } } @@ -168,9 +191,10 @@ public void testRequiredRoutingCrudApis() throws Exception { .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject()) .execute().actionGet(); ensureGreen(); + String routingValue = findNonMatchingRoutingValue("test", "1"); - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1") + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex(indexOrAlias(), "type1", "1").setRouting(routingValue).setSource("field", "value1") .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should fail"); @@ -184,7 +208,7 @@ public void testRequiredRoutingCrudApis() throws Exception { logger.info("--> verifying get with routing, should find"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } logger.info("--> deleting with no routing, should fail"); @@ -203,7 +227,7 @@ public void testRequiredRoutingCrudApis() throws Exception { assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); } try { @@ -213,7 +237,7 @@ public void testRequiredRoutingCrudApis() throws Exception { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); } - client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); + client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting(routingValue).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get(); client().admin().indices().prepareRefresh().execute().actionGet(); for (int i = 0; i < 5; i++) { @@ -224,12 +248,12 @@ public void testRequiredRoutingCrudApis() throws Exception { assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - GetResponse getResponse = client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet(); + GetResponse getResponse = client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getSourceAsMap().get("field"), equalTo("value2")); } - client().prepareDelete(indexOrAlias(), "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + client().prepareDelete(indexOrAlias(), "type1", "1").setRouting(routingValue).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { try { @@ -239,7 +263,7 @@ public void testRequiredRoutingCrudApis() throws Exception { assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(false)); } } @@ -251,7 +275,6 @@ public void testRequiredRoutingBulk() throws Exception { .endObject().endObject()) .execute().actionGet(); ensureGreen(); - { BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1") .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet(); @@ -320,19 +343,20 @@ public void testRequiredRoutingBulk() throws Exception { } public void testRequiredRoutingMappingVariousAPIs() throws Exception { + client().admin().indices().prepareCreate("test").addAlias(new Alias("alias")) .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject()) .execute().actionGet(); ensureGreen(); - - logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1").get(); - logger.info("--> indexing with id [2], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "2").setRouting("0").setSource("field", "value2") + String routingValue = findNonMatchingRoutingValue("test", "1"); + logger.info("--> indexing with id [1], and routing [{}]", routingValue); + client().prepareIndex(indexOrAlias(), "type1", "1").setRouting(routingValue).setSource("field", "value1").get(); + logger.info("--> indexing with id [2], and routing [{}]", routingValue); + client().prepareIndex(indexOrAlias(), "type1", "2").setRouting(routingValue).setSource("field", "value2") .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with id [1] with routing [0], should succeed"); - assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); + assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting(routingValue).execute().actionGet().isExists(), equalTo(true)); logger.info("--> verifying get with id [1], with no routing, should fail"); try { @@ -345,7 +369,7 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { logger.info("--> verifying explain with id [2], with routing [0], should succeed"); ExplainResponse explainResponse = client().prepareExplain(indexOrAlias(), "type1", "2") .setQuery(QueryBuilders.matchAllQuery()) - .setRouting("0").get(); + .setRouting(routingValue).get(); assertThat(explainResponse.isExists(), equalTo(true)); assertThat(explainResponse.isMatch(), equalTo(true)); @@ -359,7 +383,7 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { } logger.info("--> verifying term vector with id [1], with routing [0], should succeed"); - TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "type1", "1").setRouting("0").get(); + TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "type1", "1").setRouting(routingValue).get(); assertThat(termVectorsResponse.isExists(), equalTo(true)); assertThat(termVectorsResponse.getId(), equalTo("1")); @@ -370,7 +394,7 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]")); } - UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0") + UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting(routingValue) .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get(); assertThat(updateResponse.getId(), equalTo("1")); assertThat(updateResponse.getVersion(), equalTo(2L)); From b02b9f37aca730fd04e219fb675f4b9d2a250391 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 21 Nov 2017 22:17:22 +0100 Subject: [PATCH 11/17] fix SharedSignificantTermsTestMethods tests --- .../aggregations/bucket/SharedSignificantTermsTestMethods.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index e5081481859ab..6db737793fe7c 100644 --- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -50,7 +50,7 @@ public class SharedSignificantTermsTestMethods { public static void aggregateAndCheckFromSeveralShards(ESIntegTestCase testCase) throws ExecutionException, InterruptedException { String type = ESTestCase.randomBoolean() ? "text" : "keyword"; - String settings = "{\"index.number_of_shards\": 7, \"index.number_of_replicas\": 0}"; + String settings = "{\"index.number_of_shards\": 7, \"index.number_of_routing_shards\": 7, \"index.number_of_replicas\": 0}"; index01Docs(type, settings, testCase); testCase.ensureGreen(); testCase.logClusterState(); From 8176378b0bd69b37002dc555016d0e628fec58a4 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 22 Nov 2017 07:58:20 +0100 Subject: [PATCH 12/17] fix SimpleRoutingIT again --- .../routing/SimpleRoutingIT.java | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java index d5f805d49a702..84caed948a2be 100644 --- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -151,9 +151,9 @@ public void testSimpleSearchRouting() { assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - String secondRouting = findNonMatchingRoutingValue("test", "2"); - logger.info("--> indexing with id [2], and routing [{}]", secondRouting); - client().prepareIndex("test", "type1", "2").setRouting(secondRouting).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); + String secondRoutingValue = "1"; + logger.info("--> indexing with id [{}], and routing [{}]", routingValue, secondRoutingValue); + client().prepareIndex("test", "type1", routingValue).setRouting(secondRoutingValue).setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { @@ -167,22 +167,22 @@ public void testSimpleSearchRouting() { assertThat(client().prepareSearch().setSize(0).setRouting(routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> search with {} routing, should find one", secondRouting); + logger.info("--> search with {} routing, should find one", secondRoutingValue); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting(secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); - assertThat(client().prepareSearch().setSize(0).setRouting(secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setRouting("1").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); + assertThat(client().prepareSearch().setSize(0).setRouting(secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(1L)); } - logger.info("--> search with {},{} routings , should find two", routingValue, secondRouting); + logger.info("--> search with {},{} routings , should find two", routingValue, "1"); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting(routingValue, secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); - assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRouting).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setRouting(routingValue, secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRoutingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } - logger.info("--> search with {},{},{} routings , should find two", routingValue, secondRouting, routingValue); + logger.info("--> search with {},{},{} routings , should find two", routingValue, secondRoutingValue, routingValue); for (int i = 0; i < 5; i++) { - assertThat(client().prepareSearch().setRouting(routingValue, secondRouting, routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); - assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRouting,routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setRouting(routingValue, secondRoutingValue, routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); + assertThat(client().prepareSearch().setSize(0).setRouting(routingValue, secondRoutingValue,routingValue).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().getHits().getTotalHits(), equalTo(2L)); } } @@ -345,7 +345,8 @@ public void testRequiredRoutingBulk() throws Exception { public void testRequiredRoutingMappingVariousAPIs() throws Exception { client().admin().indices().prepareCreate("test").addAlias(new Alias("alias")) - .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject()) + .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("_routing").field("required", true).endObject().endObject().endObject()) .execute().actionGet(); ensureGreen(); String routingValue = findNonMatchingRoutingValue("test", "1"); @@ -429,8 +430,8 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { assertThat(multiGetResponse.getResponses()[1].getFailure().getMessage(), equalTo("routing is required for [test]/[type1]/[2]")); MultiTermVectorsResponse multiTermVectorsResponse = client().prepareMultiTermVectors() - .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").routing("0")) - .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").routing("0")).get(); + .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").routing(routingValue)) + .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").routing(routingValue)).get(); assertThat(multiTermVectorsResponse.getResponses().length, equalTo(2)); assertThat(multiTermVectorsResponse.getResponses()[0].getId(), equalTo("1")); assertThat(multiTermVectorsResponse.getResponses()[0].isFailed(), equalTo(false)); From 1f46ce3a881a77d8904667cb5253e2b6fb832d95 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 22 Nov 2017 08:50:13 +0100 Subject: [PATCH 13/17] use a factor but incompatible one --- .../resources/rest-api-spec/test/indices.split/10_basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 923a6540b1398..ee40cc36347b9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -232,4 +232,4 @@ setup: body: settings: index.number_of_replicas: 0 - index.number_of_shards: 3 + index.number_of_shards: 6 From a6616cdf0bdf531d27e3245bdee45ad4638d9087 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 22 Nov 2017 13:23:48 +0100 Subject: [PATCH 14/17] bound num routing shards to 1024 --- .../cluster/metadata/MetaDataCreateIndexService.java | 10 +++++++--- .../metadata/MetaDataCreateIndexServiceTests.java | 11 +++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 5cf8ee065a883..d47a917ae2fa5 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -735,9 +735,13 @@ public static int calculateNumRoutingShards(int numShards, Version indexVersionC if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) { // only select this automatically for indices that are created on or after 7.0 this will prevent this new behaviour // until we have a fully upgraded cluster see {@link IndexMetaDataE# - int base = 9; // logBase2(512) - final int minNumSplits = 1; - return numShards * 1 << Math.max(minNumSplits, (base - (int) (Math.log(numShards) / Math.log(2)))); + // We use as a default number of routing shards the higher number that can be expressed + // as {@code numShards * 2^x`} that is less than or equal to the maximum number of shards: 1024. + int log2MaxNumShards = 10; // logBase2(1024) + int log2NumShards = 32 - Integer.numberOfLeadingZeros(numShards - 1); // ceil(logBase2(numShards)) + int numSplits = log2MaxNumShards - log2NumShards; + numSplits = Math.max(1, numSplits); // Ensure the index can be split at least once + return numShards * 1 << numSplits; } else { return numShards; } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 97fa4e960914f..d69818567a3ef 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -300,8 +300,8 @@ private void validateIndexName(String indexName, String errorMessage) { } public void testCalculateNumRoutingShards() { - assertEquals(512, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT)); - assertEquals(512, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(1, Version.CURRENT)); + assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(2, Version.CURRENT)); assertEquals(768, MetaDataCreateIndexService.calculateNumRoutingShards(3, Version.CURRENT)); assertEquals(576, MetaDataCreateIndexService.calculateNumRoutingShards(9, Version.CURRENT)); assertEquals(1024, MetaDataCreateIndexService.calculateNumRoutingShards(512, Version.CURRENT)); @@ -317,6 +317,13 @@ public void testCalculateNumRoutingShards() { for (int i = 0; i < 1000; i++) { int randomNumShards = randomIntBetween(1, 10000); int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards, Version.CURRENT); + if (numRoutingShards <= 1024) { + assertTrue("numShards: " + randomNumShards, randomNumShards < 513); + assertTrue("numRoutingShards: " + numRoutingShards, numRoutingShards > 512); + } else { + assertEquals("numShards: " + randomNumShards, numRoutingShards / 2, randomNumShards); + } + double ratio = numRoutingShards / randomNumShards; int intRatio = (int) ratio; assertEquals(ratio, (double)(intRatio), 0.0d); From b2a9a0846ca958e62b86f79b67f73ad19704bc4b Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 22 Nov 2017 13:26:00 +0100 Subject: [PATCH 15/17] fix comments --- .../cluster/metadata/MetaDataCreateIndexService.java | 4 +++- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index d47a917ae2fa5..01783060c0b8a 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -734,7 +734,9 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma public static int calculateNumRoutingShards(int numShards, Version indexVersionCreated) { if (indexVersionCreated.onOrAfter(Version.V_7_0_0_alpha1)) { // only select this automatically for indices that are created on or after 7.0 this will prevent this new behaviour - // until we have a fully upgraded cluster see {@link IndexMetaDataE# + // until we have a fully upgraded cluster. Additionally it will make integratin testing easier since mixed clusters + // will always have the behavior of the min node in the cluster. + // // We use as a default number of routing shards the higher number that can be expressed // as {@code numShards * 2^x`} that is less than or equal to the maximum number of shards: 1024. int log2MaxNumShards = 10; // logBase2(1024) diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index d69818567a3ef..6074102cde313 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -330,7 +330,7 @@ public void testCalculateNumRoutingShards() { assertTrue(1 < ratio); assertTrue(ratio <= 1024); assertEquals(0, intRatio % 2); - assertEquals("ration is not a power of two", intRatio, Integer.highestOneBit(intRatio)); + assertEquals("ratio is not a power of two", intRatio, Integer.highestOneBit(intRatio)); } } } From dccad0bf3eb526102cc992fd5222043ec659451b Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 22 Nov 2017 14:14:02 +0100 Subject: [PATCH 16/17] fix tests for new hashing --- .../metrics/geocentroid-aggregation.asciidoc | 4 ++-- docs/reference/cat/segments.asciidoc | 4 ++-- docs/reference/how-to/recipes/stemming.asciidoc | 8 ++++---- docs/reference/mapping/params/normalizer.asciidoc | 12 ++++++------ docs/reference/query-dsl/percolate-query.asciidoc | 12 ++++++------ docs/reference/search/search-shards.asciidoc | 4 ++-- docs/reference/search/validate.asciidoc | 7 +++---- .../test/lang_mustache/30_search_template.yml | 2 +- 8 files changed, 26 insertions(+), 27 deletions(-) diff --git a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc index 9aafcee56bc90..59cadf1518eba 100644 --- a/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geocentroid-aggregation.asciidoc @@ -61,7 +61,7 @@ The response for the above aggregation: "centroid": { "location": { "lat": 51.00982963806018, - "lon": 3.9662130922079086 + "lon": 3.9662131061777472 }, "count": 6 } @@ -114,7 +114,7 @@ The response for the above aggregation: "centroid": { "location": { "lat": 52.371655656024814, - "lon": 4.909563269466162 + "lon": 4.909563297405839 }, "count": 3 } diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc index 2fa7e309bccf4..88fb18b363745 100644 --- a/docs/reference/cat/segments.asciidoc +++ b/docs/reference/cat/segments.asciidoc @@ -17,8 +17,8 @@ might look like: ["source","txt",subs="attributes,callouts"] -------------------------------------------------- index shard prirep ip segment generation docs.count docs.deleted size size.memory committed searchable version compound -test 2 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true -test1 2 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true +test 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true +test1 4 p 127.0.0.1 _0 0 1 0 3kb 2042 false true {lucene_version} true -------------------------------------------------- // TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ _cat] diff --git a/docs/reference/how-to/recipes/stemming.asciidoc b/docs/reference/how-to/recipes/stemming.asciidoc index c8e0c2a109e60..49d9f8bafa305 100644 --- a/docs/reference/how-to/recipes/stemming.asciidoc +++ b/docs/reference/how-to/recipes/stemming.asciidoc @@ -90,19 +90,19 @@ GET index/_search { "_index": "index", "_type": "type", - "_id": "1", + "_id": "2", "_score": 0.2876821, "_source": { - "body": "Ski resort" + "body": "A pair of skis" } }, { "_index": "index", "_type": "type", - "_id": "2", + "_id": "1", "_score": 0.2876821, "_source": { - "body": "A pair of skis" + "body": "Ski resort" } } ] diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc index 1b3d753e04125..44a27dab7589d 100644 --- a/docs/reference/mapping/params/normalizer.asciidoc +++ b/docs/reference/mapping/params/normalizer.asciidoc @@ -80,24 +80,24 @@ both index and query time. }, "hits": { "total": 2, - "max_score": 0.6931472, + "max_score": 0.2876821, "hits": [ { "_index": "index", "_type": "type", - "_id": "1", - "_score": 0.6931472, + "_id": "2", + "_score": 0.2876821, "_source": { - "foo": "BÀR" + "foo": "bar" } }, { "_index": "index", "_type": "type", - "_id": "2", + "_id": "1", "_score": 0.2876821, "_source": { - "foo": "bar" + "foo": "BÀR" } } ] diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index f5d779340754d..a2c91d697fddb 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -383,18 +383,18 @@ This will yield the following response. { "_index": "my-index", "_type": "doc", - "_id": "4", + "_id": "3", "_score": 0.5753642, "_source": { "query": { "match": { - "message": "lazy dog" + "message": "brown fox" } } }, "highlight": { "message": [ - "The quick brown fox jumps over the lazy dog" <1> + "The quick brown fox jumps over the lazy dog" <1> ] }, "fields" : { @@ -404,18 +404,18 @@ This will yield the following response. { "_index": "my-index", "_type": "doc", - "_id": "3", + "_id": "4", "_score": 0.5753642, "_source": { "query": { "match": { - "message": "brown fox" + "message": "lazy dog" } } }, "highlight": { "message": [ - "The quick brown fox jumps over the lazy dog" <1> + "The quick brown fox jumps over the lazy dog" <1> ] }, "fields" : { diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc index 2acb2f8aa2b2e..1a7c45545769a 100644 --- a/docs/reference/search/search-shards.asciidoc +++ b/docs/reference/search/search-shards.asciidoc @@ -100,7 +100,7 @@ And specifying the same request, this time with a routing value: [source,js] -------------------------------------------------- -GET /twitter/_search_shards?routing=foo,baz +GET /twitter/_search_shards?routing=foo,bar -------------------------------------------------- // CONSOLE // TEST[s/^/PUT twitter\n/] @@ -120,7 +120,7 @@ This will yield the following result: "index": "twitter", "node": "JklnKbD7Tyqi9TP3_Q_tBg", "primary": true, - "shard": 1, + "shard": 2, "state": "STARTED", "allocation_id": {"id":"fMju3hd1QHWmWrIgFnI4Ww"}, "relocating_node": null diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index d330264fd58f0..a9de086cbbb3a 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -239,20 +239,19 @@ Response: "index": "twitter", "shard": 2, "valid": true, - "explanation": "user:kimchy" + "explanation": "user:kimchy~2" }, { "index": "twitter", "shard": 3, "valid": true, - "explanation": "user:kimchy~2" - + "explanation": "(user:kimchi)^0.8333333" }, { "index": "twitter", "shard": 4, "valid": true, - "explanation": "(user:kimchi)^0.8333333" + "explanation": "user:kimchy" } ] } diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml index b3875982b2eb9..53b52cb3b8bc7 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml @@ -113,7 +113,7 @@ - match: { hits.total: 1 } - length: { hits.hits: 1 } - - match: { hits.hits.0._explanation.description: "weight(otherField:foo in 2) [PerFieldSimilarity], result of:" } + - match: { hits.hits.0._explanation.description: "weight(otherField:foo in 1) [PerFieldSimilarity], result of:" } - do: search_template: From 4db1b973795d379c9d8c6476a8e831accea39703 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 23 Nov 2017 08:49:44 +0100 Subject: [PATCH 17/17] add note to migration guide --- docs/reference/migration/migrate_7_0/indices.asciidoc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/reference/migration/migrate_7_0/indices.asciidoc b/docs/reference/migration/migrate_7_0/indices.asciidoc index de16498c84a53..92f56a2ddbb17 100644 --- a/docs/reference/migration/migrate_7_0/indices.asciidoc +++ b/docs/reference/migration/migrate_7_0/indices.asciidoc @@ -36,4 +36,12 @@ To safeguard against creating too many tokens, the difference between `max_shing `min_shingle_size` in `ShingleTokenFilter` has been limited to 3. This default limit can be changed with the index setting `index.max_shingle_diff`. Note that if the limit is exceeded a error is thrown only for new indices. For existing pre-7.0 indices, a deprecation -warning is logged. \ No newline at end of file +warning is logged. + +==== Document distribution changes + +Indices created with version `7.0.0` onwards will have an automatic `index.number_of_routing_shards` +value set. This might change how documents are distributed across shards depending on how many +shards the index has. In order to maintain the exact same distribution as a pre `7.0.0` index, the +`index.number_of_routing_shards` must be set to the `index.number_of_shards` at index creation time. +Note: if the number of routing shards equals the number of shards `_split` operations are not supported. \ No newline at end of file