diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 1ef6534616f18..9572bbb35c7e4 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -24,6 +24,8 @@ === Enhancements +<> ({pull}30255[#30255]) + === Bug Fixes Fail snapshot operations early when creating or deleting a snapshot on a repository that has been diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc index 2dfc2b4f617fa..ac1813a6fba84 100644 --- a/docs/reference/indices/shrink-index.asciidoc +++ b/docs/reference/indices/shrink-index.asciidoc @@ -121,8 +121,13 @@ POST my_source_index/_shrink/my_target_index NOTE: Mappings may not be specified in the `_shrink` request. -NOTE: By default, with the exception of `index.analysis`, `index.similarity`, and `index.sort` settings, index settings on the source -index are not copied during a shrink operation. +NOTE: By default, with the exception of `index.analysis`, `index.similarity`, +and `index.sort` settings, index settings on the source index are not copied +during a shrink operation. With the exception of non-copyable settings, settings +from the source index can be copied to the target index by adding the URL +parameter `copy_settings=true` to the request. + +deprecated[7.0.0, `copy_settings` will default to `true` in 8.x and will be removed in 9.0.0] [float] === Monitoring the shrink process diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index 8285fa4fa448a..094e5e127184f 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -177,8 +177,13 @@ POST my_source_index/_split/my_target_index NOTE: Mappings may not be specified in the `_split` request. -NOTE: By default, with the exception of `index.analysis`, `index.similarity`, and `index.sort` settings, index settings on the source -index are not copied during a shrink operation. +NOTE: By default, with the exception of `index.analysis`, `index.similarity`, +and `index.sort` settings, index settings on the source index are not copied +during a split operation. With the exception of non-copyable settings, settings +from the source index can be copied to the target index by adding the URL +parameter `copy_settings=true` to the request. + +deprecated[7.0.0, `copy_settings` will default to `true` in 8.x and will be removed in 9.0.0] [float] === Monitoring the split process diff --git a/docs/reference/migration/migrate_7_0/api.asciidoc b/docs/reference/migration/migrate_7_0/api.asciidoc index 831245847619e..91adf7160b5a0 100644 --- a/docs/reference/migration/migrate_7_0/api.asciidoc +++ b/docs/reference/migration/migrate_7_0/api.asciidoc @@ -61,8 +61,24 @@ backwards compatibility. Backwards support for the `suggest` metric was deprecated in 6.3.0 and now removed in 7.0.0. [[remove-field-caps-body]] -==== In the fields capabilities API, `fields` can no longer be provided in the request body. In the past, `fields` could be provided either as a parameter, or as part of the request body. Specifying `fields` in the request body as opposed to a parameter was deprecated in 6.4.0, and is now unsupported in 7.0.0. + +[[copy-source-settings-on-resize]] +==== Copying source settings during shrink/split operations + +In prior versions of Elasticsearch, resize operations (shrink/split) would only +copy `index.analysis`, `index.similarity`, and `index.sort` settings from the +source index. Elasticsearch 7.0.0 introduces a request parameter `copy_settings` +which will copy all index settings from the source except for non-copyable index +settings. This parameter defaults to `false` in 7.x, is immediately deprecated +in 7.0.0, will only be able to be set to `true` in 8.x, and will be removed in +9.0.0. Note than when this parameter is used it means that all copyable settings +will be copied; this includes the index blocks that must be put in place for a +resize operation, and any allocation settings put in place in preparation for +executing the resize operation. If you use this parameter, you will either have +to follow up the operation with a request to adjust to the desired settings on +the target index, or send the desired value of these settings with the resize +operation. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json index 5ef943eacba6c..f92421b79ae91 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json @@ -18,6 +18,10 @@ } }, "params": { + "copy_settings": { + "type" : "boolean", + "description" : "whether or not to copy settings from the source index (defaults to false)" + }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json index a79fa7b708269..2c14fced28c36 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json @@ -18,6 +18,10 @@ } }, "params": { + "copy_settings": { + "type" : "boolean", + "description" : "whether or not to copy settings from the source index (defaults to false)" + }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml new file mode 100644 index 0000000000000..d85a4cb1dd707 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml @@ -0,0 +1,94 @@ +--- +"Copy settings during shrink index": + - skip: + version: " - 6.99.99" + reason: copy_settings did not exist prior to 7.0.0 + features: "warnings" + + - do: + cluster.state: {} + + # get master node id + - set: { master_node: master } + + - do: + indices.create: + index: source + wait_for_active_shards: 1 + body: + settings: + # ensure everything is allocated on the master node + index.routing.allocation.include._id: $master + index.number_of_replicas: 0 + index.merge.scheduler.max_merge_count: 4 + + # make it read-only + - do: + indices.put_settings: + index: source + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source + + # now we do a actual shrink and copy settings + - do: + indices.shrink: + index: "source" + target: "copy-settings-target" + wait_for_active_shards: 1 + master_timeout: 10s + copy_settings: true + body: + settings: + index.number_of_replicas: 0 + index.merge.scheduler.max_thread_count: 2 + warnings: + - "parameter [copy_settings] is deprecated but was [true]" + + - do: + cluster.health: + wait_for_status: green + + - do: + indices.get_settings: + index: "copy-settings-target" + + # settings should be copied + - match: { copy-settings-target.settings.index.merge.scheduler.max_merge_count: "4" } + - match: { copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } + - match: { copy-settings-target.settings.index.blocks.write: "true" } + - match: { copy-settings-target.settings.index.routing.allocation.include._id: $master } + + # now we do a actual shrink and do not copy settings + - do: + indices.shrink: + index: "source" + target: "no-copy-settings-target" + wait_for_active_shards: 1 + master_timeout: 10s + copy_settings: false + body: + settings: + index.number_of_replicas: 0 + index.merge.scheduler.max_thread_count: 2 + warnings: + - "parameter [copy_settings] is deprecated but was [false]" + + - do: + cluster.health: + wait_for_status: green + + - do: + indices.get_settings: + index: "no-copy-settings-target" + + # only the request setting should be copied + - is_false: no-copy-settings-target.settings.index.merge.scheduler.max_merge_count + - match: { no-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } + - is_false: no-copy-settings-target.settings.index.blocks.write + - is_false: no-copy-settings-target.settings.index.routing.allocation.include._id diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml new file mode 100644 index 0000000000000..1bb0a52307df2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml @@ -0,0 +1,98 @@ +--- +"Copy settings during split index": + - skip: + version: " - 6.99.99" + reason: copy_settings did not exist prior to 7.0.0 + features: "warnings" + + - do: + cluster.state: {} + + # get master node id + - set: { master_node: master } + + - do: + indices.create: + index: source + wait_for_active_shards: 1 + body: + settings: + # ensure everything is allocated on the master node + index.routing.allocation.include._id: $master + index.number_of_replicas: 0 + index.number_of_shards: 1 + index.number_of_routing_shards: 4 + index.merge.scheduler.max_merge_count: 4 + + # make it read-only + - do: + indices.put_settings: + index: source + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source + + # now we do a actual split and copy settings + - do: + indices.split: + index: "source" + target: "copy-settings-target" + wait_for_active_shards: 1 + master_timeout: 10s + copy_settings: true + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 2 + index.merge.scheduler.max_thread_count: 2 + warnings: + - "parameter [copy_settings] is deprecated but was [true]" + + - do: + cluster.health: + wait_for_status: green + + - do: + indices.get_settings: + index: "copy-settings-target" + + # settings should be copied + - match: { copy-settings-target.settings.index.merge.scheduler.max_merge_count: "4" } + - match: { copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } + - match: { copy-settings-target.settings.index.blocks.write: "true" } + - match: { copy-settings-target.settings.index.routing.allocation.include._id: $master } + + # now we do a actual shrink and do not copy settings + - do: + indices.split: + index: "source" + target: "no-copy-settings-target" + wait_for_active_shards: 1 + master_timeout: 10s + copy_settings: false + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 2 + index.merge.scheduler.max_thread_count: 2 + warnings: + - "parameter [copy_settings] is deprecated but was [false]" + + - do: + cluster.health: + wait_for_status: green + + - do: + indices.get_settings: + index: "no-copy-settings-target" + + # only the request setting should be copied + - is_false: no-copy-settings-target.settings.index.merge.scheduler.max_merge_count + - match: { no-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" } + - is_false: no-copy-settings-target.settings.index.blocks.write + - is_false: no-copy-settings-target.settings.index.routing.allocation.include._id diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index 4e2e257887512..f2e07e29bad6e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -45,6 +45,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private final String providedName; private Index recoverFrom; private ResizeType resizeType; + private boolean copySettings; private IndexMetaData.State state = IndexMetaData.State.OPEN; @@ -112,6 +113,11 @@ public CreateIndexClusterStateUpdateRequest resizeType(ResizeType resizeType) { return this; } + public CreateIndexClusterStateUpdateRequest copySettings(final boolean copySettings) { + this.copySettings = copySettings; + return this; + } + public TransportMessage originalMessage() { return originalMessage; } @@ -170,4 +176,9 @@ public ActiveShardCount waitForActiveShards() { public ResizeType resizeType() { return resizeType; } + + public boolean copySettings() { + return copySettings; + } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index 79600674f4ade..77b3945db09d0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.action.admin.indices.shrink; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -55,6 +56,7 @@ public class ResizeRequest extends AcknowledgedRequest implements private CreateIndexRequest targetIndexRequest; private String sourceIndex; private ResizeType type = ResizeType.SHRINK; + private boolean copySettings = false; ResizeRequest() {} @@ -96,6 +98,11 @@ public void readFrom(StreamInput in) throws IOException { } else { type = ResizeType.SHRINK; // BWC this used to be shrink only } + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + copySettings = in.readBoolean(); + } else { + copySettings = false; + } } @Override @@ -106,6 +113,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(ResizeAction.COMPATIBILITY_VERSION)) { out.writeEnum(type); } + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeBoolean(copySettings); + } } @Override @@ -177,6 +187,14 @@ public ResizeType getResizeType() { return type; } + public void setCopySettings(final boolean copySettings) { + this.copySettings = copySettings; + } + + public boolean getCopySettings() { + return copySettings; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 28fc994a4677e..834ef15ce264d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -178,19 +178,19 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi settingsBuilder.put("index.number_of_shards", numShards); targetIndex.settings(settingsBuilder); - return new CreateIndexClusterStateUpdateRequest(targetIndex, - cause, targetIndex.index(), targetIndexName) - // mappings are updated on the node when creating in the shards, this prevents race-conditions since all mapping must be - // applied once we took the snapshot and if somebody messes things up and switches the index read/write and adds docs we miss - // the mappings for everything is corrupted and hard to debug - .ackTimeout(targetIndex.timeout()) - .masterNodeTimeout(targetIndex.masterNodeTimeout()) - .settings(targetIndex.settings()) - .aliases(targetIndex.aliases()) - .customs(targetIndex.customs()) - .waitForActiveShards(targetIndex.waitForActiveShards()) - .recoverFrom(metaData.getIndex()) - .resizeType(resizeRequest.getResizeType()); + return new CreateIndexClusterStateUpdateRequest(targetIndex, cause, targetIndex.index(), targetIndexName) + // mappings are updated on the node when creating in the shards, this prevents race-conditions since all mapping must be + // applied once we took the snapshot and if somebody messes things up and switches the index read/write and adds docs we + // miss the mappings for everything is corrupted and hard to debug + .ackTimeout(targetIndex.timeout()) + .masterNodeTimeout(targetIndex.masterNodeTimeout()) + .settings(targetIndex.settings()) + .aliases(targetIndex.aliases()) + .customs(targetIndex.customs()) + .waitForActiveShards(targetIndex.waitForActiveShards()) + .recoverFrom(metaData.getIndex()) + .resizeType(resizeRequest.getResizeType()) + .copySettings(resizeRequest.getCopySettings()); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 37c27fb9b7bd7..166aad3ecaa12 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -219,9 +219,19 @@ private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request, Settings build = updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); indexScopedSettings.validate(build, true); // we do validate here - index setting must be consistent request.settings(build); - clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]", - new IndexCreationTask(logger, allocationService, request, listener, indicesService, aliasValidator, xContentRegistry, settings, - this::validate)); + clusterService.submitStateUpdateTask( + "create-index [" + request.index() + "], cause [" + request.cause() + "]", + new IndexCreationTask( + logger, + allocationService, + request, + listener, + indicesService, + aliasValidator, + xContentRegistry, + settings, + this::validate, + indexScopedSettings)); } interface IndexValidator { @@ -238,11 +248,12 @@ static class IndexCreationTask extends AckedClusterStateUpdateTask listener, IndicesService indicesService, AliasValidator aliasValidator, NamedXContentRegistry xContentRegistry, - Settings settings, IndexValidator validator) { + Settings settings, IndexValidator validator, IndexScopedSettings indexScopedSettings) { super(Priority.URGENT, request, listener); this.request = request; this.logger = logger; @@ -252,6 +263,7 @@ static class IndexCreationTask extends AckedClusterStateUpdateTask templates = MetaDataIndexTemplateService.findTemplates(currentState.metaData(), request.index()); + List templates = + MetaDataIndexTemplateService.findTemplates(currentState.metaData(), request.index()); Map customs = new HashMap<>(); @@ -402,7 +415,14 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (recoverFromIndex != null) { assert request.resizeType() != null; prepareResizeIndexSettings( - currentState, mappings.keySet(), indexSettingsBuilder, recoverFromIndex, request.index(), request.resizeType()); + currentState, + mappings.keySet(), + indexSettingsBuilder, + recoverFromIndex, + request.index(), + request.resizeType(), + request.copySettings(), + indexScopedSettings); } final Settings actualIndexSettings = indexSettingsBuilder.build(); tmpImdBuilder.settings(actualIndexSettings); @@ -673,8 +693,15 @@ static IndexMetaData validateResize(ClusterState state, String sourceIndex, return sourceMetaData; } - static void prepareResizeIndexSettings(ClusterState currentState, Set mappingKeys, Settings.Builder indexSettingsBuilder, - Index resizeSourceIndex, String resizeIntoName, ResizeType type) { + static void prepareResizeIndexSettings( + final ClusterState currentState, + final Set mappingKeys, + final Settings.Builder indexSettingsBuilder, + final Index resizeSourceIndex, + final String resizeIntoName, + final ResizeType type, + final boolean copySettings, + final IndexScopedSettings indexScopedSettings) { final IndexMetaData sourceMetaData = currentState.metaData().index(resizeSourceIndex.getName()); if (type == ResizeType.SHRINK) { final List nodesToAllocateOn = validateShrinkIndex(currentState, resizeSourceIndex.getName(), @@ -695,15 +722,33 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma throw new IllegalStateException("unknown resize type is " + type); } - final Predicate sourceSettingsPredicate = - (s) -> (s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort.")) - && indexSettingsBuilder.keys().contains(s) == false; + final Settings.Builder builder = Settings.builder(); + if (copySettings) { + // copy all settings and non-copyable settings and settings that have already been set (e.g., from the request) + for (final String key : sourceMetaData.getSettings().keySet()) { + final Setting setting = indexScopedSettings.get(key); + if (setting == null) { + assert indexScopedSettings.isPrivateSetting(key) : key; + } else if (setting.getProperties().contains(Setting.Property.NotCopyableOnResize)) { + continue; + } + // do not override settings that have already been set (for example, from the request) + if (indexSettingsBuilder.keys().contains(key)) { + continue; + } + builder.copy(key, sourceMetaData.getSettings()); + } + } else { + final Predicate sourceSettingsPredicate = + (s) -> (s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort.")) + && indexSettingsBuilder.keys().contains(s) == false; + builder.put(sourceMetaData.getSettings().filter(sourceSettingsPredicate)); + } + indexSettingsBuilder - // now copy all similarity / analysis / sort settings - this overrides all settings from the user unless they - // wanna add extra settings .put(IndexMetaData.SETTING_VERSION_CREATED, sourceMetaData.getCreationVersion()) .put(IndexMetaData.SETTING_VERSION_UPGRADED, sourceMetaData.getUpgradedVersion()) - .put(sourceMetaData.getSettings().filter(sourceSettingsPredicate)) + .put(builder.build()) .put(IndexMetaData.SETTING_ROUTING_PARTITION_SIZE, sourceMetaData.getRoutingPartitionSize()) .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index bd6bba7b784cd..debd0f59a2ea2 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 9d4ee53aa1aa9..d9e42a67671c6 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -114,7 +114,13 @@ public enum Property { /** * Index scope */ - IndexScope + IndexScope, + + /** + * Mark this setting as not copyable during an index resize (shrink or split). This property can only be applied to settings that + * also have {@link Property#IndexScope}. + */ + NotCopyableOnResize } private final Key key; @@ -142,10 +148,15 @@ private Setting(Key key, @Nullable Setting fallbackSetting, Function propertiesAsSet = EnumSet.copyOf(Arrays.asList(properties)); + if (propertiesAsSet.contains(Property.Dynamic) && propertiesAsSet.contains(Property.Final)) { throw new IllegalArgumentException("final setting [" + key + "] cannot be dynamic"); } + if (propertiesAsSet.contains(Property.NotCopyableOnResize) && propertiesAsSet.contains(Property.IndexScope) == false) { + throw new IllegalArgumentException( + "non-index-scoped setting [" + key + "] can not have property [" + Property.NotCopyableOnResize + "]"); + } + this.properties = propertiesAsSet; } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java index 3d0158cf95f0f..e6c994a85c35d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -46,6 +47,19 @@ public abstract class RestResizeHandler extends BaseRestHandler { public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index")); resizeRequest.setResizeType(getResizeType()); + final String rawCopySettings = request.param("copy_settings"); + final boolean copySettings; + if (rawCopySettings == null) { + copySettings = resizeRequest.getCopySettings(); + } else { + deprecationLogger.deprecated("parameter [copy_settings] is deprecated but was [" + rawCopySettings + "]"); + if (rawCopySettings.length() == 0) { + copySettings = true; + } else { + copySettings = Booleans.parseBoolean(rawCopySettings); + } + } + resizeRequest.setCopySettings(copySettings); request.applyContentParser(resizeRequest::fromXContent); resizeRequest.timeout(request.paramAsTime("timeout", resizeRequest.timeout())); resizeRequest.masterNodeTimeout(request.paramAsTime("master_timeout", resizeRequest.masterNodeTimeout())); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 49df78565d315..ad36457bde505 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -388,8 +389,7 @@ private ClusterState executeTask() throws Exception { setupRequest(); final MetaDataCreateIndexService.IndexCreationTask task = new MetaDataCreateIndexService.IndexCreationTask( logger, allocationService, request, listener, indicesService, aliasValidator, xContentRegistry, clusterStateSettings.build(), - validator - ); + validator, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); return task.execute(state); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 28fbfaefe6d06..d5f3d71d7ee26 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.ClusterName; @@ -34,22 +35,27 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.gateway.TestGatewayAllocator; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; -import static java.util.Collections.min; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; @@ -228,90 +234,146 @@ public void testValidateSplitIndex() { Settings.builder().put("index.number_of_shards", targetShards).build()); } - public void testResizeIndexSettings() { - String indexName = randomAlphaOfLength(10); - List versions = Arrays.asList(VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random()), - VersionUtils.randomVersion(random())); + public void testPrepareResizeIndexSettings() { + final List versions = Arrays.asList(VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random())); versions.sort(Comparator.comparingLong(l -> l.id)); - Version version = versions.get(0); - Version minCompat = versions.get(1); - Version upgraded = versions.get(2); - // create one that won't fail - ClusterState clusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0, - Settings.builder() - .put("index.blocks.write", true) - .put("index.similarity.default.type", "BM25") - .put("index.version.created", version) - .put("index.version.upgraded", upgraded) - .put("index.version.minimum_compatible", minCompat.luceneVersion.toString()) - .put("index.analysis.analyzer.default.tokenizer", "keyword") - .build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) - .build(); - AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, - Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), - new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); - - RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); - // now we start the shard - routingTable = service.applyStartedShards(clusterState, - routingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + final Version version = versions.get(0); + final Version upgraded = versions.get(1); + final Settings indexSettings = + Settings.builder() + .put("index.version.created", version) + .put("index.version.upgraded", upgraded) + .put("index.similarity.default.type", "BM25") + .put("index.analysis.analyzer.default.tokenizer", "keyword") + .build(); + runPrepareResizeIndexSettingsTest( + indexSettings, + Settings.EMPTY, + Collections.emptyList(), + randomBoolean(), + settings -> { + assertThat("similarity settings must be copied", settings.get("index.similarity.default.type"), equalTo("BM25")); + assertThat( + "analysis settings must be copied", + settings.get("index.analysis.analyzer.default.tokenizer"), + equalTo("keyword")); + assertThat(settings.get("index.routing.allocation.initial_recovery._id"), equalTo("node1")); + assertThat(settings.get("index.allocation.max_retries"), equalTo("1")); + assertThat(settings.getAsVersion("index.version.created", null), equalTo(version)); + assertThat(settings.getAsVersion("index.version.upgraded", null), equalTo(upgraded)); + }); + } - { - final Settings.Builder builder = Settings.builder(); - builder.put("index.number_of_shards", 1); - MetaDataCreateIndexService.prepareResizeIndexSettings( - clusterState, - Collections.emptySet(), - builder, - clusterState.metaData().index(indexName).getIndex(), - "target", - ResizeType.SHRINK); - final Settings settings = builder.build(); - assertThat("similarity settings must be copied", settings.get("index.similarity.default.type"), equalTo("BM25")); - assertThat( - "analysis settings must be copied", settings.get("index.analysis.analyzer.default.tokenizer"), equalTo("keyword")); - assertThat(settings.get("index.routing.allocation.initial_recovery._id"), equalTo("node1")); - assertThat(settings.get("index.allocation.max_retries"), equalTo("1")); - assertThat(settings.getAsVersion("index.version.created", null), equalTo(version)); - assertThat(settings.getAsVersion("index.version.upgraded", null), equalTo(upgraded)); - } + public void testPrepareResizeIndexSettingsCopySettings() { + final int maxMergeCount = randomIntBetween(1, 16); + final int maxThreadCount = randomIntBetween(1, 16); + final Setting nonCopyableExistingIndexSetting = + Setting.simpleString("index.non_copyable.existing", Setting.Property.IndexScope, Setting.Property.NotCopyableOnResize); + final Setting nonCopyableRequestIndexSetting = + Setting.simpleString("index.non_copyable.request", Setting.Property.IndexScope, Setting.Property.NotCopyableOnResize); + runPrepareResizeIndexSettingsTest( + Settings.builder() + .put("index.merge.scheduler.max_merge_count", maxMergeCount) + .put("index.non_copyable.existing", "existing") + .build(), + Settings.builder() + .put("index.blocks.write", (String) null) + .put("index.merge.scheduler.max_thread_count", maxThreadCount) + .put("index.non_copyable.request", "request") + .build(), + Arrays.asList(nonCopyableExistingIndexSetting, nonCopyableRequestIndexSetting), + true, + settings -> { + assertNull(settings.getAsBoolean("index.blocks.write", null)); + assertThat(settings.get("index.routing.allocation.require._name"), equalTo("node1")); + assertThat(settings.getAsInt("index.merge.scheduler.max_merge_count", null), equalTo(maxMergeCount)); + assertThat(settings.getAsInt("index.merge.scheduler.max_thread_count", null), equalTo(maxThreadCount)); + assertNull(settings.get("index.non_copyable.existing")); + assertThat(settings.get("index.non_copyable.request"), equalTo("request")); + }); + } + public void testPrepareResizeIndexSettingsAnalysisSettings() { // analysis settings from the request are not overwritten - { - final Settings.Builder builder = Settings.builder(); - builder.put("index.number_of_shards", 1); - builder.put("index.analysis.analyzer.default.tokenizer", "whitespace"); - MetaDataCreateIndexService.prepareResizeIndexSettings( - clusterState, - Collections.emptySet(), - builder, - clusterState.metaData().index(indexName).getIndex(), - "target", - ResizeType.SHRINK); - final Settings settings = builder.build(); - assertThat( - "analysis settings are not overwritten", - settings.get("index.analysis.analyzer.default.tokenizer"), - equalTo("whitespace")); - } + runPrepareResizeIndexSettingsTest( + Settings.EMPTY, + Settings.builder().put("index.analysis.analyzer.default.tokenizer", "whitespace").build(), + Collections.emptyList(), + randomBoolean(), + settings -> + assertThat( + "analysis settings are not overwritten", + settings.get("index.analysis.analyzer.default.tokenizer"), + equalTo("whitespace")) + ); + } + + public void testPrepareResizeIndexSettingsSimilaritySettings() { // similarity settings from the request are not overwritten - { - final Settings.Builder builder = Settings.builder(); - builder.put("index.number_of_shards", 1); - builder.put("index.similarity.default.type", "DFR"); - MetaDataCreateIndexService.prepareResizeIndexSettings( - clusterState, - Collections.emptySet(), - builder, - clusterState.metaData().index(indexName).getIndex(), - "target", - ResizeType.SHRINK); - final Settings settings = builder.build(); - assertThat("similarity settings are not overwritten", settings.get("index.similarity.default.type"), equalTo("DFR")); - } + runPrepareResizeIndexSettingsTest( + Settings.EMPTY, + Settings.builder().put("index.similarity.sim.type", "DFR").build(), + Collections.emptyList(), + randomBoolean(), + settings -> + assertThat("similarity settings are not overwritten", settings.get("index.similarity.sim.type"), equalTo("DFR"))); + + } + + private void runPrepareResizeIndexSettingsTest( + final Settings sourceSettings, + final Settings requestSettings, + final Collection> additionalIndexScopedSettings, + final boolean copySettings, + final Consumer consumer) { + final String indexName = randomAlphaOfLength(10); + + final Settings indexSettings = Settings.builder() + .put("index.blocks.write", true) + .put("index.routing.allocation.require._name", "node1") + .put(sourceSettings) + .build(); + + final ClusterState initialClusterState = + ClusterState + .builder(createClusterState(indexName, randomIntBetween(2, 10), 0, indexSettings)) + .nodes(DiscoveryNodes.builder().add(newNode("node1"))) + .build(); + + final AllocationService service = new AllocationService( + Settings.builder().build(), + new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), + new BalancedShardsAllocator(Settings.EMPTY), + EmptyClusterInfoService.INSTANCE); + + final RoutingTable initialRoutingTable = service.reroute(initialClusterState, "reroute").routingTable(); + final ClusterState routingTableClusterState = ClusterState.builder(initialClusterState).routingTable(initialRoutingTable).build(); + + // now we start the shard + final RoutingTable routingTable = service.applyStartedShards( + routingTableClusterState, + initialRoutingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); + final ClusterState clusterState = ClusterState.builder(routingTableClusterState).routingTable(routingTable).build(); + + final Settings.Builder indexSettingsBuilder = Settings.builder().put("index.number_of_shards", 1).put(requestSettings); + final Set> settingsSet = + Stream.concat( + IndexScopedSettings.BUILT_IN_INDEX_SETTINGS.stream(), + additionalIndexScopedSettings.stream()) + .collect(Collectors.toSet()); + MetaDataCreateIndexService.prepareResizeIndexSettings( + clusterState, + Collections.emptySet(), + indexSettingsBuilder, + clusterState.metaData().index(indexName).getIndex(), + "target", + ResizeType.SHRINK, + copySettings, + new IndexScopedSettings(Settings.EMPTY, settingsSet)); + consumer.accept(indexSettingsBuilder.build()); } private DiscoveryNode newNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java index 187c0e21b4d42..1ab92526e3130 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java @@ -722,12 +722,19 @@ public void testRejectNullProperties() { assertThat(ex.getMessage(), containsString("properties cannot be null for setting")); } - public void testRejectConflictProperties() { + public void testRejectConflictingDynamicAndFinalProperties() { IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> Setting.simpleString("foo.bar", Property.Final, Property.Dynamic)); assertThat(ex.getMessage(), containsString("final setting [foo.bar] cannot be dynamic")); } + public void testRejectNonIndexScopedNotCopyableOnResizeSetting() { + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> Setting.simpleString("foo.bar", Property.NotCopyableOnResize)); + assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [NotCopyableOnResize]"))); + } + public void testTimeValue() { final TimeValue random = TimeValue.parseTimeValue(randomTimeValue(), "test"); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java new file mode 100644 index 0000000000000..75071309458cc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; + +import java.io.IOException; +import java.util.Collections; + +import static org.mockito.Mockito.mock; + +public class RestResizeHandlerTests extends ESTestCase { + + public void testShrinkCopySettingsDeprecated() throws IOException { + final RestResizeHandler.RestShrinkIndexAction handler = + new RestResizeHandler.RestShrinkIndexAction(Settings.EMPTY, mock(RestController.class)); + final String copySettings = randomFrom("true", "false"); + final FakeRestRequest request = + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withParams(Collections.singletonMap("copy_settings", copySettings)) + .withPath("source/_shrink/target") + .build(); + handler.prepareRequest(request, mock(NodeClient.class)); + assertWarnings("parameter [copy_settings] is deprecated but was [" + copySettings + "]"); + } + + public void testSplitCopySettingsDeprecated() throws IOException { + final RestResizeHandler.RestSplitIndexAction handler = + new RestResizeHandler.RestSplitIndexAction(Settings.EMPTY, mock(RestController.class)); + final String copySettings = randomFrom("true", "false"); + final FakeRestRequest request = + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withParams(Collections.singletonMap("copy_settings", copySettings)) + .withPath("source/_split/target") + .build(); + handler.prepareRequest(request, mock(NodeClient.class)); + assertWarnings("parameter [copy_settings] is deprecated but was [" + copySettings + "]"); + } + +}