diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java index 730399481f72b..a96b39357ac1d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java @@ -28,6 +28,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesResponse; import java.io.IOException; @@ -378,4 +380,47 @@ public Cancellable deleteAsync(DeleteSnapshotRequest deleteSnapshotRequest, Requ SnapshotRequestConverters::deleteSnapshot, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } + + /** + * Get a list of features which can be included in a snapshot as feature states. + * See Get Snapshottable + * Features API on elastic.co + * + * @param getFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetSnapshottableFeaturesResponse getFeatures(GetSnapshottableFeaturesRequest getFeaturesRequest, RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity( + getFeaturesRequest, + SnapshotRequestConverters::getSnapshottableFeatures, + options, + GetSnapshottableFeaturesResponse::parse, + emptySet() + ); + } + + /** + * Asynchronously get a list of features which can be included in a snapshot as feature states. + * See Get Snapshottable + * Features API on elastic.co + * + * @param getFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable getFeaturesAsync(GetSnapshottableFeaturesRequest getFeaturesRequest, RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity( + getFeaturesRequest, + SnapshotRequestConverters::getSnapshottableFeatures, + options, + GetSnapshottableFeaturesResponse::parse, + listener, + emptySet() + ); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java index 31383d0c351bc..21dc404036886 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; import org.elasticsearch.common.Strings; import java.io.IOException; @@ -190,4 +191,13 @@ static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) { request.addParameters(parameters.asMap()); return request; } + + static Request getSnapshottableFeatures(GetSnapshottableFeaturesRequest getSnapshottableFeaturesRequest) { + String endpoint = "/_snapshottable_features"; + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + RequestConverters.Params parameters = new RequestConverters.Params(); + parameters.withMasterTimeout(getSnapshottableFeaturesRequest.masterNodeTimeout()); + request.addParameters(parameters.asMap()); + return request; + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java new file mode 100644 index 0000000000000..458c3f5720440 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.snapshots; + +import org.elasticsearch.client.TimedRequest; + +/** + * A {@link TimedRequest} to get the list of features available to be included in snapshots in the cluster. + */ +public class GetSnapshottableFeaturesRequest extends TimedRequest { +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java new file mode 100644 index 0000000000000..049eba6b051b8 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.snapshots; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.List; +import java.util.Objects; + +public class GetSnapshottableFeaturesResponse { + + private final List features; + + private static final ParseField FEATURES = new ParseField("features"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "snapshottable_features_response", true, (a, ctx) -> new GetSnapshottableFeaturesResponse((List) a[0]) + ); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), SnapshottableFeature::parse, FEATURES); + } + + public GetSnapshottableFeaturesResponse(List features) { + this.features = features; + } + + public List getFeatures() { + return features; + } + + public static GetSnapshottableFeaturesResponse parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof GetSnapshottableFeaturesResponse) == false) return false; + GetSnapshottableFeaturesResponse that = (GetSnapshottableFeaturesResponse) o; + return getFeatures().equals(that.getFeatures()); + } + + @Override + public int hashCode() { + return Objects.hash(getFeatures()); + } + + public static class SnapshottableFeature { + + private final String featureName; + private final String description; + + private static final ParseField FEATURE_NAME = new ParseField("name"); + private static final ParseField DESCRIPTION = new ParseField("description"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "feature", true, (a, ctx) -> new SnapshottableFeature((String) a[0], (String) a[1]) + ); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), FEATURE_NAME, ObjectParser.ValueType.STRING); + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), DESCRIPTION, ObjectParser.ValueType.STRING); + } + + public SnapshottableFeature(String featureName, String description) { + this.featureName = featureName; + this.description = description; + } + + public static SnapshottableFeature parse(XContentParser parser, Void ctx) { + return PARSER.apply(parser, ctx); + } + + public String getFeatureName() { + return featureName; + } + + public String getDescription() { + return description; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof SnapshottableFeature) == false) return false; + SnapshottableFeature feature = (SnapshottableFeature) o; + return Objects.equals(getFeatureName(), feature.getFeatureName()); + } + + @Override + public int hashCode() { + return Objects.hash(getFeatureName()); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index fe093c81c4faf..aa7879b43d181 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -28,6 +28,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; +import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; @@ -39,12 +41,16 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; public class SnapshotIT extends ESRestHighLevelClientTestCase { @@ -150,6 +156,14 @@ public void testCreateSnapshot() throws Exception { } request.partial(randomBoolean()); request.includeGlobalState(randomBoolean()); + final List featureStates = randomFrom( + List.of( + Collections.emptyList(), + Collections.singletonList(TASKS_FEATURE_NAME), + Collections.singletonList(NO_FEATURE_STATES_VALUE) + ) + ); + request.featureStates(featureStates); CreateSnapshotResponse response = createTestSnapshot(request); assertEquals(waitForCompletion ? RestStatus.OK : RestStatus.ACCEPTED, response.status()); @@ -262,9 +276,14 @@ public void testRestoreSnapshot() throws IOException { assertFalse("index [" + testIndex + "] should have been deleted", indexExists(testIndex)); RestoreSnapshotRequest request = new RestoreSnapshotRequest(testRepository, testSnapshot); + request.indices(testIndex); request.waitForCompletion(true); request.renamePattern(testIndex); request.renameReplacement(restoredIndex); + if (randomBoolean()) { + request.includeGlobalState(true); + request.featureStates(Collections.singletonList(NO_FEATURE_STATES_VALUE)); + } RestoreSnapshotResponse response = execute(request, highLevelClient().snapshot()::restore, highLevelClient().snapshot()::restoreAsync); @@ -364,6 +383,18 @@ public void testCloneSnapshot() throws IOException { assertTrue(response.isAcknowledged()); } + public void testGetFeatures() throws IOException { + GetSnapshottableFeaturesRequest request = new GetSnapshottableFeaturesRequest(); + + GetSnapshottableFeaturesResponse response = execute(request, + highLevelClient().snapshot()::getFeatures, highLevelClient().snapshot()::getFeaturesAsync); + + assertThat(response, notNullValue()); + assertThat(response.getFeatures(), notNullValue()); + assertThat(response.getFeatures().size(), greaterThan(1)); + assertTrue(response.getFeatures().stream().anyMatch(feature -> "tasks".equals(feature.getFeatureName()))); + } + private static Map randomUserMetadata() { if (randomBoolean()) { return null; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java new file mode 100644 index 0000000000000..0b899af725c7b --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.snapshots; + +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.is; + +public class GetSnapshottableFeaturesResponseTests extends AbstractResponseTestCase< + org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse, + GetSnapshottableFeaturesResponse> { + + @Override + protected org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse createServerTestInstance( + XContentType xContentType + ) { + return new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse( + randomList( + 10, + () -> new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse.SnapshottableFeature( + randomAlphaOfLengthBetween(4, 10), + randomAlphaOfLengthBetween(5, 10) + ) + ) + ); + } + + @Override + protected GetSnapshottableFeaturesResponse doParseToClientInstance(XContentParser parser) throws IOException { + return GetSnapshottableFeaturesResponse.parse(parser); + } + + @Override + protected void assertInstances( + org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse serverTestInstance, + GetSnapshottableFeaturesResponse clientInstance + ) { + assertNotNull(serverTestInstance.getSnapshottableFeatures()); + assertNotNull(serverTestInstance.getSnapshottableFeatures()); + + assertThat(clientInstance.getFeatures(), hasSize(serverTestInstance.getSnapshottableFeatures().size())); + + Map clientFeatures = clientInstance.getFeatures() + .stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); + Map serverFeatures = serverTestInstance.getSnapshottableFeatures() + .stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); + + assertThat(clientFeatures.entrySet(), everyItem(is(in(serverFeatures.entrySet())))); + } +} diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index a0a80cefd35d6..bcad8e399211b 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -99,20 +99,35 @@ argument is provided, the snapshot only includes the specified data streams and + -- (Optional, Boolean) -If `true`, the current cluster state is included in the snapshot. +If `true`, the current global state is included in the snapshot. Defaults to `true`. -The cluster state includes: +The global state includes: * Persistent cluster settings * Index templates * Legacy index templates * Ingest pipelines * {ilm-init} lifecycle policies +* Data stored in system indices, such as Watches and task records (configurable via `feature_states`) -- + IMPORTANT: By default, the entire snapshot will fail if one or more indices included in the snapshot do not have all primary shards available. You can change this behavior by setting <> to `true`. +[[create-snapshot-api-feature-states]] +`feature_states`:: +(Optional, array of strings) +A list of feature states to be included in this snapshot. A list of features +available for inclusion in the snapshot and their descriptions be can be +retrieved using the <>. +Each feature state includes one or more system indices containing data necessary +for the function of that feature. Providing an empty array will include no feature +states in the snapshot, regardless of the value of `include_global_state`. ++ +By default, all available feature states will be included in the snapshot if +`include_global_state` is `true`, or no feature states if `include_global_state` +is `false`. + include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `metadata`:: @@ -163,6 +178,7 @@ The API returns the following response: "version": , "indices": [], "data_streams": [], + "feature_states": [], "include_global_state": false, "metadata": { "taken_by": "user123", diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index 241283a29d4e4..35a9a0e8d4611 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -122,6 +122,15 @@ List of <> included in the snapshot. (Boolean) Indicates whether the current cluster state is included in the snapshot. +[[get-snapshot-api-feature-states]] +`feature_states`:: +(array) +List of feature states which were included when the snapshot was taken, +including the list of system indices included as part of the feature state. The +`feature_name` field of each can be used in the `feature_states` parameter when +restoring the snapshot to restore a subset of feature states. Only present if +the snapshot includes one or more feature states. + `start_time`:: (string) Date timestamp of when the snapshot creation process started. @@ -218,6 +227,7 @@ The API returns the following response: "version": , "indices": [], "data_streams": [], + "feature_states": [], "include_global_state": true, "state": "SUCCESS", "start_time": "2020-07-06T21:55:18.129Z", diff --git a/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc new file mode 100644 index 0000000000000..6515a03936586 --- /dev/null +++ b/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc @@ -0,0 +1,56 @@ +[[get-snapshottable-features-api]] +=== Get Snapshottable Features API +++++ +Get snapshottable features +++++ + +Gets a list of features which can be included in snapshots using the +<> when creating a +snapshot. + +[source,console] +----------------------------------- +GET /_snapshottable_features +----------------------------------- + +[[get-snapshottable-features-api-request]] +==== {api-request-title} + +`GET /_snapshottable_features` + + +[[get-snapshottable-features-api-desc]] +==== {api-description-title} + +You can use the get snapshottable features API to determine which feature states +to include when <>. By default, all +feature states are included in a snapshot if that snapshot includes the global +state, or none if it does not. + +A feature state includes one or more system indices necessary for a given +feature to function. In order to ensure data integrity, all system indices that +comprise a feature state are snapshotted and restored together. + +The features listed by this API are a combination of built-in features and +features defined by plugins. In order for a feature's state to be listed in this +API and recognized as a valid feature state by the create snapshot API, the +plugin which defines that feature must be installed on the master node. + +==== {api-examples-title} + +[source,console-result] +---- +{ + "features": [ + { + "name": "tasks", + "description": "Manages task results" + }, + { + "name": "kibana", + "description": "Manages Kibana configuration and reports" + } + ] +} +---- +// TESTRESPONSE[skip:response differs between default distro and OSS] diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc index 0f348148f07ee..af3be7cea1834 100644 --- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc @@ -129,21 +129,33 @@ indices. + -- (Optional, Boolean) -If `false`, the cluster state is not restored. Defaults to `false`. +If `false`, the global state is not restored. Defaults to `false`. -If `true`, the current cluster state is included in the restore operation. +If `true`, the current global state is included in the restore operation. -The cluster state includes: +The global state includes: * Persistent cluster settings * Index templates * Legacy index templates * Ingest pipelines * {ilm-init} lifecycle policies +* For snapshots taken after 7.12.0, data stored in system indices, such as Watches and task records, replacing any existing configuration (configurable via `feature_states`) -- + IMPORTANT: By default, the entire restore operation will fail if one or more indices included in the snapshot do not have all primary shards available. You can change this behavior by setting <> to `true`. +[[restore-snapshot-api-feature-states]] +`feature_states`:: +(Optional, array of strings) +A comma-separated list of feature states you wish to restore. Each feature state contains one or more system indices. The list of feature states +available in a given snapshot are returned by the <>. Note that feature +states restored this way will completely replace any existing configuration, rather than returning an error if the system index already exists. +Providing an empty array will restore no feature states, regardless of the value of `include_global_state`. ++ +By default, all available feature states will be restored if `include_global_state` is `true`, and no feature states will be restored if +`include_global_state` is `false`. + [[restore-snapshot-api-index-settings]] `index_settings`:: (Optional, string) diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index cf70d3bcb2eab..2691f56fc786d 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -36,3 +36,4 @@ include::get-snapshot-api.asciidoc[] include::get-snapshot-status-api.asciidoc[] include::restore-snapshot-api.asciidoc[] include::delete-snapshot-api.asciidoc[] +include::get-snapshottable-features-api.asciidoc[] diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 907856f53050a..f889fe5053f8b 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -32,6 +32,9 @@ By default, all data streams and indices in the snapshot are restored, but the c supports <>. To include the global cluster state, set `include_global_state` to `true` in the restore request body. +Because all indices in the snapshot are restored by default, all system indices will be restored +by default as well. + [WARNING] ==== Each data stream requires a matching @@ -88,7 +91,7 @@ POST /_snapshot/my_backup/snapshot_1/_restore // TEST[continued] <1> By default, `include_global_state` is `false`, meaning the snapshot's -cluster state is not restored. +cluster state and feature states are not restored. + If `true`, the snapshot's persistent settings, index templates, ingest pipelines, and {ilm-init} policies are restored into the current cluster. This diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index ddc2812dbe280..5723ffde7ec9f 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -77,8 +77,10 @@ The snapshot process starts immediately for the primary shards that have been st relocation or initialization of shards to complete before snapshotting them. Besides creating a copy of each data stream and index, the snapshot process can also store global cluster metadata, which includes persistent -cluster settings and templates. The transient settings and registered snapshot repositories are not stored as part of -the snapshot. +cluster settings, templates, and data stored in system indices, such as Watches and task records, regardless of whether those system +indices are named in the `indices` section of the request. The <> can be used to +select a subset of system indices to be included in the snapshot. The transient settings and registered snapshot repositories are not stored +as part of the snapshot. While a snapshot of a particular shard is being created, this shard cannot be moved to another node, which can interfere with rebalancing and allocation @@ -101,7 +103,7 @@ the snapshot. IMPORTANT: The global cluster state includes the cluster's index templates, such as those <>. If your snapshot includes data streams, we recommend storing the -cluster state as part of the snapshot. This lets you later restored any +global state as part of the snapshot. This lets you later restored any templates required for a data stream. By default, the entire snapshot will fail if one or more indices participating in the snapshot do not have @@ -125,4 +127,4 @@ PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E ----------------------------------- // TEST[continued] -NOTE: You can also create snapshots that are copies of part of an existing snapshot using the <>. \ No newline at end of file +NOTE: You can also create snapshots that are copies of part of an existing snapshot using the <>. diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index 2afb2b058ec53..5dc532efbf1a1 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -64,6 +64,16 @@ public Collection getSystemIndexDescriptors(Settings sett .collect(Collectors.toUnmodifiableList()); } + @Override + public String getFeatureName() { + return "kibana"; + } + + @Override + public String getFeatureDescription() { + return "Manages Kibana configuration and reports"; + } + @Override public List getRestHandlers( Settings settings, diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java index 639b8e93c423d..84fd235d71b1d 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -130,6 +130,16 @@ public Collection getSystemIndexDescriptors(Settings sett return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests")); } + @Override + public String getFeatureName() { + return SystemIndexRestIT.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "test plugin"; + } + public static class AddDocRestHandler extends BaseRestHandler { @Override public boolean allowSystemIndexAccessByDefault() { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_features.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_features.json new file mode 100644 index 0000000000000..76b340d329dd8 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_features.json @@ -0,0 +1,29 @@ +{ + "snapshot.get_features":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html", + "description":"Returns a list of features which can be snapshotted in this cluster." + }, + "stability":"stable", + "visibility":"public", + "headers":{ + "accept": [ "application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_snapshottable_features", + "methods":[ + "GET" + ] + } + ] + }, + "params":{ + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + } + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml new file mode 100644 index 0000000000000..6d0567a72e312 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.features/10_basic.yml @@ -0,0 +1,8 @@ +--- +"Get Features": + - skip: + features: contains + version: " - 7.99.99" # Adjust this after backport + reason: "This API was added in 7.12.0" + - do: { snapshot.get_features: {}} + - contains: {'features': {'name': 'tasks'}} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java index d4979a1f1cbf9..47657f6f336f2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java @@ -84,6 +84,16 @@ public List getActionFilters() { public Collection getSystemIndexDescriptors(Settings settings) { return List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, "System index for [" + getTestClass().getName() + ']')); } + + @Override + public String getFeatureName() { + return ClusterInfoServiceIT.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "test plugin"; + } } public static class BlockingActionFilter extends org.elasticsearch.action.support.ActionFilter.Simple { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index a3ebff40d4d8f..059d0f7c5ea6c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -709,8 +709,8 @@ public ClusterState.Custom randomCreate(String name) { SnapshotsInProgressSerializationTests.randomState(ImmutableOpenMap.of()), Collections.emptyList(), Collections.emptyList(), - Math.abs(randomLong()), - randomIntBetween(0, 1000), + Collections.emptyList(), + Math.abs(randomLong()), randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, SnapshotInfoTests.randomUserMetadata(), diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java index d4be0d3a2e432..dfcd4a90ee174 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexPlugin.java @@ -23,4 +23,14 @@ public class TestSystemIndexPlugin extends Plugin implements SystemIndexPlugin { public Collection getSystemIndexDescriptors(Settings settings) { return List.of(new TestSystemIndexDescriptor()); } + + @Override + public String getFeatureName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return this.getClass().getCanonicalName(); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java new file mode 100644 index 0000000000000..a617b672a57ca --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java @@ -0,0 +1,957 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.snapshots; + +import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.MockLogAppender; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.not; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase { + + public static final String REPO_NAME = "test-repo"; + + private List dataNodes = null; + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(SystemIndexTestPlugin.class); + plugins.add(AnotherSystemIndexTestPlugin.class); + plugins.add(AssociatedIndicesTestPlugin.class); + return plugins; + } + + @Before + public void setup() { + internalCluster().startMasterOnlyNodes(2); + dataNodes = internalCluster().startDataOnlyNodes(2); + } + + /** + * Test that if a snapshot includes system indices and we restore global state, + * with no reference to feature state, the system indices are restored too. + */ + public void testRestoreSystemIndicesAsGlobalState() { + createRepository(REPO_NAME, "fs"); + // put a document in a system index + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + createFullSnapshot(REPO_NAME, "test-snap"); + + // add another document + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore snapshot with global state, without closing the system index + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify only the original document is restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + /** + * If we take a snapshot with includeGlobalState set to false, are system indices included? + */ + public void testSnapshotWithoutGlobalState() { + createRepository(REPO_NAME, "fs"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "system index doc"); + indexDoc("not-a-system-index", "1", "purpose", "non system index doc"); + + // run a snapshot without global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(false) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // check snapshot info for for which + clusterAdmin().prepareGetRepositories(REPO_NAME).get(); + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots(REPO_NAME).get() + .getSnapshots(REPO_NAME).stream() + .map(SnapshotInfo::indices) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + + assertThat("not-a-system-index", in(snapshottedIndices)); + // TODO: without global state the system index shouldn't be snapshotted (8.0 & later only) + // assertThat(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, not(in(snapshottedIndices))); + } + + /** + * Test that we can snapshot feature states by name. + */ + public void testSnapshotByFeature() { + createRepository(REPO_NAME, "fs"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot by feature + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIncludeGlobalState(false) + .setWaitForCompletion(true) + .setFeatureStates(SystemIndexTestPlugin.class.getSimpleName(), AnotherSystemIndexTestPlugin.class.getSimpleName()) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add some other documents + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore indices as global state without closing the index + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify only the original document is restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + /** + * Take a snapshot with global state but don't restore system indexes. By + * default, snapshot restorations ignore global state. This means that, + * for now, the system index is treated as part of the snapshot and must be + * handled explicitly. Otherwise, as in this test, there will be an + * exception. + */ + public void testDefaultRestoreOnlyRegularIndices() { + createRepository(REPO_NAME, "fs"); + final String regularIndex = "test-idx"; + + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // restore indices by feature, with only the regular index named explicitly + SnapshotRestoreException exception = expectThrows(SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .get()); + + assertThat(exception.getMessage(), containsString( + "cannot restore index [" + + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + + "] because an open index with same name already exists")); + } + + /** + * Take a snapshot with global state but restore features by state. + */ + public void testRestoreByFeature() { + createRepository(REPO_NAME, "fs"); + final String regularIndex = "test-idx"; + + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add some other documents + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // Delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // restore indices by feature, with only the regular index named explicitly + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIndices(regularIndex) + .setFeatureStates("SystemIndexTestPlugin") + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify that the restored system index has only one document + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + + // but the non-requested feature should still have its new document + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + } + + /** + * Test that if a feature state has associated indices, they are included in the snapshot + * when that feature state is selected. + */ + public void testSnapshotAndRestoreAssociatedIndices() { + createRepository(REPO_NAME, "fs"); + final String regularIndex = "regular-idx"; + + // put documents into a regular index as well as the system index and associated index of a feature + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME); + + // snapshot + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) + .setWaitForCompletion(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // verify the correctness of the snapshot + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots(REPO_NAME).get() + .getSnapshots(REPO_NAME).stream() + .map(SnapshotInfo::indices) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME)); + assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME)); + + // add some other documents + indexDoc(regularIndex, "2", "purpose", "post-snapshot doc"); + indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(regularIndex), equalTo(2L)); + assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // And delete the associated index so we can restore it + assertAcked(client().admin().indices().prepareDelete(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME).get()); + + // restore the feature state and its associated index + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME) + .setWaitForCompletion(true) + .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName()) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify only the original document is restored + assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + assertThat(getDocCount(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME), equalTo(1L)); + } + + /** + * Check that if we request a feature not in the snapshot, we get an error. + */ + public void testRestoreFeatureNotInSnapshot() { + createRepository(REPO_NAME, "fs"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + final String fakeFeatureStateName = "NonExistentTestPlugin"; + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setFeatureStates("SystemIndexTestPlugin", fakeFeatureStateName) + .get()); + + assertThat(exception.getMessage(), + containsString("requested feature states [[" + fakeFeatureStateName + "]] are not present in snapshot")); + } + + /** + * Check that directly requesting a system index in a restore request logs a deprecation warning. + * @throws IllegalAccessException if something goes wrong with the mock log appender + */ + public void testRestoringSystemIndexByNameIsDeprecated() throws IllegalAccessException { + createRepository(REPO_NAME, "fs"); + // put a document in system index + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Delete the index so we can restore it without requesting the feature state + assertAcked(client().admin().indices().prepareDelete(SystemIndexTestPlugin.SYSTEM_INDEX_NAME).get()); + + // Set up a mock log appender to watch for the log message we expect + MockLogAppender mockLogAppender = new MockLogAppender(); + Loggers.addAppender(LogManager.getLogger("org.elasticsearch.deprecation.snapshots.RestoreService"), mockLogAppender); + mockLogAppender.start(); + mockLogAppender.addExpectation(new MockLogAppender.SeenEventExpectation( + "restore-system-index-from-snapshot", + "org.elasticsearch.deprecation.snapshots.RestoreService", + DeprecationLogger.DEPRECATION, + "Restoring system indices by name is deprecated. Use feature states instead. System indices: [.test-system-idx]")); + + // restore system index by name, rather than feature state + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIndices(SystemIndexTestPlugin.SYSTEM_INDEX_NAME) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // Check that the message was logged and remove log appender + mockLogAppender.assertAllExpectationsMatched(); + mockLogAppender.stop(); + Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.deprecation.snapshots.RestoreService"), mockLogAppender); + + // verify only the original document is restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + /** + * Check that if a system index matches a rename pattern in a restore request, it's not renamed + */ + public void testSystemIndicesCannotBeRenamed() { + createRepository(REPO_NAME, "fs"); + final String nonSystemIndex = ".test-non-system-index"; + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(nonSystemIndex, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + assertAcked(client().admin().indices().prepareDelete(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, nonSystemIndex).get()); + + // Restore using a rename pattern that matches both the regular and the system index + clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .setRenamePattern(".test-(.+)") + .setRenameReplacement(".test-restored-$1") + .get(); + + // The original system index and the renamed normal index should exist + assertTrue("System index not renamed", indexExists(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)); + assertTrue("Non-system index was renamed", indexExists(".test-restored-non-system-index")); + + // The original normal index should still be deleted, and there shouldn't be a renamed version of the system index + assertFalse("Renamed system index doesn't exist", indexExists(".test-restored-system-index")); + assertFalse("Original non-system index doesn't exist", indexExists(nonSystemIndex)); + } + + /** + * If the list of feature states to restore is left unspecified and we are restoring global state, + * all feature states should be restored. + */ + public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() { + createRepository(REPO_NAME, "fs"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add another document + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore indices as global state a null list of feature states + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify that the system index is destroyed + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L)); + } + + /** + * If the list of feature states to restore contains only "none" and we are restoring global state, + * no feature states should be restored. + * + * In this test, we explicitly request a regular index to avoid any confusion over the meaning of + * "all indices." + */ + public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStates() { + createRepository(REPO_NAME, "fs"); + String regularIndex = "my-index"; + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, regularIndex); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // add another document + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + assertAcked(client().admin().indices().prepareDelete(regularIndex).get()); + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + + // restore regular index, with global state and an empty list of feature states + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .setFeatureStates(new String[]{ randomFrom("none", "NONE") }) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // verify that the system index still has the updated document, i.e. has not been restored + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + } + + /** + * If the list of feature states to restore contains only "none" and we are restoring global state, + * no feature states should be restored. However, for backwards compatibility, if no index is + * specified, system indices are included in "all indices." In this edge case, we get an error + * saying that the system index must be closed, because here it is included in "all indices." + */ + public void testRestoreSystemIndicesAsGlobalStateWithEmptyListOfFeatureStatesNoIndicesSpecified() { + createRepository(REPO_NAME, "fs"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // restore indices as global state without closing the index + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .setFeatureStates(new String[]{ randomFrom("none", "NONE") }) + .get()); + + assertThat(exception.getMessage(), containsString("cannot restore index [" + SystemIndexTestPlugin.SYSTEM_INDEX_NAME + + "] because an open index with same name already exists in the cluster.")); + } + + /** + * When a feature state is restored, all indices that are part of that feature state should be deleted, then the indices in + * the snapshot should be restored. + * + * However, other feature states should be unaffected. + */ + public void testAllSystemIndicesAreRemovedWhenThatFeatureStateIsRestored() { + createRepository(REPO_NAME, "fs"); + // Create a system index we'll snapshot and restore + final String systemIndexInSnapshot = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1"; + indexDoc(systemIndexInSnapshot, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "*"); + + // And one we'll snapshot but not restore + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + + // And a regular index so we can avoid matching all indices on the restore + final String regularIndex = "regular-index"; + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Now index another doc and create another index in the same pattern as the first index + final String systemIndexNotInSnapshot = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-2"; + indexDoc(systemIndexInSnapshot, "2", "purpose", "post-snapshot doc"); + indexDoc(systemIndexNotInSnapshot, "1", "purpose", "post-snapshot doc"); + + // Add another doc to the second system index, so we can be sure it hasn't been touched + indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(systemIndexInSnapshot, systemIndexNotInSnapshot, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // Delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // restore the snapshot + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setFeatureStates("SystemIndexTestPlugin") + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // The index we created after the snapshot should be gone + assertFalse(indexExists(systemIndexNotInSnapshot)); + // And the first index should have a single doc + assertThat(getDocCount(systemIndexInSnapshot), equalTo(1L)); + // And the system index whose state we didn't restore shouldn't have been touched and still have 2 docs + assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + } + + public void testSystemIndexAliasesAreAlwaysRestored() { + createRepository(REPO_NAME, "fs"); + // Create a system index + final String systemIndexName = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1"; + indexDoc(systemIndexName, "1", "purpose", "pre-snapshot doc"); + + // And a regular index + // And a regular index so we can avoid matching all indices on the restore + final String regularIndex = "regular-index"; + final String regularAlias = "regular-alias"; + indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc"); + + // And make sure they both have aliases + final String systemIndexAlias = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-alias"; + assertAcked(client().admin().indices().prepareAliases() + .addAlias(systemIndexName, systemIndexAlias) + .addAlias(regularIndex, regularAlias).get()); + + // run a snapshot including global state + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // And delete both the indices + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex, systemIndexName)); + + // Now restore the snapshot with no aliases + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setFeatureStates("SystemIndexTestPlugin") + .setWaitForCompletion(true) + .setRestoreGlobalState(false) + .setIncludeAliases(false) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // The regular index should exist + assertTrue(indexExists(regularIndex)); + assertFalse(indexExists(regularAlias)); + // And the system index, queried by alias, should have a doc + assertTrue(indexExists(systemIndexName)); + assertTrue(indexExists(systemIndexAlias)); + assertThat(getDocCount(systemIndexAlias), equalTo(1L)); + + } + + /** + * Tests that the special "none" feature state name cannot be combined with other + * feature state names, and an error occurs if it's tried. + */ + public void testNoneFeatureStateMustBeAlone() { + createRepository(REPO_NAME, "fs"); + // put a document in a system index + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // run a snapshot including global state + IllegalArgumentException createEx = expectThrows( + IllegalArgumentException.class, + () -> clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(randomBoolean()) + .setFeatureStates("SystemIndexTestPlugin", "none", "AnotherSystemIndexTestPlugin") + .get() + ); + assertThat(createEx.getMessage(), equalTo("the feature_states value [none] indicates that no feature states should be " + + "snapshotted, but other feature states were requested: [SystemIndexTestPlugin, none, AnotherSystemIndexTestPlugin]")); + + // create a successful snapshot with global state/all features + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + SnapshotRestoreException restoreEx = expectThrows( + SnapshotRestoreException.class, + () -> clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setRestoreGlobalState(randomBoolean()) + .setFeatureStates("SystemIndexTestPlugin", "none") + .get() + ); + assertThat( + restoreEx.getMessage(), + allOf( // the order of the requested feature states is non-deterministic so just check that it includes most of the right stuff + containsString( + "the feature_states value [none] indicates that no feature states should be restored, but other feature states were " + + "requested:" + ), + containsString("SystemIndexTestPlugin") + ) + ); + } + + /** + * Tests that using the special "none" feature state value creates a snapshot with no feature states included + */ + public void testNoneFeatureStateOnCreation() { + createRepository(REPO_NAME, "fs"); + final String regularIndex = "test-idx"; + + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .setFeatureStates(randomFrom("none", "NONE")) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Verify that the system index was not included + Set snapshottedIndices = clusterAdmin().prepareGetSnapshots(REPO_NAME).get() + .getSnapshots(REPO_NAME).stream() + .map(SnapshotInfo::indices) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + + assertThat(snapshottedIndices, allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)))); + } + + public void testNoneFeatureStateOnRestore() { + createRepository(REPO_NAME, "fs"); + final String regularIndex = "test-idx"; + + indexDoc(regularIndex, "1", "purpose", "create an index that can be restored"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // Create a snapshot + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setIncludeGlobalState(true) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Index another doc into the system index + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + // And delete the regular index so we can restore it + assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex)); + + // Restore the snapshot specifying the regular index and "none" for feature states + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(regularIndex) + .setWaitForCompletion(true) + .setRestoreGlobalState(randomBoolean()) + .setFeatureStates(randomFrom("none", "NONE")) + .get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + // The regular index should only have one doc + assertThat(getDocCount(regularIndex), equalTo(1L)); + // But the system index shouldn't have been touched + assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L)); + } + + /** + * This test checks a piece of BWC logic, and so should be removed when we block restoring system indices by name. + * + * This test checks whether it's possible to change the name of a system index when it's restored by name (rather than by feature state) + */ + public void testCanRenameSystemIndicesIfRestoredByIndexName() { + createRepository(REPO_NAME, "fs"); + indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc"); + refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME); + + // snapshot including our system index + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, "test-snap") + .setWaitForCompletion(true) + .setIncludeGlobalState(false) + .get(); + assertSnapshotSuccess(createSnapshotResponse); + + // Now restore it with a rename + clusterAdmin().prepareRestoreSnapshot(REPO_NAME, "test-snap") + .setIndices(SystemIndexTestPlugin.SYSTEM_INDEX_NAME) + .setWaitForCompletion(true) + .setRestoreGlobalState(false) + .setFeatureStates(NO_FEATURE_STATES_VALUE) + .setRenamePattern(".test-(.+)") + .setRenameReplacement("restored-$1") + .get(); + + assertTrue("The renamed system index should be present", indexExists("restored-system-idx")); + assertTrue("The original index should still be present", indexExists(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)); + } + + /** + * Ensures that if we can only capture a partial snapshot of a system index, then the feature state associated with that index is + * not included in the snapshot, because it would not be safe to restore that feature state. + */ + public void testPartialSnapshotsOfSystemIndexRemovesFeatureState() throws Exception { + final String partialIndexName = SystemIndexTestPlugin.SYSTEM_INDEX_NAME; + final String fullIndexName = AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME; + + createRepositoryNoVerify(REPO_NAME, "mock"); + + // Creating the index that we'll get a partial snapshot of with a bunch of shards + assertAcked(prepareCreate(partialIndexName, 0, indexSettingsNoReplicas(6))); + indexDoc(partialIndexName, "1", "purpose", "pre-snapshot doc"); + // And another one with the default + indexDoc(fullIndexName, "1", "purpose", "pre-snapshot doc"); + ensureGreen(); + + // Stop a random data node so we lose a shard from the partial index + internalCluster().stopRandomDataNode(); + assertBusy(() -> assertEquals(ClusterHealthStatus.RED, client().admin().cluster().prepareHealth().get().getStatus()), + 30, TimeUnit.SECONDS); + + // Get ready to block + blockMasterFromFinalizingSnapshotOnIndexFile(REPO_NAME); + + // Start a snapshot and wait for it to hit the block, then kill the master to force a failover + final String partialSnapName = "test-partial-snap"; + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(REPO_NAME, partialSnapName) + .setIncludeGlobalState(true) + .setWaitForCompletion(false) + .setPartial(true) + .get(); + assertThat(createSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + waitForBlock(internalCluster().getMasterName(), REPO_NAME); + internalCluster().stopCurrentMasterNode(); + + // Now get the snapshot and do our checks + assertBusy(() -> { + GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() + .prepareGetSnapshots(REPO_NAME).setSnapshots(partialSnapName).get(); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots(REPO_NAME).get(0); + assertNotNull(snapshotInfo); + assertThat(snapshotInfo.failedShards(), lessThan(snapshotInfo.totalShards())); + List statesInSnapshot = snapshotInfo.featureStates().stream() + .map(SnapshotFeatureInfo::getPluginName) + .collect(Collectors.toList()); + assertThat(statesInSnapshot, not(hasItem((new SystemIndexTestPlugin()).getFeatureName()))); + assertThat(statesInSnapshot, hasItem((new AnotherSystemIndexTestPlugin()).getFeatureName())); + }); + } + + public void testParallelIndexDeleteRemovesFeatureState() throws Exception { + final String indexToBeDeleted = SystemIndexTestPlugin.SYSTEM_INDEX_NAME; + final String fullIndexName = AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME; + final String nonsystemIndex = "nonsystem-idx"; + + // Stop one data node so we only have one data node to start with + internalCluster().stopNode(dataNodes.get(1)); + dataNodes.remove(1); + + createRepositoryNoVerify(REPO_NAME, "mock"); + + // Creating the index that we'll get a partial snapshot of with a bunch of shards + assertAcked(prepareCreate(indexToBeDeleted, 0, indexSettingsNoReplicas(6))); + indexDoc(indexToBeDeleted, "1", "purpose", "pre-snapshot doc"); + // And another one with the default + indexDoc(fullIndexName, "1", "purpose", "pre-snapshot doc"); + + // Now start up a new node and create an index that should get allocated to it + dataNodes.add(internalCluster().startDataOnlyNode()); + createIndexWithContent( + nonsystemIndex, + indexSettingsNoReplicas(2).put("index.routing.allocation.require._name", dataNodes.get(1)).build() + ); + refresh(); + ensureGreen(); + + logger.info("--> Created indices, blocking repo on new data node..."); + blockDataNode(REPO_NAME, dataNodes.get(1)); + + // Start a snapshot - need to do this async because some blocks will block this call + logger.info("--> Blocked repo, starting snapshot..."); + final String partialSnapName = "test-partial-snap"; + ActionFuture createSnapshotFuture = clusterAdmin().prepareCreateSnapshot(REPO_NAME, partialSnapName) + .setIndices(nonsystemIndex) + .setIncludeGlobalState(true) + .setWaitForCompletion(true) + .setPartial(true) + .execute(); + + logger.info("--> Started snapshot, waiting for block..."); + waitForBlock(dataNodes.get(1), REPO_NAME); + + logger.info("--> Repo hit block, deleting the index..."); + assertAcked(cluster().client().admin().indices().prepareDelete(indexToBeDeleted)); + + logger.info("--> Index deleted, unblocking repo..."); + unblockNode(REPO_NAME, dataNodes.get(1)); + + logger.info("--> Repo unblocked, checking that snapshot finished..."); + CreateSnapshotResponse createSnapshotResponse = createSnapshotFuture.actionGet(); + logger.info(createSnapshotResponse.toString()); + assertThat(createSnapshotResponse.status(), equalTo(RestStatus.OK)); + + logger.info("--> All operations complete, running assertions"); + SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertNotNull(snapshotInfo); + assertThat(snapshotInfo.indices(), not(hasItem(indexToBeDeleted))); + List statesInSnapshot = snapshotInfo.featureStates().stream() + .map(SnapshotFeatureInfo::getPluginName) + .collect(Collectors.toList()); + assertThat(statesInSnapshot, not(hasItem((new SystemIndexTestPlugin()).getFeatureName()))); + assertThat(statesInSnapshot, hasItem((new AnotherSystemIndexTestPlugin()).getFeatureName())); + } + + private void assertSnapshotSuccess(CreateSnapshotResponse createSnapshotResponse) { + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), + equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + } + + private long getDocCount(String indexName) { + return client().admin().indices().prepareStats(indexName).get().getPrimaries().getDocs().getCount(); + } + + public static class SystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_INDEX_NAME = ".test-system-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME + "*", "System indices for tests")); + } + + @Override + public String getFeatureName() { + return SystemIndexTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "A simple test plugin"; + } + } + + public static class AnotherSystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_INDEX_NAME = ".another-test-system-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System indices for tests")); + } + + @Override + public String getFeatureName() { + return AnotherSystemIndexTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "Another simple test plugin"; + } + } + + public static class AssociatedIndicesTestPlugin extends Plugin implements SystemIndexPlugin { + + public static final String SYSTEM_INDEX_NAME = ".third-test-system-idx"; + public static final String ASSOCIATED_INDEX_NAME = ".associated-idx"; + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return Collections.singletonList(new SystemIndexDescriptor(SYSTEM_INDEX_NAME, "System & associated indices for tests")); + } + + @Override + public Collection getAssociatedIndexPatterns() { + return Collections.singletonList(ASSOCIATED_INDEX_NAME); + } + + @Override + public String getFeatureName() { + return AssociatedIndicesTestPlugin.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "Another simple test plugin"; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index cdb56efe022d9..d4ff96be8db45 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -58,6 +58,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.delete.TransportDeleteSnapshotAction; +import org.elasticsearch.action.admin.cluster.snapshots.features.SnapshottableFeaturesAction; +import org.elasticsearch.action.admin.cluster.snapshots.features.TransportSnapshottableFeaturesAction; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsAction; import org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotAction; @@ -282,6 +284,7 @@ import org.elasticsearch.rest.action.admin.cluster.RestRemoteClusterInfoAction; import org.elasticsearch.rest.action.admin.cluster.RestRestoreSnapshotAction; import org.elasticsearch.rest.action.admin.cluster.RestSnapshotsStatusAction; +import org.elasticsearch.rest.action.admin.cluster.RestSnapshottableFeaturesAction; import org.elasticsearch.rest.action.admin.cluster.RestVerifyRepositoryAction; import org.elasticsearch.rest.action.admin.cluster.dangling.RestDeleteDanglingIndexAction; import org.elasticsearch.rest.action.admin.cluster.dangling.RestImportDanglingIndexAction; @@ -497,6 +500,7 @@ public void reg actions.register(CloneSnapshotAction.INSTANCE, TransportCloneSnapshotAction.class); actions.register(RestoreSnapshotAction.INSTANCE, TransportRestoreSnapshotAction.class); actions.register(SnapshotsStatusAction.INSTANCE, TransportSnapshotsStatusAction.class); + actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class); actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class); actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class); @@ -646,6 +650,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestRestoreSnapshotAction()); registerHandler.accept(new RestDeleteSnapshotAction()); registerHandler.accept(new RestSnapshotsStatusAction()); + registerHandler.accept(new RestSnapshottableFeaturesAction()); registerHandler.accept(new RestGetIndicesAction()); registerHandler.accept(new RestIndicesStatsAction()); registerHandler.accept(new RestIndicesSegmentsAction()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index a86ba4f88bf2c..ceb5111528455 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; import java.util.Arrays; @@ -64,6 +65,8 @@ public class CreateSnapshotRequest extends MasterNodeRequest MAXIMUM_METADATA_BYTES) { validationException = addValidationError("metadata must be smaller than 1024 bytes, but was [" + metadataSize + "]", @@ -337,6 +349,28 @@ public CreateSnapshotRequest userMetadata(Map userMetadata) { return this; } + /** + * @return Which plugin states should be included in the snapshot + */ + public String[] featureStates() { + return featureStates; + } + + /** + * @param featureStates The plugin states to be included in the snapshot + */ + public CreateSnapshotRequest featureStates(String[] featureStates) { + this.featureStates = featureStates; + return this; + } + + /** + * @param featureStates The plugin states to be included in the snapshot + */ + public CreateSnapshotRequest featureStates(List featureStates) { + return featureStates(featureStates.toArray(EMPTY_ARRAY)); + } + /** * Parses snapshot definition. * @@ -355,6 +389,12 @@ public CreateSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } + } else if (name.equals("feature_states")) { + if (entry.getValue() instanceof List) { + featureStates((List) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed feature_states section, should be an array of strings"); + } } else if (name.equals("partial")) { partial(nodeBooleanValue(entry.getValue(), "partial")); } else if (name.equals("include_global_state")) { @@ -380,6 +420,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(index); } builder.endArray(); + if (featureStates != null) { + builder.startArray("feature_states"); + for (String plugin : featureStates) { + builder.value(plugin); + } + builder.endArray(); + } builder.field("partial", partial); builder.field("include_global_state", includeGlobalState); if (indicesOptions != null) { @@ -407,6 +454,7 @@ public boolean equals(Object o) { Objects.equals(repository, that.repository) && Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && + Arrays.equals(featureStates, that.featureStates) && Objects.equals(masterNodeTimeout, that.masterNodeTimeout) && Objects.equals(userMetadata, that.userMetadata); } @@ -416,6 +464,7 @@ public int hashCode() { int result = Objects.hash(snapshot, repository, indicesOptions, partial, includeGlobalState, waitForCompletion, userMetadata); result = 31 * result + Arrays.hashCode(indices); + result = 31 * result + Arrays.hashCode(featureStates); return result; } @@ -426,6 +475,7 @@ public String toString() { ", repository='" + repository + '\'' + ", indices=" + (indices == null ? null : Arrays.asList(indices)) + ", indicesOptions=" + indicesOptions + + ", featureStates=" + Arrays.asList(featureStates) + ", partial=" + partial + ", includeGlobalState=" + includeGlobalState + ", waitForCompletion=" + waitForCompletion + diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java index 2fe7033c85ab2..355060834d8f3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java @@ -111,4 +111,15 @@ public CreateSnapshotRequestBuilder setIncludeGlobalState(boolean includeGlobalS request.includeGlobalState(includeGlobalState); return this; } + + /** + * Provide a list of features whose state indices should be included in the snapshot + * + * @param featureStates A list of feature names + * @return this builder + */ + public CreateSnapshotRequestBuilder setFeatureStates(String... featureStates) { + request.featureStates(featureStates); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java new file mode 100644 index 0000000000000..545f5c7fbdd7a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesRequest.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.features; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class GetSnapshottableFeaturesRequest extends MasterNodeRequest { + + public GetSnapshottableFeaturesRequest() { + + } + + public GetSnapshottableFeaturesRequest(StreamInput in) throws IOException { + super(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java new file mode 100644 index 0000000000000..a2048bab29c58 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponse.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.features; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class GetSnapshottableFeaturesResponse extends ActionResponse implements ToXContentObject { + + private final List snapshottableFeatures; + + public GetSnapshottableFeaturesResponse(List features) { + this.snapshottableFeatures = Collections.unmodifiableList(features); + } + + public GetSnapshottableFeaturesResponse(StreamInput in) throws IOException { + super(in); + snapshottableFeatures = in.readList(SnapshottableFeature::new); + } + + public List getSnapshottableFeatures() { + return snapshottableFeatures; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(snapshottableFeatures); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.startArray("features"); + for (SnapshottableFeature feature : snapshottableFeatures) { + builder.value(feature); + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof GetSnapshottableFeaturesResponse) == false) return false; + GetSnapshottableFeaturesResponse that = (GetSnapshottableFeaturesResponse) o; + return snapshottableFeatures.equals(that.snapshottableFeatures); + } + + @Override + public int hashCode() { + return Objects.hash(snapshottableFeatures); + } + + public static class SnapshottableFeature implements Writeable, ToXContentObject { + + private final String featureName; + private final String description; + + public SnapshottableFeature(String featureName, String description) { + this.featureName = featureName; + this.description = description; + } + + public SnapshottableFeature(StreamInput in) throws IOException { + featureName = in.readString(); + description = in.readString(); + } + + public String getFeatureName() { + return featureName; + } + + public String getDescription() { + return description; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(featureName); + out.writeString(description); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", featureName); + builder.field("description", description); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof SnapshottableFeature) == false) return false; + SnapshottableFeature feature = (SnapshottableFeature) o; + return Objects.equals(getFeatureName(), feature.getFeatureName()); + } + + @Override + public int hashCode() { + return Objects.hash(getFeatureName()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java new file mode 100644 index 0000000000000..38bf7afd6b505 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/SnapshottableFeaturesAction.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.features; + +import org.elasticsearch.action.ActionType; + +public class SnapshottableFeaturesAction extends ActionType { + + public static final SnapshottableFeaturesAction INSTANCE = new SnapshottableFeaturesAction(); + public static final String NAME = "cluster:admin/snapshot/features/get"; + + private SnapshottableFeaturesAction() { + super(NAME, GetSnapshottableFeaturesResponse::new); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java new file mode 100644 index 0000000000000..62f32bc460f0e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportSnapshottableFeaturesAction.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.features; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.stream.Collectors; + +public class TransportSnapshottableFeaturesAction extends TransportMasterNodeAction { + + private final SystemIndices systemIndices; + + @Inject + public TransportSnapshottableFeaturesAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + SystemIndices systemIndices) { + super(SnapshottableFeaturesAction.NAME, transportService, clusterService, threadPool, actionFilters, + GetSnapshottableFeaturesRequest::new, indexNameExpressionResolver, GetSnapshottableFeaturesResponse::new, + ThreadPool.Names.SAME); + this.systemIndices = systemIndices; + } + + @Override + protected void masterOperation(Task task, GetSnapshottableFeaturesRequest request, ClusterState state, + ActionListener listener) throws Exception { + listener.onResponse(new GetSnapshottableFeaturesResponse(systemIndices.getFeatures().entrySet().stream() + .map(featureEntry -> new GetSnapshottableFeaturesResponse.SnapshottableFeature( + featureEntry.getKey(), + featureEntry.getValue().getDescription())) + .collect(Collectors.toList()))); + } + + @Override + protected ClusterBlockException checkBlock(GetSnapshottableFeaturesRequest request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index fedccc0854450..ea62ee86a9fb1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -279,7 +279,8 @@ private static List buildSimpleSnapshotInfos(final Set for (SnapshotId snapshotId : toResolve) { final List indices = snapshotsToIndices.getOrDefault(snapshotId, Collections.emptyList()); CollectionUtil.timSort(indices); - snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), repositoryData.getSnapshotState(snapshotId))); + snapshotInfos.add(new SnapshotInfo(snapshotId, indices, Collections.emptyList(), Collections.emptyList(), + repositoryData.getSnapshotState(snapshotId))); } CollectionUtil.timSort(snapshotInfos); return Collections.unmodifiableList(snapshotInfos); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index f2d412d7baab9..0498472dd8d91 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -28,6 +28,7 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.snapshots.SnapshotsService.FEATURE_STATES_VERSION; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; @@ -42,6 +43,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest featureStates) { + return featureStates(featureStates.toArray(Strings.EMPTY_ARRAY)); + } + /** * Parses restore definition * @@ -466,6 +500,12 @@ public RestoreSnapshotRequest source(Map source) { } else { throw new IllegalArgumentException("malformed indices section, should be an array of strings"); } + } else if (name.equals("feature_states")) { + if (entry.getValue() instanceof List) { + featureStates((List) entry.getValue()); + } else { + throw new IllegalArgumentException("malformed feature_states section, should be an array of strings"); + } } else if (name.equals("partial")) { partial(nodeBooleanValue(entry.getValue(), "partial")); } else if (name.equals("include_global_state")) { @@ -530,6 +570,13 @@ private void toXContentFragment(XContentBuilder builder, Params params) throws I if (renameReplacement != null) { builder.field("rename_replacement", renameReplacement); } + if (featureStates != null && featureStates.length > 0) { + builder.startArray("feature_states"); + for (String plugin : featureStates) { + builder.value(plugin); + } + builder.endArray(); + } builder.field("include_global_state", includeGlobalState); builder.field("partial", partial); builder.field("include_aliases", includeAliases); @@ -565,6 +612,7 @@ public boolean equals(Object o) { Objects.equals(repository, that.repository) && Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && + Arrays.equals(featureStates, that.featureStates) && Objects.equals(renamePattern, that.renamePattern) && Objects.equals(renameReplacement, that.renameReplacement) && Objects.equals(indexSettings, that.indexSettings) && @@ -579,6 +627,7 @@ public int hashCode() { includeGlobalState, partial, includeAliases, indexSettings, snapshotUuid, skipOperatorOnlyState); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(ignoreIndexSettings); + result = 31 * result + Arrays.hashCode(featureStates); return result; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java index b3dd357f12ae0..ceab46e73fc63 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java @@ -223,4 +223,12 @@ public RestoreSnapshotRequestBuilder setIgnoreIndexSettings(List ignoreI request.ignoreIndexSettings(ignoreIndexSettings); return this; } + + /** + * Sets the list of features whose states should be restored as part of this snapshot + */ + public RestoreSnapshotRequestBuilder setFeatureStates(String... featureStates) { + request.featureStates(featureStates); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 090b6797fe500..54fb1d20a80a0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -95,6 +95,7 @@ public class ClusterModule extends AbstractModule { private final AllocationDeciders allocationDeciders; private final AllocationService allocationService; private final List clusterPlugins; + private final MetadataDeleteIndexService metadataDeleteIndexService; // pkg private for tests final Collection deciderList; final ShardsAllocator shardsAllocator; @@ -108,6 +109,7 @@ public ClusterModule(Settings settings, ClusterService clusterService, List getNamedWriteables() { @@ -248,13 +250,17 @@ public AllocationService getAllocationService() { return allocationService; } + public MetadataDeleteIndexService getMetadataDeleteIndexService() { + return metadataDeleteIndexService; + } + @Override protected void configure() { bind(GatewayAllocator.class).asEagerSingleton(); bind(AllocationService.class).toInstance(allocationService); bind(ClusterService.class).toInstance(clusterService); bind(NodeConnectionsService.class).asEagerSingleton(); - bind(MetadataDeleteIndexService.class).asEagerSingleton(); + bind(MetadataDeleteIndexService.class).toInstance(metadataDeleteIndexService); bind(MetadataIndexStateService.class).asEagerSingleton(); bind(MetadataMappingService.class).asEagerSingleton(); bind(MetadataIndexAliasesService.class).asEagerSingleton(); diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 3bfb26e1aae13..12906d755666c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -25,10 +25,11 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.repositories.IndexId; -import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.repositories.RepositoryOperation; +import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.snapshots.InFlightShardSnapshotStates; import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotFeatureInfo; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotsService; @@ -42,6 +43,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.snapshots.SnapshotsService.FEATURE_STATES_VERSION; + /** * Meta data about snapshots that are currently executing */ @@ -82,12 +85,11 @@ public String toString() { * will be in state {@link State#SUCCESS} right away otherwise it will be in state {@link State#STARTED}. */ public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, boolean partial, List indices, - List dataStreams, long startTime, long repositoryStateId, - ImmutableOpenMap shards, Map userMetadata, - Version version) { + List dataStreams, long startTime, long repositoryStateId, ImmutableOpenMap shards, + Map userMetadata, Version version, List featureStates) { return new SnapshotsInProgress.Entry(snapshot, includeGlobalState, partial, completed(shards.values()) ? State.SUCCESS : State.STARTED, - indices, dataStreams, startTime, repositoryStateId, shards, null, userMetadata, version); + indices, dataStreams, featureStates, startTime, repositoryStateId, shards, null, userMetadata, version); } /** @@ -104,8 +106,8 @@ public static Entry startedEntry(Snapshot snapshot, boolean includeGlobalState, public static Entry startClone(Snapshot snapshot, SnapshotId source, List indices, long startTime, long repositoryStateId, Version version) { return new SnapshotsInProgress.Entry(snapshot, true, false, State.STARTED, indices, Collections.emptyList(), - startTime, repositoryStateId, ImmutableOpenMap.of(), null, Collections.emptyMap(), version, source, - ImmutableOpenMap.of()); + Collections.emptyList(), startTime, repositoryStateId, ImmutableOpenMap.of(), null, Collections.emptyMap(), version, source, + ImmutableOpenMap.of()); } public static class Entry implements Writeable, ToXContent, RepositoryOperation { @@ -119,6 +121,7 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation private final ImmutableOpenMap shards; private final List indices; private final List dataStreams; + private final List featureStates; private final long startTime; private final long repositoryStateId; // see #useShardGenerations @@ -141,24 +144,25 @@ public static class Entry implements Writeable, ToXContent, RepositoryOperation // visible for testing, use #startedEntry and copy constructors in production code public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - List dataStreams, long startTime, long repositoryStateId, + List dataStreams, List featureStates, long startTime, long repositoryStateId, ImmutableOpenMap shards, String failure, Map userMetadata, Version version) { - this(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, repositoryStateId, shards, failure, - userMetadata, version, null, ImmutableOpenMap.of()); + this(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, startTime, repositoryStateId, shards, + failure, userMetadata, version, null, ImmutableOpenMap.of()); } private Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - List dataStreams, long startTime, long repositoryStateId, - ImmutableOpenMap shards, String failure, Map userMetadata, - Version version, @Nullable SnapshotId source, - @Nullable ImmutableOpenMap clones) { + List dataStreams, List featureStates, long startTime, long repositoryStateId, + ImmutableOpenMap shards, String failure, Map userMetadata, + Version version, @Nullable SnapshotId source, + @Nullable ImmutableOpenMap clones) { this.state = state; this.snapshot = snapshot; this.includeGlobalState = includeGlobalState; this.partial = partial; this.indices = indices; this.dataStreams = dataStreams; + this.featureStates = Collections.unmodifiableList(featureStates); this.startTime = startTime; this.shards = shards; this.repositoryStateId = repositoryStateId; @@ -195,6 +199,11 @@ private Entry(StreamInput in) throws IOException { source = null; clones = ImmutableOpenMap.of(); } + if (in.getVersion().onOrAfter(FEATURE_STATES_VERSION)) { + featureStates = Collections.unmodifiableList(in.readList(SnapshotFeatureInfo::new)); + } else { + featureStates = Collections.emptyList(); + } } private static boolean assertShardsConsistent(SnapshotId source, State state, List indices, @@ -229,8 +238,8 @@ assert hasFailures(clones) == false || state == State.FAILED public Entry withRepoGen(long newRepoGen) { assert newRepoGen > repositoryStateId : "Updated repository generation [" + newRepoGen + "] must be higher than current generation [" + repositoryStateId + "]"; - return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, newRepoGen, shards, failure, - userMetadata, version, source, clones); + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, startTime, newRepoGen, + shards, failure, userMetadata, version, source, clones); } public Entry withClones(ImmutableOpenMap updatedClones) { @@ -239,8 +248,8 @@ public Entry withClones(ImmutableOpenMap } return new Entry(snapshot, includeGlobalState, partial, completed(updatedClones.values()) ? (hasFailures(updatedClones) ? State.FAILED : State.SUCCESS) : - state, indices, dataStreams, startTime, repositoryStateId, shards, failure, userMetadata, version, source, - updatedClones); + state, indices, dataStreams, featureStates, startTime, repositoryStateId, shards, failure, userMetadata, + version, source, updatedClones); } /** @@ -276,8 +285,8 @@ public Entry abort() { } public Entry fail(ImmutableOpenMap shards, State state, String failure) { - return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, repositoryStateId, shards, - failure, userMetadata, version, source, clones); + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, featureStates, startTime, + repositoryStateId, shards, failure, userMetadata, version, source, clones); } /** @@ -290,8 +299,8 @@ public Entry fail(ImmutableOpenMap shards, State s */ public Entry withShardStates(ImmutableOpenMap shards) { if (completed(shards.values())) { - return new Entry(snapshot, includeGlobalState, partial, State.SUCCESS, indices, dataStreams, startTime, repositoryStateId, - shards, failure, userMetadata, version); + return new Entry(snapshot, includeGlobalState, partial, State.SUCCESS, indices, dataStreams, featureStates, + startTime, repositoryStateId, shards, failure, userMetadata, version); } return withStartedShards(shards); } @@ -302,7 +311,7 @@ public Entry withShardStates(ImmutableOpenMap shar */ public Entry withStartedShards(ImmutableOpenMap shards) { final SnapshotsInProgress.Entry updated = new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, - startTime, repositoryStateId, shards, failure, userMetadata, version); + featureStates, startTime, repositoryStateId, shards, failure, userMetadata, version); assert updated.state().completed() == false && completed(updated.shards().values()) == false : "Only running snapshots allowed but saw [" + updated + "]"; return updated; @@ -349,6 +358,10 @@ public List dataStreams() { return dataStreams; } + public List featureStates() { + return featureStates; + } + @Override public long repositoryStateId() { return repositoryStateId; @@ -399,6 +412,7 @@ public boolean equals(Object o) { if (version.equals(entry.version) == false) return false; if (Objects.equals(source, ((Entry) o).source) == false) return false; if (clones.equals(((Entry) o).clones) == false) return false; + if (featureStates.equals(entry.featureStates) == false) return false; return true; } @@ -419,6 +433,7 @@ public int hashCode() { result = 31 * result + version.hashCode(); result = 31 * result + (source == null ? 0 : source.hashCode()); result = 31 * result + clones.hashCode(); + result = 31 * result + featureStates.hashCode(); return result; } @@ -461,6 +476,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endArray(); + builder.startArray(FEATURE_STATES); + { + for (SnapshotFeatureInfo featureState : featureStates) { + featureState.toXContent(builder, params); + } + } + builder.endArray(); if (isClone()) { builder.field(SOURCE, source); builder.startArray(CLONES); @@ -503,6 +525,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(source); out.writeMap(clones); } + if (out.getVersion().onOrAfter(FEATURE_STATES_VERSION)) { + out.writeList(featureStates); + } } @Override @@ -804,6 +829,7 @@ public void writeTo(StreamOutput out) throws IOException { private static final String INDEX = "index"; private static final String SHARD = "shard"; private static final String NODE = "node"; + private static final String FEATURE_STATES = "feature_states"; @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 6bc2cf1d5d438..1a40e915e39c2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -125,6 +125,11 @@ public String[] concreteIndexNames(ClusterState state, IndicesOptions options, I return concreteIndexNames(context, request.indices()); } + public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, IndicesOptions options, String... indexExpressions) { + Context context = new Context(state, options, true); + return concreteIndexNames(context, indexExpressions); + } + public List dataStreamNames(ClusterState state, IndicesOptions options, String... indexExpressions) { // Allow system index access - they'll be filtered out below as there's no such thing (yet) as system data streams Context context = new Context(state, options, false, false, true, true, true); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 3fddd19f7d158..4f79f8a5a212e 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -866,19 +866,20 @@ public void afterIndexShardClosed(ShardId shardId, IndexShard indexShard, Settin * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)} */ @Override - public void deleteUnassignedIndex(String reason, IndexMetadata metadata, ClusterState clusterState) { + public void deleteUnassignedIndex(String reason, IndexMetadata oldIndexMetadata, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { - String indexName = metadata.getIndex().getName(); + Index index = oldIndexMetadata.getIndex(); try { - if (clusterState.metadata().hasIndex(indexName)) { - final IndexMetadata index = clusterState.metadata().index(indexName); - throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of " + - "the cluster state [" + index.getIndexUUID() + "] [" + metadata.getIndexUUID() + "]"); + if (clusterState.metadata().hasIndex(index)) { + final IndexMetadata currentMetadata = clusterState.metadata().index(index); + throw new IllegalStateException("Can't delete unassigned index store for [" + index.getName() + "] - it's still part " + + "of the cluster state [" + currentMetadata.getIndexUUID() + "] [" + + oldIndexMetadata.getIndexUUID() + "]"); } - deleteIndexStore(reason, metadata); + deleteIndexStore(reason, oldIndexMetadata); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", - metadata.getIndex(), reason), e); + oldIndexMetadata.getIndex(), reason), e); } } } diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 728d110a59899..d880416868ba0 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -14,10 +14,14 @@ import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Objects; @@ -188,6 +192,26 @@ public boolean matchesIndexPattern(String index) { return indexPatternAutomaton.run(index); } + /** + * Retrieves a list of all indices which match this descriptor's pattern. + * + * This cannot be done via {@link org.elasticsearch.cluster.metadata.IndexNameExpressionResolver} because that class can only handle + * simple wildcard expressions, but system index name patterns may use full Lucene regular expression syntax, + * + * @param metadata The current metadata to get the list of matching indices from + * @return A list of index names that match this descriptor + */ + public List getMatchingIndices(Metadata metadata) { + ArrayList matchingIndices = new ArrayList<>(); + metadata.indices().keysIt().forEachRemaining(indexName -> { + if (matchesIndexPattern(indexName)) { + matchingIndices.add(indexName); + } + }); + + return Collections.unmodifiableList(matchingIndices); + } + /** * @return A short description of the purpose of this system index. */ diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index cd97756b16429..9eda85e963db5 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -16,9 +16,10 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.index.Index; -import org.elasticsearch.tasks.TaskResultsService; +import org.elasticsearch.snapshots.SnapshotsService; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -28,6 +29,7 @@ import static java.util.stream.Collectors.toUnmodifiableList; import static org.elasticsearch.tasks.TaskResultsService.TASKS_DESCRIPTOR; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; /** * This class holds the {@link SystemIndexDescriptor} objects that represent system indices the @@ -35,19 +37,18 @@ * to reduce the locations within the code that need to deal with {@link SystemIndexDescriptor}s. */ public class SystemIndices { - private static final Map> SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of( - TaskResultsService.class.getName(), List.of(TASKS_DESCRIPTOR) + private static final Map SERVER_SYSTEM_INDEX_DESCRIPTORS = Map.of( + TASKS_FEATURE_NAME, new Feature("Manages task results", List.of(TASKS_DESCRIPTOR)) ); private final CharacterRunAutomaton runAutomaton; - private final Collection systemIndexDescriptors; - - public SystemIndices(Map> pluginAndModulesDescriptors) { - final Map> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); - checkForOverlappingPatterns(descriptorsMap); - this.systemIndexDescriptors = descriptorsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toUnmodifiableList()); - checkForDuplicateAliases(this.systemIndexDescriptors); - this.runAutomaton = buildCharacterRunAutomaton(systemIndexDescriptors); + private final Map featureDescriptors; + + public SystemIndices(Map pluginAndModulesDescriptors) { + featureDescriptors = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); + checkForOverlappingPatterns(featureDescriptors); + checkForDuplicateAliases(this.getSystemIndexDescriptors()); + this.runAutomaton = buildCharacterRunAutomaton(featureDescriptors); } private void checkForDuplicateAliases(Collection descriptors) { @@ -97,7 +98,8 @@ public boolean isSystemIndex(String indexName) { * @throws IllegalStateException if multiple descriptors match the name */ public @Nullable SystemIndexDescriptor findMatchingDescriptor(String name) { - final List matchingDescriptors = systemIndexDescriptors.stream() + final List matchingDescriptors = featureDescriptors.values().stream() + .flatMap(feature -> feature.getIndexDescriptors().stream()) .filter(descriptor -> descriptor.matchesIndexPattern(name)) .collect(toUnmodifiableList()); @@ -120,8 +122,13 @@ public boolean isSystemIndex(String indexName) { } } - private static CharacterRunAutomaton buildCharacterRunAutomaton(Collection descriptors) { - Optional automaton = descriptors.stream() + public Map getFeatures() { + return featureDescriptors; + } + + private static CharacterRunAutomaton buildCharacterRunAutomaton(Map descriptors) { + Optional automaton = descriptors.values().stream() + .flatMap(feature -> feature.getIndexDescriptors().stream()) .map(descriptor -> SystemIndexDescriptor.buildAutomaton(descriptor.getIndexPattern(), descriptor.getAliasName())) .reduce(Operations::union); return new CharacterRunAutomaton(MinimizationOperations.minimize(automaton.orElse(Automata.makeEmpty()), Integer.MAX_VALUE)); @@ -134,9 +141,9 @@ private static CharacterRunAutomaton buildCharacterRunAutomaton(Collection> sourceToDescriptors) { + static void checkForOverlappingPatterns(Map sourceToDescriptors) { List> sourceDescriptorPair = sourceToDescriptors.entrySet().stream() - .flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) + .flatMap(entry -> entry.getValue().getIndexDescriptors().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message .collect(Collectors.toUnmodifiableList()); @@ -165,14 +172,12 @@ private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false; } - private static Map> buildSystemIndexDescriptorMap( - Map> pluginAndModulesMap) { - final Map> map = - new HashMap<>(pluginAndModulesMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size()); - map.putAll(pluginAndModulesMap); + private static Map buildSystemIndexDescriptorMap(Map featuresMap) { + final Map map = new HashMap<>(featuresMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size()); + map.putAll(featuresMap); // put the server items last since we expect less of them - SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, descriptors) -> { - if (map.putIfAbsent(source, descriptors) != null) { + SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, feature) -> { + if (map.putIfAbsent(source, feature) != null) { throw new IllegalArgumentException("plugin or module attempted to define the same source [" + source + "] as a built-in system index"); } @@ -181,6 +186,43 @@ private static Map> buildSystemIndexDe } Collection getSystemIndexDescriptors() { - return this.systemIndexDescriptors; + return this.featureDescriptors.values().stream() + .flatMap(f -> f.getIndexDescriptors().stream()) + .collect(Collectors.toList()); + } + + public static void validateFeatureName(String name, String plugin) { + if (SnapshotsService.NO_FEATURE_STATES_VALUE.equalsIgnoreCase(name)) { + throw new IllegalArgumentException("feature name cannot be reserved name [\"" + SnapshotsService.NO_FEATURE_STATES_VALUE + + "\"], but was for plugin [" + plugin + "]"); + } + } + + public static class Feature { + private final String description; + private final Collection indexDescriptors; + private final Collection associatedIndexPatterns; + + public Feature(String description, Collection indexDescriptors, Collection associatedIndexPatterns) { + this.description = description; + this.indexDescriptors = indexDescriptors; + this.associatedIndexPatterns = associatedIndexPatterns; + } + + public Feature(String description, Collection indexDescriptors) { + this(description, indexDescriptors, Collections.emptyList()); + } + + public String getDescription() { + return description; + } + + public Collection getIndexDescriptors() { + return indexDescriptors; + } + + public Collection getAssociatedIndexPatterns() { + return associatedIndexPatterns; + } } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index f0ecdf761a8d6..cb51b0d7ccae8 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -99,7 +99,6 @@ import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.ShardLimitValidator; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexManager; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -496,13 +495,19 @@ protected Node(final Environment initialEnvironment, .flatMap(m -> m.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final Map> systemIndexDescriptorMap = pluginsService + final Map featuresMap = pluginsService .filterPlugins(SystemIndexPlugin.class) .stream() + .peek(plugin -> SystemIndices.validateFeatureName(plugin.getFeatureName(), plugin.getClass().getCanonicalName())) .collect(Collectors.toUnmodifiableMap( - plugin -> plugin.getClass().getSimpleName(), - plugin -> plugin.getSystemIndexDescriptors(settings))); - final SystemIndices systemIndices = new SystemIndices(systemIndexDescriptorMap); + plugin -> plugin.getFeatureName(), + plugin -> new SystemIndices.Feature( + plugin.getFeatureDescription(), + plugin.getSystemIndexDescriptors(settings), + plugin.getAssociatedIndexPatterns() + )) + ); + final SystemIndices systemIndices = new SystemIndices(featuresMap); final SystemIndexManager systemIndexManager = new SystemIndexManager(systemIndices, client); clusterService.addListener(systemIndexManager); @@ -592,11 +597,13 @@ protected Node(final Environment initialEnvironment, RepositoriesService repositoryService = repositoriesModule.getRepositoryService(); repositoriesServiceReference.set(repositoryService); SnapshotsService snapshotsService = new SnapshotsService(settings, clusterService, - clusterModule.getIndexNameExpressionResolver(), repositoryService, transportService, actionModule.getActionFilters()); + clusterModule.getIndexNameExpressionResolver(), repositoryService, transportService, actionModule.getActionFilters(), + systemIndices.getFeatures()); SnapshotShardsService snapshotShardsService = new SnapshotShardsService(settings, clusterService, repositoryService, transportService, indicesService); RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(), - metadataCreateIndexService, indexMetadataVerifier, shardLimitValidator); + metadataCreateIndexService, clusterModule.getMetadataDeleteIndexService(), indexMetadataVerifier, + shardLimitValidator, systemIndices); final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor(settings, clusterService::state, clusterService.getClusterSettings(), client, threadPool::relativeTimeInMillis, rerouteService); clusterInfoService.addListener(diskThresholdMonitor::onNewInfo); diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java index 861677e61e58f..c3a4a56f24ba1 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -29,4 +29,24 @@ public interface SystemIndexPlugin extends ActionPlugin { default Collection getSystemIndexDescriptors(Settings settings) { return Collections.emptyList(); } + + /** + * @return The name of the feature, as used for specifying feature states in snapshot creation and restoration. + */ + String getFeatureName(); + + /** + * @return A description of the feature, as used for the Get Snapshottable Features API. + */ + String getFeatureDescription(); + + /** + * Returns a list of index patterns for "associated indices": indices which depend on this plugin's system indices, but are not + * themselves system indices. + * + * @return A list of index patterns which depend on the contents of this plugin's system indices, but are not themselves system indices + */ + default Collection getAssociatedIndexPatterns() { + return Collections.emptyList(); + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java new file mode 100644 index 0000000000000..50092106dd32b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshottableFeaturesAction.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.rest.action.admin.cluster; + +import org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesRequest; +import org.elasticsearch.action.admin.cluster.snapshots.features.SnapshottableFeaturesAction; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.List; + +public class RestSnapshottableFeaturesAction extends BaseRestHandler { + @Override + public List routes() { + return List.of(new Route(RestRequest.Method.GET, "/_snapshottable_features")); + } + + @Override + public String getName() { + return "get_snapshottable_features"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + final GetSnapshottableFeaturesRequest req = new GetSnapshottableFeaturesRequest(); + req.masterNodeTimeout(request.paramAsTime("master_timeout", req.masterNodeTimeout())); + + return restChannel -> { + client.execute(SnapshottableFeaturesAction.INSTANCE, req, new RestToXContentListener<>(restChannel)); + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index b7d924da2310d..926dc140b7251 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -55,6 +56,8 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ClusterSettings; @@ -66,6 +69,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.ShardLimitValidator; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; @@ -81,6 +85,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; @@ -99,6 +104,8 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.elasticsearch.snapshots.SnapshotUtils.filterIndices; +import static org.elasticsearch.snapshots.SnapshotsService.FEATURE_STATES_VERSION; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; /** * Service responsible for restoring snapshots @@ -123,6 +130,7 @@ public class RestoreService implements ClusterStateApplier { private static final Logger logger = LogManager.getLogger(RestoreService.class); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestoreService.class); public static final Setting REFRESH_REPO_UUID_ON_RESTORE_SETTING = Setting.boolSetting( "snapshot.refresh_repo_uuid_on_restore", @@ -158,27 +166,40 @@ public class RestoreService implements ClusterStateApplier { private final IndexMetadataVerifier indexMetadataVerifier; + private final MetadataDeleteIndexService metadataDeleteIndexService; + private final ShardLimitValidator shardLimitValidator; private final ClusterSettings clusterSettings; + private final SystemIndices systemIndices; + private volatile boolean refreshRepositoryUuidOnRestore; private static final CleanRestoreStateTaskExecutor cleanRestoreStateTaskExecutor = new CleanRestoreStateTaskExecutor(); - public RestoreService(ClusterService clusterService, RepositoriesService repositoriesService, - AllocationService allocationService, MetadataCreateIndexService createIndexService, - IndexMetadataVerifier indexMetadataVerifier, ShardLimitValidator shardLimitValidator) { + public RestoreService( + ClusterService clusterService, + RepositoriesService repositoriesService, + AllocationService allocationService, + MetadataCreateIndexService createIndexService, + MetadataDeleteIndexService metadataDeleteIndexService, + IndexMetadataVerifier indexMetadataVerifier, + ShardLimitValidator shardLimitValidator, + SystemIndices systemIndices + ) { this.clusterService = clusterService; this.repositoriesService = repositoriesService; this.allocationService = allocationService; this.createIndexService = createIndexService; this.indexMetadataVerifier = indexMetadataVerifier; + this.metadataDeleteIndexService = metadataDeleteIndexService; if (DiscoveryNode.isMasterNode(clusterService.getSettings())) { clusterService.addStateApplier(this); } this.clusterSettings = clusterService.getClusterSettings(); this.shardLimitValidator = shardLimitValidator; + this.systemIndices = systemIndices; this.refreshRepositoryUuidOnRestore = REFRESH_REPO_UUID_ON_RESTORE_SETTING.get(clusterService.getSettings()); clusterService.getClusterSettings().addSettingsUpdateConsumer( REFRESH_REPO_UUID_ON_RESTORE_SETTING, @@ -238,55 +259,74 @@ public void restoreSnapshot(final RestoreSnapshotRequest request, // Make sure that we can restore from this snapshot validateSnapshotRestorable(repositoryName, snapshotInfo); + // Get the global state if necessary Metadata globalMetadata = null; - // Resolve the indices from the snapshot that need to be restored - Map dataStreams; - List requestIndices = new ArrayList<>(Arrays.asList(request.indices())); - - List requestedDataStreams = filterIndices(snapshotInfo.dataStreams(), requestIndices.toArray(String[]::new), - IndicesOptions.fromOptions(true, true, true, true)); - if (requestedDataStreams.isEmpty()) { - dataStreams = new HashMap<>(); - } else { + final Metadata.Builder metadataBuilder; + if (request.includeGlobalState()) { globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); - final Map dataStreamsInSnapshot = globalMetadata.dataStreams(); - dataStreams = new HashMap<>(requestedDataStreams.size()); - for (String requestedDataStream : requestedDataStreams) { - final DataStream dataStreamInSnapshot = dataStreamsInSnapshot.get(requestedDataStream); - assert dataStreamInSnapshot != null : "DataStream [" + requestedDataStream + "] not found in snapshot"; - dataStreams.put(requestedDataStream, dataStreamInSnapshot); - } + metadataBuilder = Metadata.builder(globalMetadata); + } else { + metadataBuilder = Metadata.builder(); } - requestIndices.removeAll(dataStreams.keySet()); - Set dataStreamIndices = dataStreams.values().stream() + + List requestIndices = new ArrayList<>(Arrays.asList(request.indices())); + + // Get data stream metadata for requested data streams + Map dataStreamsToRestore = getDataStreamsToRestore(repository, snapshotId, snapshotInfo, globalMetadata, + requestIndices); + + // Remove the data streams from the list of requested indices + requestIndices.removeAll(dataStreamsToRestore.keySet()); + + // And add the backing indices + Set dataStreamIndices = dataStreamsToRestore.values().stream() .flatMap(ds -> ds.getIndices().stream()) .map(Index::getName) .collect(Collectors.toSet()); requestIndices.addAll(dataStreamIndices); - final List indicesInSnapshot = filterIndices(snapshotInfo.indices(), requestIndices.toArray(String[]::new), + // Determine system indices to restore from requested feature states + final Map> featureStatesToRestore = getFeatureStatesToRestore(request, snapshotInfo, snapshot); + final Set featureStateIndices = featureStatesToRestore.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + + // Resolve the indices that were directly requested + final List requestedIndicesInSnapshot = filterIndices(snapshotInfo.indices(), requestIndices.toArray(String[]::new), request.indicesOptions()); - final Metadata.Builder metadataBuilder; - if (request.includeGlobalState()) { - if (globalMetadata == null) { - globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); + // Combine into the final list of indices to be restored + final List requestedIndicesIncludingSystem = Stream.concat( + requestedIndicesInSnapshot.stream(), + featureStateIndices.stream() + ).distinct().collect(Collectors.toList()); + + final Set explicitlyRequestedSystemIndices = new HashSet<>(); + final List indexIdsInSnapshot = repositoryData.resolveIndices(requestedIndicesIncludingSystem); + for (IndexId indexId : indexIdsInSnapshot) { + IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); + if (snapshotIndexMetaData.isSystem()) { + if (requestedIndicesInSnapshot.contains(indexId.getName())) { + explicitlyRequestedSystemIndices.add(indexId.getName()); + } } - metadataBuilder = Metadata.builder(globalMetadata); - } else { - metadataBuilder = Metadata.builder(); + metadataBuilder.put(snapshotIndexMetaData, false); } - final List indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot); - for (IndexId indexId : indexIdsInSnapshot) { - metadataBuilder.put(repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId), false); + // log a deprecation warning if the any of the indexes to delete were included in the request and the snapshot + // is from a version that should have feature states + if (snapshotInfo.version().onOrAfter(FEATURE_STATES_VERSION) && explicitlyRequestedSystemIndices.isEmpty() == false) { + deprecationLogger.deprecate(DeprecationCategory.API, "restore-system-index-from-snapshot", + "Restoring system indices by name is deprecated. Use feature states instead. System indices: " + + explicitlyRequestedSystemIndices); } - final Metadata metadata = metadataBuilder.dataStreams(dataStreams).build(); + final Metadata metadata = metadataBuilder.dataStreams(dataStreamsToRestore).build(); // Apply renaming on index names, returning a map of names where // the key is the renamed index and the value is the original name - final Map indices = renamedIndices(request, indicesInSnapshot, dataStreamIndices); + final Map indices = renamedIndices(request, requestedIndicesIncludingSystem, dataStreamIndices, + featureStateIndices); // Now we can start the actual restore process by adding shards to be recovered in the cluster state // and updating cluster metadata (global and index) as needed @@ -306,6 +346,13 @@ public ClusterState execute(ClusterState currentState) { deletionsInProgress.getEntries().get(0) + "]"); } + // Clear out all existing indices which fall within a system index pattern being restored + final Set systemIndicesToDelete = resolveSystemIndicesToDelete( + currentState, + featureStatesToRestore.keySet() + ); + currentState = metadataDeleteIndexService.deleteIndices(currentState, systemIndicesToDelete); + // Updating cluster state ClusterState.Builder builder = ClusterState.builder(currentState); Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); @@ -355,7 +402,8 @@ public ClusterState execute(ClusterState currentState) { .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())) .timestampRange(IndexLongFieldRange.NO_SHARDS); shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState); - if (request.includeAliases() == false && snapshotIndexMetadata.getAliases().isEmpty() == false) { + if (request.includeAliases() == false && snapshotIndexMetadata.getAliases().isEmpty() == false + && isSystemIndex(snapshotIndexMetadata) == false) { // Remove all aliases - they shouldn't be restored indexMdBuilder.removeAllAliases(); } else { @@ -393,7 +441,7 @@ public ClusterState execute(ClusterState currentState) { Math.max(snapshotIndexMetadata.primaryTerm(shard), currentIndexMetadata.primaryTerm(shard))); } - if (request.includeAliases() == false) { + if (request.includeAliases() == false && isSystemIndex(snapshotIndexMetadata) == false) { // Remove all snapshot aliases if (snapshotIndexMetadata.getAliases().isEmpty() == false) { indexMdBuilder.removeAllAliases(); @@ -445,7 +493,7 @@ restoreUUID, snapshot, overallState(RestoreInProgress.State.INIT, shards), checkAliasNameConflicts(indices, aliases); Map updatedDataStreams = new HashMap<>(currentState.metadata().dataStreams()); - updatedDataStreams.putAll(dataStreams.values().stream() + updatedDataStreams.putAll(dataStreamsToRestore.values().stream() .map(ds -> updateDataStream(ds, mdBuilder, request)) .collect(Collectors.toMap(DataStream::getName, Function.identity()))); mdBuilder.dataStreams(updatedDataStreams); @@ -706,6 +754,105 @@ public void onFailure(Exception e) { } + private boolean isSystemIndex(IndexMetadata indexMetadata) { + return indexMetadata.isSystem() || systemIndices.isSystemIndex(indexMetadata.getIndex()); + } + + private Map getDataStreamsToRestore(Repository repository, SnapshotId snapshotId, SnapshotInfo snapshotInfo, + Metadata globalMetadata, List requestIndices) { + Map dataStreams; + List requestedDataStreams = filterIndices(snapshotInfo.dataStreams(), requestIndices.toArray(String[]::new), + IndicesOptions.fromOptions(true, true, true, true)); + if (requestedDataStreams.isEmpty()) { + dataStreams = Collections.emptyMap(); + } else { + if (globalMetadata == null) { + globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId); + } + final Map dataStreamsInSnapshot = globalMetadata.dataStreams(); + dataStreams = new HashMap<>(requestedDataStreams.size()); + for (String requestedDataStream : requestedDataStreams) { + final DataStream dataStreamInSnapshot = dataStreamsInSnapshot.get(requestedDataStream); + assert dataStreamInSnapshot != null : "DataStream [" + requestedDataStream + "] not found in snapshot"; + dataStreams.put(requestedDataStream, dataStreamInSnapshot); + } + } + return dataStreams; + } + + private Map> getFeatureStatesToRestore(RestoreSnapshotRequest request, SnapshotInfo snapshotInfo, + Snapshot snapshot) { + if (snapshotInfo.featureStates() == null) { + return Collections.emptyMap(); + } + final Map> snapshotFeatureStates = snapshotInfo.featureStates().stream() + .collect(Collectors.toMap(SnapshotFeatureInfo::getPluginName, SnapshotFeatureInfo::getIndices)); + + final Map> featureStatesToRestore; + final String[] requestedFeatureStates = request.featureStates(); + + if (requestedFeatureStates == null || requestedFeatureStates.length == 0) { + // Handle the default cases - defer to the global state value + if (request.includeGlobalState()) { + featureStatesToRestore = new HashMap<>(snapshotFeatureStates); + } else { + featureStatesToRestore = Collections.emptyMap(); + } + } else if (requestedFeatureStates.length == 1 && NO_FEATURE_STATES_VALUE.equalsIgnoreCase(requestedFeatureStates[0])) { + // If there's exactly one value and it's "none", include no states + featureStatesToRestore = Collections.emptyMap(); + } else { + // Otherwise, handle the list of requested feature states + final Set requestedStates = Set.of(requestedFeatureStates); + if (requestedStates.contains(NO_FEATURE_STATES_VALUE)) { + throw new SnapshotRestoreException(snapshot, "the feature_states value [" + NO_FEATURE_STATES_VALUE + + "] indicates that no feature states should be restored, but other feature states were requested: " + requestedStates); + } + if (snapshotFeatureStates.keySet().containsAll(requestedStates) == false) { + Set nonExistingRequestedStates = new HashSet<>(requestedStates); + nonExistingRequestedStates.removeAll(snapshotFeatureStates.keySet()); + throw new SnapshotRestoreException(snapshot, "requested feature states [" + nonExistingRequestedStates + + "] are not present in snapshot"); + } + featureStatesToRestore = new HashMap<>(snapshotFeatureStates); + featureStatesToRestore.keySet().retainAll(requestedStates); + } + + final List featuresNotOnThisNode = featureStatesToRestore.keySet().stream() + .filter(featureName -> systemIndices.getFeatures().containsKey(featureName) == false) + .collect(Collectors.toList()); + if (featuresNotOnThisNode.isEmpty() == false) { + throw new SnapshotRestoreException(snapshot, "requested feature states " + featuresNotOnThisNode + " are present in " + + "snapshot but those features are not installed on the current master node"); + } + return featureStatesToRestore; + } + + /** + * Resolves a set of index names that currently exist in the cluster that are part of a feature state which is about to be restored, + * and should therefore be removed prior to restoring those feature states from the snapshot. + * + * @param currentState The current cluster state + * @param featureStatesToRestore A set of feature state names that are about to be restored + * @return A set of index names that should be removed based on the feature states being restored + */ + private Set resolveSystemIndicesToDelete(ClusterState currentState, Set featureStatesToRestore) { + if (featureStatesToRestore == null) { + return Collections.emptySet(); + } + + return featureStatesToRestore.stream() + .map(featureName -> systemIndices.getFeatures().get(featureName)) + .filter(Objects::nonNull) // Features that aren't present on this node will be warned about in `getFeatureStatesToRestore` + .flatMap(feature -> feature.getIndexDescriptors().stream()) + .flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata()).stream()) + .map(indexName -> { + assert currentState.metadata().hasIndex(indexName) : "index [" + indexName + "] not found in metadata but must be present"; + return currentState.metadata().getIndices().get(indexName).getIndex(); + }) + .collect(Collectors.toUnmodifiableSet()); + } + //visible for testing static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metadata, RestoreSnapshotRequest request) { String dataStreamName = dataStream.getName(); @@ -979,10 +1126,16 @@ public static int failedShards(ImmutableOpenMap renamedIndices(RestoreSnapshotRequest request, List filteredIndices, - Set dataStreamIndices) { + Set dataStreamIndices, Set featureIndices) { Map renamedIndices = new HashMap<>(); for (String index : filteredIndices) { - String renamedIndex = renameIndex(index, request, dataStreamIndices.contains(index)); + String renamedIndex; + if (featureIndices.contains(index)) { + // Don't rename system indices + renamedIndex = index; + } else { + renamedIndex = renameIndex(index, request, dataStreamIndices.contains(index)); + } String previousIndex = renamedIndices.put(renamedIndex, index); if (previousIndex != null) { throw new SnapshotRestoreException(request.repository(), request.snapshot(), diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java new file mode 100644 index 0000000000000..419d75a7226a5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotFeatureInfo.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class SnapshotFeatureInfo implements Writeable, ToXContentObject { + final String pluginName; + final List indices; + + static final ConstructingObjectParser SNAPSHOT_FEATURE_INFO_PARSER = + new ConstructingObjectParser<>("feature_info", true, (a, name) -> { + String pluginName = (String) a[0]; + List indices = (List) a[1]; + return new SnapshotFeatureInfo(pluginName, indices); + }); + + static { + SNAPSHOT_FEATURE_INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("feature_name")); + SNAPSHOT_FEATURE_INFO_PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), new ParseField("indices")); + } + + public SnapshotFeatureInfo(String pluginName, List indices) { + this.pluginName = pluginName; + this.indices = indices; + } + + public SnapshotFeatureInfo(final StreamInput in) throws IOException { + this.pluginName = in.readString(); + this.indices = in.readStringList(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(pluginName); + out.writeStringCollection(indices); + } + + public static SnapshotFeatureInfo fromXContent(XContentParser parser) throws IOException { + return SNAPSHOT_FEATURE_INFO_PARSER.parse(parser, null); + } + + public String getPluginName() { + return pluginName; + } + + public List getIndices() { + return indices; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("feature_name", pluginName); + builder.startArray("indices"); + for (String index : indices) { + builder.value(index); + } + builder.endArray(); + } + builder.endObject(); + return builder; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof SnapshotFeatureInfo) == false) return false; + SnapshotFeatureInfo that = (SnapshotFeatureInfo) o; + return getPluginName().equals(that.getPluginName()) && + getIndices().equals(that.getIndices()); + } + + @Override + public int hashCode() { + return Objects.hash(getPluginName(), getIndices()); + } +} diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index bdb1954efa22b..d6f6ba03bd642 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -37,6 +37,8 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.snapshots.SnapshotsService.FEATURE_STATES_VERSION; + /** * Information about a snapshot */ @@ -69,6 +71,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final String SUCCESSFUL_SHARDS = "successful_shards"; private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; private static final String USER_METADATA = "metadata"; + private static final String FEATURE_STATES = "feature_states"; private static final Comparator COMPARATOR = Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId); @@ -80,6 +83,7 @@ public static final class SnapshotInfoBuilder { private String reason = null; private List indices = null; private List dataStreams = null; + private List featureStates = null; private long startTime = 0L; private long endTime = 0L; private ShardStatsBuilder shardStatsBuilder = null; @@ -112,6 +116,10 @@ private void setDataStreams(List dataStreams) { this.dataStreams = dataStreams; } + private void setFeatureStates(List featureStates) { + this.featureStates = featureStates; + } + private void setStartTime(long startTime) { this.startTime = startTime; } @@ -151,6 +159,10 @@ public SnapshotInfo build() { dataStreams = Collections.emptyList(); } + if (featureStates == null) { + featureStates = Collections.emptyList(); + } + SnapshotState snapshotState = state == null ? null : SnapshotState.valueOf(state); Version version = this.version == -1 ? Version.CURRENT : Version.fromId(this.version); @@ -161,8 +173,9 @@ public SnapshotInfo build() { shardFailures = new ArrayList<>(); } - return new SnapshotInfo(snapshotId, indices, dataStreams, snapshotState, reason, version, startTime, endTime, - totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata); + return new SnapshotInfo(snapshotId, indices, dataStreams, featureStates, reason, version, startTime, endTime, totalShards, + successfulShards, shardFailures, includeGlobalState, userMetadata, snapshotState + ); } } @@ -200,6 +213,8 @@ int getSuccessfulShards() { SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setReason, new ParseField(REASON)); SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setIndices, new ParseField(INDICES)); SNAPSHOT_INFO_PARSER.declareStringArray(SnapshotInfoBuilder::setDataStreams, new ParseField(DATA_STREAMS)); + SNAPSHOT_INFO_PARSER.declareObjectArray(SnapshotInfoBuilder::setFeatureStates, SnapshotFeatureInfo.SNAPSHOT_FEATURE_INFO_PARSER, + new ParseField(FEATURE_STATES)); SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setStartTime, new ParseField(START_TIME_IN_MILLIS)); SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setEndTime, new ParseField(END_TIME_IN_MILLIS)); SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS)); @@ -225,6 +240,8 @@ int getSuccessfulShards() { private final List dataStreams; + private final List featureStates; + private final long startTime; private final long endTime; @@ -244,33 +261,40 @@ int getSuccessfulShards() { private final List shardFailures; - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, SnapshotState state) { - this(snapshotId, indices, dataStreams, state, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null, null); + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + SnapshotState state) { + this(snapshotId, indices, dataStreams, featureStates, null, null, 0L, 0L, 0, 0, Collections.emptyList(), null, null, state); } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, SnapshotState state, Version version) { - this(snapshotId, indices, dataStreams, state, null, version, 0L, 0L, 0, 0, Collections.emptyList(), null, null); + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + Version version, SnapshotState state) { + this(snapshotId, indices, dataStreams, featureStates, null, version, 0L, 0L, 0, 0, Collections.emptyList(), null, null, state); } public SnapshotInfo(SnapshotsInProgress.Entry entry) { this(entry.snapshot().getSnapshotId(), - entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), SnapshotState.IN_PROGRESS, - null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata()); + entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), entry.featureStates(), + null, Version.CURRENT, entry.startTime(), 0L, 0, 0, Collections.emptyList(), entry.includeGlobalState(), entry.userMetadata(), + SnapshotState.IN_PROGRESS + ); } - public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, long startTime, String reason, - long endTime, int totalShards, List shardFailures, Boolean includeGlobalState, - Map userMetadata) { - this(snapshotId, indices, dataStreams, snapshotState(reason, shardFailures), reason, Version.CURRENT, - startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata); + public SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + String reason, long endTime, int totalShards, List shardFailures, Boolean includeGlobalState, + Map userMetadata, long startTime) { + this(snapshotId, indices, dataStreams, featureStates, reason, Version.CURRENT, startTime, endTime, totalShards, + totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata, snapshotState(reason, shardFailures) + ); } - SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, SnapshotState state, String reason, - Version version, long startTime, long endTime, int totalShards, int successfulShards, - List shardFailures, Boolean includeGlobalState, Map userMetadata) { + SnapshotInfo(SnapshotId snapshotId, List indices, List dataStreams, List featureStates, + String reason, Version version, long startTime, long endTime, int totalShards, int successfulShards, + List shardFailures, Boolean includeGlobalState, Map userMetadata, + SnapshotState state) { this.snapshotId = Objects.requireNonNull(snapshotId); this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices)); this.dataStreams = Collections.unmodifiableList(Objects.requireNonNull(dataStreams)); + this.featureStates = Collections.unmodifiableList(Objects.requireNonNull(featureStates)); this.state = state; this.reason = reason; this.version = version; @@ -300,6 +324,11 @@ public SnapshotInfo(final StreamInput in) throws IOException { includeGlobalState = in.readOptionalBoolean(); userMetadata = in.readMap(); dataStreams = in.readStringList(); + if (in.getVersion().before(FEATURE_STATES_VERSION)) { + featureStates = Collections.emptyList(); + } else { + featureStates = Collections.unmodifiableList(in.readList(SnapshotFeatureInfo::new)); + } } /** @@ -307,7 +336,7 @@ public SnapshotInfo(final StreamInput in) throws IOException { * all information stripped out except the snapshot id, state, and indices. */ public SnapshotInfo basic() { - return new SnapshotInfo(snapshotId, indices, Collections.emptyList(), state); + return new SnapshotInfo(snapshotId, indices, Collections.emptyList(), featureStates, state); } /** @@ -439,6 +468,10 @@ public Map userMetadata() { return userMetadata; } + public List featureStates() { + return featureStates; + } + /** * Compares two snapshots by their start time; if the start times are the same, then * compares the two snapshots by their snapshot ids. @@ -462,6 +495,7 @@ public String toString() { ", includeGlobalState=" + includeGlobalState + ", version=" + version + ", shardFailures=" + shardFailures + + ", featureStates=" + featureStates + '}'; } @@ -540,6 +574,14 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(SUCCESSFUL, successfulShards); builder.endObject(); } + if (verbose || featureStates.isEmpty() == false) { + builder.startArray(FEATURE_STATES); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); + } + builder.endArray(); + + } builder.endObject(); return builder; } @@ -577,6 +619,12 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final shardFailure.toXContent(builder, params); } builder.endArray(); + builder.startArray(FEATURE_STATES); + for (SnapshotFeatureInfo snapshotFeatureInfo : featureStates) { + builder.value(snapshotFeatureInfo); + } + builder.endArray(); + builder.endObject(); return builder; } @@ -601,6 +649,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr Boolean includeGlobalState = null; Map userMetadata = null; List shardFailures = Collections.emptyList(); + List featureStates = Collections.emptyList(); if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); } @@ -655,6 +704,12 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser)); } shardFailures = Collections.unmodifiableList(shardFailureArrayList); + } else if (FEATURE_STATES.equals(currentFieldName)) { + ArrayList snapshotFeatureInfoArrayList = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + snapshotFeatureInfoArrayList.add(SnapshotFeatureInfo.fromXContent(parser)); + } + featureStates = Collections.unmodifiableList(snapshotFeatureInfoArrayList); } else { // It was probably created by newer version - ignoring parser.skipChildren(); @@ -677,7 +732,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr return new SnapshotInfo(new SnapshotId(name, uuid), indices, dataStreams, - state, + featureStates, reason, version, startTime, @@ -686,7 +741,9 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr successfulShards, shardFailures, includeGlobalState, - userMetadata); + userMetadata, + state + ); } @Override @@ -714,6 +771,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalBoolean(includeGlobalState); out.writeMap(userMetadata); out.writeStringCollection(dataStreams); + if (out.getVersion().onOrAfter(FEATURE_STATES_VERSION)) { + out.writeList(featureStates); + } } private static SnapshotState snapshotState(final String reason, final List shardFailures) { @@ -745,12 +805,15 @@ public boolean equals(Object o) { Objects.equals(includeGlobalState, that.includeGlobalState) && Objects.equals(version, that.version) && Objects.equals(shardFailures, that.shardFailures) && - Objects.equals(userMetadata, that.userMetadata); + Objects.equals(userMetadata, that.userMetadata) && + Objects.equals(featureStates, that.featureStates); } @Override public int hashCode() { return Objects.hash(snapshotId, state, reason, indices, dataStreams, startTime, endTime, - totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata); + totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata, + featureStates); } + } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 913e99736323e..7f849ba68c043 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -69,6 +69,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; @@ -107,6 +108,7 @@ import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.action.support.IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN; import static org.elasticsearch.cluster.SnapshotsInProgress.completed; /** @@ -129,6 +131,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public static final Version OLD_SNAPSHOT_FORMAT = Version.V_7_5_0; + public static final Version FEATURE_STATES_VERSION = Version.V_8_0_0; + private static final Logger logger = LogManager.getLogger(SnapshotsService.class); public static final String UPDATE_SNAPSHOT_STATUS_ACTION_NAME = "internal:cluster/snapshot/update_snapshot_status"; @@ -153,6 +157,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public static final String CACHE_FILE_NAME = "shared_snapshot_cache"; + public static final String NO_FEATURE_STATES_VALUE = "none"; + private final ClusterService clusterService; private final IndexNameExpressionResolver indexNameExpressionResolver; @@ -184,6 +190,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private final OngoingRepositoryOperations repositoryOperations = new OngoingRepositoryOperations(); + private final Map systemIndexDescriptorMap; + /** * Setting that specifies the maximum number of allowed concurrent snapshot create and delete operations in the * cluster state. The number of concurrent operations in a cluster state is defined as the sum of the sizes of @@ -195,7 +203,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private volatile int maxConcurrentOperations; public SnapshotsService(Settings settings, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, - RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters) { + RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters, + Map systemIndexDescriptorMap) { this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.repositoriesService = repositoriesService; @@ -212,6 +221,7 @@ public SnapshotsService(Settings settings, ClusterService clusterService, IndexN clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING, i -> maxConcurrentOperations = i); } + this.systemIndexDescriptorMap = systemIndexDescriptorMap; } /** @@ -267,6 +277,59 @@ public ClusterState execute(ClusterState currentState) { // Store newSnapshot here to be processed in clusterStateProcessed List indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request)); + List featureStates = Collections.emptyList(); + final List requestedStates = Arrays.asList(request.featureStates()); + + // We should only use the feature states logic if we're sure we'll be able to finish the snapshot without a lower-version + // node taking over and causing problems. Therefore, if we're in a mixed cluster with versions that don't know how to handle + // feature states, skip all feature states logic, and if `feature_states` is explicitly configured, throw an exception. + if (currentState.nodes().getMinNodeVersion().onOrAfter(FEATURE_STATES_VERSION)) { + if (request.includeGlobalState() || requestedStates.isEmpty() == false) { + final Set featureStatesSet; + if (request.includeGlobalState() && requestedStates.isEmpty()) { + // If we're including global state and feature states aren't specified, include all of them + featureStatesSet = new HashSet<>(systemIndexDescriptorMap.keySet()); + } else if (requestedStates.size() == 1 && NO_FEATURE_STATES_VALUE.equalsIgnoreCase(requestedStates.get(0))) { + // If there's exactly one value and it's "none", include no states + featureStatesSet = Collections.emptySet(); + } else { + // Otherwise, check for "none" then use the list of requested states + if (requestedStates.contains(NO_FEATURE_STATES_VALUE)) { + throw new IllegalArgumentException("the feature_states value [" + SnapshotsService.NO_FEATURE_STATES_VALUE + + "] indicates that no feature states should be snapshotted, but other feature states were requested: " + + requestedStates); + } + featureStatesSet = new HashSet<>(requestedStates); + } + + featureStates = systemIndexDescriptorMap.keySet().stream() + .filter(feature -> featureStatesSet.contains(feature)) + .map(feature -> new SnapshotFeatureInfo(feature, resolveFeatureIndexNames(currentState, feature))) + .filter(featureInfo -> featureInfo.getIndices().isEmpty() == false) // Omit any empty featureStates + .collect(Collectors.toList()); + final Stream featureStateIndices = featureStates.stream().flatMap(feature -> feature.getIndices().stream()); + + final Stream associatedIndices = systemIndexDescriptorMap.keySet().stream() + .filter(feature -> featureStatesSet.contains(feature)) + .flatMap(feature -> resolveAssociatedIndices(currentState, feature).stream()); + + // Add all resolved indices from the feature states to the list of indices + indices = Stream.of(indices.stream(), featureStateIndices, associatedIndices) + .flatMap(s -> s) + .distinct() + .collect(Collectors.toList()); + } + } else if (requestedStates.isEmpty() == false) { + throw new SnapshotException( + new Snapshot(repositoryName, snapshotId), + "feature_states can only be used when all nodes in cluster are version [" + + FEATURE_STATES_VERSION + + "] or higher, but at least one node in this cluster is on version [" + + currentState.nodes().getMinNodeVersion() + + "]" + ); + } + final List dataStreams = indexNameExpressionResolver.dataStreamNames(currentState, request.indicesOptions(), request.indices()); @@ -291,7 +354,8 @@ public ClusterState execute(ClusterState currentState) { } newEntry = SnapshotsInProgress.startedEntry( new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), - indexIds, dataStreams, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, userMeta, version); + indexIds, dataStreams, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, + userMeta, version, featureStates); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(CollectionUtils.appendToCopy(runningSnapshots, newEntry))).build(); } @@ -316,6 +380,29 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } + private List resolveFeatureIndexNames(ClusterState currentState, String featureName) { + if (systemIndexDescriptorMap.containsKey(featureName) == false) { + throw new IllegalArgumentException("requested snapshot of feature state for unknown feature [" + featureName + "]"); + } + + final SystemIndices.Feature feature = systemIndexDescriptorMap.get(featureName); + return feature.getIndexDescriptors().stream() + .flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata()).stream()) + .collect(Collectors.toList()); + } + + private List resolveAssociatedIndices(ClusterState currentState, String featureName) { + if (systemIndexDescriptorMap.containsKey(featureName) == false) { + throw new IllegalArgumentException("requested associated indices for feature state for unknown feature [" + featureName + "]"); + } + + final SystemIndices.Feature feature = systemIndexDescriptorMap.get(featureName); + return feature.getAssociatedIndexPatterns().stream() + .flatMap(pattern -> Arrays.stream(indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(currentState, + LENIENT_EXPAND_OPEN_CLOSED_HIDDEN, pattern))) + .collect(Collectors.toList()); + } + private static void ensureSnapshotNameNotRunning(List runningSnapshots, String repositoryName, String snapshotName) { if (runningSnapshots.stream().anyMatch(s -> { @@ -1210,14 +1297,18 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met } metadataListener.whenComplete(meta -> { final Metadata metaForSnapshot = metadataForSnapshot(entry, meta); + final List finalIndices = shardGenerations.indices().stream() + .map(IndexId::getName) + .collect(Collectors.toList()); final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), - shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), + finalIndices, entry.partial() ? entry.dataStreams().stream() .filter(metaForSnapshot.dataStreams()::containsKey) .collect(Collectors.toList()) : entry.dataStreams(), - entry.startTime(), failure, threadPool.absoluteTimeInMillis(), + entry.partial() ? onlySuccessfulFeatureStates(entry, finalIndices) : entry.featureStates(), + failure, threadPool.absoluteTimeInMillis(), entry.partial() ? shardGenerations.totalShards() : entry.shards().size(), shardFailures, - entry.includeGlobalState(), entry.userMetadata()); + entry.includeGlobalState(), entry.userMetadata(), entry.startTime()); repo.finalizeSnapshot( shardGenerations, repositoryData.getGenId(), @@ -1239,6 +1330,31 @@ private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata met } } + /** + * Removes all feature states which have missing or failed shards, as they are no longer safely restorable. + * @param entry The "in progress" entry with a list of feature states and one or more failed shards. + * @param finalIndices The final list of indices in the snapshot, after any indices that were concurrently deleted are removed. + * @return The list of feature states which were completed successfully in the given entry. + */ + private List onlySuccessfulFeatureStates(SnapshotsInProgress.Entry entry, List finalIndices) { + assert entry.partial() : "should not try to filter feature states from a non-partial entry"; + + // Figure out which indices have unsuccessful shards + Set indicesWithUnsuccessfulShards = new HashSet<>(); + entry.shards().keysIt().forEachRemaining(shardId -> { + final ShardState shardState = entry.shards().get(shardId).state(); + if (shardState.failed() || shardState.completed() == false) { + indicesWithUnsuccessfulShards.add(shardId.getIndexName()); + } + }); + + // Now remove any feature states which contain any of those indices, as the feature state is not intact and not safely restorable + return entry.featureStates().stream() + .filter(stateInfo -> finalIndices.containsAll(stateInfo.getIndices())) + .filter(stateInfo -> stateInfo.getIndices().stream().anyMatch(indicesWithUnsuccessfulShards::contains) == false) + .collect(Collectors.toList()); + } + /** * Remove a snapshot from {@link #endingSnapshots} set and return its completion listeners that must be resolved. */ diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java index 8b125d95fa35c..3d2f9dfc182cc 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java @@ -45,6 +45,8 @@ public class TaskResultsService { private static final Logger logger = LogManager.getLogger(TaskResultsService.class); + public static final String TASKS_FEATURE_NAME = "tasks"; + public static final String TASK_INDEX = ".tasks"; public static final String TASK_RESULT_MAPPING_VERSION_META_FIELD = "version"; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java index 333757add027b..f40aaeeda51b1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java @@ -55,6 +55,18 @@ public void testToXContent() throws IOException { original.indices(indices); } + if (randomBoolean()) { + List featureStates = new ArrayList<>(); + int count = randomInt(3) + 1; + + for (int i = 0; i < count; ++i) { + featureStates.add(randomAlphaOfLength(randomInt(3) + 2)); + } + + original.featureStates(featureStates); + } + + if (randomBoolean()) { original.partial(randomBoolean()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java index d2a202568ebbb..62d8dbf593f7a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java @@ -10,6 +10,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.SnapshotFeatureInfo; +import org.elasticsearch.snapshots.SnapshotFeatureInfoTests; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotInfoTests; @@ -44,6 +46,9 @@ protected CreateSnapshotResponse createTestInstance() { List dataStreams = new ArrayList<>(); dataStreams.add("test0"); dataStreams.add("test1"); + + List featureStates = randomList(5, SnapshotFeatureInfoTests::randomSnapshotFeatureInfo); + String reason = "reason"; long startTime = System.currentTimeMillis(); long endTime = startTime + 10000; @@ -59,8 +64,9 @@ protected CreateSnapshotResponse createTestInstance() { boolean globalState = randomBoolean(); return new CreateSnapshotResponse( - new SnapshotInfo(snapshotId, indices, dataStreams, startTime, reason, endTime, totalShards, shardFailures, - globalState, SnapshotInfoTests.randomUserMetadata())); + new SnapshotInfo(snapshotId, indices, dataStreams, featureStates, reason, endTime, totalShards, shardFailures, + globalState, SnapshotInfoTests.randomUserMetadata(), startTime + )); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java new file mode 100644 index 0000000000000..40702a8b51a71 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.features; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.Set; +import java.util.stream.Collectors; + +public class GetSnapshottableFeaturesResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return GetSnapshottableFeaturesResponse::new; + } + + @Override + protected GetSnapshottableFeaturesResponse createTestInstance() { + return new GetSnapshottableFeaturesResponse(randomList(10, + () -> new GetSnapshottableFeaturesResponse.SnapshottableFeature( + randomAlphaOfLengthBetween(4, 10), + randomAlphaOfLengthBetween(5,10)))); + } + + @Override + protected GetSnapshottableFeaturesResponse mutateInstance(GetSnapshottableFeaturesResponse instance) throws IOException { + int minSize = 0; + if (instance.getSnapshottableFeatures().size() == 0) { + minSize = 1; + } + Set existingFeatureNames = instance.getSnapshottableFeatures().stream() + .map(feature -> feature.getFeatureName()) + .collect(Collectors.toSet()); + return new GetSnapshottableFeaturesResponse(randomList(minSize, 10, + () -> new GetSnapshottableFeaturesResponse.SnapshottableFeature( + randomValueOtherThanMany(existingFeatureNames::contains, () -> randomAlphaOfLengthBetween(4, 10)), + randomAlphaOfLengthBetween(5, 10)))); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java index 44afc5cdb38f9..f31f037deebc7 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.SnapshotFeatureInfo; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotInfoTests; @@ -33,6 +34,7 @@ import java.util.function.Predicate; import java.util.regex.Pattern; +import static org.elasticsearch.snapshots.SnapshotFeatureInfoTests.randomSnapshotFeatureInfo; import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; import static org.hamcrest.CoreMatchers.containsString; @@ -70,9 +72,11 @@ private List createSnapshotInfos() { String reason = randomBoolean() ? null : "reason"; ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2); List shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason")); + List featureInfos = randomList(0, () -> randomSnapshotFeatureInfo()); snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), Collections.singletonList("ds"), - System.currentTimeMillis(), reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), - SnapshotInfoTests.randomUserMetadata())); + featureInfos, reason, System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(), + SnapshotInfoTests.randomUserMetadata(), System.currentTimeMillis() + )); } return snapshots; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index a8a52931ad888..68db9b6b58e4e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -44,6 +44,18 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { instance.indices(indices); } + + if (randomBoolean()) { + List plugins = new ArrayList<>(); + int count = randomInt(3) + 1; + + for (int i = 0; i < count; ++i) { + plugins.add(randomAlphaOfLength(randomInt(3) + 2)); + } + + instance.featureStates(plugins); + } + if (randomBoolean()) { instance.renamePattern(randomUnicodeOfLengthBetween(1, 100)); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index a9a316f29061f..e85cafdca2fa0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -151,8 +151,10 @@ public void testDeprecationWarningNotEmittedWhenOnlyNonsystemIndexRequested() { public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemPattern() { ClusterState state = systemIndexTestClusterState(); - SystemIndices systemIndices = new SystemIndices(Collections.singletonMap(this.getTestName(), - Collections.singletonList(new SystemIndexDescriptor(".y", "an index that doesn't exist")))); + SystemIndices systemIndices = new SystemIndices(Collections.singletonMap( + this.getTestName(), + new SystemIndices.Feature("test feature", + Collections.singletonList(new SystemIndexDescriptor(".y", "an index that doesn't exist"))))); GetAliasesRequest request = new GetAliasesRequest(".y"); ImmutableOpenMap> aliases = ImmutableOpenMap.>builder() diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index b14adc7d8eee8..000b1ce3b4e14 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -243,7 +243,8 @@ public void testOnlySystem() { new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); indicesLookup.put(".bar", new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); - SystemIndices systemIndices = new SystemIndices(Map.of("plugin", List.of(new SystemIndexDescriptor(".test", "")))); + SystemIndices systemIndices = new SystemIndices( + Map.of("plugin", new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(".test", ""))))); List onlySystem = List.of(".foo", ".bar"); assertTrue(bulkAction.isOnlySystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); diff --git a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java index 8b2a92f83e12b..808eaa6d614e4 100644 --- a/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java @@ -300,7 +300,8 @@ private static ClusterState buildClusterState(String... indices) { } private AutoCreateIndex newAutoCreateIndex(Settings settings) { - SystemIndices systemIndices = new SystemIndices(Map.of("plugin", List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, "")))); + SystemIndices systemIndices = new SystemIndices(Map.of( + "plugin", new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(TEST_SYSTEM_INDEX_NAME, ""))))); return new AutoCreateIndex(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), systemIndices); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 12bc4b8c66e41..0f2e77fb18b54 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -517,7 +517,7 @@ public void testValidateDotIndex() { null, threadPool, null, - new SystemIndices(Collections.singletonMap("foo", systemIndexDescriptors)), + new SystemIndices(Collections.singletonMap("foo", new SystemIndices.Feature("test feature", systemIndexDescriptors))), false ); // Check deprecations diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index 76718a8ba383f..7c43273490bcc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -73,8 +73,8 @@ public void testDeleteSnapshotting() { Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid")); SnapshotsInProgress snaps = SnapshotsInProgress.of(List.of(new SnapshotsInProgress.Entry(snapshot, true, false, SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")), - Collections.emptyList(), System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of(), null, - SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())))); + Collections.emptyList(), Collections.emptyList(), System.currentTimeMillis(), (long) randomIntBetween(0, 1000), + ImmutableOpenMap.of(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())))); ClusterState state = ClusterState.builder(clusterState(index)) .putCustom(SnapshotsInProgress.TYPE, snaps) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index de1bcdef4dcd7..95dce9bfe556a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -386,8 +386,10 @@ private static ClusterState addSnapshotIndex(final String index, final int numSh final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5))); final SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry(snapshot, randomBoolean(), false, SnapshotsInProgress.State.INIT, - Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), randomNonNegativeLong(), randomLong(), - shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); + Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), Collections.emptyList(), + randomNonNegativeLong(), randomLong(), shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), + VersionUtils.randomVersion(random()) + ); return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(List.of(entry))).build(); } diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java index 70461b3281329..f85dd64022752 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java @@ -72,6 +72,8 @@ public class SystemIndexManagerTests extends ESTestCase { .setOrigin("FAKE_ORIGIN") .build(); + private static final SystemIndices.Feature FEATURE = new SystemIndices.Feature("a test feature", List.of(DESCRIPTOR)); + private Client client; @Before @@ -98,7 +100,9 @@ public void testManagerSkipsDescriptorsThatAreNotManaged() { .setOrigin("FAKE_ORIGIN") .build(); - SystemIndices systemIndices = new SystemIndices(Map.of("index 1", List.of(d1), "index 2", List.of(d2))); + SystemIndices systemIndices = new SystemIndices(Map.of( + "index 1", new SystemIndices.Feature("index 1 feature", List.of(d1)), + "index 2", new SystemIndices.Feature("index 2 feature", List.of(d2)))); SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final List eligibleDescriptors = manager.getEligibleDescriptors( @@ -134,7 +138,9 @@ public void testManagerSkipsDescriptorsForIndicesThatDoNotExist() { .setOrigin("FAKE_ORIGIN") .build(); - SystemIndices systemIndices = new SystemIndices(Map.of("index 1", List.of(d1), "index 2", List.of(d2))); + SystemIndices systemIndices = new SystemIndices(Map.of( + "index 1", new SystemIndices.Feature("index 1 feature", List.of(d1)), + "index 2", new SystemIndices.Feature("index 2 feature", List.of(d2))));; SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final List eligibleDescriptors = manager.getEligibleDescriptors( @@ -149,7 +155,7 @@ public void testManagerSkipsDescriptorsForIndicesThatDoNotExist() { * Check that the manager won't try to upgrade closed indices. */ public void testManagerSkipsClosedIndices() { - SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", List.of(DESCRIPTOR))); + SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", FEATURE)); SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final ClusterState.Builder clusterStateBuilder = createClusterState(IndexMetadata.State.CLOSE); @@ -161,7 +167,7 @@ public void testManagerSkipsClosedIndices() { * Check that the manager won't try to upgrade unhealthy indices. */ public void testManagerSkipsIndicesWithRedStatus() { - SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", List.of(DESCRIPTOR))); + SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", FEATURE)); SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final ClusterState.Builder clusterStateBuilder = createClusterState(); @@ -175,7 +181,7 @@ public void testManagerSkipsIndicesWithRedStatus() { * is earlier than an expected value. */ public void testManagerSkipsIndicesWithOutdatedFormat() { - SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", List.of(DESCRIPTOR))); + SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", FEATURE)); SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final ClusterState.Builder clusterStateBuilder = createClusterState(5); @@ -188,7 +194,7 @@ public void testManagerSkipsIndicesWithOutdatedFormat() { * Check that the manager won't try to upgrade indices where their mappings are already up-to-date. */ public void testManagerSkipsIndicesWithUpToDateMappings() { - SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", List.of(DESCRIPTOR))); + SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", FEATURE)); SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final ClusterState.Builder clusterStateBuilder = createClusterState(); @@ -201,7 +207,7 @@ public void testManagerSkipsIndicesWithUpToDateMappings() { * Check that the manager will try to upgrade indices where their mappings are out-of-date. */ public void testManagerProcessesIndicesWithOutdatedMappings() { - SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", List.of(DESCRIPTOR))); + SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", FEATURE)); SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final ClusterState.Builder clusterStateBuilder = createClusterState(Strings.toString(getMappings("1.0.0"))); @@ -214,7 +220,7 @@ public void testManagerProcessesIndicesWithOutdatedMappings() { * Check that the manager submits the expected request for an index whose mappings are out-of-date. */ public void testManagerSubmitsPutRequest() { - SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", List.of(DESCRIPTOR))); + SystemIndices systemIndices = new SystemIndices(Map.of("MyIndex", FEATURE)); SystemIndexManager manager = new SystemIndexManager(systemIndices, client); final ClusterState.Builder clusterStateBuilder = createClusterState(Strings.toString(getMappings("1.0.0"))); diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java index 16fadd3870ff0..23b48b4284035 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndicesTests.java @@ -8,14 +8,13 @@ package org.elasticsearch.indices; -import org.elasticsearch.tasks.TaskResultsService; import org.elasticsearch.test.ESTestCase; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME; import static org.elasticsearch.tasks.TaskResultsService.TASK_INDEX; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -34,9 +33,10 @@ public void testBasicOverlappingPatterns() { // across tests String broadPatternSource = "AAA" + randomAlphaOfLength(5); String otherSource = "ZZZ" + randomAlphaOfLength(6); - Map> descriptors = new HashMap<>(); - descriptors.put(broadPatternSource, List.of(broadPattern)); - descriptors.put(otherSource, List.of(notOverlapping, overlapping1, overlapping2, overlapping3)); + Map descriptors = new HashMap<>(); + descriptors.put(broadPatternSource, new SystemIndices.Feature("test feature", List.of(broadPattern))); + descriptors.put(otherSource, + new SystemIndices.Feature("test 2", List.of(notOverlapping, overlapping1, overlapping2, overlapping3))); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> SystemIndices.checkForOverlappingPatterns(descriptors)); @@ -61,9 +61,9 @@ public void testComplexOverlappingPatterns() { // across tests String source1 = "AAA" + randomAlphaOfLength(5); String source2 = "ZZZ" + randomAlphaOfLength(6); - Map> descriptors = new HashMap<>(); - descriptors.put(source1, List.of(pattern1)); - descriptors.put(source2, List.of(pattern2)); + Map descriptors = new HashMap<>(); + descriptors.put(source1, new SystemIndices.Feature("test", List.of(pattern1))); + descriptors.put(source2, new SystemIndices.Feature("test", List.of(pattern2))); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> SystemIndices.checkForOverlappingPatterns(descriptors)); @@ -83,8 +83,8 @@ public void testBuiltInSystemIndices() { } public void testPluginCannotOverrideBuiltInSystemIndex() { - Map> pluginMap = Map.of( - TaskResultsService.class.getName(), List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index")) + Map pluginMap = Map.of( + TASKS_FEATURE_NAME, new SystemIndices.Feature("test", List.of(new SystemIndexDescriptor(TASK_INDEX, "Task Result Index"))) ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SystemIndices(pluginMap)); assertThat(e.getMessage(), containsString("plugin or module attempted to define the same source")); @@ -92,7 +92,9 @@ public void testPluginCannotOverrideBuiltInSystemIndex() { public void testPatternWithSimpleRange() { - final SystemIndices systemIndices = new SystemIndices(Map.of("test", List.of(new SystemIndexDescriptor(".test-[abc]", "")))); + final SystemIndices systemIndices = new SystemIndices(Map.of( + "test", new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(".test-[abc]", ""))) + )); assertThat(systemIndices.isSystemIndex(".test-a"), equalTo(true)); assertThat(systemIndices.isSystemIndex(".test-b"), equalTo(true)); @@ -105,7 +107,9 @@ public void testPatternWithSimpleRange() { } public void testPatternWithSimpleRangeAndRepeatOperator() { - final SystemIndices systemIndices = new SystemIndices(Map.of("test", List.of(new SystemIndexDescriptor(".test-[a]+", "")))); + final SystemIndices systemIndices = new SystemIndices(Map.of( + "test", new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(".test-[a]+", ""))) + )); assertThat(systemIndices.isSystemIndex(".test-a"), equalTo(true)); assertThat(systemIndices.isSystemIndex(".test-aa"), equalTo(true)); @@ -115,7 +119,9 @@ public void testPatternWithSimpleRangeAndRepeatOperator() { } public void testPatternWithComplexRange() { - final SystemIndices systemIndices = new SystemIndices(Map.of("test", List.of(new SystemIndexDescriptor(".test-[a-c]", "")))); + final SystemIndices systemIndices = new SystemIndices(Map.of( + "test", new SystemIndices.Feature("test feature", List.of(new SystemIndexDescriptor(".test-[a-c]", ""))) + )); assertThat(systemIndices.isSystemIndex(".test-a"), equalTo(true)); assertThat(systemIndices.isSystemIndex(".test-b"), equalTo(true)); @@ -134,9 +140,9 @@ public void testOverlappingDescriptorsWithRanges() { SystemIndexDescriptor pattern1 = new SystemIndexDescriptor(".test-[ab]*", ""); SystemIndexDescriptor pattern2 = new SystemIndexDescriptor(".test-a*", ""); - Map> descriptors = new HashMap<>(); - descriptors.put(source1, List.of(pattern1)); - descriptors.put(source2, List.of(pattern2)); + Map descriptors = new HashMap<>(); + descriptors.put(source1, new SystemIndices.Feature("source 1", List.of(pattern1))); + descriptors.put(source2, new SystemIndices.Feature("source 2", List.of(pattern2))); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> SystemIndices.checkForOverlappingPatterns(descriptors)); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index edbf259780f08..ad4442dfff486 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -171,9 +171,21 @@ public void testSnapshotWithConflictingName() throws Exception { shardGenerations, RepositoryData.EMPTY_REPO_GEN, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), - new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream() - .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), 0L, null, 1L, 6, - Collections.emptyList(), true, Collections.emptyMap()), + new SnapshotInfo( + snapshot.getSnapshotId(), + shardGenerations.indices().stream() + .map(IndexId::getName) + .collect(Collectors.toList()), + Collections.emptyList(), + Collections.emptyList(), + null, + 1L, + 6, + Collections.emptyList(), + true, + Collections.emptyMap(), + 0L + ), Version.CURRENT, Function.identity(), f)); IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, () -> snapshotShard(shard, snapshotWithSameName, repository)); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java new file mode 100644 index 0000000000000..b16aa56293cf9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.snapshots; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +public class SnapshotFeatureInfoTests extends AbstractSerializingTestCase { + @Override + protected SnapshotFeatureInfo doParseInstance(XContentParser parser) throws IOException { + return SnapshotFeatureInfo.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return SnapshotFeatureInfo::new; + } + + @Override + protected SnapshotFeatureInfo createTestInstance() { + return randomSnapshotFeatureInfo(); + } + + public static SnapshotFeatureInfo randomSnapshotFeatureInfo() { + String feature = randomAlphaOfLengthBetween(5,20); + List indices = randomList(1, 10, () -> randomAlphaOfLengthBetween(5, 20)); + return new SnapshotFeatureInfo(feature, indices); + } + + @Override + protected SnapshotFeatureInfo mutateInstance(SnapshotFeatureInfo instance) throws IOException { + if (randomBoolean()) { + return new SnapshotFeatureInfo(randomValueOtherThan(instance.getPluginName(), () -> randomAlphaOfLengthBetween(5, 20)), + instance.getIndices()); + } else { + return new SnapshotFeatureInfo(instance.getPluginName(), + randomList(1, 10, () -> randomValueOtherThanMany(instance.getIndices()::contains, + () -> randomAlphaOfLengthBetween(5, 20)))); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java index 4ec9ea1f8bf82..e16005373aca2 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,8 +49,9 @@ protected SnapshotInfo createTestInstance() { Map userMetadata = randomUserMetadata(); - return new SnapshotInfo(snapshotId, indices, dataStreams, startTime, reason, endTime, totalShards, shardFailures, - includeGlobalState, userMetadata); + return new SnapshotInfo(snapshotId, indices, dataStreams, Collections.emptyList(), reason, endTime, totalShards, shardFailures, + includeGlobalState, userMetadata, startTime + ); } @Override @@ -64,29 +66,36 @@ protected SnapshotInfo mutateInstance(SnapshotInfo instance) { SnapshotId snapshotId = new SnapshotId( randomValueOtherThan(instance.snapshotId().getName(), () -> randomAlphaOfLength(5)), randomValueOtherThan(instance.snapshotId().getUUID(), () -> randomAlphaOfLength(5))); - return new SnapshotInfo(snapshotId, instance.indices(), instance.dataStreams(), instance.startTime(), instance.reason(), + return new SnapshotInfo(snapshotId, instance.indices(), instance.dataStreams(), Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - instance.userMetadata()); + instance.userMetadata(), instance.startTime() + ); case 1: int indicesSize = randomValueOtherThan(instance.indices().size(), () -> randomIntBetween(1, 10)); List indices = Arrays.asList(randomArray(indicesSize, indicesSize, String[]::new, () -> randomAlphaOfLengthBetween(2, 20))); - return new SnapshotInfo(instance.snapshotId(), indices, instance.dataStreams(), instance.startTime(), instance.reason(), + return new SnapshotInfo(instance.snapshotId(), indices, instance.dataStreams(), Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - instance.userMetadata()); + instance.userMetadata(), instance.startTime() + ); case 2: return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), - randomValueOtherThan(instance.startTime(), ESTestCase::randomNonNegativeLong), instance.reason(), - instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - instance.userMetadata()); + Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), + instance.includeGlobalState(), instance.userMetadata(), randomValueOtherThan(instance.startTime(), + ESTestCase::randomNonNegativeLong) + ); case 3: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), randomValueOtherThan(instance.reason(), () -> randomAlphaOfLengthBetween(5, 15)), instance.endTime(), - instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata()); + instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), + instance.startTime() + ); case 4: return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), - instance.startTime(), instance.reason(), randomValueOtherThan(instance.endTime(), ESTestCase::randomNonNegativeLong), - instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata()); + Collections.emptyList(), instance.reason(), randomValueOtherThan(instance.endTime(), ESTestCase::randomNonNegativeLong), + instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata(), + instance.startTime() + ); case 5: int totalShards = randomValueOtherThan(instance.totalShards(), () -> randomIntBetween(0, 100)); int failedShards = randomIntBetween(0, totalShards); @@ -99,23 +108,27 @@ protected SnapshotInfo mutateInstance(SnapshotInfo instance) { return new SnapshotShardFailure(randomAlphaOfLengthBetween(5, 10), shardId, randomAlphaOfLengthBetween(5, 10)); })); - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), instance.reason(), instance.endTime(), totalShards, shardFailures, instance.includeGlobalState(), - instance.userMetadata()); + instance.userMetadata(), instance.startTime() + ); case 6: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), - Boolean.FALSE.equals(instance.includeGlobalState()), instance.userMetadata()); + Boolean.FALSE.equals(instance.includeGlobalState()), instance.userMetadata(), instance.startTime() + ); case 7: - return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), instance.startTime(), + return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.dataStreams(), Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), - randomValueOtherThan(instance.userMetadata(), SnapshotInfoTests::randomUserMetadata)); + randomValueOtherThan(instance.userMetadata(), SnapshotInfoTests::randomUserMetadata), instance.startTime() + ); case 8: List dataStreams = randomValueOtherThan(instance.dataStreams(), () -> Arrays.asList(randomArray(1, 10, String[]::new, () -> randomAlphaOfLengthBetween(2, 20)))); return new SnapshotInfo(instance.snapshotId(), instance.indices(), dataStreams, - instance.startTime(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), - instance.includeGlobalState(), instance.userMetadata()); + Collections.emptyList(), instance.reason(), instance.endTime(), instance.totalShards(), instance.shardFailures(), + instance.includeGlobalState(), instance.userMetadata(), instance.startTime() + ); default: throw new IllegalArgumentException("invalid randomization case"); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 1f5c53e9271c2..155824dfb682d 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1449,7 +1449,7 @@ protected NamedWriteableRegistry writeableRegistry() { ); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService(settings, clusterService, indexNameExpressionResolver, repositoriesService, - transportService, actionFilters); + transportService, actionFilters, Collections.emptyMap()); nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); @@ -1562,13 +1562,14 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi final RestoreService restoreService = new RestoreService( clusterService, repositoriesService, allocationService, metadataCreateIndexService, + new MetadataDeleteIndexService(settings, clusterService, allocationService), new IndexMetadataVerifier( settings, namedXContentRegistry, mapperRegistry, indexScopedSettings, null), - shardLimitValidator - ); + shardLimitValidator, + systemIndices); actions.put(PutMappingAction.INSTANCE, new TransportPutMappingAction(transportService, clusterService, threadPool, metadataMappingService, actionFilters, indexNameExpressionResolver, new RequestValidators<>(Collections.emptyList()), diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java index e81d1490f514d..e6aff993d1b84 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -72,9 +72,10 @@ private Entry randomSnapshot() { shardState.failed() ? randomAlphaOfLength(10) : null, "1")); } } + List featureStates = randomList(5, SnapshotFeatureInfoTests::randomSnapshotFeatureInfo); ImmutableOpenMap shards = builder.build(); - return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, - startTime, repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); + return new Entry(snapshot, includeGlobalState, partial, randomState(shards), indices, dataStreams, featureStates, + startTime, repositoryStateId, shards, null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random())); } @Override @@ -141,38 +142,40 @@ protected Custom mutateInstance(Custom instance) { } private Entry mutateEntry(Entry entry) { - switch (randomInt(7)) { + switch (randomInt(8)) { case 0: boolean includeGlobalState = entry.includeGlobalState() == false; return new Entry(entry.snapshot(), includeGlobalState, entry.partial(), entry.state(), entry.indices(), entry.dataStreams(), - entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), entry.version()); + entry.featureStates(), entry.repositoryStateId(), entry.startTime(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 1: boolean partial = entry.partial() == false; return new Entry(entry.snapshot(), entry.includeGlobalState(), partial, entry.state(), entry.indices(), entry.dataStreams(), - entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), entry.version()); + entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 2: List dataStreams = Stream.concat( entry.dataStreams().stream(), Stream.of(randomAlphaOfLength(10))) .collect(Collectors.toList()); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - dataStreams, entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), - entry.version()); + dataStreams, entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 3: long startTime = randomValueOtherThan(entry.startTime(), ESTestCase::randomLong); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), startTime, entry.repositoryStateId(), entry.shards(), entry.failure(), entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), startTime, entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 4: long repositoryStateId = randomValueOtherThan(entry.startTime(), ESTestCase::randomLong); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), entry.startTime(), repositoryStateId, entry.shards(), entry.failure(), entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), repositoryStateId, entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); case 5: String failure = randomValueOtherThan(entry.failure(), () -> randomAlphaOfLengthBetween(2, 10)); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), entry.startTime(), entry.repositoryStateId(), entry.shards(), failure, entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), failure, + entry.userMetadata(), entry.version()); case 6: List indices = entry.indices(); ImmutableOpenMap shards = entry.shards(); @@ -192,8 +195,8 @@ private Entry mutateEntry(Entry entry) { } shards = builder.build(); return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), randomState(shards), indices, - entry.dataStreams(), entry.startTime(), entry.repositoryStateId(), shards, entry.failure(), entry.userMetadata(), - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), entry.repositoryStateId(), shards, entry.failure(), + entry.userMetadata(), entry.version()); case 7: Map userMetadata = entry.userMetadata() != null ? new HashMap<>(entry.userMetadata()) : new HashMap<>(); String key = randomAlphaOfLengthBetween(2, 10); @@ -203,8 +206,17 @@ private Entry mutateEntry(Entry entry) { userMetadata.put(key, randomAlphaOfLengthBetween(2, 10)); } return new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), entry.indices(), - entry.dataStreams(), entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), userMetadata, - entry.version()); + entry.dataStreams(), entry.featureStates(), entry.startTime(), entry.repositoryStateId(), entry.shards(), + entry.failure(), userMetadata, entry.version()); + case 8: + logger.error("randomizing feature states"); + List featureStates = randomList(1, 5, + () -> randomValueOtherThanMany(entry.featureStates()::contains, SnapshotFeatureInfoTests::randomSnapshotFeatureInfo)); + final Entry newEntry = new Entry(entry.snapshot(), entry.includeGlobalState(), entry.partial(), entry.state(), + entry.indices(), + entry.dataStreams(), featureStates, entry.startTime(), entry.repositoryStateId(), entry.shards(), entry.failure(), + entry.userMetadata(), entry.version()); + return newEntry; default: throw new IllegalArgumentException("invalid randomization case"); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 651961e8624a8..e7109516ca2b9 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -379,8 +379,8 @@ private static ClusterState applyUpdates(ClusterState state, SnapshotsService.Sh private static SnapshotsInProgress.Entry snapshotEntry(Snapshot snapshot, List indexIds, ImmutableOpenMap shards) { - return SnapshotsInProgress.startedEntry(snapshot, randomBoolean(), randomBoolean(), indexIds, Collections.emptyList(), - 1L, randomNonNegativeLong(), shards, Collections.emptyMap(), Version.CURRENT); + return SnapshotsInProgress.startedEntry(snapshot, randomBoolean(), randomBoolean(), indexIds, Collections.emptyList(), 1L, + randomNonNegativeLong(), shards, Collections.emptyMap(), Version.CURRENT, Collections.emptyList()); } private static SnapshotsInProgress.Entry cloneEntry( diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java index 67ec1a3a3847b..c4250fb728526 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java @@ -148,7 +148,7 @@ blobStoreContext, random())) { // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. repository.finalizeSnapshot(ShardGenerations.EMPTY, RepositoryData.EMPTY_REPO_GEN, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - 0L, null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap()), + Collections.emptyList(), null, 1L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f)); // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. @@ -156,7 +156,7 @@ blobStoreContext, random())) { () -> PlainActionFuture.get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), + Collections.emptyList(), null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f))); assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n but: was <5>")); @@ -165,7 +165,7 @@ blobStoreContext, random())) { PlainActionFuture.get(f -> repository.finalizeSnapshot(ShardGenerations.EMPTY, 0L, Metadata.EMPTY_METADATA, new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - 0L, null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap()), + Collections.emptyList(), null, 2L, 5, Collections.emptyList(), true, Collections.emptyMap(), 0L), Version.CURRENT, Function.identity(), f)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 658d7a5acae6a..8554de99e685b 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -79,6 +79,7 @@ import java.util.stream.StreamSupport; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY; +import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -382,6 +383,7 @@ protected SnapshotInfo createSnapshot(String repositoryName, String snapshot, Li .prepareCreateSnapshot(repositoryName, snapshot) .setIndices(indices.toArray(Strings.EMPTY_ARRAY)) .setWaitForCompletion(true) + .setFeatureStates(NO_FEATURE_STATES_VALUE) // Exclude all feature states to ensure only specified indices are included .get(); final SnapshotInfo snapshotInfo = response.getSnapshotInfo(); @@ -437,9 +439,9 @@ protected void addBwCFailedSnapshot(String repoName, String snapshotName, Map adding old version FAILED snapshot [{}] to repository [{}]", snapshotId, repoName); final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshotId, Collections.emptyList(), Collections.emptyList(), - SnapshotState.FAILED, "failed on purpose", - SnapshotsService.OLD_SNAPSHOT_FORMAT, 0L,0L, 0, 0, Collections.emptyList(), - randomBoolean(), metadata); + Collections.emptyList(), "failed on purpose", SnapshotsService.OLD_SNAPSHOT_FORMAT, 0L, 0L, 0, 0, Collections.emptyList(), + randomBoolean(), metadata, SnapshotState.FAILED + ); PlainActionFuture.get(f -> repo.finalizeSnapshot( ShardGenerations.EMPTY, getRepositoryData(repoName).getGenId(), state.metadata(), snapshotInfo, SnapshotsService.OLD_SNAPSHOT_FORMAT, Function.identity(), f)); diff --git a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java index 4d946067b79b7..fdc2abb909617 100644 --- a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java +++ b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java @@ -48,6 +48,16 @@ public Collection getSystemIndexDescriptors(Settings sett return List.of(AsyncTaskIndexService.getSystemIndexDescriptor()); } + @Override + public String getFeatureName() { + return "async_search"; + } + + @Override + public String getFeatureDescription() { + return "Manages results of async searches"; + } + @Override public Collection createComponents( Client client, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index ef5d17b352114..c44e064641154 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -182,8 +182,9 @@ public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) { ArrayList indices = new ArrayList<>(indicesMap.size()); indicesMap.keysIt().forEachRemaining(indices::add); - return new SnapshotInfo(snapshotId, indices, new ArrayList<>(metadata.dataStreams().keySet()), SnapshotState.SUCCESS, - response.getState().getNodes().getMaxNodeVersion()); + return new SnapshotInfo(snapshotId, indices, new ArrayList<>(metadata.dataStreams().keySet()), Collections.emptyList(), + response.getState().getNodes().getMaxNodeVersion(), SnapshotState.SUCCESS + ); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index c396456b9bb9d..dacb5af3f0a12 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -228,9 +228,9 @@ public void testRestoreMinmal() throws IOException { Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), new SnapshotInfo(snapshotId, shardGenerations.indices().stream() - .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), 0L, null, 1L, - shardGenerations.totalShards(), - Collections.emptyList(), true, Collections.emptyMap()), + .map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), Collections.emptyList(), null, 1L, + shardGenerations.totalShards(), Collections.emptyList(), true, Collections.emptyMap(), 0L + ), Version.CURRENT, Function.identity(), finFuture); finFuture.actionGet(); }); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index a7758ae071c28..3aae4c0d5917b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -564,4 +564,14 @@ public Map getMetadataMappers() { .flatMap (map -> map.entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } + + @Override + public String getFeatureName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return this.getClass().getCanonicalName(); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java index 5ca58cac912ed..3e486d46a5cf1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/AsyncTaskServiceTests.java @@ -63,6 +63,16 @@ public static class TestPlugin extends Plugin implements SystemIndexPlugin { public Collection getSystemIndexDescriptors(Settings settings) { return List.of(AsyncTaskIndexService.getSystemIndexDescriptor()); } + + @Override + public String getFeatureName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return this.getClass().getCanonicalName(); + } } public void testEnsuredAuthenticatedUserIsSame() throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java index a14b043a1c555..7d2c6a319363f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotRetentionConfigurationTests.java @@ -288,13 +288,8 @@ private SnapshotInfo makeInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-" + randomAlphaOfLength(3), "uuid"), Collections.singletonList("foo"), Collections.singletonList("bar"), - startTime, - null, - startTime + between(1, 10000), - totalShards, - new ArrayList<>(), - false, - meta); + Collections.emptyList(), null, startTime + between(1, 10000), totalShards, new ArrayList<>(), false, meta, startTime + ); assertThat(snapInfo.state(), equalTo(SnapshotState.SUCCESS)); return snapInfo; } @@ -320,13 +315,9 @@ private SnapshotInfo makeFailureInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - startTime, - "forced-failure", - startTime + between(1, 10000), - totalShards, - failures, - randomBoolean(), - meta); + Collections.emptyList(), + "forced-failure", startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime + ); assertThat(snapInfo.state(), equalTo(SnapshotState.FAILED)); return snapInfo; } @@ -344,13 +335,8 @@ private SnapshotInfo makePartialInfo(long startTime) { SnapshotInfo snapInfo = new SnapshotInfo(new SnapshotId("snap-fail-" + randomAlphaOfLength(3), "uuid-fail"), Collections.singletonList("foo-fail"), Collections.singletonList("bar-fail"), - startTime, - null, - startTime + between(1, 10000), - totalShards, - failures, - randomBoolean(), - meta); + Collections.emptyList(), null, startTime + between(1, 10000), totalShards, failures, randomBoolean(), meta, startTime + ); assertThat(snapInfo.state(), equalTo(SnapshotState.PARTIAL)); return snapInfo; } diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java index 4d97513137595..27b43fa554a2a 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -116,6 +116,7 @@ private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo SnapshotsInProgress.State.SUCCESS, Collections.emptyList(), List.of(dataStreamName), + Collections.emptyList(), 0, 1, ImmutableOpenMap.of(), diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index 5079319e9e4ee..cf1cb00e1c0fe 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -240,4 +240,14 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(ENRICH_INDEX_PATTERN, "Contains data to support enrich ingest processors.") ); } + + @Override + public String getFeatureName() { + return "enrich"; + } + + @Override + public String getFeatureDescription() { + return "Manages data related to Enrich policies"; + } } diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 475960a7153eb..dad4be17c80ad 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -31,4 +31,14 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(".fleet-actions*", "Fleet actions") ); } + + @Override + public String getFeatureName() { + return "fleet"; + } + + @Override + public String getFeatureDescription() { + return "Manages configuration for Fleet"; + } } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java index 6dee0f5d9ee52..7998e4a5dd597 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java @@ -245,14 +245,9 @@ public void testPartialFailureSnapshot() throws Exception { new SnapshotId(req.snapshot(), "uuid"), Arrays.asList(req.indices()), Collections.emptyList(), - startTime, - "snapshot started", - endTime, - 3, - Collections.singletonList( + Collections.emptyList(), "snapshot started", endTime, 3, Collections.singletonList( new SnapshotShardFailure("nodeId", new ShardId("index", "uuid", 0), "forced failure")), - req.includeGlobalState(), - req.userMetadata() + req.includeGlobalState(), req.userMetadata(), startTime )); })) { final AtomicBoolean historyStoreCalled = new AtomicBoolean(false); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index afa3cd4eef55c..631a9aceec02c 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -111,35 +111,37 @@ public void testSnapshotEligibleForDeletion() { // Test when user metadata is null SnapshotInfo info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), - Collections.emptyList(),0L, null, 1L, 1, Collections.emptyList(), true, null); + Collections.emptyList(), Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, null, 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); // Test when no retention is configured info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - 0L, null, 1L, 1, Collections.emptyList(), true, null); + Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, null, 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyWithNoRetentionMap), equalTo(false)); // Test when user metadata is a map that doesn't contain "policy" info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - 0L, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("foo", "bar")); + Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("foo", "bar"), 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); // Test with an ancient snapshot that should be expunged info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - 0L, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy")); + Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy"), 0L); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(true)); // Test with a snapshot that's start date is old enough to be expunged (but the finish date is not) long time = System.currentTimeMillis() - TimeValue.timeValueDays(30).millis() - 1; info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - time, null, time + TimeValue.timeValueDays(4).millis(), 1, Collections.emptyList(), - true, Collections.singletonMap("policy", "policy")); + Collections.emptyList(), null, time + TimeValue.timeValueDays(4).millis(), 1, Collections.emptyList(), true, + Collections.singletonMap("policy", "policy"), time + ); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(true)); // Test with a fresh snapshot that should not be expunged info = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), Collections.emptyList(), - System.currentTimeMillis(), null, System.currentTimeMillis() + 1, - 1, Collections.emptyList(), true, Collections.singletonMap("policy", "policy")); + Collections.emptyList(), null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, + Collections.singletonMap("policy", "policy"), System.currentTimeMillis() + ); assertThat(SnapshotRetentionTask.snapshotEligibleForDeletion(info, mkInfos.apply(info), policyMap), equalTo(false)); } @@ -165,10 +167,12 @@ private void retentionTaskTest(final boolean deletionSuccess) throws Exception { ClusterServiceUtils.setState(clusterService, state); final SnapshotInfo eligibleSnapshot = new SnapshotInfo(new SnapshotId("name", "uuid"), Collections.singletonList("index"), - Collections.emptyList(), 0L, null, 1L, 1, Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), Collections.emptyList(), null, 1L, 1, Collections.emptyList(), true, + Collections.singletonMap("policy", policyId), 0L); final SnapshotInfo ineligibleSnapshot = new SnapshotInfo(new SnapshotId("name2", "uuid2"), Collections.singletonList("index"), - Collections.emptyList(), System.currentTimeMillis(), null, System.currentTimeMillis() + 1, 1, - Collections.emptyList(), true, Collections.singletonMap("policy", policyId)); + Collections.emptyList(), Collections.emptyList(), null, System.currentTimeMillis() + 1, 1, Collections.emptyList(), true, + Collections.singletonMap("policy", policyId), System.currentTimeMillis() + ); Set deleted = ConcurrentHashMap.newKeySet(); Set deletedSnapshotsInHistory = ConcurrentHashMap.newKeySet(); diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index 5fc59607bba77..f7b9b90467f20 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -176,4 +176,14 @@ private XContentBuilder getIndexMappings() { throw new UncheckedIOException("Failed to build " + LOGSTASH_CONCRETE_INDEX_NAME + " index mappings", e); } } + + @Override + public String getFeatureName() { + return "logstash_management"; + } + + @Override + public String getFeatureDescription() { + return "Enables Logstash Central Management pipeline storage"; + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 6dda8ee2fba5f..74a2fd36cce40 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1196,6 +1196,16 @@ public static SystemIndexDescriptor getInferenceIndexSecurityDescriptor() { .build(); } + @Override + public String getFeatureName() { + return "machine_learning"; + } + + @Override + public String getFeatureDescription() { + return "Provides anomaly detection and forecasting functionality"; + } + @Override public BreakerSettings getCircuitBreaker(Settings settings) { return BreakerSettings.updateFromSettings( diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java index 5efc8df21f9f8..c91734e83871a 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java @@ -10,12 +10,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import java.nio.file.Path; import java.util.Collection; -public class LocalStateSearchableSnapshots extends LocalStateCompositeXPackPlugin { +public class LocalStateSearchableSnapshots extends LocalStateCompositeXPackPlugin implements SystemIndexPlugin { private final SearchableSnapshots plugin; @@ -36,4 +37,14 @@ protected XPackLicenseState getLicenseState() { public Collection getSystemIndexDescriptors(Settings settings) { return plugin.getSystemIndexDescriptors(settings); } + + @Override + public String getFeatureName() { + return plugin.getFeatureName(); + } + + @Override + public String getFeatureDescription() { + return plugin.getFeatureDescription(); + } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java index 3372f64f843ae..7fa3dccb4c4d7 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java @@ -91,5 +91,15 @@ public static class TestSystemIndexPlugin extends Plugin implements SystemIndexP public Collection getSystemIndexDescriptors(Settings settings) { return List.of(new SystemIndexDescriptor(INDEX_NAME, "System index for [" + getTestClass().getName() + ']')); } + + @Override + public String getFeatureName() { + return SearchableSnapshotsSystemIndicesIntegTests.class.getSimpleName(); + } + + @Override + public String getFeatureDescription() { + return "test plugin"; + } } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 1b76d6b3312b8..60ccf155b9780 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -335,6 +335,16 @@ public Collection getSystemIndexDescriptors(Settings sett ); } + @Override + public String getFeatureName() { + return "searchable_snapshots"; + } + + @Override + public String getFeatureDescription() { + return "Manages caches and configuration for searchable snapshots"; + } + @Override public Map getDirectoryFactories() { return Map.of(SNAPSHOT_DIRECTORY_FACTORY_KEY, (indexSettings, shardPath) -> { diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java index 65289a375b33a..b485c6c72d1a1 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java @@ -259,6 +259,7 @@ public void testSnapshotOfSearchableSnapshot() throws Exception { try (XContentBuilder builder = jsonBuilder()) { builder.startObject(); builder.field("indices", restoredIndexName); + builder.field("include_global_state", "false"); builder.endObject(); snapshotRequest.setEntity(new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON)); } diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 61b43ac12d278..ec6daa1b44b77 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -81,6 +81,7 @@ public class Constants { "cluster:admin/snapshot/restore", "cluster:admin/snapshot/status", "cluster:admin/snapshot/status[nodes]", + "cluster:admin/snapshot/features/get", "cluster:admin/tasks/cancel", "cluster:admin/transform/delete", "cluster:admin/transform/preview", diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java new file mode 100644 index 0000000000000..c3967c799c224 --- /dev/null +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.integration; + +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.SecuritySettingsSourceField; +import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; +import org.hamcrest.Matchers; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.nio.file.Path; + +import static org.elasticsearch.test.SecuritySettingsSource.TEST_SUPERUSER; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class SecurityFeatureStateIntegTests extends AbstractPrivilegeTestCase { + + private static final String LOCAL_TEST_USER_NAME = "feature_state_user"; + private static final String LOCAL_TEST_USER_PASSWORD = "my_password"; + private static Path repositoryLocation; + + @BeforeClass + public static void setupRepositoryPath() { + repositoryLocation = createTempDir(); + } + + @AfterClass + public static void cleanupRepositoryPath() { + repositoryLocation = null; + } + + @Override + protected boolean addMockHttpTransport() { + return false; // enable http + } + + @Override + protected Settings nodeSettings() { + return Settings.builder().put(super.nodeSettings()) + .put("path.repo", repositoryLocation) + .build(); + } + + /** + * Test that, when the security system index is restored as a feature state, + * the security plugin's listeners detect the state change and reload native + * realm privileges. + * + * We use the admin client to handle snapshots and the rest API to manage + * security roles and users. We use the native realm instead of the file + * realm because this test relies on dynamically changing privileges. + */ + public void testSecurityFeatureStateSnapshotAndRestore() throws Exception { + // set up a snapshot repository + final String repositoryName = "test-repo"; + client().admin().cluster().preparePutRepository(repositoryName) + .setType("fs") + .setSettings(Settings.builder().put("location", repositoryLocation)) + .get(); + + // create a new role + final String roleName = "extra_role"; + final Request createRoleRequest = new Request("PUT", "/_security/role/" + roleName); + createRoleRequest.addParameter("refresh", "wait_for"); + createRoleRequest.setJsonEntity("{" + + " \"indices\": [" + + " {" + + " \"names\": [ \"test_index\" ]," + + " \"privileges\" : [ \"create\", \"create_index\", \"create_doc\" ]" + + " }" + + " ]" + + "}"); + performSuperuserRequest(createRoleRequest); + + // create a test user + final Request createUserRequest = new Request("PUT", "/_security/user/" + LOCAL_TEST_USER_NAME); + createUserRequest.addParameter("refresh", "wait_for"); + createUserRequest.setJsonEntity("{" + + " \"password\": \"" + LOCAL_TEST_USER_PASSWORD + "\"," + + " \"roles\": [ \"" + roleName + "\" ]" + + "}"); + performSuperuserRequest(createUserRequest); + + // test user posts a document + final Request postTestDocument1 = new Request("POST", "/test_index/_doc"); + postTestDocument1.setJsonEntity("{\"message\": \"before snapshot\"}"); + performTestUserRequest(postTestDocument1); + + // snapshot state + final String snapshotName = "security-state"; + client().admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName) + .setIndices("test_index") + .setFeatureStates("LocalStateSecurity") + .get(); + waitForSnapshotToFinish(repositoryName, snapshotName); + + // modify user's roles + final Request modifyUserRequest = new Request("PUT", "/_security/user/" + LOCAL_TEST_USER_NAME); + modifyUserRequest.addParameter("refresh", "wait_for"); + modifyUserRequest.setJsonEntity("{\"roles\": [] }"); + performSuperuserRequest(modifyUserRequest); + + // new user has lost privileges and can't post a document + final Request postDocumentRequest2 = new Request("POST", "/test_index/_doc"); + postDocumentRequest2.setJsonEntity("{\"message\": \"between snapshot and restore\"}"); + ResponseException exception = expectThrows(ResponseException.class, () -> performTestUserRequest(postDocumentRequest2)); + + assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(403)); + assertThat(exception.getMessage(), + containsString("action [" + IndexAction.NAME + "] is unauthorized for user [" + LOCAL_TEST_USER_NAME + "]")); + + client().admin().indices().prepareClose("test_index").get(); + + // restore state + client().admin().cluster().prepareRestoreSnapshot(repositoryName, snapshotName) + .setFeatureStates("LocalStateSecurity") + .setIndices("test_index") + .setWaitForCompletion(true) + .get(); + + // user has privileges again + final Request postDocumentRequest3 = new Request("POST", "/test_index/_doc"); + postDocumentRequest3.setJsonEntity("{\"message\": \"after restore\"}"); + performTestUserRequest(postDocumentRequest3); + } + + private Response performSuperuserRequest(Request request) throws Exception { + String token = UsernamePasswordToken.basicAuthHeaderValue( + TEST_SUPERUSER, new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())); + return performAuthenticatedRequest(request, token); + } + + private Response performTestUserRequest(Request request) throws Exception { + String token = UsernamePasswordToken.basicAuthHeaderValue( + LOCAL_TEST_USER_NAME, new SecureString(LOCAL_TEST_USER_PASSWORD.toCharArray())); + return performAuthenticatedRequest(request, token); + } + + private Response performAuthenticatedRequest(Request request, String token) throws Exception { + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Authorization", token); + request.setOptions(options); + return getRestClient().performRequest(request); + } + + private void waitForSnapshotToFinish(String repo, String snapshot) throws Exception { + assertBusy(() -> { + SnapshotsStatusResponse response = client().admin().cluster().prepareSnapshotStatus(repo).setSnapshots(snapshot).get(); + assertThat(response.getSnapshots().get(0).getState(), is(SnapshotsInProgress.State.SUCCESS)); + // The status of the snapshot in the repository can become SUCCESS before it is fully finalized in the cluster state so wait for + // it to disappear from the cluster state as well + SnapshotsInProgress snapshotsInProgress = + client().admin().cluster().state(new ClusterStateRequest()).get().getState().custom(SnapshotsInProgress.TYPE); + assertThat(snapshotsInProgress.entries(), Matchers.empty()); + }); + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 80d7e9ce5cc39..0eb105ca5c688 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -217,10 +217,10 @@ import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor; +import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.operator.OperatorOnlyRegistry; import org.elasticsearch.xpack.security.operator.OperatorPrivileges; import org.elasticsearch.xpack.security.operator.OperatorPrivileges.OperatorPrivilegesService; -import org.elasticsearch.xpack.security.operator.FileOperatorUsersStore; import org.elasticsearch.xpack.security.rest.SecurityRestFilter; import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction; import org.elasticsearch.xpack.security.rest.action.RestDelegatePkiAuthenticationAction; @@ -1773,5 +1773,13 @@ private static XContentBuilder getTokenIndexMappings() { } } -} + @Override + public String getFeatureName() { + return "security"; + } + @Override + public String getFeatureDescription() { + return "Manages configuration for Security features, such as users and roles"; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java index 719e0f90e1b9c..6fbf64c5376eb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java @@ -206,7 +206,9 @@ public void expireAll() { public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { if (lastSuccessfulAuthCache != null) { - if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState)) { + if (isMoveFromRedToNonRed(previousState, currentState) + || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false) { expireAll(); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java index 642e24b4f1585..3f7643762e7d1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealm.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; @@ -43,7 +44,9 @@ protected void doAuthenticate(UsernamePasswordToken token, ActionListener> listener, List> listener) { } public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { - if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) || - previousState.isIndexUpToDate != currentState.isIndexUpToDate) { + if (isMoveFromRedToNonRed(previousState, currentState) + || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false + || previousState.isIndexUpToDate != currentState.isIndexUpToDate) { invalidateAll(); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java index 869969af2c4cc..3b55d029d314e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java @@ -57,6 +57,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; @@ -419,7 +420,9 @@ private static String toDocId(String application, String name) { } public void onSecurityIndexStateChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { - if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) + if (isMoveFromRedToNonRed(previousState, currentState) + || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false || previousState.isIndexUpToDate != currentState.isIndexUpToDate) { invalidateAll(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java index 4a487cae15360..9c68e92d9331c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistry.java @@ -9,6 +9,7 @@ import java.util.Collection; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; @@ -34,6 +35,7 @@ public void registerCacheInvalidator(String name, CacheInvalidator cacheInvalida public void onSecurityIndexStageChange(SecurityIndexManager.State previousState, SecurityIndexManager.State currentState) { if (isMoveFromRedToNonRed(previousState, currentState) || isIndexDeleted(previousState, currentState) + || Objects.equals(previousState.indexUUID, currentState.indexUUID) == false || previousState.isIndexUpToDate != currentState.isIndexUpToDate) { cacheInvalidators.values().forEach(CacheInvalidator::invalidateAll); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index b84f40547bf23..e74f62937a9a9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -189,8 +189,9 @@ public void clusterChanged(ClusterChangedEvent event) { final IndexRoutingTable routingTable = event.state().getRoutingTable().index(indexMetadata.getIndex()); indexHealth = new ClusterIndexHealth(indexMetadata, routingTable).getStatus(); } + final String indexUUID = indexMetadata != null ? indexMetadata.getIndexUUID() : null; final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion, - concreteIndexName, indexHealth, indexState, event.state().nodes().getMinNodeVersion()); + concreteIndexName, indexHealth, indexState, event.state().nodes().getMinNodeVersion(), indexUUID); this.indexState = newState; if (newState.equals(previousState) == false) { @@ -414,7 +415,7 @@ public static boolean isIndexDeleted(State previousState, State currentState) { * State of the security index. */ public static class State { - public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null, null); + public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null, null, null); public final Instant creationTime; public final boolean isIndexUpToDate; public final boolean indexAvailable; @@ -424,10 +425,11 @@ public static class State { public final ClusterHealthStatus indexHealth; public final IndexMetadata.State indexState; public final Version minimumNodeVersion; + public final String indexUUID; public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailable, boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth, - IndexMetadata.State indexState, Version minimumNodeVersion) { + IndexMetadata.State indexState, Version minimumNodeVersion, String indexUUID) { this.creationTime = creationTime; this.isIndexUpToDate = isIndexUpToDate; this.indexAvailable = indexAvailable; @@ -437,6 +439,7 @@ public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailab this.indexHealth = indexHealth; this.indexState = indexState; this.minimumNodeVersion = minimumNodeVersion; + this.indexUUID = indexUUID; } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index b4fc0d5a0abce..1840ac4bf522b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2029,6 +2029,6 @@ private void setCompletedToTrue(AtomicBoolean completed) { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null); + Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null, "my_uuid"); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index a5e087e1cd8a0..3556f826c9e33 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -31,7 +31,7 @@ public class NativeRealmTests extends ESTestCase { private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { return new SecurityIndexManager.State( - Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null); + Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetadata.State.OPEN, null, "my_uuid"); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index f657662ccf11c..ebdcfd6f2c399 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -151,7 +151,8 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null); + Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null, "my_uuid" + ); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index dcccf991c2841..4b57ce3547dcf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -810,7 +810,8 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( - Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null); + Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null, + "my_uuid"); } public void testCacheClearOnIndexHealthChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index fc9a3824893d1..adab224deb4b6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -611,7 +611,7 @@ private SecurityIndexManager.State dummyState( String concreteSecurityIndexName, boolean isIndexUpToDate, ClusterHealthStatus healthStatus) { return new SecurityIndexManager.State( Instant.now(), isIndexUpToDate, true, true, null, - concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null + concreteSecurityIndexName, healthStatus, IndexMetadata.State.OPEN, null, "my_uuid" ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index 46c444afd738d..137fedbddf5e0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -49,7 +49,7 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( final SecurityIndexManager.State previousState = SecurityIndexManager.State.UNRECOVERED_STATE; final SecurityIndexManager.State currentState = new SecurityIndexManager.State( Instant.now(), true, true, true, Version.CURRENT, - ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, null); + ".security", ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, null, "my_uuid"); cacheInvalidatorRegistry.onSecurityIndexStageChange(previousState, currentState); verify(invalidator1).invalidateAll(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 54a2b3fd56974..3c36c03186518 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -370,4 +370,14 @@ public Collection getSystemIndexDescriptors(Settings sett throw new UncheckedIOException(e); } } + + @Override + public String getFeatureName() { + return "transform"; + } + + @Override + public String getFeatureDescription() { + return "Manages configuration and state for transforms"; + } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 7ba91d93bd90f..e7b1ef0aa81cb 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -700,4 +700,14 @@ public Collection getSystemIndexDescriptors(Settings sett new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution") ); } + + @Override + public String getFeatureName() { + return "watcher"; + } + + @Override + public String getFeatureDescription() { + return "Manages Watch definitions and state"; + } }