Skip to content

Commit 37123d7

Browse files
dakronedakrone
committed
Auto convert shared_cache SBIs to only use frozen tier (elastic#71014)
This commit converts the index metadata of searchable snapshot indices using the `shared_cache` storage type to: - Remove all the `index.routing.allocation.(include|exclude|require)._tier` settings - Sets `index.routing.allocation.include._tier_preference` to `data_frozen` automatically when the index metadata is read This is in preperation to enforcing that the `_tier_preference` setting is always set to `data_frozen` for shared cache SBIs. Relates to elastic#70846, elastic#71013, elastic#70786, elastic#70141
1 parent df003fa commit 37123d7

File tree

3 files changed

+128
-9
lines changed

3 files changed

+128
-9
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,11 @@ public IndexMetadataVerifier(Settings settings, NamedXContentRegistry xContentRe
7171
public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, Version minimumIndexCompatibilityVersion) {
7272
checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion);
7373

74-
// we have to run this first otherwise in we try to create IndexSettings
75-
// with broken settings and fail in checkMappingsCompatibility
76-
IndexMetadata newMetadata = archiveBrokenIndexSettings(indexMetadata);
74+
// First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen
75+
IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata);
76+
// Next we have to run this otherwise if we try to create IndexSettings
77+
// with broken settings it would fail in checkMappingsCompatibility
78+
newMetadata = archiveBrokenIndexSettings(newMetadata);
7779
checkMappingsCompatibility(newMetadata);
7880
return newMetadata;
7981
}
@@ -183,4 +185,30 @@ IndexMetadata archiveBrokenIndexSettings(IndexMetadata indexMetadata) {
183185
return indexMetadata;
184186
}
185187
}
188+
189+
/**
190+
* Convert shared_cache searchable snapshot indices to only specify
191+
* _tier_preference: data_frozen, removing any pre-existing tier allocation rules.
192+
*/
193+
IndexMetadata convertSharedCacheTierPreference(IndexMetadata indexMetadata) {
194+
final Settings settings = indexMetadata.getSettings();
195+
// Only remove these settings for a shared_cache searchable snapshot
196+
if ("snapshot".equals(settings.get("index.store.type", "")) && settings.getAsBoolean("index.store.snapshot.partial", false)) {
197+
final Settings.Builder settingsBuilder = Settings.builder().put(settings);
198+
// Clear any allocation rules other than preference for tier
199+
settingsBuilder.remove("index.routing.allocation.include._tier");
200+
settingsBuilder.remove("index.routing.allocation.exclude._tier");
201+
settingsBuilder.remove("index.routing.allocation.require._tier");
202+
// Override the tier preference to be only on frozen nodes, regardless of its current setting
203+
settingsBuilder.put("index.routing.allocation.include._tier_preference", "data_frozen");
204+
final Settings newSettings = settingsBuilder.build();
205+
if (settings.equals(newSettings)) {
206+
return indexMetadata;
207+
} else {
208+
return IndexMetadata.builder(indexMetadata).settings(newSettings).build();
209+
}
210+
} else {
211+
return indexMetadata;
212+
}
213+
}
186214
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataConversionTests.java

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.cluster.metadata;
9+
10+
import org.elasticsearch.common.UUIDs;
11+
import org.elasticsearch.common.settings.IndexScopedSettings;
12+
import org.elasticsearch.common.settings.Settings;
13+
import org.elasticsearch.index.IndexModule;
14+
import org.elasticsearch.index.mapper.MapperRegistry;
15+
import org.elasticsearch.plugins.MapperPlugin;
16+
import org.elasticsearch.test.ESTestCase;
17+
import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
18+
19+
import java.util.Collections;
20+
21+
import static org.elasticsearch.test.VersionUtils.randomIndexCompatibleVersion;
22+
import static org.hamcrest.Matchers.equalTo;
23+
24+
public class IndexMetadataConversionTests extends ESTestCase {
25+
26+
public void testConvertSearchableSnapshotSettings() {
27+
IndexMetadataVerifier service = getIndexMetadataVerifier();
28+
IndexMetadata src = newIndexMeta("foo", Settings.EMPTY);
29+
IndexMetadata indexMetadata = service.convertSharedCacheTierPreference(src);
30+
assertSame(indexMetadata, src);
31+
32+
// A full_copy searchable snapshot (settings should be untouched)
33+
src = newIndexMeta("foo", Settings.builder()
34+
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
35+
.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
36+
.put("index.routing.allocation.include._tier", "data_hot")
37+
.put("index.routing.allocation.exclude._tier", "data_warm")
38+
.put("index.routing.allocation.require._tier", "data_hot")
39+
.put("index.routing.allocation.include._tier_preference", "data_cold")
40+
.build());
41+
indexMetadata = service.convertSharedCacheTierPreference(src);
42+
assertSame(indexMetadata, src);
43+
44+
// A shared_cache searchable snapshot with valid settings (metadata should be untouched)
45+
src = newIndexMeta("foo", Settings.builder()
46+
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
47+
.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
48+
.put("index.routing.allocation.include._tier_preference", "data_frozen")
49+
.build());
50+
indexMetadata = service.convertSharedCacheTierPreference(src);
51+
assertSame(indexMetadata, src);
52+
53+
// A shared_cache searchable snapshot (should have its settings converted)
54+
src = newIndexMeta("foo", Settings.builder()
55+
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
56+
.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), true)
57+
.put("index.routing.allocation.include._tier", "data_hot")
58+
.put("index.routing.allocation.exclude._tier", "data_warm")
59+
.put("index.routing.allocation.require._tier", "data_hot")
60+
.put("index.routing.allocation.include._tier_preference", "data_frozen,data_cold")
61+
.build());
62+
indexMetadata = service.convertSharedCacheTierPreference(src);
63+
assertNotSame(indexMetadata, src);
64+
Settings newSettings = indexMetadata.getSettings();
65+
assertNull(newSettings.get("index.routing.allocation.include._tier"));
66+
assertNull(newSettings.get("index.routing.allocation.exclude._tier"));
67+
assertNull(newSettings.get("index.routing.allocation.require._tier"));
68+
assertThat(newSettings.get("index.routing.allocation.include._tier_preference"), equalTo("data_frozen"));
69+
}
70+
71+
private IndexMetadataVerifier getIndexMetadataVerifier() {
72+
return new IndexMetadataVerifier(
73+
Settings.EMPTY,
74+
xContentRegistry(),
75+
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(),
76+
MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
77+
null
78+
);
79+
}
80+
81+
public static IndexMetadata newIndexMeta(String name, Settings indexSettings) {
82+
final Settings settings = Settings.builder()
83+
.put(IndexMetadata.SETTING_VERSION_CREATED, randomIndexCompatibleVersion(random()))
84+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5))
85+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))
86+
.put(IndexMetadata.SETTING_CREATION_DATE, randomNonNegativeLong())
87+
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random()))
88+
.put(indexSettings)
89+
.build();
90+
final IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(name).settings(settings);
91+
if (randomBoolean()) {
92+
indexMetadataBuilder.state(IndexMetadata.State.CLOSE);
93+
}
94+
return indexMetadataBuilder.build();
95+
}
96+
}

‎x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -502,12 +502,7 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception {
502502
expectedReplicas = 0;
503503
}
504504
final String expectedDataTiersPreference;
505-
if (randomBoolean()) {
506-
expectedDataTiersPreference = String.join(",", randomSubsetOf(DataTier.ALL_DATA_TIERS));
507-
indexSettingsBuilder.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, expectedDataTiersPreference);
508-
} else {
509-
expectedDataTiersPreference = getDataTiersPreference(MountSearchableSnapshotRequest.Storage.SHARED_CACHE);
510-
}
505+
expectedDataTiersPreference = getDataTiersPreference(MountSearchableSnapshotRequest.Storage.SHARED_CACHE);
511506

512507
indexSettingsBuilder.put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.ZERO);
513508
final AtomicBoolean statsWatcherRunning = new AtomicBoolean(true);

0 commit comments

Comments
 (0)