Skip to content

Commit 0da111d

Browse files
danhermanndanhermann
authored
Use legacy backing index names for data streams in mixed clusters (elastic#66400) (elastic#66468)
1 parent 675bfd4 commit 0da111d

File tree

9 files changed

+53
-19
lines changed

9 files changed

+53
-19
lines changed

‎build.gradle

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,8 @@ tasks.register("verifyVersions") {
183183
* after the backport of the backcompat code is complete.
184184
*/
185185

186-
boolean bwc_tests_enabled = false
187-
final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/issues/66370" /* place a PR link here when committing bwc changes */
186+
boolean bwc_tests_enabled = true
187+
final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
188188
if (bwc_tests_enabled == false) {
189189
if (bwc_tests_disabled_issue.isEmpty()) {
190190
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.action.admin.indices.rollover;
2121

22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
2324
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
2425
import org.elasticsearch.action.support.ActiveShardCount;
@@ -148,10 +149,11 @@ private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstra
148149
boolean silent, boolean onlyValidate) throws Exception {
149150
lookupTemplateForDataStream(dataStreamName, currentState.metadata());
150151

152+
final Version minNodeVersion = currentState.nodes().getMinNodeVersion();
151153
final DataStream ds = dataStream.getDataStream();
152154
final IndexMetadata originalWriteIndex = dataStream.getWriteIndex();
153-
final String newWriteIndexName = DataStream.getDefaultBackingIndexName(ds.getName(), ds.getGeneration() + 1);
154-
ds.rollover(new Index(newWriteIndexName, "uuid")); // just for validation
155+
final String newWriteIndexName = DataStream.getDefaultBackingIndexName(ds.getName(), ds.getGeneration() + 1, minNodeVersion);
156+
ds.rollover(new Index(newWriteIndexName, "uuid"), minNodeVersion); // just for validation
155157
createIndexService.validateIndexName(newWriteIndexName, currentState); // fails if the index already exists
156158
if (onlyValidate) {
157159
return new RolloverResult(newWriteIndexName, originalWriteIndex.getIndex().getName(), currentState);
@@ -160,7 +162,7 @@ private RolloverResult rolloverDataStream(ClusterState currentState, IndexAbstra
160162
CreateIndexClusterStateUpdateRequest createIndexClusterStateRequest =
161163
prepareDataStreamCreateIndexRequest(dataStreamName, newWriteIndexName, createIndexRequest);
162164
ClusterState newState = createIndexService.applyCreateIndexRequest(currentState, createIndexClusterStateRequest, silent,
163-
(builder, indexMetadata) -> builder.put(ds.rollover(indexMetadata.getIndex())));
165+
(builder, indexMetadata) -> builder.put(ds.rollover(indexMetadata.getIndex(), minNodeVersion)));
164166

165167
RolloverInfo rolloverInfo = new RolloverInfo(dataStreamName, metConditions, threadPool.absoluteTimeInMillis());
166168
newState = ClusterState.builder(newState)

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,8 @@ public boolean isReplicated() {
126126
* backing indices on data streams. See {@link #getDefaultBackingIndexName}.
127127
* @return new {@code DataStream} instance with the rollover operation applied
128128
*/
129-
public DataStream rollover(Index newWriteIndex) {
130-
assert newWriteIndex.getName().equals(getDefaultBackingIndexName(name, generation + 1));
129+
public DataStream rollover(Index newWriteIndex, Version minNodeVersion) {
130+
assert newWriteIndex.getName().equals(getDefaultBackingIndexName(name, generation + 1, minNodeVersion));
131131
if (replicated) {
132132
throw new IllegalArgumentException("data stream [" + name + "] cannot be rolled over, " +
133133
"because it is a replicated data stream");
@@ -190,7 +190,11 @@ public DataStream promoteDataStream() {
190190
* @return backing index name
191191
*/
192192
public static String getDefaultBackingIndexName(String dataStreamName, long generation) {
193-
return getDefaultBackingIndexName(dataStreamName, generation, System.currentTimeMillis());
193+
return getDefaultBackingIndexName(dataStreamName, generation, System.currentTimeMillis(), Version.CURRENT);
194+
}
195+
196+
public static String getDefaultBackingIndexName(String dataStreamName, long generation, Version minNodeVersion) {
197+
return getDefaultBackingIndexName(dataStreamName, generation, System.currentTimeMillis(), minNodeVersion);
194198
}
195199

196200
/**
@@ -207,6 +211,22 @@ public static String getDefaultBackingIndexName(String dataStreamName, long gene
207211
generation);
208212
}
209213

214+
public static String getDefaultBackingIndexName(String dataStreamName, long generation, long epochMillis, Version minNodeVersion) {
215+
if (minNodeVersion.onOrAfter(NEW_FEATURES_VERSION)) {
216+
return String.format(Locale.ROOT,
217+
BACKING_INDEX_PREFIX + "%s-%s-%06d",
218+
dataStreamName,
219+
DATE_FORMATTER.formatMillis(epochMillis),
220+
generation);
221+
} else {
222+
return getLegacyDefaultBackingIndexName(dataStreamName, generation);
223+
}
224+
}
225+
226+
public static String getLegacyDefaultBackingIndexName(String dataStreamName, long generation) {
227+
return String.format(Locale.ROOT, BACKING_INDEX_PREFIX + "%s-%06d", dataStreamName, generation);
228+
}
229+
210230
public DataStream(StreamInput in) throws IOException {
211231
this(in.readString(), new TimestampField(in), in.readList(Index::new), in.readVLong(),
212232
in.getVersion().onOrAfter(NEW_FEATURES_VERSION) ? in.readMap(): null,

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,8 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn
164164
ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, currentState.metadata());
165165

166166
if (writeIndex == null) {
167-
String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
167+
String firstBackingIndexName =
168+
DataStream.getDefaultBackingIndexName(dataStreamName, 1, currentState.nodes().getMinNodeVersion());
168169
CreateIndexClusterStateUpdateRequest createIndexRequest =
169170
new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName)
170171
.dataStreamName(dataStreamName)

‎server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.elasticsearch.cluster.metadata;
2020

21+
import org.elasticsearch.Version;
2122
import org.elasticsearch.cluster.DataStreamTestHelper;
2223
import org.elasticsearch.common.UUIDs;
2324
import org.elasticsearch.common.io.stream.Writeable;
@@ -54,7 +55,21 @@ protected DataStream createTestInstance() {
5455
public void testRollover() {
5556
DataStream ds = DataStreamTestHelper.randomInstance().promoteDataStream();
5657
Index newWriteIndex = new Index(getDefaultBackingIndexName(ds.getName(), ds.getGeneration() + 1), UUIDs.randomBase64UUID(random()));
57-
DataStream rolledDs = ds.rollover(newWriteIndex);
58+
DataStream rolledDs = ds.rollover(newWriteIndex, DataStream.NEW_FEATURES_VERSION);
59+
60+
assertThat(rolledDs.getName(), equalTo(ds.getName()));
61+
assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField()));
62+
assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1));
63+
assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1));
64+
assertTrue(rolledDs.getIndices().containsAll(ds.getIndices()));
65+
assertTrue(rolledDs.getIndices().contains(newWriteIndex));
66+
}
67+
68+
public void testRolloverWithLegacyBackingIndexNames() {
69+
DataStream ds = DataStreamTestHelper.randomInstance().promoteDataStream();
70+
Index newWriteIndex = new Index(getDefaultBackingIndexName(ds.getName(), ds.getGeneration() + 1, Version.V_7_10_0),
71+
UUIDs.randomBase64UUID(random()));
72+
DataStream rolledDs = ds.rollover(newWriteIndex, Version.V_7_10_0);
5873

5974
assertThat(rolledDs.getName(), equalTo(ds.getName()));
6075
assertThat(rolledDs.getTimeStampField(), equalTo(ds.getTimeStampField()));

‎test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,4 @@ public static String backingIndexPattern(String dataStreamName, long generation)
183183
return String.format(Locale.ROOT, "\\.ds-%s-(\\d{4}\\.\\d{2}\\.\\d{2}-)?%06d",dataStreamName, generation);
184184
}
185185

186-
public static String getLegacyDefaultBackingIndexName(String dataStreamName, long generation) {
187-
return String.format(Locale.ROOT, ".ds-%s-%06d", dataStreamName, generation);
188-
}
189186
}

‎x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
package org.elasticsearch.xpack.autoscaling.storage;
88

9+
import org.elasticsearch.Version;
910
import org.elasticsearch.cluster.ClusterInfo;
1011
import org.elasticsearch.cluster.ClusterState;
1112
import org.elasticsearch.cluster.DiskUsage;
@@ -448,7 +449,7 @@ private SingleForecast forecast(IndexAbstraction.DataStream stream, long forecas
448449
long size = Math.min(avgSizeCeil, scaledTotalSize - (avgSizeCeil * i));
449450
assert size > 0;
450451
newIndices.put(newIndex, size);
451-
dataStream = dataStream.rollover(newIndex.getIndex());
452+
dataStream = dataStream.rollover(newIndex.getIndex(), Version.CURRENT);
452453
}
453454

454455
return new SingleForecast(newIndices, dataStream);

‎x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.elasticsearch.client.Response;
1515
import org.elasticsearch.client.ResponseException;
1616
import org.elasticsearch.client.RestClient;
17-
import org.elasticsearch.cluster.DataStreamTestHelper;
1817
import org.elasticsearch.cluster.metadata.DataStream;
1918
import org.elasticsearch.common.Strings;
2019
import org.elasticsearch.common.settings.Settings;
@@ -975,7 +974,7 @@ public void testDataStreams() throws Exception {
975974
assertEquals(1, indices.size());
976975
assertEquals(getOldClusterVersion().onOrAfter(Version.V_7_11_0)
977976
? DataStream.getDefaultBackingIndexName("ds", 1)
978-
: DataStreamTestHelper.getLegacyDefaultBackingIndexName("ds", 1),
977+
: DataStream.getLegacyDefaultBackingIndexName("ds", 1),
979978
indices.get(0).get("index_name"));
980979
assertNumHits("ds", 1, 1);
981980
}

‎x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import org.elasticsearch.Version;
1010
import org.elasticsearch.client.Request;
1111
import org.elasticsearch.client.Response;
12-
import org.elasticsearch.cluster.DataStreamTestHelper;
1312
import org.elasticsearch.cluster.metadata.DataStream;
1413
import org.elasticsearch.common.Booleans;
1514
import org.hamcrest.Matchers;
@@ -67,7 +66,7 @@ public void testDataStreams() throws IOException {
6766
if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) {
6867
// include legacy name and date-named indices with today +/-1 in case of clock skew
6968
List<String> expectedIndices = org.elasticsearch.common.collect.List.of(
70-
"{\"_index\":\"" + DataStreamTestHelper.getLegacyDefaultBackingIndexName("logs-foobar", 2) + "\"}",
69+
"{\"_index\":\"" + DataStream.getLegacyDefaultBackingIndexName("logs-foobar", 2) + "\"}",
7170
"{\"_index\":\"" + DataStream.getDefaultBackingIndexName("logs-foobar", 2, nowMillis) + "\"}",
7271
"{\"_index\":\"" + DataStream.getDefaultBackingIndexName("logs-foobar", 2, nowMillis + 86400000) + "\"}",
7372
"{\"_index\":\"" + DataStream.getDefaultBackingIndexName("logs-foobar", 2, nowMillis - 86400000) + "\"}"
@@ -78,7 +77,7 @@ public void testDataStreams() throws IOException {
7877
} else {
7978
// include legacy name and date-named indices with today +/-1 in case of clock skew
8079
List<String> expectedIndices = org.elasticsearch.common.collect.List.of(
81-
"{\"_index\":\"" + DataStreamTestHelper.getLegacyDefaultBackingIndexName("logs-foobar", 3) + "\"}",
80+
"{\"_index\":\"" + DataStream.getLegacyDefaultBackingIndexName("logs-foobar", 3) + "\"}",
8281
"{\"_index\":\"" + DataStream.getDefaultBackingIndexName("logs-foobar", 3, nowMillis) + "\"}",
8382
"{\"_index\":\"" + DataStream.getDefaultBackingIndexName("logs-foobar", 3, nowMillis + 86400000) + "\"}",
8483
"{\"_index\":\"" + DataStream.getDefaultBackingIndexName("logs-foobar", 3, nowMillis - 86400000) + "\"}"

0 commit comments

Comments
 (0)