From 2a93c8a73ae6b4d04cebce52bb3d0fbf7c4c1978 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 10 Apr 2025 21:37:43 +0200 Subject: [PATCH 1/2] Allow a tsdb data stream to rolled over to a logsdb data stream and the other way around. This doesn't make much sense. However, if a data stream's index mode differs from the index mode of most recent backing index, then this can cause confusion. Typically, misconfiguration is a reason this can happen. Related to #126637 --- .../cluster/metadata/DataStream.java | 4 +++ .../cluster/metadata/DataStreamTests.java | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index c09f1d48f1583..98e78903b41e0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -610,6 +610,10 @@ public DataStream unsafeRollover( } else if (dsIndexMode == IndexMode.LOGSDB && (indexModeFromTemplate == null || indexModeFromTemplate == IndexMode.STANDARD)) { // Allow downgrading a time series data stream to a regular data stream dsIndexMode = null; + } else if (dsIndexMode == IndexMode.TIME_SERIES && indexModeFromTemplate == IndexMode.LOGSDB) { + dsIndexMode = IndexMode.LOGSDB; + } else if (dsIndexMode == IndexMode.LOGSDB && indexModeFromTemplate == IndexMode.TIME_SERIES) { + dsIndexMode = IndexMode.TIME_SERIES; } List backingIndices = new ArrayList<>(this.backingIndices.indices); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index ee911edd1789d..fc0b9a291549b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -277,6 +277,39 @@ public void testRolloverUpgradeToLogsdbDataStream() { assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.LOGSDB)); } + public void testRolloverFromTSdbToLogsdb() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.TIME_SERIES).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.LOGSDB, null); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.LOGSDB)); + } + + public void testRolloverFromLogsdbToTsdb() { + DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB).build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + var newCoordinates = ds.nextWriteIndexAndGeneration(project, ds.getDataComponent()); + + var rolledDs = ds.rollover( + new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), + newCoordinates.v2(), + IndexMode.TIME_SERIES, + null + ); + assertThat(rolledDs.getName(), equalTo(ds.getName())); + assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); + assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1)); + assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); + assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); + assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES)); + } + public void testRolloverDowngradeFromTsdbToRegularDataStream() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.TIME_SERIES).build(); final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); From 078968bd8d5aa85babc74797343ab8bdc044eb4c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 11 Apr 2025 11:52:03 +0200 Subject: [PATCH 2/2] added warning logging --- .../java/org/elasticsearch/cluster/metadata/DataStream.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 4b37331ad584d..607e3a8a07efb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -8,6 +8,8 @@ */ package org.elasticsearch.cluster.metadata; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; @@ -71,6 +73,8 @@ public final class DataStream implements SimpleDiffable, ToXContentObject, IndexAbstraction { + private static final Logger LOGGER = LogManager.getLogger(DataStream.class); + public static final boolean FAILURE_STORE_FEATURE_FLAG = new FeatureFlag("failure_store").isEnabled(); public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0; public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0; @@ -612,8 +616,10 @@ public DataStream unsafeRollover( dsIndexMode = null; } else if (dsIndexMode == IndexMode.TIME_SERIES && indexModeFromTemplate == IndexMode.LOGSDB) { dsIndexMode = IndexMode.LOGSDB; + LOGGER.warn("Changing [{}] index mode from [{}] to [{}]", name, indexModeFromTemplate, dsIndexMode); } else if (dsIndexMode == IndexMode.LOGSDB && indexModeFromTemplate == IndexMode.TIME_SERIES) { dsIndexMode = IndexMode.TIME_SERIES; + LOGGER.warn("Changing [{}] index mode from [{}] to [{}]", name, indexModeFromTemplate, dsIndexMode); } List backingIndices = new ArrayList<>(this.backingIndices.indices);