Skip to content

Commit 5c43302

Browse files
martijnvgmartijnvg
authored
Fix data stream alias validation. (#81040)
In case of restoring a snapshot, it is possible to overwrite an existing data stream with a data stream alias from a snapshot. This change fixes this by improving the generic duplicate name validation. On top of this the lack of data stream alias validation in Metadata.Builder#build() method resulted in cases where data stream aliases could be added for existing index aliases, data streams or indices with the same name. Closes #80972
1 parent e20fe6d commit 5c43302

File tree

4 files changed

+305
-4
lines changed

4 files changed

+305
-4
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1587,17 +1587,23 @@ public Metadata build(boolean builtIndicesLookupEagerly) {
15871587
indexMetadata.getAliases().keysIt().forEachRemaining(allAliases::add);
15881588
}
15891589

1590+
final ArrayList<String> duplicates = new ArrayList<>();
15901591
final Set<String> allDataStreams = new HashSet<>();
15911592
DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE);
15921593
if (dataStreamMetadata != null) {
15931594
for (DataStream dataStream : dataStreamMetadata.dataStreams().values()) {
15941595
allDataStreams.add(dataStream.getName());
15951596
}
1597+
// Adding data stream aliases:
1598+
for (String dataStreamAlias : dataStreamMetadata.getDataStreamAliases().keySet()) {
1599+
if (allAliases.add(dataStreamAlias) == false) {
1600+
duplicates.add("data stream alias and indices alias have the same name (" + dataStreamAlias + ")");
1601+
}
1602+
}
15961603
}
15971604

15981605
final Set<String> aliasDuplicatesWithIndices = new HashSet<>(allAliases);
15991606
aliasDuplicatesWithIndices.retainAll(allIndices);
1600-
ArrayList<String> duplicates = new ArrayList<>();
16011607
if (aliasDuplicatesWithIndices.isEmpty() == false) {
16021608
// iterate again and constructs a helpful message
16031609
for (ObjectCursor<IndexMetadata> cursor : indices.values()) {
@@ -1613,12 +1619,19 @@ public Metadata build(boolean builtIndicesLookupEagerly) {
16131619
aliasDuplicatesWithDataStreams.retainAll(allDataStreams);
16141620
if (aliasDuplicatesWithDataStreams.isEmpty() == false) {
16151621
// iterate again and constructs a helpful message
1616-
for (ObjectCursor<IndexMetadata> cursor : indices.values()) {
1617-
for (String alias : aliasDuplicatesWithDataStreams) {
1622+
for (String alias : aliasDuplicatesWithDataStreams) {
1623+
// reported var avoids adding a message twice if an index alias has the same name as a data stream.
1624+
boolean reported = false;
1625+
for (ObjectCursor<IndexMetadata> cursor : indices.values()) {
16181626
if (cursor.value.getAliases().containsKey(alias)) {
16191627
duplicates.add(alias + " (alias of " + cursor.value.getIndex() + ") conflicts with data stream");
1628+
reported = true;
16201629
}
16211630
}
1631+
// This is for adding an error message for when a data steam alias has the same name as a data stream.
1632+
if (reported == false && dataStreamMetadata != null && dataStreamMetadata.dataStreams().containsKey(alias)) {
1633+
duplicates.add("data stream alias and data stream have the same name (" + alias + ")");
1634+
}
16221635
}
16231636
}
16241637

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1215,6 +1215,71 @@ public void testBuildIndicesLookupForDataStreamAliases() {
12151215
assertThat(value.getAliases(), nullValue());
12161216
}
12171217

1218+
public void testDataStreamAliasValidation() {
1219+
Metadata.Builder b = Metadata.builder();
1220+
addDataStream("my-alias", b);
1221+
b.put("my-alias", "my-alias", null, null);
1222+
var e = expectThrows(IllegalStateException.class, b::build);
1223+
assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
1224+
1225+
b = Metadata.builder();
1226+
addDataStream("d1", b);
1227+
addDataStream("my-alias", b);
1228+
b.put("my-alias", "d1", null, null);
1229+
e = expectThrows(IllegalStateException.class, b::build);
1230+
assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
1231+
1232+
b = Metadata.builder();
1233+
b.put(
1234+
IndexMetadata.builder("index1")
1235+
.settings(
1236+
Settings.builder()
1237+
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
1238+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
1239+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
1240+
)
1241+
.putAlias(new AliasMetadata.Builder("my-alias"))
1242+
);
1243+
1244+
addDataStream("d1", b);
1245+
b.put("my-alias", "d1", null, null);
1246+
e = expectThrows(IllegalStateException.class, b::build);
1247+
assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)"));
1248+
}
1249+
1250+
public void testDataStreamAliasValidationRestoreScenario() {
1251+
Metadata.Builder b = Metadata.builder();
1252+
b.dataStreams(
1253+
Map.of("my-alias", createDataStream("my-alias")),
1254+
Map.of("my-alias", new DataStreamAlias("my-alias", List.of("my-alias"), null, null))
1255+
);
1256+
var e = expectThrows(IllegalStateException.class, b::build);
1257+
assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
1258+
1259+
b = Metadata.builder();
1260+
b.dataStreams(
1261+
Map.of("d1", createDataStream("d1"), "my-alias", createDataStream("my-alias")),
1262+
Map.of("my-alias", new DataStreamAlias("my-alias", List.of("d1"), null, null))
1263+
);
1264+
e = expectThrows(IllegalStateException.class, b::build);
1265+
assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
1266+
1267+
b = Metadata.builder();
1268+
b.put(
1269+
IndexMetadata.builder("index1")
1270+
.settings(
1271+
Settings.builder()
1272+
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
1273+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
1274+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
1275+
)
1276+
.putAlias(new AliasMetadata.Builder("my-alias"))
1277+
);
1278+
b.dataStreams(Map.of("d1", createDataStream("d1")), Map.of("my-alias", new DataStreamAlias("my-alias", List.of("d1"), null, null)));
1279+
e = expectThrows(IllegalStateException.class, b::build);
1280+
assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)"));
1281+
}
1282+
12181283
private void addDataStream(String name, Metadata.Builder b) {
12191284
int numBackingIndices = randomIntBetween(1, 4);
12201285
List<Index> indices = new ArrayList<>(numBackingIndices);
@@ -1226,6 +1291,16 @@ private void addDataStream(String name, Metadata.Builder b) {
12261291
b.put(new DataStream(name, createTimestampField("@timestamp"), indices));
12271292
}
12281293

1294+
private DataStream createDataStream(String name) {
1295+
int numBackingIndices = randomIntBetween(1, 4);
1296+
List<Index> indices = new ArrayList<>(numBackingIndices);
1297+
for (int j = 1; j <= numBackingIndices; j++) {
1298+
IndexMetadata idx = createBackingIndex(name, j).build();
1299+
indices.add(idx.getIndex());
1300+
}
1301+
return new DataStream(name, createTimestampField("@timestamp"), indices);
1302+
}
1303+
12291304
public void testIndicesLookupRecordsDataStreamForBackingIndices() {
12301305
final int numIndices = randomIntBetween(2, 5);
12311306
final int numBackingIndices = randomIntBetween(2, 5);

‎x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

Lines changed: 183 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,12 @@
1414
import org.elasticsearch.action.ActionRequestBuilder;
1515
import org.elasticsearch.action.DocWriteRequest;
1616
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
17+
import org.elasticsearch.action.admin.indices.alias.Alias;
1718
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
1819
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
1920
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
2021
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
22+
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
2123
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
2224
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
2325
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
@@ -58,6 +60,7 @@
5860
import org.elasticsearch.index.mapper.MapperParsingException;
5961
import org.elasticsearch.index.query.TermQueryBuilder;
6062
import org.elasticsearch.indices.InvalidAliasNameException;
63+
import org.elasticsearch.indices.InvalidIndexNameException;
6164
import org.elasticsearch.plugins.Plugin;
6265
import org.elasticsearch.rest.RestStatus;
6366
import org.elasticsearch.search.SearchHit;
@@ -1531,6 +1534,174 @@ public void testSegmentsSortedOnTimestampDesc() throws Exception {
15311534
assertTrue(timestamp2 > timestamp3);
15321535
}
15331536

1537+
public void testCreateDataStreamWithSameNameAsIndexAlias() throws Exception {
1538+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("my-index").alias(new Alias("my-alias"));
1539+
assertAcked(client().admin().indices().create(createIndexRequest).actionGet());
1540+
1541+
// Important detail: create template with data stream template after the index has been created
1542+
DataStreamIT.putComposableIndexTemplate("my-template", List.of("my-*"));
1543+
1544+
var request = new CreateDataStreamAction.Request("my-alias");
1545+
var e = expectThrows(IllegalStateException.class, () -> client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1546+
assertThat(e.getMessage(), containsString("[my-alias (alias of ["));
1547+
assertThat(e.getMessage(), containsString("]) conflicts with data stream"));
1548+
}
1549+
1550+
public void testCreateDataStreamWithSameNameAsIndex() throws Exception {
1551+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("my-index").alias(new Alias("my-alias"));
1552+
assertAcked(client().admin().indices().create(createIndexRequest).actionGet());
1553+
1554+
// Important detail: create template with data stream template after the index has been created
1555+
DataStreamIT.putComposableIndexTemplate("my-template", List.of("my-*"));
1556+
1557+
var request = new CreateDataStreamAction.Request("my-index");
1558+
var e = expectThrows(IllegalStateException.class, () -> client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1559+
assertThat(e.getMessage(), containsString("data stream [my-index] conflicts with index"));
1560+
}
1561+
1562+
public void testCreateDataStreamWithSameNameAsDataStreamAlias() throws Exception {
1563+
{
1564+
DataStreamIT.putComposableIndexTemplate("my-template", List.of("my-*"));
1565+
var request = new CreateDataStreamAction.Request("my-ds");
1566+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1567+
var aliasesAddRequest = new IndicesAliasesRequest();
1568+
aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("my-ds").aliases("my-alias"));
1569+
assertAcked(client().admin().indices().aliases(aliasesAddRequest).actionGet());
1570+
1571+
var request2 = new CreateDataStreamAction.Request("my-alias");
1572+
var e = expectThrows(
1573+
IllegalStateException.class,
1574+
() -> client().execute(CreateDataStreamAction.INSTANCE, request2).actionGet()
1575+
);
1576+
assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
1577+
}
1578+
{
1579+
assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet());
1580+
DataStreamIT.putComposableIndexTemplate(
1581+
"my-template",
1582+
null,
1583+
List.of("my-*"),
1584+
null,
1585+
null,
1586+
Map.of("my-alias", AliasMetadata.builder("my-alias").build())
1587+
);
1588+
var request = new CreateDataStreamAction.Request("my-ds");
1589+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1590+
1591+
var request2 = new CreateDataStreamAction.Request("my-alias");
1592+
var e = expectThrows(
1593+
IllegalStateException.class,
1594+
() -> client().execute(CreateDataStreamAction.INSTANCE, request2).actionGet()
1595+
);
1596+
assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)"));
1597+
}
1598+
}
1599+
1600+
public void testCreateDataStreamAliasWithSameNameAsIndexAlias() throws Exception {
1601+
{
1602+
DataStreamIT.putComposableIndexTemplate("my-template", List.of("logs-*"));
1603+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("es-logs").alias(new Alias("logs"));
1604+
assertAcked(client().admin().indices().create(createIndexRequest).actionGet());
1605+
1606+
var request = new CreateDataStreamAction.Request("logs-es");
1607+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1608+
IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
1609+
aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
1610+
var e = expectThrows(IllegalStateException.class, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet());
1611+
assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (logs)"));
1612+
}
1613+
{
1614+
assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet());
1615+
DataStreamIT.putComposableIndexTemplate(
1616+
"my-template",
1617+
null,
1618+
List.of("logs-*"),
1619+
null,
1620+
null,
1621+
Map.of("logs", AliasMetadata.builder("logs").build())
1622+
);
1623+
1624+
var request = new CreateDataStreamAction.Request("logs-es");
1625+
var e = expectThrows(IllegalStateException.class, () -> client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1626+
assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (logs)"));
1627+
}
1628+
}
1629+
1630+
public void testCreateDataStreamAliasWithSameNameAsIndex() throws Exception {
1631+
DataStreamIT.putComposableIndexTemplate("my-template", List.of("logs-*"));
1632+
1633+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs");
1634+
assertAcked(client().admin().indices().create(createIndexRequest).actionGet());
1635+
1636+
{
1637+
var request = new CreateDataStreamAction.Request("logs-es");
1638+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1639+
IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
1640+
aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
1641+
var e = expectThrows(InvalidAliasNameException.class, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet());
1642+
assertThat(
1643+
e.getMessage(),
1644+
equalTo("Invalid alias name [logs]: an index or data stream exists with the same name as the alias")
1645+
);
1646+
}
1647+
{
1648+
assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).actionGet());
1649+
var e = expectThrows(
1650+
IllegalArgumentException.class,
1651+
() -> DataStreamIT.putComposableIndexTemplate(
1652+
"my-template",
1653+
null,
1654+
List.of("logs-*"),
1655+
null,
1656+
null,
1657+
Map.of("logs", AliasMetadata.builder("logs").build())
1658+
)
1659+
);
1660+
assertThat(
1661+
e.getCause().getMessage(),
1662+
equalTo("Invalid alias name [logs]: an index or data stream exists with the same name as the alias")
1663+
);
1664+
}
1665+
}
1666+
1667+
public void testCreateIndexWithSameNameAsDataStreamAlias() throws Exception {
1668+
DataStreamIT.putComposableIndexTemplate("my-template", List.of("logs-*"));
1669+
1670+
var request = new CreateDataStreamAction.Request("logs-es");
1671+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1672+
IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
1673+
aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
1674+
assertAcked(client().admin().indices().aliases(aliasesAddRequest).actionGet());
1675+
1676+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs");
1677+
var e = expectThrows(InvalidIndexNameException.class, () -> client().admin().indices().create(createIndexRequest).actionGet());
1678+
assertThat(e.getMessage(), equalTo("Invalid index name [logs], already exists as alias"));
1679+
}
1680+
1681+
public void testCreateIndexAliasWithSameNameAsDataStreamAlias() throws Exception {
1682+
DataStreamIT.putComposableIndexTemplate("my-template", List.of("logs-*"));
1683+
1684+
var request = new CreateDataStreamAction.Request("logs-es");
1685+
assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
1686+
IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest();
1687+
aliasesAddRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("logs-es").aliases("logs"));
1688+
assertAcked(client().admin().indices().aliases(aliasesAddRequest).actionGet());
1689+
1690+
{
1691+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("my-index").alias(new Alias("logs"));
1692+
var e = expectThrows(IllegalStateException.class, () -> client().admin().indices().create(createIndexRequest).actionGet());
1693+
assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (logs)"));
1694+
}
1695+
{
1696+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("my-index");
1697+
assertAcked(client().admin().indices().create(createIndexRequest).actionGet());
1698+
IndicesAliasesRequest addAliasRequest = new IndicesAliasesRequest();
1699+
addAliasRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index("my-index").aliases("logs"));
1700+
var e = expectThrows(IllegalStateException.class, () -> client().admin().indices().aliases(addAliasRequest).actionGet());
1701+
assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (logs)"));
1702+
}
1703+
}
1704+
15341705
private static void verifyResolvability(String dataStream, ActionRequestBuilder<?, ?> requestBuilder, boolean fail) {
15351706
verifyResolvability(dataStream, requestBuilder, fail, 0);
15361707
}
@@ -1723,12 +1894,23 @@ static void putComposableIndexTemplate(
17231894
List<String> patterns,
17241895
@Nullable Settings settings,
17251896
@Nullable Map<String, Object> metadata
1897+
) throws IOException {
1898+
putComposableIndexTemplate(id, mappings, patterns, settings, metadata, null);
1899+
}
1900+
1901+
static void putComposableIndexTemplate(
1902+
String id,
1903+
@Nullable String mappings,
1904+
List<String> patterns,
1905+
@Nullable Settings settings,
1906+
@Nullable Map<String, Object> metadata,
1907+
@Nullable Map<String, AliasMetadata> aliases
17261908
) throws IOException {
17271909
PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(id);
17281910
request.indexTemplate(
17291911
new ComposableIndexTemplate(
17301912
patterns,
1731-
new Template(settings, mappings == null ? null : new CompressedXContent(mappings), null),
1913+
new Template(settings, mappings == null ? null : new CompressedXContent(mappings), aliases),
17321914
null,
17331915
null,
17341916
null,

0 commit comments

Comments
 (0)