From 52c94029e50d2e55f157753a8b86334f6a9d14e6 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 7 Jun 2021 16:10:52 -0500 Subject: [PATCH 01/22] wip --- .../cluster/metadata/AliasMetadata.java | 36 ++++++++++++++-- .../metadata/ComposableIndexTemplate.java | 42 ++++++++++++++++--- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index 4ac6200e0a738..a96276f7294ec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -32,6 +32,7 @@ import java.util.Set; import static java.util.Collections.emptySet; +import static org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_ALIAS_VERSION; public class AliasMetadata extends AbstractDiffable implements ToXContentFragment { @@ -51,8 +52,11 @@ public class AliasMetadata extends AbstractDiffable implements To @Nullable private final Boolean isHidden; + @Nullable + private final Boolean isDataStream; + private AliasMetadata(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex, - @Nullable Boolean isHidden) { + @Nullable Boolean isHidden, @Nullable Boolean isDataStream) { this.alias = alias; this.filter = filter; this.indexRouting = indexRouting; @@ -64,11 +68,12 @@ private AliasMetadata(String alias, CompressedXContent filter, String indexRouti } this.writeIndex = writeIndex; this.isHidden = isHidden; + this.isDataStream = isDataStream; } private AliasMetadata(AliasMetadata aliasMetadata, String alias) { this(alias, aliasMetadata.filter(), aliasMetadata.indexRouting(), aliasMetadata.searchRouting(), aliasMetadata.writeIndex(), - aliasMetadata.isHidden); + aliasMetadata.isHidden, aliasMetadata.isDataStream); } public String alias() { @@ -120,6 +125,11 @@ public Boolean isHidden() { return isHidden; } + @Nullable + public Boolean isDataStream() { + return isDataStream; + } + public static Builder builder(String alias) { return new Builder(alias); } @@ -148,6 +158,7 @@ public boolean equals(Object o) { if (Objects.equals(searchRouting, that.searchRouting) == false) return false; if (Objects.equals(writeIndex, that.writeIndex) == false) return false; if (Objects.equals(isHidden, that.isHidden) == false) return false; + if (Objects.equals(isDataStream, that.isDataStream) == false) return false; return true; } @@ -159,6 +170,7 @@ public int hashCode() { result = 31 * result + (indexRouting != null ? indexRouting.hashCode() : 0); result = 31 * result + (searchRouting != null ? searchRouting.hashCode() : 0); result = 31 * result + (writeIndex != null ? writeIndex.hashCode() : 0); + result = 31 * result + (isDataStream != null ? isDataStream.hashCode() : 0); return result; } @@ -185,6 +197,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalBoolean(writeIndex()); out.writeOptionalBoolean(isHidden); + if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + out.writeOptionalBoolean(isDataStream); + } } public AliasMetadata(StreamInput in) throws IOException { @@ -208,6 +223,7 @@ public AliasMetadata(StreamInput in) throws IOException { } writeIndex = in.readOptionalBoolean(); isHidden = in.readOptionalBoolean(); + isDataStream = in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION) ? in.readOptionalBoolean() : false; } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -249,6 +265,9 @@ public static class Builder { @Nullable private Boolean isHidden; + @Nullable + private Boolean isDataStream; + public Builder(String alias) { this.alias = alias; } @@ -310,8 +329,13 @@ public Builder isHidden(@Nullable Boolean isHidden) { return this; } + public Builder isDataStream(@Nullable Boolean isDataStream) { + this.isDataStream = isDataStream; + return this; + } + public AliasMetadata build() { - return new AliasMetadata(alias, filter, indexRouting, searchRouting, writeIndex, isHidden); + return new AliasMetadata(alias, filter, indexRouting, searchRouting, writeIndex, isHidden, isDataStream); } public static void toXContent(AliasMetadata aliasMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -341,6 +365,10 @@ public static void toXContent(AliasMetadata aliasMetadata, XContentBuilder build builder.field("is_hidden", aliasMetadata.isHidden()); } + if (aliasMetadata.isDataStream != null) { + builder.field("is_data_stream", aliasMetadata.isDataStream()); + } + builder.endObject(); } @@ -384,6 +412,8 @@ public static AliasMetadata fromXContent(XContentParser parser) throws IOExcepti builder.writeIndex(parser.booleanValue()); } else if ("is_hidden".equals(currentFieldName)) { builder.isHidden(parser.booleanValue()); + } else if ("is_data_stream".equals(currentFieldName)) { + builder.isDataStream(parser.booleanValue()); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 932ec4551ef06..ee4dcc94f0534 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; @@ -256,12 +257,21 @@ public String toString() { public static class DataStreamTemplate implements Writeable, ToXContentObject { + public static final Version DATA_STREAM_ALIAS_VERSION = Version.V_8_0_0; + private static final ParseField HIDDEN = new ParseField("hidden"); + private static final ParseField ALIASES = new ParseField("aliases"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_stream_template", false, - a -> new DataStreamTemplate(a[0] != null && (boolean) a[0])); + args -> { + boolean hidden = args[0] != null && (boolean) args[0]; + @SuppressWarnings("unchecked") + Map aliases = (Map) args[1]; + return new DataStreamTemplate(hidden, aliases); + } + ); static { PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN); @@ -269,16 +279,23 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { private final boolean hidden; + @Nullable + private final Map aliases; + public DataStreamTemplate() { - this(false); + this(false, null); } - public DataStreamTemplate(boolean hidden) { + public DataStreamTemplate(boolean hidden, Map aliases) { this.hidden = hidden; + this.aliases = aliases; } DataStreamTemplate(StreamInput in) throws IOException { hidden = in.readBoolean(); + aliases = in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION) + ? in.readBoolean() ? in.readMap(StreamInput::readString, AliasMetadata::new) : null + : null; } public String getTimestampField() { @@ -300,12 +317,27 @@ public boolean isHidden() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(hidden); + if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + if (aliases != null) { + out.writeBoolean(true); + out.writeMap(aliases, StreamOutput::writeString, (innerOut, alias) -> alias.writeTo(innerOut)); + } else { + out.writeBoolean(false); + } + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("hidden", hidden); + if (aliases != null) { + builder.startObject("aliases"); + for (var alias : aliases.values()) { + alias.toXContent(builder, params); + } + builder.endObject(); + } builder.endObject(); return builder; } @@ -315,12 +347,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; DataStreamTemplate that = (DataStreamTemplate) o; - return hidden == that.hidden; + return hidden == that.hidden && Objects.equals(aliases, that.aliases); } @Override public int hashCode() { - return Objects.hash(hidden); + return Objects.hash(hidden, aliases); } } From ebdbce5bb21ad1a5063d1c139f50ec476a658f01 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 7 Jun 2021 16:25:03 -0500 Subject: [PATCH 02/22] fix test compile error --- .../elasticsearch/xpack/datastreams/DataStreamsStatsTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java index 3a8a8aab26e87..ea19d4634df30 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java @@ -242,7 +242,7 @@ private String createDataStream(boolean hidden) throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(hidden), + new ComposableIndexTemplate.DataStreamTemplate(hidden, null), null ); assertTrue( From cc8c7c4dc2ea097f5953324f950f60159a615a8f Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 7 Jun 2021 16:36:54 -0500 Subject: [PATCH 03/22] add missing parser --- .../cluster/metadata/ComposableIndexTemplate.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index ee4dcc94f0534..0198868ddd9f8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -24,6 +25,7 @@ import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -275,6 +277,14 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { static { PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + Map aliasMap = new HashMap<>(); + while ((p.nextToken()) != XContentParser.Token.END_OBJECT) { + AliasMetadata alias = AliasMetadata.Builder.fromXContent(p); + aliasMap.put(alias.alias(), alias); + } + return aliasMap; + }, ALIASES); } private final boolean hidden; From 5d21f2f307c4d3213c0a60c448a465262987162f Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 7 Jun 2021 16:48:47 -0500 Subject: [PATCH 04/22] bwc --- .../org/elasticsearch/cluster/metadata/AliasMetadata.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index a96276f7294ec..2ea016073e947 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -223,7 +223,9 @@ public AliasMetadata(StreamInput in) throws IOException { } writeIndex = in.readOptionalBoolean(); isHidden = in.readOptionalBoolean(); - isDataStream = in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION) ? in.readOptionalBoolean() : false; + if (in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { + isDataStream = in.readOptionalBoolean(); + } } public static Diff readDiffFrom(StreamInput in) throws IOException { From 738b9e60f07c923320bff9ae717e811aa65224d0 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 7 Jun 2021 16:57:21 -0500 Subject: [PATCH 05/22] fix compilation --- .../java/org/elasticsearch/cluster/metadata/AliasMetadata.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index 2ea016073e947..e2d60610f8d38 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -225,6 +225,8 @@ public AliasMetadata(StreamInput in) throws IOException { isHidden = in.readOptionalBoolean(); if (in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { isDataStream = in.readOptionalBoolean(); + } else { + isDataStream = null; } } From b0848d6ba5e324caeeb6f1147f74adf7aced9c7a Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 7 Jun 2021 17:07:15 -0500 Subject: [PATCH 06/22] unused import --- .../elasticsearch/cluster/metadata/ComposableIndexTemplate.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 0198868ddd9f8..da4b1103708e7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; From 36847c8b4a4cc67a56cd56627cc0873b3c5db45d Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 7 Jun 2021 17:18:53 -0500 Subject: [PATCH 07/22] fix test --- .../elasticsearch/cluster/metadata/DataStreamTemplateTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java index 65e6efcdca6e4..cd2372d440cc0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java @@ -32,7 +32,7 @@ protected DataStreamTemplate createTestInstance() { } public static DataStreamTemplate randomInstance() { - return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean()); + return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), null); } } From 001b0d51d6e9904db1e75d0b85f075fb999a6094 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 14 Jul 2021 16:26:49 -0500 Subject: [PATCH 08/22] fix merge error --- .../xpack/core/ilm/LifecyclePolicyUtilsTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java index 319799e10650b..c8129cd4c6625 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java @@ -92,7 +92,7 @@ public void testCalculateUsage() { new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate", new ComposableIndexTemplate(Collections.singletonList("myds"), new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null), - null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false))))) + null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false, null))))) .build()) .build(); assertThat(LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), @@ -133,7 +133,7 @@ public void testCalculateUsage() { new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate", new ComposableIndexTemplate(Collections.singletonList("myds"), new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null), - null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false))))) + null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false, null))))) .build()) .build(); assertThat(LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), From 39973f625834c6929599da64e5aebcf298365fb5 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Fri, 16 Jul 2021 07:44:48 -0500 Subject: [PATCH 09/22] auto-create alias when data stream is created --- .../cluster/metadata/ComposableIndexTemplate.java | 5 +++++ .../cluster/metadata/MetadataCreateDataStreamService.java | 7 +++++++ .../cluster/metadata/MetadataIndexTemplateService.java | 6 +++--- .../indices/rollover/MetadataRolloverServiceTests.java | 2 +- .../metadata/MetadataIndexTemplateServiceTests.java | 4 ++-- .../elasticsearch/xpack/datastreams/DataStreamsRestIT.java | 2 +- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index e14f09d198541..85aed10ed459a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -311,6 +312,10 @@ public String getTimestampField() { return FIXED_TIMESTAMP_FIELD; } + public Collection getAliases() { + return aliases == null ? null : aliases.values(); + } + /** * @return a mapping snippet for a backing index with `_data_stream_timestamp` meta field mapper properly configured. */ diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index e5abf16a74016..348244f997d3c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -222,6 +222,13 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName, writeIndex.getIndex().getName(), Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray())); + + if (template.getDataStreamTemplate().getAliases() != null) { + for (var alias : template.getDataStreamTemplate().getAliases()) { + builder.put(alias.getAlias(), dataStreamName, alias.writeIndex()); + } + } + return ClusterState.builder(currentState).metadata(builder).build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index bd34d5adf9d24..cbea658a5f791 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1146,10 +1146,10 @@ static List> resolveAliases(final ComposableIndexTemp .map(Template::aliases) .ifPresent(aliases::add); - // A template that creates data streams can't also create aliases. - // (otherwise we end up with aliases pointing to backing indices of data streams) + // A template that creates data streams may not specify index aliases. Data stream + // aliases are defined separately in the `data_stream` object on the template. if (aliases.size() > 0 && template.getDataStreamTemplate() != null) { - throw new IllegalArgumentException("template [" + templateName + "] has alias and data stream definitions"); + throw new IllegalArgumentException("template [" + templateName + "] may not specify both data streams and index alias(es)"); } // Aliases are applied in order, but subsequent alias configuration from the same name is diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 26ec11805bcfb..09fc92c2f040d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -705,7 +705,7 @@ clusterService, indicesService, allocationService, new AliasValidator(), shardLi () -> rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, randomBoolean(), false) ); - assertThat(e.getMessage(), equalTo("template [template] has alias and data stream definitions")); + assertThat(e.getMessage(), equalTo("template [template] may not specify both data streams and index alias(es)")); } finally { testThreadPool.shutdown(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index ab0402d6bc906..bedf7a31d4ca4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1219,7 +1219,7 @@ public void testResolveAliasesDataStreams() throws Exception { .build(); Exception e = expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state1.metadata(), "1")); - assertThat(e.getMessage(), equalTo("template [1] has alias and data stream definitions")); + assertThat(e.getMessage(), equalTo("template [1] may not specify both data streams and index alias(es)")); // index template can't have data streams and a component template with an aliases ComponentTemplate componentTemplate = new ComponentTemplate(new Template(null, null, a1), null, null); @@ -1229,7 +1229,7 @@ public void testResolveAliasesDataStreams() throws Exception { .metadata(Metadata.builder().put("1", it).put("c1", componentTemplate).build()) .build(); e = expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state2.metadata(), "1")); - assertThat(e.getMessage(), equalTo("template [1] has alias and data stream definitions")); + assertThat(e.getMessage(), equalTo("template [1] may not specify both data streams and index alias(es)")); } public void testAddInvalidTemplate() throws Exception { diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 47c80dba46775..45aed6ce11c9a 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -90,7 +90,7 @@ public void testAddingIndexTemplateWithAliasesAndDataStream() { String body = "{\"index_patterns\":[\"mypattern*\"],\"data_stream\":{},\"template\":{\"aliases\":{\"my-alias\":{}}}}"; putComposableIndexTemplateRequest.setJsonEntity(body); Exception e = expectThrows(ResponseException.class, () -> client().performRequest(putComposableIndexTemplateRequest)); - assertThat(e.getMessage(), containsString("template [my-template] has alias and data stream definitions")); + assertThat(e.getMessage(), containsString("template [my-template] may not specify both data streams and index alias(es)")); } public void testDataStreamAliases() throws Exception { From e12bd60b6902446daf7bed55971746a2d0f3439d Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 11:21:29 -0500 Subject: [PATCH 10/22] remove AliasMetadata::isDataStream --- .../cluster/metadata/AliasMetadata.java | 32 +++---------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index 1221aa52d858b..463dbbb34897a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -11,7 +11,6 @@ import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -24,6 +23,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.Nullable; import java.io.IOException; import java.util.Collections; @@ -32,7 +32,6 @@ import java.util.Set; import static java.util.Collections.emptySet; -import static org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_ALIAS_VERSION; public class AliasMetadata extends AbstractDiffable implements ToXContentFragment { @@ -52,11 +51,8 @@ public class AliasMetadata extends AbstractDiffable implements To @Nullable private final Boolean isHidden; - @Nullable - private final Boolean isDataStream; - private AliasMetadata(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex, - @Nullable Boolean isHidden, @Nullable Boolean isDataStream) { + @Nullable Boolean isHidden) { this.alias = alias; this.filter = filter; this.indexRouting = indexRouting; @@ -68,12 +64,11 @@ private AliasMetadata(String alias, CompressedXContent filter, String indexRouti } this.writeIndex = writeIndex; this.isHidden = isHidden; - this.isDataStream = isDataStream; } private AliasMetadata(AliasMetadata aliasMetadata, String alias) { this(alias, aliasMetadata.filter(), aliasMetadata.indexRouting(), aliasMetadata.searchRouting(), aliasMetadata.writeIndex(), - aliasMetadata.isHidden, aliasMetadata.isDataStream); + aliasMetadata.isHidden); } public String alias() { @@ -125,11 +120,6 @@ public Boolean isHidden() { return isHidden; } - @Nullable - public Boolean isDataStream() { - return isDataStream; - } - public static Builder builder(String alias) { return new Builder(alias); } @@ -158,7 +148,6 @@ public boolean equals(Object o) { if (Objects.equals(searchRouting, that.searchRouting) == false) return false; if (Objects.equals(writeIndex, that.writeIndex) == false) return false; if (Objects.equals(isHidden, that.isHidden) == false) return false; - if (Objects.equals(isDataStream, that.isDataStream) == false) return false; return true; } @@ -170,7 +159,6 @@ public int hashCode() { result = 31 * result + (indexRouting != null ? indexRouting.hashCode() : 0); result = 31 * result + (searchRouting != null ? searchRouting.hashCode() : 0); result = 31 * result + (writeIndex != null ? writeIndex.hashCode() : 0); - result = 31 * result + (isDataStream != null ? isDataStream.hashCode() : 0); return result; } @@ -197,9 +185,6 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalBoolean(writeIndex()); out.writeOptionalBoolean(isHidden); - if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { - out.writeOptionalBoolean(isDataStream); - } } public AliasMetadata(StreamInput in) throws IOException { @@ -223,11 +208,6 @@ public AliasMetadata(StreamInput in) throws IOException { } writeIndex = in.readOptionalBoolean(); isHidden = in.readOptionalBoolean(); - if (in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { - isDataStream = in.readOptionalBoolean(); - } else { - isDataStream = null; - } } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -339,7 +319,7 @@ public Builder isDataStream(@Nullable Boolean isDataStream) { } public AliasMetadata build() { - return new AliasMetadata(alias, filter, indexRouting, searchRouting, writeIndex, isHidden, isDataStream); + return new AliasMetadata(alias, filter, indexRouting, searchRouting, writeIndex, isHidden); } public static void toXContent(AliasMetadata aliasMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -369,10 +349,6 @@ public static void toXContent(AliasMetadata aliasMetadata, XContentBuilder build builder.field("is_hidden", aliasMetadata.isHidden()); } - if (aliasMetadata.isDataStream != null) { - builder.field("is_data_stream", aliasMetadata.isDataStream()); - } - builder.endObject(); } From 1399903643f16dd3d9b1bcb0baecd7cfed2a423a Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 14:20:42 -0500 Subject: [PATCH 11/22] reuse existing alias definitions --- .../metadata/ComposableIndexTemplate.java | 46 ++--------- .../MetadataCreateDataStreamService.java | 4 +- .../metadata/MetadataCreateIndexService.java | 32 +++++--- .../MetadataIndexTemplateService.java | 6 -- .../MetadataRolloverServiceTests.java | 76 ------------------- .../metadata/DataStreamTemplateTests.java | 2 +- .../MetadataIndexTemplateServiceTests.java | 25 ------ .../core/ilm/LifecyclePolicyUtilsTests.java | 4 +- .../xpack/datastreams/DataStreamsRestIT.java | 8 -- .../datastreams/DataStreamsStatsTests.java | 2 +- 10 files changed, 36 insertions(+), 169 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 85aed10ed459a..a448aeede6df6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -8,23 +8,21 @@ package org.elasticsearch.cluster.metadata; -import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -259,8 +257,6 @@ public String toString() { public static class DataStreamTemplate implements Writeable, ToXContentObject { - public static final Version DATA_STREAM_ALIAS_VERSION = Version.V_8_0_0; - private static final ParseField HIDDEN = new ParseField("hidden"); private static final ParseField ALIASES = new ParseField("aliases"); @@ -269,9 +265,7 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { false, args -> { boolean hidden = args[0] != null && (boolean) args[0]; - @SuppressWarnings("unchecked") - Map aliases = (Map) args[1]; - return new DataStreamTemplate(hidden, aliases); + return new DataStreamTemplate(hidden); } ); @@ -289,33 +283,22 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { private final boolean hidden; - @Nullable - private final Map aliases; - public DataStreamTemplate() { - this(false, null); + this(false); } - public DataStreamTemplate(boolean hidden, Map aliases) { + public DataStreamTemplate(boolean hidden) { this.hidden = hidden; - this.aliases = aliases; } DataStreamTemplate(StreamInput in) throws IOException { hidden = in.readBoolean(); - aliases = in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION) - ? in.readBoolean() ? in.readMap(StreamInput::readString, AliasMetadata::new) : null - : null; } public String getTimestampField() { return FIXED_TIMESTAMP_FIELD; } - public Collection getAliases() { - return aliases == null ? null : aliases.values(); - } - /** * @return a mapping snippet for a backing index with `_data_stream_timestamp` meta field mapper properly configured. */ @@ -331,27 +314,12 @@ public boolean isHidden() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(hidden); - if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) { - if (aliases != null) { - out.writeBoolean(true); - out.writeMap(aliases, StreamOutput::writeString, (innerOut, alias) -> alias.writeTo(innerOut)); - } else { - out.writeBoolean(false); - } - } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("hidden", hidden); - if (aliases != null) { - builder.startObject("aliases"); - for (var alias : aliases.values()) { - alias.toXContent(builder, params); - } - builder.endObject(); - } builder.endObject(); return builder; } @@ -361,12 +329,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; DataStreamTemplate that = (DataStreamTemplate) o; - return hidden == that.hidden && Objects.equals(aliases, that.aliases); + return hidden == that.hidden; } @Override public int hashCode() { - return Objects.hash(hidden, aliases); + return Objects.hash(hidden); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 348244f997d3c..6283591fd0fad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -223,8 +223,8 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn writeIndex.getIndex().getName(), Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray())); - if (template.getDataStreamTemplate().getAliases() != null) { - for (var alias : template.getDataStreamTemplate().getAliases()) { + if (template.template().aliases() != null) { + for (var alias : template.template().aliases().values()) { builder.put(alias.getAlias(), dataStreamName, alias.writeIndex()); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 1c0adeb62199a..83db8ced0eb10 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -504,7 +504,8 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu logger.debug("applying create index request using composable template [{}]", templateName); ComposableIndexTemplate template = currentState.getMetadata().templatesV2().get(templateName); - if (request.dataStreamName() == null && template.getDataStreamTemplate() != null) { + final boolean isDataStream = template.getDataStreamTemplate() != null; + if (isDataStream && request.dataStreamName() == null) { throw new IllegalArgumentException("cannot create index with name [" + request.index() + "], because it matches with template [" + templateName + "] that creates data streams only, " + "use create data stream api instead"); @@ -519,14 +520,27 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); - return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, - indexService -> resolveAndValidateAliases(request.index(), request.aliases(), - MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), - // the context is only used for validation so it's fine to pass fake values for the - // shard id and the current timestamp - aliasValidator, xContentRegistry, indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - indexService.dateMathExpressionResolverAt(request.getNameResolvedAt())), - Collections.singletonList(templateName), metadataTransformer); + return applyCreateIndexWithTemporaryService( + currentState, + request, + silent, + null, + tmpImd, + mappings, + indexService -> resolveAndValidateAliases( + request.index(), + isDataStream ? Set.of() : request.aliases(), + isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), + currentState.metadata(), + aliasValidator, + xContentRegistry, + // the context is used ony for validation so it's fine to pass fake values for the shard id and the current timestamp + indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), + indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()) + ), + Collections.singletonList(templateName), + metadataTransformer + ); } private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterState currentState, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index cbea658a5f791..de9201a0f839c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1146,12 +1146,6 @@ static List> resolveAliases(final ComposableIndexTemp .map(Template::aliases) .ifPresent(aliases::add); - // A template that creates data streams may not specify index aliases. Data stream - // aliases are defined separately in the `data_stream` object on the template. - if (aliases.size() > 0 && template.getDataStreamTemplate() != null) { - throw new IllegalArgumentException("template [" + templateName + "] may not specify both data streams and index alias(es)"); - } - // Aliases are applied in order, but subsequent alias configuration from the same name is // ignored, so in order for the order to be correct, alias configuration should be in order // of precedence (with the index template first) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 09fc92c2f040d..e8371dd4d9dea 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -635,82 +635,6 @@ protected String contentType() { } } - public void testRolloverDataStreamWorksWithTemplateThatAlsoCreatesAliases() throws Exception { - final DataStream dataStream = DataStreamTestHelper.randomInstance() - // ensure no replicate data stream - .promoteDataStream(); - ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*")) - .template(new Template(null, null, Map.of("my-alias", AliasMetadata.newAliasMetadataBuilder("my-alias").build()))) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()).build(); - Metadata.Builder builder = Metadata.builder(); - builder.put("template", template); - for (Index index : dataStream.getIndices()) { - builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); - } - builder.put(dataStream); - final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build(); - - ThreadPool testThreadPool = new TestThreadPool(getTestName()); - try { - DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder( - "@timestamp", - DateFieldMapper.Resolution.MILLISECONDS, - null, - ScriptCompiler.NONE, - false, - Version.CURRENT).build(new ContentPath()); - MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class); - when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp"); - MetadataFieldMapper mockedTimestampField = new MetadataFieldMapper(mockedTimestampFieldType) { - @Override - protected String contentType() { - return null; - } - }; - MetadataFieldMapper[] metadataFieldMappers = {new MetadataIndexTemplateServiceTests.MetadataTimestampFieldMapper(true)}; - RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc", Version.CURRENT); - root.add(new DateFieldMapper.Builder(dataStream.getTimeStampField().getName(), DateFieldMapper.Resolution.MILLISECONDS, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, true, Version.CURRENT)); - Mapping mapping = new Mapping(root.build(new ContentPath("")), metadataFieldMappers, Collections.emptyMap()); - MappingLookup mappingLookup = MappingLookup.fromMappers( - mapping, - List.of(mockedTimestampField, dateFieldMapper), - List.of(), - List.of()); - ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); - Environment env = mock(Environment.class); - when(env.sharedDataFile()).thenReturn(null); - AllocationService allocationService = mock(AllocationService.class); - when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]); - IndicesService indicesService = mockIndicesServices(mappingLookup); - IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class); - when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg()); - - ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService); - MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, - clusterService, indicesService, allocationService, new AliasValidator(), shardLimitValidator, env, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, EmptySystemIndices.INSTANCE, false); - MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, - new AliasValidator(), null, xContentRegistry()); - MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, - mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); - - MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); - List> metConditions = Collections.singletonList(condition); - CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); - - // Ensure that a warning header is emitted - Exception e = expectThrows( - IllegalArgumentException.class, - () -> rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, - randomBoolean(), false) - ); - assertThat(e.getMessage(), equalTo("template [template] may not specify both data streams and index alias(es)")); - } finally { - testThreadPool.shutdown(); - } - } - public void testValidation() throws Exception { final String rolloverTarget; final String sourceIndexName; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java index cd2372d440cc0..65e6efcdca6e4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java @@ -32,7 +32,7 @@ protected DataStreamTemplate createTestInstance() { } public static DataStreamTemplate randomInstance() { - return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), null); + return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean()); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index bedf7a31d4ca4..651f341c3acda 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1207,31 +1207,6 @@ public void testResolveAliases() throws Exception { assertThat(resolvedAliases, equalTo(List.of(a3, a1, a2))); } - public void testResolveAliasesDataStreams() throws Exception { - Map a1 = new HashMap<>(); - a1.put("logs", AliasMetadata.newAliasMetadataBuilder("logs").build()); - - // index template can't have data streams and aliases - ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("logs-*"), - new Template(null, null, a1), null, 0L, 1L, null, new ComposableIndexTemplate.DataStreamTemplate(), null); - ClusterState state1 = ClusterState.builder(ClusterState.EMPTY_STATE) - .metadata(Metadata.builder().put("1", it).build()) - .build(); - Exception e = - expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state1.metadata(), "1")); - assertThat(e.getMessage(), equalTo("template [1] may not specify both data streams and index alias(es)")); - - // index template can't have data streams and a component template with an aliases - ComponentTemplate componentTemplate = new ComponentTemplate(new Template(null, null, a1), null, null); - it = new ComposableIndexTemplate(List.of("logs-*"), null, List.of("c1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate(), null); - ClusterState state2 = ClusterState.builder(ClusterState.EMPTY_STATE) - .metadata(Metadata.builder().put("1", it).put("c1", componentTemplate).build()) - .build(); - e = expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state2.metadata(), "1")); - assertThat(e.getMessage(), equalTo("template [1] may not specify both data streams and index alias(es)")); - } - public void testAddInvalidTemplate() throws Exception { ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("a"), null, Arrays.asList("good", "bad"), null, null, null); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java index c8129cd4c6625..319799e10650b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java @@ -92,7 +92,7 @@ public void testCalculateUsage() { new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate", new ComposableIndexTemplate(Collections.singletonList("myds"), new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null), - null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false, null))))) + null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false))))) .build()) .build(); assertThat(LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), @@ -133,7 +133,7 @@ public void testCalculateUsage() { new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate", new ComposableIndexTemplate(Collections.singletonList("myds"), new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null), - null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false, null))))) + null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false))))) .build()) .build(); assertThat(LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 45aed6ce11c9a..65eed652bcd09 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -85,14 +85,6 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { assertEquals(1, XContentMapValues.extractValue("hits.total.value", results)); } - public void testAddingIndexTemplateWithAliasesAndDataStream() { - Request putComposableIndexTemplateRequest = new Request("PUT", "/_index_template/my-template"); - String body = "{\"index_patterns\":[\"mypattern*\"],\"data_stream\":{},\"template\":{\"aliases\":{\"my-alias\":{}}}}"; - putComposableIndexTemplateRequest.setJsonEntity(body); - Exception e = expectThrows(ResponseException.class, () -> client().performRequest(putComposableIndexTemplateRequest)); - assertThat(e.getMessage(), containsString("template [my-template] may not specify both data streams and index alias(es)")); - } - public void testDataStreamAliases() throws Exception { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java index ea19d4634df30..3a8a8aab26e87 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java @@ -242,7 +242,7 @@ private String createDataStream(boolean hidden) throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(hidden, null), + new ComposableIndexTemplate.DataStreamTemplate(hidden), null ); assertTrue( From 0f5c6a95a665bb137ccb9d3805c3a2321a23c2fe Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 16:18:05 -0500 Subject: [PATCH 12/22] fix NPE --- .../cluster/metadata/MetadataCreateDataStreamService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 6283591fd0fad..d155d1d7364fa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -223,7 +223,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn writeIndex.getIndex().getName(), Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray())); - if (template.template().aliases() != null) { + if (template.template() != null && template.template().aliases() != null) { for (var alias : template.template().aliases().values()) { builder.put(alias.getAlias(), dataStreamName, alias.writeIndex()); } From 43e1d5fe676f40b57c569760b57441f4a52e3da5 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 18:25:05 -0500 Subject: [PATCH 13/22] add test --- .../MetadataCreateDataStreamServiceTests.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index a369f011ae88e..b239f200faf4e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -23,12 +23,15 @@ import org.elasticsearch.indices.SystemIndices.Feature; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateMapping; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -66,6 +69,41 @@ public void testCreateDataStream() throws Exception { assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).isSystem(), is(false)); } + public void testCreateDataStreamWithAliasFromTemplate() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "my-data-stream"; + final int aliasCount = randomIntBetween(0, 3); + Map aliases = new HashMap<>(aliasCount); + for (int k = 0; k < aliasCount; k++) { + final String aliasName = randomAlphaOfLength(6); + aliases.put(aliasName, AliasMetadata.newAliasMetadataBuilder(aliasName).build()); + } + ComposableIndexTemplate template = new ComposableIndexTemplate.Builder() + .indexPatterns(List.of(dataStreamName + "*")) + .dataStreamTemplate(new DataStreamTemplate()) + .template(new Template(null, null, aliases)) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder().put("template", template).build()) + .build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); + ClusterState newState = MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(1)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); + assertThat(newState.metadata().dataStreamAliases().size(), is(aliasCount)); + assertThat(newState.metadata().dataStreamAliases().values().stream().map(DataStreamAlias::getName).toArray(), + arrayContainingInAnyOrder (new ArrayList<>(aliases.keySet()).toArray())); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue()); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getAliases().size(), is(0)); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"), + equalTo("true")); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).isSystem(), is(false)); + } + public void testCreateSystemDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = ".system-data-stream"; From 45cc1f5009d41ab7c159ba0031d57a942ea76f20 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 18:48:15 -0500 Subject: [PATCH 14/22] remove a couple more unnecessary flags --- .../elasticsearch/cluster/metadata/AliasMetadata.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index 463dbbb34897a..1fb3337e1a817 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -249,9 +249,6 @@ public static class Builder { @Nullable private Boolean isHidden; - @Nullable - private Boolean isDataStream; - public Builder(String alias) { this.alias = alias; } @@ -313,11 +310,6 @@ public Builder isHidden(@Nullable Boolean isHidden) { return this; } - public Builder isDataStream(@Nullable Boolean isDataStream) { - this.isDataStream = isDataStream; - return this; - } - public AliasMetadata build() { return new AliasMetadata(alias, filter, indexRouting, searchRouting, writeIndex, isHidden); } @@ -392,8 +384,6 @@ public static AliasMetadata fromXContent(XContentParser parser) throws IOExcepti builder.writeIndex(parser.booleanValue()); } else if ("is_hidden".equals(currentFieldName)) { builder.isHidden(parser.booleanValue()); - } else if ("is_data_stream".equals(currentFieldName)) { - builder.isDataStream(parser.booleanValue()); } } } From 86dc1f335f15f2334f8ddc8e60ee765d365205e9 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 19:11:16 -0500 Subject: [PATCH 15/22] another test + minor reformatting --- .../MetadataCreateDataStreamService.java | 14 +++++----- .../xpack/datastreams/DataStreamsRestIT.java | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index d155d1d7364fa..321b0406e6b28 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -157,12 +157,11 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn } static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, - ClusterState currentState, - String dataStreamName, - List backingIndices, - IndexMetadata writeIndex, - SystemDataStreamDescriptor systemDataStreamDescriptor) throws Exception - { + ClusterState currentState, + String dataStreamName, + List backingIndices, + IndexMetadata writeIndex, + SystemDataStreamDescriptor systemDataStreamDescriptor) throws Exception { Objects.requireNonNull(metadataCreateIndexService); Objects.requireNonNull(currentState); Objects.requireNonNull(backingIndices); @@ -177,8 +176,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must be lowercase"); } if (dataStreamName.startsWith(DataStream.BACKING_INDEX_PREFIX)) { - throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '" - + DataStream.BACKING_INDEX_PREFIX + "'"); + throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '" + DataStream.BACKING_INDEX_PREFIX + "'"); } final boolean isSystem = systemDataStreamDescriptor != null; diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 65eed652bcd09..5c8ddcdffc3fb 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -229,6 +229,31 @@ public void testGetAliasApiFilterByDataStreamAlias() throws Exception { assertEquals(404, getAliasesResponse.get("status")); } + public void testDataStreamWithAliasFromTemplate() throws IOException { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity( + "{\"index_patterns\": [\"logs-*\"], \"aliases\": { \"\": { logs } }, \"data_stream\": {}}"); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/logs-emea/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + createDocRequest = new Request("POST", "/logs-nasa/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + Request getAliasesRequest = new Request("GET", "/_aliases"); + Map getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-emea.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-nasa.aliases", getAliasesResponse)); + + Request searchRequest = new Request("GET", "/logs/_search"); + Map searchResponse = entityAsMap(client().performRequest(searchRequest)); + assertEquals(2, XContentMapValues.extractValue("hits.total.value", searchResponse)); + } + public void testDataStreamWriteAlias() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); @@ -289,4 +314,5 @@ public void testDataStreamWriteAlias() throws IOException { ); } + } From f75425b230f7a7142ecd69d0d87d470a94b6d882 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 19:32:36 -0500 Subject: [PATCH 16/22] checkstyle --- .../cluster/metadata/MetadataCreateDataStreamService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 321b0406e6b28..83c235175a391 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -176,7 +176,8 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must be lowercase"); } if (dataStreamName.startsWith(DataStream.BACKING_INDEX_PREFIX)) { - throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '" + DataStream.BACKING_INDEX_PREFIX + "'"); + throw new IllegalArgumentException( + "data_stream [" + dataStreamName + "] must not start with '" + DataStream.BACKING_INDEX_PREFIX + "'"); } final boolean isSystem = systemDataStreamDescriptor != null; From 4edd676fb6c91d5a4d406e21f53cf543da2652d2 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 19 Jul 2021 19:59:43 -0500 Subject: [PATCH 17/22] fix test --- .../org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 5c8ddcdffc3fb..f6a56788800c9 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -233,7 +233,7 @@ public void testDataStreamWithAliasFromTemplate() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); putComposableIndexTemplateRequest.setJsonEntity( - "{\"index_patterns\": [\"logs-*\"], \"aliases\": { \"\": { logs } }, \"data_stream\": {}}"); + "{\"index_patterns\": [\"logs-*\"], \"template\": { \"aliases\": { \"logs\": {} } }, \"data_stream\": {}}"); assertOK(client().performRequest(putComposableIndexTemplateRequest)); Request createDocRequest = new Request("POST", "/logs-emea/_doc?refresh=true"); From 3b70768c03d257e813df504b2cd2e64a9a5467d4 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 21 Jul 2021 11:17:58 -0500 Subject: [PATCH 18/22] review comments --- .../cluster/metadata/AliasMetadata.java | 2 +- .../metadata/ComposableIndexTemplate.java | 19 +++---------------- .../metadata/MetadataCreateIndexService.java | 1 + .../xpack/datastreams/DataStreamsRestIT.java | 2 ++ 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index 1fb3337e1a817..43c41292865d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; +import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -23,7 +24,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.core.Nullable; import java.io.IOException; import java.util.Collections; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index a448aeede6df6..f4c9ef7625936 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -10,20 +10,19 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -263,22 +262,10 @@ public static class DataStreamTemplate implements Writeable, ToXContentObject { public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_stream_template", false, - args -> { - boolean hidden = args[0] != null && (boolean) args[0]; - return new DataStreamTemplate(hidden); - } - ); + a -> new DataStreamTemplate(a[0] != null && (boolean) a[0])); static { PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { - Map aliasMap = new HashMap<>(); - while ((p.nextToken()) != XContentParser.Token.END_OBJECT) { - AliasMetadata alias = AliasMetadata.Builder.fromXContent(p); - aliasMap.put(alias.alias(), alias); - } - return aliasMap; - }, ALIASES); } private final boolean hidden; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 83db8ced0eb10..c63d29eb2ce07 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -529,6 +529,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu mappings, indexService -> resolveAndValidateAliases( request.index(), + // data stream aliases are created separately in MetadataCreateDataStreamService::createDataStream isDataStream ? Set.of() : request.aliases(), isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index f6a56788800c9..058afdb43c53e 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; public class DataStreamsRestIT extends ESRestTestCase { @@ -246,6 +247,7 @@ public void testDataStreamWithAliasFromTemplate() throws IOException { Request getAliasesRequest = new Request("GET", "/_aliases"); Map getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertThat(getAliasesResponse.size(), is(2)); assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-emea.aliases", getAliasesResponse)); assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-nasa.aliases", getAliasesResponse)); From f3a97c86051f9f093a44d196867f65be9c09bf6a Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 21 Jul 2021 11:19:19 -0500 Subject: [PATCH 19/22] missed one spot --- .../elasticsearch/cluster/metadata/ComposableIndexTemplate.java | 1 - .../org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java | 1 - 2 files changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index f4c9ef7625936..f05f842b95869 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -257,7 +257,6 @@ public String toString() { public static class DataStreamTemplate implements Writeable, ToXContentObject { private static final ParseField HIDDEN = new ParseField("hidden"); - private static final ParseField ALIASES = new ParseField("aliases"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_stream_template", diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 058afdb43c53e..142b3241d637e 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -316,5 +316,4 @@ public void testDataStreamWriteAlias() throws IOException { ); } - } From 8d1e1e71f117542408d70d8eae40f0f91a9bfde3 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 21 Jul 2021 11:43:25 -0500 Subject: [PATCH 20/22] fix merge error --- .../cluster/metadata/MetadataCreateDataStreamService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 83c235175a391..2d432979563da 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -224,7 +224,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn if (template.template() != null && template.template().aliases() != null) { for (var alias : template.template().aliases().values()) { - builder.put(alias.getAlias(), dataStreamName, alias.writeIndex()); + builder.put(alias.getAlias(), dataStreamName, alias.writeIndex(), alias.filter().string()); } } From 33f23594d8bc8fcc1e3ab7405049c14079c35aad Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Wed, 21 Jul 2021 14:13:38 -0500 Subject: [PATCH 21/22] fix test --- .../cluster/metadata/MetadataCreateDataStreamService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 2d432979563da..e62a10f80e819 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -224,7 +224,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn if (template.template() != null && template.template().aliases() != null) { for (var alias : template.template().aliases().values()) { - builder.put(alias.getAlias(), dataStreamName, alias.writeIndex(), alias.filter().string()); + builder.put(alias.getAlias(), dataStreamName, alias.writeIndex(), alias.filter() == null ? null : alias.filter().string()); } } From 833677acdbeedf60357493e2c5e0df845af63c07 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Thu, 22 Jul 2021 12:19:00 -0500 Subject: [PATCH 22/22] review comments --- .../MetadataCreateDataStreamService.java | 14 +++++++++++--- .../MetadataCreateDataStreamServiceTests.java | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index e62a10f80e819..1d62054b737c7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -35,6 +35,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -218,16 +219,23 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn DataStream newDataStream = new DataStream(dataStreamName, timestampField, dsBackingIndices, 1L, template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden, false, isSystem); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); - logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName, - writeIndex.getIndex().getName(), - Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray())); + List aliases = new ArrayList<>(); if (template.template() != null && template.template().aliases() != null) { for (var alias : template.template().aliases().values()) { + aliases.add(alias.getAlias()); builder.put(alias.getAlias(), dataStreamName, alias.writeIndex(), alias.filter() == null ? null : alias.filter().string()); } } + logger.info( + "adding data stream [{}] with write index [{}], backing indices [{}], and aliases [{}]", + dataStreamName, + writeIndex.getIndex().getName(), + Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray()), + Strings.collectionToCommaDelimitedString(aliases) + ); + return ClusterState.builder(currentState).metadata(builder).build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index b239f200faf4e..3edadb8c35637 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -76,7 +76,12 @@ public void testCreateDataStreamWithAliasFromTemplate() throws Exception { Map aliases = new HashMap<>(aliasCount); for (int k = 0; k < aliasCount; k++) { final String aliasName = randomAlphaOfLength(6); - aliases.put(aliasName, AliasMetadata.newAliasMetadataBuilder(aliasName).build()); + var builder = AliasMetadata.newAliasMetadataBuilder(aliasName); + if (randomBoolean()) { + builder.filter(Map.of("term", Map.of("user", Map.of("value", randomAlphaOfLength(5))))); + } + builder.writeIndex(randomBoolean()); + aliases.put(aliasName, builder.build()); } ComposableIndexTemplate template = new ComposableIndexTemplate.Builder() .indexPatterns(List.of(dataStreamName + "*")) @@ -95,6 +100,15 @@ public void testCreateDataStreamWithAliasFromTemplate() throws Exception { assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); assertThat(newState.metadata().dataStreamAliases().size(), is(aliasCount)); + for (String aliasName : aliases.keySet()) { + var expectedAlias = aliases.get(aliasName); + var actualAlias = newState.metadata().dataStreamAliases().get(aliasName); + assertThat(actualAlias, is(notNullValue())); + assertThat(actualAlias.getName(), equalTo(expectedAlias.alias())); + assertThat(actualAlias.getFilter(), equalTo(expectedAlias.filter())); + assertThat(actualAlias.getWriteDataStream(), equalTo(expectedAlias.writeIndex() ? dataStreamName : null)); + } + assertThat(newState.metadata().dataStreamAliases().values().stream().map(DataStreamAlias::getName).toArray(), arrayContainingInAnyOrder (new ArrayList<>(aliases.keySet()).toArray())); assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue());