Create data stream aliases from template

server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java

@@ -32,6 +32,7 @@
 import java.util.Set;
 
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_ALIAS_VERSION;
 
 public class AliasMetadata extends AbstractDiffable<AliasMetadata> implements ToXContentFragment {
 
@@ -51,8 +52,11 @@ public class AliasMetadata extends AbstractDiffable<AliasMetadata> implements To
     @Nullable
     private final Boolean isHidden;
 
+    @Nullable
+    private final Boolean isDataStream;
+
     private AliasMetadata(String alias, CompressedXContent filter, String indexRouting, String searchRouting, Boolean writeIndex,
-                          @Nullable Boolean isHidden) {
+                          @Nullable Boolean isHidden, @Nullable Boolean isDataStream) {
         this.alias = alias;
         this.filter = filter;
         this.indexRouting = indexRouting;
@@ -64,11 +68,12 @@ private AliasMetadata(String alias, CompressedXContent filter, String indexRouti
         }
         this.writeIndex = writeIndex;
         this.isHidden = isHidden;
+        this.isDataStream = isDataStream;
     }
 
     private AliasMetadata(AliasMetadata aliasMetadata, String alias) {
         this(alias, aliasMetadata.filter(), aliasMetadata.indexRouting(), aliasMetadata.searchRouting(), aliasMetadata.writeIndex(),
-            aliasMetadata.isHidden);
+            aliasMetadata.isHidden, aliasMetadata.isDataStream);
     }
 
     public String alias() {
@@ -120,6 +125,11 @@ public Boolean isHidden() {
         return isHidden;
     }
 
+    @Nullable
+    public Boolean isDataStream() {
+        return isDataStream;
+    }
+
     public static Builder builder(String alias) {
         return new Builder(alias);
     }
@@ -148,6 +158,7 @@ public boolean equals(Object o) {
         if (Objects.equals(searchRouting, that.searchRouting) == false) return false;
         if (Objects.equals(writeIndex, that.writeIndex) == false) return false;
         if (Objects.equals(isHidden, that.isHidden) == false) return false;
+        if (Objects.equals(isDataStream, that.isDataStream) == false) return false;
 
         return true;
     }
@@ -159,6 +170,7 @@ public int hashCode() {
         result = 31 * result + (indexRouting != null ? indexRouting.hashCode() : 0);
         result = 31 * result + (searchRouting != null ? searchRouting.hashCode() : 0);
         result = 31 * result + (writeIndex != null ? writeIndex.hashCode() : 0);
+        result = 31 * result + (isDataStream != null ? isDataStream.hashCode() : 0);
         return result;
     }
 
@@ -185,6 +197,9 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeOptionalBoolean(writeIndex());
         out.writeOptionalBoolean(isHidden);
+        if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) {
+            out.writeOptionalBoolean(isDataStream);
+        }
     }
 
     public AliasMetadata(StreamInput in) throws IOException {
@@ -208,6 +223,11 @@ public AliasMetadata(StreamInput in) throws IOException {
         }
         writeIndex = in.readOptionalBoolean();
         isHidden = in.readOptionalBoolean();
+        if (in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) {
+            isDataStream = in.readOptionalBoolean();
+        } else {
+            isDataStream = null;
+        }
     }
 
     public static Diff<AliasMetadata> readDiffFrom(StreamInput in) throws IOException {
@@ -249,6 +269,9 @@ public static class Builder {
         @Nullable
         private Boolean isHidden;
 
+        @Nullable
+        private Boolean isDataStream;
+
         public Builder(String alias) {
             this.alias = alias;
         }
@@ -310,8 +333,13 @@ public Builder isHidden(@Nullable Boolean isHidden) {
             return this;
         }
 
+        public Builder isDataStream(@Nullable Boolean isDataStream) {
+            this.isDataStream = isDataStream;
+            return this;
+        }
+
         public AliasMetadata build() {
-            return new AliasMetadata(alias, filter, indexRouting, searchRouting, writeIndex, isHidden);
+            return new AliasMetadata(alias, filter, indexRouting, searchRouting, writeIndex, isHidden, isDataStream);
         }
 
         public static void toXContent(AliasMetadata aliasMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException {
@@ -341,6 +369,10 @@ public static void toXContent(AliasMetadata aliasMetadata, XContentBuilder build
                 builder.field("is_hidden", aliasMetadata.isHidden());
             }
 
+            if (aliasMetadata.isDataStream != null) {
+                builder.field("is_data_stream", aliasMetadata.isDataStream());
+            }
+
             builder.endObject();
         }
 
@@ -384,6 +416,8 @@ public static AliasMetadata fromXContent(XContentParser parser) throws IOExcepti
                         builder.writeIndex(parser.booleanValue());
                     } else if ("is_hidden".equals(currentFieldName)) {
                         builder.isHidden(parser.booleanValue());
+                    } else if ("is_data_stream".equals(currentFieldName)) {
+                        builder.isDataStream(parser.booleanValue());
                     }
                 }
             }

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.core.Nullable;
@@ -23,6 +24,8 @@
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -256,35 +259,63 @@ public String toString() {
 
     public static class DataStreamTemplate implements Writeable, ToXContentObject {
 
+        public static final Version DATA_STREAM_ALIAS_VERSION = Version.V_8_0_0;
+
         private static final ParseField HIDDEN = new ParseField("hidden");
+        private static final ParseField ALIASES = new ParseField("aliases");
 
         public static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(
             "data_stream_template",
             false,
-            a -> new DataStreamTemplate(a[0] != null && (boolean) a[0]));
+            args -> {
+                boolean hidden = args[0] != null && (boolean) args[0];
+                @SuppressWarnings("unchecked")
+                Map<String, AliasMetadata> aliases = (Map<String, AliasMetadata>) args[1];
+                return new DataStreamTemplate(hidden, aliases);
+            }
+        );
 
         static {
             PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN);
+            PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+                Map<String, AliasMetadata> aliasMap = new HashMap<>();
+                while ((p.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    AliasMetadata alias = AliasMetadata.Builder.fromXContent(p);
+                    aliasMap.put(alias.alias(), alias);
+                }
+                return aliasMap;
+            }, ALIASES);
         }
 
         private final boolean hidden;
 
+        @Nullable
+        private final Map<String, AliasMetadata> aliases;
+
         public DataStreamTemplate() {
-            this(false);
+            this(false, null);
         }
 
-        public DataStreamTemplate(boolean hidden) {
+        public DataStreamTemplate(boolean hidden, Map<String, AliasMetadata> aliases) {
             this.hidden = hidden;
+            this.aliases = aliases;
         }
 
         DataStreamTemplate(StreamInput in) throws IOException {
             hidden = in.readBoolean();
+            aliases = in.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)
+                ? in.readBoolean() ? in.readMap(StreamInput::readString, AliasMetadata::new) : null
+                : null;
         }
 
         public String getTimestampField() {
             return FIXED_TIMESTAMP_FIELD;
         }
 
+        public Collection<AliasMetadata> getAliases() {
+            return aliases == null ? null : aliases.values();
+        }
+
         /**
          * @return a mapping snippet for a backing index with `_data_stream_timestamp` meta field mapper properly configured.
          */
@@ -300,12 +331,27 @@ public boolean isHidden() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(hidden);
+            if (out.getVersion().onOrAfter(DATA_STREAM_ALIAS_VERSION)) {
+                if (aliases != null) {
+                    out.writeBoolean(true);
+                    out.writeMap(aliases, StreamOutput::writeString, (innerOut, alias) -> alias.writeTo(innerOut));
+                } else {
+                    out.writeBoolean(false);
+                }
+            }
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field("hidden", hidden);
+            if (aliases != null) {
+                builder.startObject("aliases");
+                for (var alias : aliases.values()) {
+                    alias.toXContent(builder, params);
+                }
+                builder.endObject();
+            }
             builder.endObject();
             return builder;
         }
@@ -315,12 +361,12 @@ public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             DataStreamTemplate that = (DataStreamTemplate) o;
-            return hidden == that.hidden;
+            return hidden == that.hidden && Objects.equals(aliases, that.aliases);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(hidden);
+            return Objects.hash(hidden, aliases);
         }
     }
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java

@@ -222,6 +222,13 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn
         logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName,
             writeIndex.getIndex().getName(),
             Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray()));
+
+        if (template.getDataStreamTemplate().getAliases() != null) {
+            for (var alias : template.getDataStreamTemplate().getAliases()) {
+                builder.put(alias.getAlias(), dataStreamName, alias.writeIndex());
+            }
+        }
+
         return ClusterState.builder(currentState).metadata(builder).build();
     }
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -1146,10 +1146,10 @@ static List<Map<String, AliasMetadata>> resolveAliases(final ComposableIndexTemp
             .map(Template::aliases)
             .ifPresent(aliases::add);
 
-        // A template that creates data streams can't also create aliases.
-        // (otherwise we end up with aliases pointing to backing indices of data streams)
+        // A template that creates data streams may not specify index aliases. Data stream
+        // aliases are defined separately in the `data_stream` object on the template.
         if (aliases.size() > 0 && template.getDataStreamTemplate() != null) {
-            throw new IllegalArgumentException("template [" + templateName + "] has alias and data stream definitions");
+            throw new IllegalArgumentException("template [" + templateName + "] may not specify both data streams and index alias(es)");
         }
 
         // Aliases are applied in order, but subsequent alias configuration from the same name is

server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

@@ -705,7 +705,7 @@ clusterService, indicesService, allocationService, new AliasValidator(), shardLi
                 () -> rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions,
                     randomBoolean(), false)
             );
-            assertThat(e.getMessage(), equalTo("template [template] has alias and data stream definitions"));
+            assertThat(e.getMessage(), equalTo("template [template] may not specify both data streams and index alias(es)"));
         } finally {
             testThreadPool.shutdown();
         }

server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java

@@ -32,7 +32,7 @@ protected DataStreamTemplate createTestInstance() {
     }
 
     public static DataStreamTemplate randomInstance() {
-        return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean());
+        return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), null);
     }
 
 }

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -1219,7 +1219,7 @@ public void testResolveAliasesDataStreams() throws Exception {
             .build();
         Exception e =
             expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state1.metadata(), "1"));
-        assertThat(e.getMessage(), equalTo("template [1] has alias and data stream definitions"));
+        assertThat(e.getMessage(), equalTo("template [1] may not specify both data streams and index alias(es)"));
 
         // index template can't have data streams and a component template with an aliases
         ComponentTemplate componentTemplate = new ComponentTemplate(new Template(null, null, a1), null, null);
@@ -1229,7 +1229,7 @@ public void testResolveAliasesDataStreams() throws Exception {
             .metadata(Metadata.builder().put("1", it).put("c1", componentTemplate).build())
             .build();
         e = expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state2.metadata(), "1"));
-        assertThat(e.getMessage(), equalTo("template [1] has alias and data stream definitions"));
+        assertThat(e.getMessage(), equalTo("template [1] may not specify both data streams and index alias(es)"));
     }
 
     public void testAddInvalidTemplate() throws Exception {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java

@@ -92,7 +92,7 @@ public void testCalculateUsage() {
                         new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate",
                             new ComposableIndexTemplate(Collections.singletonList("myds"),
                                 new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null),
-                                null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false)))))
+                                null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false, null)))))
                     .build())
                 .build();
             assertThat(LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"),
@@ -133,7 +133,7 @@ public void testCalculateUsage() {
                         new ComposableIndexTemplateMetadata(Collections.singletonMap("mytemplate",
                             new ComposableIndexTemplate(Collections.singletonList("myds"),
                                 new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null),
-                                null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false)))))
+                                null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate(false, null)))))
                     .build())
                 .build();
             assertThat(LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"),

x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java

@@ -90,7 +90,7 @@ public void testAddingIndexTemplateWithAliasesAndDataStream() {
         String body = "{\"index_patterns\":[\"mypattern*\"],\"data_stream\":{},\"template\":{\"aliases\":{\"my-alias\":{}}}}";
         putComposableIndexTemplateRequest.setJsonEntity(body);
         Exception e = expectThrows(ResponseException.class, () -> client().performRequest(putComposableIndexTemplateRequest));
-        assertThat(e.getMessage(), containsString("template [my-template] has alias and data stream definitions"));
+        assertThat(e.getMessage(), containsString("template [my-template] may not specify both data streams and index alias(es)"));
     }
 
     public void testDataStreamAliases() throws Exception {

x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java

@@ -242,7 +242,7 @@ private String createDataStream(boolean hidden) throws Exception {
             null,
             null,
             null,
-            new ComposableIndexTemplate.DataStreamTemplate(hidden),
+            new ComposableIndexTemplate.DataStreamTemplate(hidden, null),
             null
         );
         assertTrue(