Skip to content

Commit 6cfb724

Browse files
talevynot-napoleon
talevy
and
not-napoleon
authored
Add RollupV2 cluster metadata behind feature-flag (#64680) (#64892)
this commit sets the foundation for following rollup-v2-related commits to master. The intention of this metadata is to be used by both the upcoming RollupV2 action that will create new indices along with their associated cluster metadata. This metadata is to be used by the SearchService when filtering shards in the can-match phase to decide which of the indices belonging to an original index should be queried. Co-authored-by: Mark Tozzi <[email protected]>
1 parent c3bf972 commit 6cfb724

File tree

7 files changed

+638
-0
lines changed

7 files changed

+638
-0
lines changed

‎server/src/main/java/org/elasticsearch/cluster/ClusterModule.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.elasticsearch.cluster.metadata.MetadataMappingService;
3636
import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
3737
import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
38+
import org.elasticsearch.cluster.metadata.RollupMetadata;
3839
import org.elasticsearch.cluster.routing.DelayedAllocationService;
3940
import org.elasticsearch.cluster.routing.allocation.AllocationService;
4041
import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
@@ -76,6 +77,7 @@
7677
import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
7778
import org.elasticsearch.persistent.PersistentTasksNodeService;
7879
import org.elasticsearch.plugins.ClusterPlugin;
80+
import org.elasticsearch.rollup.RollupV2;
7981
import org.elasticsearch.script.ScriptMetadata;
8082
import org.elasticsearch.snapshots.SnapshotsInfoService;
8183
import org.elasticsearch.tasks.Task;
@@ -143,6 +145,10 @@ public static List<Entry> getNamedWriteables() {
143145
registerMetadataCustom(entries, ComposableIndexTemplateMetadata.TYPE, ComposableIndexTemplateMetadata::new,
144146
ComposableIndexTemplateMetadata::readDiffFrom);
145147
registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
148+
149+
if (RollupV2.ROLLUPV2_FEATURE_FLAG_REGISTERED != null && RollupV2.ROLLUPV2_FEATURE_FLAG_REGISTERED) {
150+
registerMetadataCustom(entries, RollupMetadata.TYPE, RollupMetadata::new, RollupMetadata::readDiffFrom);
151+
}
146152
// Task Status (not Diffable)
147153
entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
148154
return entries;
@@ -196,6 +202,10 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
196202
ComposableIndexTemplateMetadata::fromXContent));
197203
entries.add(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(DataStreamMetadata.TYPE),
198204
DataStreamMetadata::fromXContent));
205+
if (RollupV2.ROLLUPV2_FEATURE_FLAG_REGISTERED != null && RollupV2.ROLLUPV2_FEATURE_FLAG_REGISTERED) {
206+
entries.add(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(RollupMetadata.TYPE),
207+
RollupMetadata::fromXContent));
208+
}
199209
return entries;
200210
}
201211

‎server/src/main/java/org/elasticsearch/cluster/metadata/RollupGroup.java

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.cluster.metadata;
21+
22+
import org.elasticsearch.cluster.AbstractDiffable;
23+
import org.elasticsearch.cluster.Diff;
24+
import org.elasticsearch.common.ParseField;
25+
import org.elasticsearch.common.Strings;
26+
import org.elasticsearch.common.io.stream.StreamInput;
27+
import org.elasticsearch.common.io.stream.StreamOutput;
28+
import org.elasticsearch.common.time.WriteableZoneId;
29+
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
30+
import org.elasticsearch.common.xcontent.ToXContentObject;
31+
import org.elasticsearch.common.xcontent.XContentBuilder;
32+
import org.elasticsearch.common.xcontent.XContentParser;
33+
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
34+
35+
import java.io.IOException;
36+
import java.util.HashMap;
37+
import java.util.List;
38+
import java.util.Map;
39+
import java.util.Objects;
40+
41+
/**
42+
* Object representing information about rollup-v2 indices and their respective original-indexes. These objects
43+
* also include information about their capabilities, like which date-intervals and date-timezones they are configured
44+
* with. Used by {@link RollupMetadata}.
45+
*
46+
* The information in this class will be used to decide which index within the <code>group</code> will be chosen
47+
* for a specific aggregation. For example, if there are two indices with different intervals (`1h`, `1d`) and
48+
* a date-histogram aggregation request is sent for daily intervals, then the index with the associated `1d` interval
49+
* will be chosen.
50+
*/
51+
public class RollupGroup extends AbstractDiffable<RollupGroup> implements ToXContentObject {
52+
private static final ParseField GROUP_FIELD = new ParseField("group");
53+
private static final ParseField DATE_INTERVAL_FIELD = new ParseField("interval");
54+
private static final ParseField DATE_TIMEZONE_FIELD = new ParseField("timezone");
55+
56+
// the list of indices part of this rollup group
57+
private List<String> group;
58+
// a map from index-name to the date interval used in the associated index
59+
private Map<String, DateHistogramInterval> dateInterval;
60+
// a map from index-name to timezone used in the associated index
61+
private Map<String, WriteableZoneId> dateTimezone;
62+
63+
@SuppressWarnings("unchecked")
64+
public static final ConstructingObjectParser<RollupGroup, Void> PARSER =
65+
new ConstructingObjectParser<>("rollup_group", false,
66+
a -> new RollupGroup((List<String>) a[0], (Map<String, DateHistogramInterval>) a[1], (Map<String, WriteableZoneId>) a[2]));
67+
68+
static {
69+
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), GROUP_FIELD);
70+
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
71+
Map<String, DateHistogramInterval> intervalMap = new HashMap<>();
72+
73+
while (p.nextToken() != XContentParser.Token.END_OBJECT) {
74+
String name = p.currentName();
75+
p.nextToken();
76+
String expression = p.text();
77+
intervalMap.put(name, new DateHistogramInterval(expression));
78+
}
79+
return intervalMap;
80+
}, DATE_INTERVAL_FIELD);
81+
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
82+
Map<String, WriteableZoneId> zoneMap = new HashMap<>();
83+
while (p.nextToken() != XContentParser.Token.END_OBJECT) {
84+
String name = p.currentName();
85+
p.nextToken();
86+
String timezone = p.text();
87+
zoneMap.put(name, WriteableZoneId.of(timezone));
88+
}
89+
return zoneMap;
90+
}, DATE_TIMEZONE_FIELD);
91+
}
92+
93+
public RollupGroup(List<String> group, Map<String, DateHistogramInterval> dateInterval, Map<String, WriteableZoneId> dateTimezone) {
94+
this.group = group;
95+
this.dateInterval = dateInterval;
96+
this.dateTimezone = dateTimezone;
97+
}
98+
99+
public RollupGroup(StreamInput in) throws IOException {
100+
this.group = in.readStringList();
101+
this.dateInterval = in.readMap(StreamInput::readString, DateHistogramInterval::new);
102+
this.dateTimezone = in.readMap(StreamInput::readString, WriteableZoneId::new);
103+
}
104+
105+
106+
public static RollupGroup fromXContent(XContentParser parser) throws IOException {
107+
return PARSER.parse(parser, null);
108+
}
109+
110+
public void add(String name, DateHistogramInterval interval, WriteableZoneId timezone) {
111+
group.add(name);
112+
dateInterval.put(name, interval);
113+
dateTimezone.put(name, timezone);
114+
}
115+
116+
public void remove(String name) {
117+
group.remove(name);
118+
dateInterval.remove(name);
119+
dateTimezone.remove(name);
120+
}
121+
122+
public boolean contains(String name) {
123+
return group.contains(name);
124+
}
125+
126+
public DateHistogramInterval getDateInterval(String name) {
127+
return dateInterval.get(name);
128+
}
129+
130+
public WriteableZoneId getDateTimezone(String name) {
131+
return dateTimezone.get(name);
132+
}
133+
134+
public List<String> getIndices() {
135+
return group;
136+
}
137+
138+
static Diff<RollupGroup> readDiffFrom(StreamInput in) throws IOException {
139+
return AbstractDiffable.readDiffFrom(RollupGroup::new, in);
140+
}
141+
142+
public static RollupGroup parse(XContentParser parser) {
143+
return PARSER.apply(parser, null);
144+
}
145+
146+
@Override
147+
public String toString() {
148+
return Strings.toString(this);
149+
}
150+
151+
@Override
152+
public void writeTo(StreamOutput out) throws IOException {
153+
out.writeStringCollection(group);
154+
out.writeMap(dateInterval, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
155+
out.writeMap(dateTimezone, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
156+
}
157+
158+
@Override
159+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
160+
return builder
161+
.startObject()
162+
.field(GROUP_FIELD.getPreferredName(), group)
163+
.field(DATE_INTERVAL_FIELD.getPreferredName(), dateInterval)
164+
.field(DATE_TIMEZONE_FIELD.getPreferredName(), dateTimezone)
165+
.endObject();
166+
}
167+
168+
@Override
169+
public boolean equals(Object o) {
170+
if (this == o) return true;
171+
if (o == null || getClass() != o.getClass()) return false;
172+
RollupGroup that = (RollupGroup) o;
173+
return group.equals(that.group) &&
174+
dateInterval.equals(that.dateInterval) &&
175+
dateTimezone.equals(that.dateTimezone);
176+
}
177+
178+
@Override
179+
public int hashCode() {
180+
return Objects.hash(group, dateInterval, dateTimezone);
181+
}
182+
}

0 commit comments

Comments
 (0)