Skip to content

Commit ab5a11c

Browse files
droberts195droberts195
committed
[ML] Create the ML annotations index (#36731)
The ML UI now provides the ability for users to annotate time periods with arbitrary text to add insight to what happened. This change makes the backend create the index for these annotations, together with read and write aliases to make future upgrades possible without adding complexity to the UI. It also adds read and write permission to the index for all ML users (not just admins). The spec for the index is in https://github.com/elastic/kibana/pull/26034/files#diff-c5c6ac3dbb0e7c91b6d127aa06121b2cR7 Relates #33376 Relates elastic/kibana#26034
1 parent c2084de commit ab5a11c

File tree

8 files changed

+556
-10
lines changed

8 files changed

+556
-10
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java

+244
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ml.annotations;
7+
8+
import org.elasticsearch.common.ParseField;
9+
import org.elasticsearch.common.io.stream.StreamInput;
10+
import org.elasticsearch.common.io.stream.StreamOutput;
11+
import org.elasticsearch.common.io.stream.Writeable;
12+
import org.elasticsearch.common.xcontent.ObjectParser;
13+
import org.elasticsearch.common.xcontent.ToXContentObject;
14+
import org.elasticsearch.common.xcontent.XContentBuilder;
15+
import org.elasticsearch.xpack.core.ml.job.config.Job;
16+
import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
17+
18+
import java.io.IOException;
19+
import java.util.Date;
20+
import java.util.Objects;
21+
22+
public class Annotation implements ToXContentObject, Writeable {
23+
24+
public static final ParseField ANNOTATION = new ParseField("annotation");
25+
public static final ParseField CREATE_TIME = new ParseField("create_time");
26+
public static final ParseField CREATE_USERNAME = new ParseField("create_username");
27+
public static final ParseField TIMESTAMP = new ParseField("timestamp");
28+
public static final ParseField END_TIMESTAMP = new ParseField("end_timestamp");
29+
public static final ParseField MODIFIED_TIME = new ParseField("modified_time");
30+
public static final ParseField MODIFIED_USERNAME = new ParseField("modified_username");
31+
public static final ParseField TYPE = new ParseField("type");
32+
33+
public static final ObjectParser<Annotation, Void> PARSER = new ObjectParser<>(TYPE.getPreferredName(), true, Annotation::new);
34+
35+
static {
36+
PARSER.declareString(Annotation::setAnnotation, ANNOTATION);
37+
PARSER.declareField(Annotation::setCreateTime,
38+
p -> TimeUtils.parseTimeField(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE);
39+
PARSER.declareString(Annotation::setCreateUsername, CREATE_USERNAME);
40+
PARSER.declareField(Annotation::setTimestamp,
41+
p -> TimeUtils.parseTimeField(p, TIMESTAMP.getPreferredName()), TIMESTAMP, ObjectParser.ValueType.VALUE);
42+
PARSER.declareField(Annotation::setEndTimestamp,
43+
p -> TimeUtils.parseTimeField(p, END_TIMESTAMP.getPreferredName()), END_TIMESTAMP, ObjectParser.ValueType.VALUE);
44+
PARSER.declareString(Annotation::setJobId, Job.ID);
45+
PARSER.declareField(Annotation::setModifiedTime,
46+
p -> TimeUtils.parseTimeField(p, MODIFIED_TIME.getPreferredName()), MODIFIED_TIME, ObjectParser.ValueType.VALUE);
47+
PARSER.declareString(Annotation::setModifiedUsername, MODIFIED_USERNAME);
48+
PARSER.declareString(Annotation::setType, TYPE);
49+
}
50+
51+
private String annotation;
52+
private Date createTime;
53+
private String createUsername;
54+
private Date timestamp;
55+
private Date endTimestamp;
56+
/**
57+
* Unlike most ML classes, this may be <code>null</code> or wildcarded
58+
*/
59+
private String jobId;
60+
private Date modifiedTime;
61+
private String modifiedUsername;
62+
private String type;
63+
64+
private Annotation() {
65+
}
66+
67+
public Annotation(String annotation, Date createTime, String createUsername, Date timestamp, Date endTimestamp, String jobId,
68+
Date modifiedTime, String modifiedUsername, String type) {
69+
this.annotation = Objects.requireNonNull(annotation);
70+
this.createTime = Objects.requireNonNull(createTime);
71+
this.createUsername = Objects.requireNonNull(createUsername);
72+
this.timestamp = Objects.requireNonNull(timestamp);
73+
this.endTimestamp = endTimestamp;
74+
this.jobId = jobId;
75+
this.modifiedTime = modifiedTime;
76+
this.modifiedUsername = modifiedUsername;
77+
this.type = Objects.requireNonNull(type);
78+
}
79+
80+
public Annotation(StreamInput in) throws IOException {
81+
annotation = in.readString();
82+
createTime = new Date(in.readLong());
83+
createUsername = in.readString();
84+
timestamp = new Date(in.readLong());
85+
if (in.readBoolean()) {
86+
endTimestamp = new Date(in.readLong());
87+
} else {
88+
endTimestamp = null;
89+
}
90+
jobId = in.readOptionalString();
91+
if (in.readBoolean()) {
92+
modifiedTime = new Date(in.readLong());
93+
} else {
94+
modifiedTime = null;
95+
}
96+
modifiedUsername = in.readOptionalString();
97+
type = in.readString();
98+
}
99+
100+
@Override
101+
public void writeTo(StreamOutput out) throws IOException {
102+
out.writeString(annotation);
103+
out.writeLong(createTime.getTime());
104+
out.writeString(createUsername);
105+
out.writeLong(timestamp.getTime());
106+
if (endTimestamp != null) {
107+
out.writeBoolean(true);
108+
out.writeLong(endTimestamp.getTime());
109+
} else {
110+
out.writeBoolean(false);
111+
}
112+
out.writeOptionalString(jobId);
113+
if (modifiedTime != null) {
114+
out.writeBoolean(true);
115+
out.writeLong(modifiedTime.getTime());
116+
} else {
117+
out.writeBoolean(false);
118+
}
119+
out.writeOptionalString(modifiedUsername);
120+
out.writeString(type);
121+
122+
}
123+
124+
public String getAnnotation() {
125+
return annotation;
126+
}
127+
128+
public void setAnnotation(String annotation) {
129+
this.annotation = Objects.requireNonNull(annotation);
130+
}
131+
132+
public Date getCreateTime() {
133+
return createTime;
134+
}
135+
136+
public void setCreateTime(Date createTime) {
137+
this.createTime = Objects.requireNonNull(createTime);
138+
}
139+
140+
public String getCreateUsername() {
141+
return createUsername;
142+
}
143+
144+
public void setCreateUsername(String createUsername) {
145+
this.createUsername = Objects.requireNonNull(createUsername);
146+
}
147+
148+
public Date getTimestamp() {
149+
return timestamp;
150+
}
151+
152+
public void setTimestamp(Date timestamp) {
153+
this.timestamp = Objects.requireNonNull(timestamp);
154+
}
155+
156+
public Date getEndTimestamp() {
157+
return endTimestamp;
158+
}
159+
160+
public void setEndTimestamp(Date endTimestamp) {
161+
this.endTimestamp = endTimestamp;
162+
}
163+
164+
public String getJobId() {
165+
return jobId;
166+
}
167+
168+
public void setJobId(String jobId) {
169+
this.jobId = jobId;
170+
}
171+
172+
public Date getModifiedTime() {
173+
return modifiedTime;
174+
}
175+
176+
public void setModifiedTime(Date modifiedTime) {
177+
this.modifiedTime = modifiedTime;
178+
}
179+
180+
public String getModifiedUsername() {
181+
return modifiedUsername;
182+
}
183+
184+
public void setModifiedUsername(String modifiedUsername) {
185+
this.modifiedUsername = modifiedUsername;
186+
}
187+
188+
public String getType() {
189+
return type;
190+
}
191+
192+
public void setType(String type) {
193+
this.type = Objects.requireNonNull(type);
194+
}
195+
196+
@Override
197+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
198+
builder.startObject();
199+
builder.field(ANNOTATION.getPreferredName(), annotation);
200+
builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.getTime());
201+
builder.field(CREATE_USERNAME.getPreferredName(), createUsername);
202+
builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime());
203+
if (endTimestamp != null) {
204+
builder.timeField(END_TIMESTAMP.getPreferredName(), END_TIMESTAMP.getPreferredName() + "_string", endTimestamp.getTime());
205+
}
206+
if (jobId != null) {
207+
builder.field(Job.ID.getPreferredName(), jobId);
208+
}
209+
if (modifiedTime != null) {
210+
builder.timeField(MODIFIED_TIME.getPreferredName(), MODIFIED_TIME.getPreferredName() + "_string", modifiedTime.getTime());
211+
}
212+
if (modifiedUsername != null) {
213+
builder.field(MODIFIED_USERNAME.getPreferredName(), modifiedUsername);
214+
}
215+
builder.field(TYPE.getPreferredName(), type);
216+
builder.endObject();
217+
return builder;
218+
}
219+
220+
@Override
221+
public int hashCode() {
222+
return Objects.hash(annotation, createTime, createUsername, timestamp, endTimestamp, jobId, modifiedTime, modifiedUsername, type);
223+
}
224+
225+
@Override
226+
public boolean equals(Object obj) {
227+
if (obj == this) {
228+
return true;
229+
}
230+
if (obj == null || getClass() != obj.getClass()) {
231+
return false;
232+
}
233+
Annotation other = (Annotation) obj;
234+
return Objects.equals(annotation, other.annotation) &&
235+
Objects.equals(createTime, other.createTime) &&
236+
Objects.equals(createUsername, other.createUsername) &&
237+
Objects.equals(timestamp, other.timestamp) &&
238+
Objects.equals(endTimestamp, other.endTimestamp) &&
239+
Objects.equals(jobId, other.jobId) &&
240+
Objects.equals(modifiedTime, other.modifiedTime) &&
241+
Objects.equals(modifiedUsername, other.modifiedUsername) &&
242+
Objects.equals(type, other.type);
243+
}
244+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ml.annotations;
7+
8+
import org.elasticsearch.ResourceAlreadyExistsException;
9+
import org.elasticsearch.action.ActionListener;
10+
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
11+
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
12+
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
13+
import org.elasticsearch.action.support.master.AcknowledgedResponse;
14+
import org.elasticsearch.client.Client;
15+
import org.elasticsearch.cluster.ClusterState;
16+
import org.elasticsearch.cluster.metadata.AliasOrIndex;
17+
import org.elasticsearch.cluster.metadata.IndexMetaData;
18+
import org.elasticsearch.cluster.routing.UnassignedInfo;
19+
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.common.unit.TimeValue;
21+
import org.elasticsearch.common.xcontent.XContentBuilder;
22+
import org.elasticsearch.xpack.core.ml.MachineLearningField;
23+
import org.elasticsearch.xpack.core.ml.job.config.Job;
24+
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
25+
26+
import java.io.IOException;
27+
import java.util.SortedMap;
28+
29+
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
30+
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
31+
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
32+
33+
public class AnnotationIndex {
34+
35+
public static final String READ_ALIAS_NAME = ".ml-annotations-read";
36+
public static final String WRITE_ALIAS_NAME = ".ml-annotations-write";
37+
// Exposed for testing, but always use the aliases in non-test code
38+
public static final String INDEX_NAME = ".ml-annotations-6";
39+
40+
/**
41+
* Create the .ml-annotations index with correct mappings.
42+
* This index is read and written by the UI results views,
43+
* so needs to exist when there might be ML results to view.
44+
*/
45+
public static void createAnnotationsIndex(Settings settings, Client client, ClusterState state,
46+
final ActionListener<Boolean> finalListener) {
47+
48+
final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
49+
final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
50+
.addAlias(INDEX_NAME, READ_ALIAS_NAME)
51+
.addAlias(INDEX_NAME, WRITE_ALIAS_NAME).request();
52+
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
53+
ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
54+
client.admin().indices()::aliases);
55+
}, finalListener::onFailure);
56+
57+
// Only create the index or aliases if some other ML index exists - saves clutter if ML is never used.
58+
SortedMap<String, AliasOrIndex> mlLookup = state.getMetaData().getAliasAndIndexLookup().tailMap(".ml");
59+
if (mlLookup.isEmpty() == false && mlLookup.firstKey().startsWith(".ml")) {
60+
61+
// Create the annotations index if it doesn't exist already.
62+
if (mlLookup.containsKey(INDEX_NAME) == false) {
63+
64+
final TimeValue delayedNodeTimeOutSetting;
65+
// Whether we are using native process is a good way to detect whether we are in dev / test mode:
66+
if (MachineLearningField.AUTODETECT_PROCESS.get(settings)) {
67+
delayedNodeTimeOutSetting = UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(settings);
68+
} else {
69+
delayedNodeTimeOutSetting = TimeValue.ZERO;
70+
}
71+
72+
CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
73+
try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
74+
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, annotationsMapping);
75+
createIndexRequest.settings(Settings.builder()
76+
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
77+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
78+
.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting));
79+
80+
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
81+
ActionListener.<CreateIndexResponse>wrap(
82+
r -> createAliasListener.onResponse(r.isAcknowledged()),
83+
e -> {
84+
// Possible that the index was created while the request was executing,
85+
// so we need to handle that possibility
86+
if (e instanceof ResourceAlreadyExistsException) {
87+
// Create the alias
88+
createAliasListener.onResponse(true);
89+
} else {
90+
finalListener.onFailure(e);
91+
}
92+
}
93+
), client.admin().indices()::create);
94+
} catch (IOException e) {
95+
finalListener.onFailure(e);
96+
}
97+
return;
98+
}
99+
100+
// Recreate the aliases if they've gone even though the index still exists.
101+
if (mlLookup.containsKey(READ_ALIAS_NAME) == false || mlLookup.containsKey(WRITE_ALIAS_NAME) == false) {
102+
createAliasListener.onResponse(true);
103+
return;
104+
}
105+
}
106+
107+
// Nothing to do, but respond to the listener
108+
finalListener.onResponse(false);
109+
}
110+
111+
public static XContentBuilder annotationsMapping() throws IOException {
112+
return jsonBuilder()
113+
.startObject()
114+
.startObject(ElasticsearchMappings.DOC_TYPE)
115+
.startObject(ElasticsearchMappings.PROPERTIES)
116+
.startObject(Annotation.ANNOTATION.getPreferredName())
117+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.TEXT)
118+
.endObject()
119+
.startObject(Annotation.CREATE_TIME.getPreferredName())
120+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
121+
.endObject()
122+
.startObject(Annotation.CREATE_USERNAME.getPreferredName())
123+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
124+
.endObject()
125+
.startObject(Annotation.TIMESTAMP.getPreferredName())
126+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
127+
.endObject()
128+
.startObject(Annotation.END_TIMESTAMP.getPreferredName())
129+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
130+
.endObject()
131+
.startObject(Job.ID.getPreferredName())
132+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
133+
.endObject()
134+
.startObject(Annotation.MODIFIED_TIME.getPreferredName())
135+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
136+
.endObject()
137+
.startObject(Annotation.MODIFIED_USERNAME.getPreferredName())
138+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
139+
.endObject()
140+
.startObject(Annotation.TYPE.getPreferredName())
141+
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
142+
.endObject()
143+
.endObject()
144+
.endObject()
145+
.endObject();
146+
}
147+
}

0 commit comments

Comments
 (0)