Skip to content

Commit 532a720

Browse files
droberts195droberts195
authored
[ML] Skeleton estimate_model_memory endpoint for anomaly detection (elastic#53386)
This is a partial implementation of an endpoint for anomaly detector model memory estimation. It is not complete, lacking docs, HLRC and sensible numbers for many anomaly detector configurations. These will be added in a followup PR in time for 7.7 feature freeze. A skeleton endpoint is useful now because it allows work on the UI side of the change to commence. The skeleton endpoint handles the same cases that the old UI code used to handle, and produces very similar estimates for these cases. Backport of elastic#53333
1 parent ac72193 commit 532a720

File tree

7 files changed

+739
-0
lines changed

7 files changed

+739
-0
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ml.action;
7+
8+
import org.elasticsearch.action.ActionRequest;
9+
import org.elasticsearch.action.ActionRequestBuilder;
10+
import org.elasticsearch.action.ActionRequestValidationException;
11+
import org.elasticsearch.action.ActionResponse;
12+
import org.elasticsearch.action.ActionType;
13+
import org.elasticsearch.client.ElasticsearchClient;
14+
import org.elasticsearch.common.ParseField;
15+
import org.elasticsearch.common.io.stream.StreamInput;
16+
import org.elasticsearch.common.io.stream.StreamOutput;
17+
import org.elasticsearch.common.unit.ByteSizeValue;
18+
import org.elasticsearch.common.xcontent.ObjectParser;
19+
import org.elasticsearch.common.xcontent.ToXContentObject;
20+
import org.elasticsearch.common.xcontent.XContentBuilder;
21+
import org.elasticsearch.common.xcontent.XContentParser;
22+
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
23+
import org.elasticsearch.xpack.core.ml.job.config.Job;
24+
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
25+
26+
import java.io.IOException;
27+
import java.util.Collections;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
import java.util.Objects;
31+
32+
public class EstimateModelMemoryAction extends ActionType<EstimateModelMemoryAction.Response> {
33+
34+
public static final EstimateModelMemoryAction INSTANCE = new EstimateModelMemoryAction();
35+
public static final String NAME = "cluster:admin/xpack/ml/job/estimate_model_memory";
36+
37+
private EstimateModelMemoryAction() {
38+
super(NAME, Response::new);
39+
}
40+
41+
public static class Request extends ActionRequest {
42+
43+
public static final ParseField ANALYSIS_CONFIG = Job.ANALYSIS_CONFIG;
44+
public static final ParseField OVERALL_CARDINALITY = new ParseField("overall_cardinality");
45+
public static final ParseField MAX_BUCKET_CARDINALITY = new ParseField("max_bucket_cardinality");
46+
47+
public static final ObjectParser<Request, Void> PARSER =
48+
new ObjectParser<>(NAME, EstimateModelMemoryAction.Request::new);
49+
50+
static {
51+
PARSER.declareObject(Request::setAnalysisConfig, (p, c) -> AnalysisConfig.STRICT_PARSER.apply(p, c).build(), ANALYSIS_CONFIG);
52+
PARSER.declareObject(Request::setOverallCardinality,
53+
(p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, OVERALL_CARDINALITY)),
54+
OVERALL_CARDINALITY);
55+
PARSER.declareObject(Request::setMaxBucketCardinality,
56+
(p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, MAX_BUCKET_CARDINALITY)),
57+
MAX_BUCKET_CARDINALITY);
58+
}
59+
60+
public static Request parseRequest(XContentParser parser) {
61+
return PARSER.apply(parser, null);
62+
}
63+
64+
private AnalysisConfig analysisConfig;
65+
private Map<String, Long> overallCardinality = Collections.emptyMap();
66+
private Map<String, Long> maxBucketCardinality = Collections.emptyMap();
67+
68+
public Request() {
69+
super();
70+
}
71+
72+
public Request(StreamInput in) throws IOException {
73+
super(in);
74+
this.analysisConfig = in.readBoolean() ? new AnalysisConfig(in) : null;
75+
this.overallCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
76+
this.maxBucketCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
77+
}
78+
79+
@Override
80+
public void writeTo(StreamOutput out) throws IOException {
81+
super.writeTo(out);
82+
if (analysisConfig != null) {
83+
out.writeBoolean(true);
84+
analysisConfig.writeTo(out);
85+
} else {
86+
out.writeBoolean(false);
87+
}
88+
out.writeMap(overallCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
89+
out.writeMap(maxBucketCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
90+
}
91+
92+
@Override
93+
public ActionRequestValidationException validate() {
94+
if (analysisConfig == null) {
95+
ActionRequestValidationException e = new ActionRequestValidationException();
96+
e.addValidationError("[" + ANALYSIS_CONFIG.getPreferredName() + "] was not specified");
97+
return e;
98+
}
99+
return null;
100+
}
101+
102+
public AnalysisConfig getAnalysisConfig() {
103+
return analysisConfig;
104+
}
105+
106+
public void setAnalysisConfig(AnalysisConfig analysisConfig) {
107+
this.analysisConfig = ExceptionsHelper.requireNonNull(analysisConfig, ANALYSIS_CONFIG);
108+
}
109+
110+
public Map<String, Long> getOverallCardinality() {
111+
return overallCardinality;
112+
}
113+
114+
public void setOverallCardinality(Map<String, Long> overallCardinality) {
115+
this.overallCardinality =
116+
Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(overallCardinality, OVERALL_CARDINALITY));
117+
}
118+
119+
public Map<String, Long> getMaxBucketCardinality() {
120+
return maxBucketCardinality;
121+
}
122+
123+
public void setMaxBucketCardinality(Map<String, Long> maxBucketCardinality) {
124+
this.maxBucketCardinality =
125+
Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(maxBucketCardinality, MAX_BUCKET_CARDINALITY));
126+
}
127+
128+
private static long parseNonNegativeLong(XContentParser parser, ParseField enclosingField) throws IOException {
129+
long value = parser.longValue();
130+
if (value < 0) {
131+
throw ExceptionsHelper.badRequestException("[{}] contained negative cardinality [{}]",
132+
enclosingField.getPreferredName(), value);
133+
}
134+
return value;
135+
}
136+
}
137+
138+
public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
139+
140+
public RequestBuilder(ElasticsearchClient client, EstimateModelMemoryAction action) {
141+
super(client, action, new Request());
142+
}
143+
}
144+
145+
public static class Response extends ActionResponse implements ToXContentObject {
146+
147+
private static final ParseField MODEL_MEMORY_ESTIMATE = new ParseField("model_memory_estimate");
148+
149+
private final ByteSizeValue modelMemoryEstimate;
150+
151+
public Response(ByteSizeValue modelMemoryEstimate) {
152+
this.modelMemoryEstimate = Objects.requireNonNull(modelMemoryEstimate);
153+
}
154+
155+
public Response(StreamInput in) throws IOException {
156+
modelMemoryEstimate = new ByteSizeValue(in);
157+
}
158+
159+
@Override
160+
public void writeTo(StreamOutput out) throws IOException {
161+
modelMemoryEstimate.writeTo(out);
162+
}
163+
164+
@Override
165+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
166+
builder.startObject();
167+
builder.field(MODEL_MEMORY_ESTIMATE.getPreferredName(), modelMemoryEstimate.getStringRep());
168+
builder.endObject();
169+
return builder;
170+
}
171+
172+
public ByteSizeValue getModelMemoryEstimate() {
173+
return modelMemoryEstimate;
174+
}
175+
}
176+
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
7373
import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
7474
import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAction;
75+
import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
7576
import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
7677
import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction;
7778
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
@@ -143,6 +144,7 @@
143144
import org.elasticsearch.xpack.ml.action.TransportDeleteJobAction;
144145
import org.elasticsearch.xpack.ml.action.TransportDeleteModelSnapshotAction;
145146
import org.elasticsearch.xpack.ml.action.TransportDeleteTrainedModelAction;
147+
import org.elasticsearch.xpack.ml.action.TransportEstimateModelMemoryAction;
146148
import org.elasticsearch.xpack.ml.action.TransportEvaluateDataFrameAction;
147149
import org.elasticsearch.xpack.ml.action.TransportExplainDataFrameAnalyticsAction;
148150
import org.elasticsearch.xpack.ml.action.TransportFinalizeJobExecutionAction;
@@ -278,6 +280,7 @@
278280
import org.elasticsearch.xpack.ml.rest.job.RestCloseJobAction;
279281
import org.elasticsearch.xpack.ml.rest.job.RestDeleteForecastAction;
280282
import org.elasticsearch.xpack.ml.rest.job.RestDeleteJobAction;
283+
import org.elasticsearch.xpack.ml.rest.job.RestEstimateModelMemoryAction;
281284
import org.elasticsearch.xpack.ml.rest.job.RestFlushJobAction;
282285
import org.elasticsearch.xpack.ml.rest.job.RestForecastJobAction;
283286
import org.elasticsearch.xpack.ml.rest.job.RestGetJobStatsAction;
@@ -745,6 +748,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
745748
new RestFlushJobAction(),
746749
new RestValidateDetectorAction(),
747750
new RestValidateJobConfigAction(),
751+
new RestEstimateModelMemoryAction(),
748752
new RestGetCategoriesAction(),
749753
new RestGetModelSnapshotsAction(),
750754
new RestRevertModelSnapshotAction(),
@@ -819,6 +823,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
819823
new ActionHandler<>(FlushJobAction.INSTANCE, TransportFlushJobAction.class),
820824
new ActionHandler<>(ValidateDetectorAction.INSTANCE, TransportValidateDetectorAction.class),
821825
new ActionHandler<>(ValidateJobConfigAction.INSTANCE, TransportValidateJobConfigAction.class),
826+
new ActionHandler<>(EstimateModelMemoryAction.INSTANCE, TransportEstimateModelMemoryAction.class),
822827
new ActionHandler<>(GetCategoriesAction.INSTANCE, TransportGetCategoriesAction.class),
823828
new ActionHandler<>(GetModelSnapshotsAction.INSTANCE, TransportGetModelSnapshotsAction.class),
824829
new ActionHandler<>(RevertModelSnapshotAction.INSTANCE, TransportRevertModelSnapshotAction.class),

0 commit comments

Comments
 (0)