Skip to content

Commit f349130

Browse files
droberts195droberts195
authored
[ML] Skeleton estimate_model_memory endpoint for anomaly detection (#53333)
This is a partial implementation of an endpoint for anomaly detector model memory estimation. It is not complete, lacking docs, HLRC and sensible numbers for many anomaly detector configurations. These will be added in a followup PR in time for 7.7 feature freeze. A skeleton endpoint is useful now because it allows work on the UI side of the change to commence. The skeleton endpoint handles the same cases that the old UI code used to handle, and produces very similar estimates for these cases. Relates #53219
1 parent b9529d0 commit f349130

File tree

7 files changed

+739
-0
lines changed

7 files changed

+739
-0
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateModelMemoryAction.java

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ml.action;
7+
8+
import org.elasticsearch.action.ActionRequest;
9+
import org.elasticsearch.action.ActionRequestBuilder;
10+
import org.elasticsearch.action.ActionRequestValidationException;
11+
import org.elasticsearch.action.ActionResponse;
12+
import org.elasticsearch.action.ActionType;
13+
import org.elasticsearch.client.ElasticsearchClient;
14+
import org.elasticsearch.common.ParseField;
15+
import org.elasticsearch.common.io.stream.StreamInput;
16+
import org.elasticsearch.common.io.stream.StreamOutput;
17+
import org.elasticsearch.common.unit.ByteSizeValue;
18+
import org.elasticsearch.common.xcontent.ObjectParser;
19+
import org.elasticsearch.common.xcontent.ToXContentObject;
20+
import org.elasticsearch.common.xcontent.XContentBuilder;
21+
import org.elasticsearch.common.xcontent.XContentParser;
22+
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
23+
import org.elasticsearch.xpack.core.ml.job.config.Job;
24+
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
25+
26+
import java.io.IOException;
27+
import java.util.Collections;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
import java.util.Objects;
31+
32+
public class EstimateModelMemoryAction extends ActionType<EstimateModelMemoryAction.Response> {
33+
34+
public static final EstimateModelMemoryAction INSTANCE = new EstimateModelMemoryAction();
35+
public static final String NAME = "cluster:admin/xpack/ml/job/estimate_model_memory";
36+
37+
private EstimateModelMemoryAction() {
38+
super(NAME, Response::new);
39+
}
40+
41+
public static class Request extends ActionRequest {
42+
43+
public static final ParseField ANALYSIS_CONFIG = Job.ANALYSIS_CONFIG;
44+
public static final ParseField OVERALL_CARDINALITY = new ParseField("overall_cardinality");
45+
public static final ParseField MAX_BUCKET_CARDINALITY = new ParseField("max_bucket_cardinality");
46+
47+
public static final ObjectParser<Request, Void> PARSER =
48+
new ObjectParser<>(NAME, EstimateModelMemoryAction.Request::new);
49+
50+
static {
51+
PARSER.declareObject(Request::setAnalysisConfig, (p, c) -> AnalysisConfig.STRICT_PARSER.apply(p, c).build(), ANALYSIS_CONFIG);
52+
PARSER.declareObject(Request::setOverallCardinality,
53+
(p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, OVERALL_CARDINALITY)),
54+
OVERALL_CARDINALITY);
55+
PARSER.declareObject(Request::setMaxBucketCardinality,
56+
(p, c) -> p.map(HashMap::new, parser -> Request.parseNonNegativeLong(parser, MAX_BUCKET_CARDINALITY)),
57+
MAX_BUCKET_CARDINALITY);
58+
}
59+
60+
public static Request parseRequest(XContentParser parser) {
61+
return PARSER.apply(parser, null);
62+
}
63+
64+
private AnalysisConfig analysisConfig;
65+
private Map<String, Long> overallCardinality = Collections.emptyMap();
66+
private Map<String, Long> maxBucketCardinality = Collections.emptyMap();
67+
68+
public Request() {
69+
super();
70+
}
71+
72+
public Request(StreamInput in) throws IOException {
73+
super(in);
74+
this.analysisConfig = in.readBoolean() ? new AnalysisConfig(in) : null;
75+
this.overallCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
76+
this.maxBucketCardinality = in.readMap(StreamInput::readString, StreamInput::readVLong);
77+
}
78+
79+
@Override
80+
public void writeTo(StreamOutput out) throws IOException {
81+
super.writeTo(out);
82+
if (analysisConfig != null) {
83+
out.writeBoolean(true);
84+
analysisConfig.writeTo(out);
85+
} else {
86+
out.writeBoolean(false);
87+
}
88+
out.writeMap(overallCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
89+
out.writeMap(maxBucketCardinality, StreamOutput::writeString, StreamOutput::writeVLong);
90+
}
91+
92+
@Override
93+
public ActionRequestValidationException validate() {
94+
if (analysisConfig == null) {
95+
ActionRequestValidationException e = new ActionRequestValidationException();
96+
e.addValidationError("[" + ANALYSIS_CONFIG.getPreferredName() + "] was not specified");
97+
return e;
98+
}
99+
return null;
100+
}
101+
102+
public AnalysisConfig getAnalysisConfig() {
103+
return analysisConfig;
104+
}
105+
106+
public void setAnalysisConfig(AnalysisConfig analysisConfig) {
107+
this.analysisConfig = ExceptionsHelper.requireNonNull(analysisConfig, ANALYSIS_CONFIG);
108+
}
109+
110+
public Map<String, Long> getOverallCardinality() {
111+
return overallCardinality;
112+
}
113+
114+
public void setOverallCardinality(Map<String, Long> overallCardinality) {
115+
this.overallCardinality =
116+
Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(overallCardinality, OVERALL_CARDINALITY));
117+
}
118+
119+
public Map<String, Long> getMaxBucketCardinality() {
120+
return maxBucketCardinality;
121+
}
122+
123+
public void setMaxBucketCardinality(Map<String, Long> maxBucketCardinality) {
124+
this.maxBucketCardinality =
125+
Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(maxBucketCardinality, MAX_BUCKET_CARDINALITY));
126+
}
127+
128+
private static long parseNonNegativeLong(XContentParser parser, ParseField enclosingField) throws IOException {
129+
long value = parser.longValue();
130+
if (value < 0) {
131+
throw ExceptionsHelper.badRequestException("[{}] contained negative cardinality [{}]",
132+
enclosingField.getPreferredName(), value);
133+
}
134+
return value;
135+
}
136+
}
137+
138+
public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
139+
140+
public RequestBuilder(ElasticsearchClient client, EstimateModelMemoryAction action) {
141+
super(client, action, new Request());
142+
}
143+
}
144+
145+
public static class Response extends ActionResponse implements ToXContentObject {
146+
147+
private static final ParseField MODEL_MEMORY_ESTIMATE = new ParseField("model_memory_estimate");
148+
149+
private final ByteSizeValue modelMemoryEstimate;
150+
151+
public Response(ByteSizeValue modelMemoryEstimate) {
152+
this.modelMemoryEstimate = Objects.requireNonNull(modelMemoryEstimate);
153+
}
154+
155+
public Response(StreamInput in) throws IOException {
156+
modelMemoryEstimate = new ByteSizeValue(in);
157+
}
158+
159+
@Override
160+
public void writeTo(StreamOutput out) throws IOException {
161+
modelMemoryEstimate.writeTo(out);
162+
}
163+
164+
@Override
165+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
166+
builder.startObject();
167+
builder.field(MODEL_MEMORY_ESTIMATE.getPreferredName(), modelMemoryEstimate.getStringRep());
168+
builder.endObject();
169+
return builder;
170+
}
171+
172+
public ByteSizeValue getModelMemoryEstimate() {
173+
return modelMemoryEstimate;
174+
}
175+
}
176+
}

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
7474
import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
7575
import org.elasticsearch.xpack.core.ml.action.DeleteTrainedModelAction;
76+
import org.elasticsearch.xpack.core.ml.action.EstimateModelMemoryAction;
7677
import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
7778
import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction;
7879
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
@@ -144,6 +145,7 @@
144145
import org.elasticsearch.xpack.ml.action.TransportDeleteJobAction;
145146
import org.elasticsearch.xpack.ml.action.TransportDeleteModelSnapshotAction;
146147
import org.elasticsearch.xpack.ml.action.TransportDeleteTrainedModelAction;
148+
import org.elasticsearch.xpack.ml.action.TransportEstimateModelMemoryAction;
147149
import org.elasticsearch.xpack.ml.action.TransportEvaluateDataFrameAction;
148150
import org.elasticsearch.xpack.ml.action.TransportExplainDataFrameAnalyticsAction;
149151
import org.elasticsearch.xpack.ml.action.TransportFinalizeJobExecutionAction;
@@ -281,6 +283,7 @@
281283
import org.elasticsearch.xpack.ml.rest.job.RestCloseJobAction;
282284
import org.elasticsearch.xpack.ml.rest.job.RestDeleteForecastAction;
283285
import org.elasticsearch.xpack.ml.rest.job.RestDeleteJobAction;
286+
import org.elasticsearch.xpack.ml.rest.job.RestEstimateModelMemoryAction;
284287
import org.elasticsearch.xpack.ml.rest.job.RestFlushJobAction;
285288
import org.elasticsearch.xpack.ml.rest.job.RestForecastJobAction;
286289
import org.elasticsearch.xpack.ml.rest.job.RestGetJobStatsAction;
@@ -733,6 +736,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
733736
new RestFlushJobAction(),
734737
new RestValidateDetectorAction(),
735738
new RestValidateJobConfigAction(),
739+
new RestEstimateModelMemoryAction(),
736740
new RestGetCategoriesAction(),
737741
new RestGetModelSnapshotsAction(),
738742
new RestRevertModelSnapshotAction(),
@@ -811,6 +815,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
811815
new ActionHandler<>(FlushJobAction.INSTANCE, TransportFlushJobAction.class),
812816
new ActionHandler<>(ValidateDetectorAction.INSTANCE, TransportValidateDetectorAction.class),
813817
new ActionHandler<>(ValidateJobConfigAction.INSTANCE, TransportValidateJobConfigAction.class),
818+
new ActionHandler<>(EstimateModelMemoryAction.INSTANCE, TransportEstimateModelMemoryAction.class),
814819
new ActionHandler<>(GetCategoriesAction.INSTANCE, TransportGetCategoriesAction.class),
815820
new ActionHandler<>(GetModelSnapshotsAction.INSTANCE, TransportGetModelSnapshotsAction.class),
816821
new ActionHandler<>(RevertModelSnapshotAction.INSTANCE, TransportRevertModelSnapshotAction.class),

0 commit comments

Comments
 (0)