Skip to content

Commit df574e5

Browse files
przemekwitekprzemekwitek
authored
[7.x] Implement ml/data_frame/analytics/_estimate_memory_usage API endpoint (#45188) (#45510)
1 parent 84bf98e commit df574e5

File tree

42 files changed

+1882
-190
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1882
-190
lines changed

‎docs/reference/ml/df-analytics/apis/estimate-memory-usage-dfanalytics.asciidoc

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
[role="xpack"]
2+
[testenv="platinum"]
3+
[[estimate-memory-usage-dfanalytics]]
4+
=== Estimate memory usage API
5+
6+
[subs="attributes"]
7+
++++
8+
<titleabbrev>Estimate memory usage for {dfanalytics-jobs}</titleabbrev>
9+
++++
10+
11+
Estimates memory usage for the given {dataframe-analytics-config}.
12+
13+
experimental[]
14+
15+
[[ml-estimate-memory-usage-dfanalytics-request]]
16+
==== {api-request-title}
17+
18+
`POST _ml/data_frame/analytics/_estimate_memory_usage`
19+
20+
[[ml-estimate-memory-usage-dfanalytics-prereq]]
21+
==== {api-prereq-title}
22+
23+
* You must have `monitor_ml` privilege to use this API. For more
24+
information, see {stack-ov}/security-privileges.html[Security privileges] and
25+
{stack-ov}/built-in-roles.html[Built-in roles].
26+
27+
[[ml-estimate-memory-usage-dfanalytics-desc]]
28+
==== {api-description-title}
29+
30+
This API estimates memory usage for the given {dataframe-analytics-config} before the {dfanalytics-job} is even created.
31+
32+
Serves as an advice on how to set `model_memory_limit` when creating {dfanalytics-job}.
33+
34+
[[ml-estimate-memory-usage-dfanalytics-request-body]]
35+
==== {api-request-body-title}
36+
37+
`data_frame_analytics_config`::
38+
(Required, object) Intended configuration of {dfanalytics-job}. For more information, see
39+
<<ml-dfanalytics-resources>>.
40+
Note that `id` and `dest` don't need to be provided in the context of this API.
41+
42+
[[ml-estimate-memory-usage-dfanalytics-results]]
43+
==== {api-response-body-title}
44+
45+
`expected_memory_usage_with_one_partition`::
46+
(string) Estimated memory usage under the assumption that the whole {dfanalytics} should happen in memory
47+
(i.e. without overflowing to disk).
48+
49+
`expected_memory_usage_with_max_partitions`::
50+
(string) Estimated memory usage under the assumption that overflowing to disk is allowed during {dfanalytics}.
51+
`expected_memory_usage_with_max_partitions` is usually smaller than `expected_memory_usage_with_one_partition`
52+
as using disk allows to limit the main memory needed to perform {dfanalytics}.
53+
54+
[[ml-estimate-memory-usage-dfanalytics-example]]
55+
==== {api-examples-title}
56+
57+
[source,js]
58+
--------------------------------------------------
59+
POST _ml/data_frame/analytics/_estimate_memory_usage
60+
{
61+
"data_frame_analytics_config": {
62+
"source": {
63+
"index": "logdata"
64+
},
65+
"analysis": {
66+
"outlier_detection": {}
67+
}
68+
}
69+
}
70+
--------------------------------------------------
71+
// CONSOLE
72+
// TEST[skip:TBD]
73+
74+
The API returns the following results:
75+
76+
[source,js]
77+
----
78+
{
79+
"expected_memory_usage_with_one_partition": "128MB",
80+
"expected_memory_usage_with_max_partitions": "32MB"
81+
}
82+
----
83+
// TESTRESPONSE

‎docs/reference/ml/df-analytics/apis/index.asciidoc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ You can use the following APIs to perform {ml} {dfanalytics} activities.
1212
* <<start-dfanalytics,Start {dfanalytics-jobs}>>
1313
* <<stop-dfanalytics,Stop {dfanalytics-jobs}>>
1414
* <<evaluate-dfanalytics,Evaluate {dfanalytics}>>
15+
* <<estimate-memory-usage-dfanalytics,Estimate memory usage for {dfanalytics}>>
1516

1617
See also <<ml-apis>>.
1718

@@ -21,10 +22,11 @@ include::put-dfanalytics.asciidoc[]
2122
include::delete-dfanalytics.asciidoc[]
2223
//EVALUATE
2324
include::evaluate-dfanalytics.asciidoc[]
25+
//ESTIMATE_MEMORY_USAGE
26+
include::estimate-memory-usage-dfanalytics.asciidoc[]
2427
//GET
2528
include::get-dfanalytics.asciidoc[]
2629
include::get-dfanalytics-stats.asciidoc[]
2730
//SET/START/STOP
2831
include::start-dfanalytics.asciidoc[]
2932
include::stop-dfanalytics.asciidoc[]
30-

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
import org.elasticsearch.xpack.core.ml.action.DeleteForecastAction;
9797
import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
9898
import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
99+
import org.elasticsearch.xpack.core.ml.action.EstimateMemoryUsageAction;
99100
import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
100101
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
101102
import org.elasticsearch.xpack.core.ml.action.FindFileStructureAction;
@@ -347,6 +348,7 @@ public List<ActionType<? extends ActionResponse>> getClientActions() {
347348
StartDataFrameAnalyticsAction.INSTANCE,
348349
StopDataFrameAnalyticsAction.INSTANCE,
349350
EvaluateDataFrameAction.INSTANCE,
351+
EstimateMemoryUsageAction.INSTANCE,
350352
// security
351353
ClearRealmCacheAction.INSTANCE,
352354
ClearRolesCacheAction.INSTANCE,

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EstimateMemoryUsageAction.java

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.ml.action;
7+
8+
import org.elasticsearch.action.ActionRequest;
9+
import org.elasticsearch.action.ActionRequestValidationException;
10+
import org.elasticsearch.action.ActionResponse;
11+
import org.elasticsearch.action.ActionType;
12+
import org.elasticsearch.common.Nullable;
13+
import org.elasticsearch.common.ParseField;
14+
import org.elasticsearch.common.io.stream.StreamInput;
15+
import org.elasticsearch.common.io.stream.StreamOutput;
16+
import org.elasticsearch.common.unit.ByteSizeValue;
17+
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
18+
import org.elasticsearch.common.xcontent.ObjectParser;
19+
import org.elasticsearch.common.xcontent.ToXContentObject;
20+
import org.elasticsearch.common.xcontent.XContentBuilder;
21+
import org.elasticsearch.common.xcontent.XContentParser;
22+
import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
23+
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
24+
25+
import java.io.IOException;
26+
import java.util.Objects;
27+
28+
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
29+
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
30+
31+
public class EstimateMemoryUsageAction extends ActionType<EstimateMemoryUsageAction.Response> {
32+
33+
public static final EstimateMemoryUsageAction INSTANCE = new EstimateMemoryUsageAction();
34+
public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/estimate_memory_usage";
35+
36+
private EstimateMemoryUsageAction() {
37+
super(NAME, EstimateMemoryUsageAction.Response::new);
38+
}
39+
40+
public static class Request extends ActionRequest implements ToXContentObject {
41+
42+
private static final ParseField DATA_FRAME_ANALYTICS_CONFIG = new ParseField("data_frame_analytics_config");
43+
44+
private static final ConstructingObjectParser<EstimateMemoryUsageAction.Request, Void> PARSER =
45+
new ConstructingObjectParser<>(
46+
NAME,
47+
args -> {
48+
DataFrameAnalyticsConfig.Builder configBuilder = (DataFrameAnalyticsConfig.Builder) args[0];
49+
DataFrameAnalyticsConfig config = configBuilder.buildForMemoryEstimation();
50+
return new EstimateMemoryUsageAction.Request(config);
51+
});
52+
53+
static {
54+
PARSER.declareObject(constructorArg(), DataFrameAnalyticsConfig.STRICT_PARSER, DATA_FRAME_ANALYTICS_CONFIG);
55+
}
56+
57+
public static EstimateMemoryUsageAction.Request parseRequest(XContentParser parser) {
58+
return PARSER.apply(parser, null);
59+
}
60+
61+
private final DataFrameAnalyticsConfig config;
62+
63+
public Request(DataFrameAnalyticsConfig config) {
64+
this.config = ExceptionsHelper.requireNonNull(config, DATA_FRAME_ANALYTICS_CONFIG);
65+
}
66+
67+
public Request(StreamInput in) throws IOException {
68+
super(in);
69+
this.config = new DataFrameAnalyticsConfig(in);
70+
}
71+
72+
@Override
73+
public ActionRequestValidationException validate() {
74+
return null;
75+
}
76+
77+
public DataFrameAnalyticsConfig getConfig() {
78+
return config;
79+
}
80+
81+
@Override
82+
public void writeTo(StreamOutput out) throws IOException {
83+
super.writeTo(out);
84+
config.writeTo(out);
85+
}
86+
87+
@Override
88+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
89+
builder.startObject();
90+
builder.field(DATA_FRAME_ANALYTICS_CONFIG.getPreferredName(), config);
91+
builder.endObject();
92+
return builder;
93+
}
94+
95+
@Override
96+
public boolean equals(Object other) {
97+
if (this == other) {
98+
return true;
99+
}
100+
if (other == null || getClass() != other.getClass()) {
101+
return false;
102+
}
103+
104+
Request that = (Request) other;
105+
return Objects.equals(config, that.config);
106+
}
107+
108+
@Override
109+
public int hashCode() {
110+
return Objects.hash(config);
111+
}
112+
}
113+
114+
public static class Response extends ActionResponse implements ToXContentObject {
115+
116+
public static final ParseField TYPE = new ParseField("memory_usage_estimation_result");
117+
118+
public static final ParseField EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION =
119+
new ParseField("expected_memory_usage_with_one_partition");
120+
public static final ParseField EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS =
121+
new ParseField("expected_memory_usage_with_max_partitions");
122+
123+
static final ConstructingObjectParser<Response, Void> PARSER =
124+
new ConstructingObjectParser<>(
125+
TYPE.getPreferredName(),
126+
args -> new Response((ByteSizeValue) args[0], (ByteSizeValue) args[1]));
127+
128+
static {
129+
PARSER.declareField(
130+
optionalConstructorArg(),
131+
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION.getPreferredName()),
132+
EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION,
133+
ObjectParser.ValueType.VALUE);
134+
PARSER.declareField(
135+
optionalConstructorArg(),
136+
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS.getPreferredName()),
137+
EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS,
138+
ObjectParser.ValueType.VALUE);
139+
}
140+
141+
private final ByteSizeValue expectedMemoryUsageWithOnePartition;
142+
private final ByteSizeValue expectedMemoryUsageWithMaxPartitions;
143+
144+
public Response(@Nullable ByteSizeValue expectedMemoryUsageWithOnePartition,
145+
@Nullable ByteSizeValue expectedMemoryUsageWithMaxPartitions) {
146+
this.expectedMemoryUsageWithOnePartition = expectedMemoryUsageWithOnePartition;
147+
this.expectedMemoryUsageWithMaxPartitions = expectedMemoryUsageWithMaxPartitions;
148+
}
149+
150+
public Response(StreamInput in) throws IOException {
151+
super(in);
152+
this.expectedMemoryUsageWithOnePartition = in.readOptionalWriteable(ByteSizeValue::new);
153+
this.expectedMemoryUsageWithMaxPartitions = in.readOptionalWriteable(ByteSizeValue::new);
154+
}
155+
156+
public ByteSizeValue getExpectedMemoryUsageWithOnePartition() {
157+
return expectedMemoryUsageWithOnePartition;
158+
}
159+
160+
public ByteSizeValue getExpectedMemoryUsageWithMaxPartitions() {
161+
return expectedMemoryUsageWithMaxPartitions;
162+
}
163+
164+
@Override
165+
public void writeTo(StreamOutput out) throws IOException {
166+
out.writeOptionalWriteable(expectedMemoryUsageWithOnePartition);
167+
out.writeOptionalWriteable(expectedMemoryUsageWithMaxPartitions);
168+
}
169+
170+
@Override
171+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
172+
builder.startObject();
173+
if (expectedMemoryUsageWithOnePartition != null) {
174+
builder.field(
175+
EXPECTED_MEMORY_USAGE_WITH_ONE_PARTITION.getPreferredName(), expectedMemoryUsageWithOnePartition.getStringRep());
176+
}
177+
if (expectedMemoryUsageWithMaxPartitions != null) {
178+
builder.field(
179+
EXPECTED_MEMORY_USAGE_WITH_MAX_PARTITIONS.getPreferredName(), expectedMemoryUsageWithMaxPartitions.getStringRep());
180+
}
181+
builder.endObject();
182+
return builder;
183+
}
184+
185+
@Override
186+
public boolean equals(Object other) {
187+
if (this == other) {
188+
return true;
189+
}
190+
if (other == null || getClass() != other.getClass()) {
191+
return false;
192+
}
193+
194+
Response that = (Response) other;
195+
return Objects.equals(expectedMemoryUsageWithOnePartition, that.expectedMemoryUsageWithOnePartition)
196+
&& Objects.equals(expectedMemoryUsageWithMaxPartitions, that.expectedMemoryUsageWithMaxPartitions);
197+
}
198+
199+
@Override
200+
public int hashCode() {
201+
return Objects.hash(expectedMemoryUsageWithOnePartition, expectedMemoryUsageWithMaxPartitions);
202+
}
203+
}
204+
}

0 commit comments

Comments
 (0)