Skip to content

Commit 4e3d5a0

Browse files
benwtrentbenwtrent
committed
[ML] add GET _cat/ml/datafeeds (elastic#51500)
This adds GET _cat/ml/datafeeds && _cat/ml/datafeeds/{datafeed_id}
1 parent 8fa4a40 commit 4e3d5a0

File tree

5 files changed

+333
-2
lines changed

5 files changed

+333
-2
lines changed

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@
254254
import org.elasticsearch.xpack.ml.rest.calendar.RestPostCalendarEventAction;
255255
import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarAction;
256256
import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarJobAction;
257+
import org.elasticsearch.xpack.ml.rest.cat.RestCatDatafeedsAction;
257258
import org.elasticsearch.xpack.ml.rest.cat.RestCatJobsAction;
258259
import org.elasticsearch.xpack.ml.rest.datafeeds.RestDeleteDatafeedAction;
259260
import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedStatsAction;
@@ -784,7 +785,8 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
784785
new RestGetTrainedModelsStatsAction(restController),
785786
new RestPutTrainedModelAction(restController),
786787
// CAT Handlers
787-
new RestCatJobsAction(restController)
788+
new RestCatJobsAction(restController),
789+
new RestCatDatafeedsAction(restController)
788790
);
789791
}
790792

‎x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatDatafeedsAction.java

+148
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.ml.rest.cat;
7+
8+
import org.elasticsearch.client.node.NodeClient;
9+
import org.elasticsearch.cluster.node.DiscoveryNode;
10+
import org.elasticsearch.common.Strings;
11+
import org.elasticsearch.common.Table;
12+
import org.elasticsearch.common.unit.TimeValue;
13+
import org.elasticsearch.rest.RestController;
14+
import org.elasticsearch.rest.RestRequest;
15+
import org.elasticsearch.rest.RestResponse;
16+
import org.elasticsearch.rest.action.RestResponseListener;
17+
import org.elasticsearch.rest.action.cat.AbstractCatAction;
18+
import org.elasticsearch.rest.action.cat.RestTable;
19+
import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
20+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
21+
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats;
22+
23+
import static org.elasticsearch.rest.RestRequest.Method.GET;
24+
25+
public class RestCatDatafeedsAction extends AbstractCatAction {
26+
27+
public RestCatDatafeedsAction(RestController controller) {
28+
controller.registerHandler(GET, "_cat/ml/datafeeds/{" + DatafeedConfig.ID.getPreferredName() + "}", this);
29+
controller.registerHandler(GET, "_cat/ml/datafeeds", this);
30+
}
31+
32+
@Override
33+
public String getName() {
34+
return "cat_ml_get_datafeeds_action";
35+
}
36+
37+
@Override
38+
protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient client) {
39+
String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName());
40+
if (Strings.isNullOrEmpty(datafeedId)) {
41+
datafeedId = GetDatafeedsStatsAction.ALL;
42+
}
43+
GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId);
44+
request.setAllowNoDatafeeds(restRequest.paramAsBoolean(GetDatafeedsStatsAction.Request.ALLOW_NO_DATAFEEDS.getPreferredName(),
45+
request.allowNoDatafeeds()));
46+
return channel -> client.execute(GetDatafeedsStatsAction.INSTANCE, request, new RestResponseListener<>(channel) {
47+
@Override
48+
public RestResponse buildResponse(GetDatafeedsStatsAction.Response getDatafeedsStatsRespons) throws Exception {
49+
return RestTable.buildResponse(buildTable(restRequest, getDatafeedsStatsRespons), channel);
50+
}
51+
});
52+
}
53+
54+
@Override
55+
protected void documentation(StringBuilder sb) {
56+
sb.append("/_cat/ml/datafeeds\n");
57+
sb.append("/_cat/ml/datafeeds/{datafeed_id}\n");
58+
}
59+
60+
@Override
61+
protected Table getTableWithHeader(RestRequest request) {
62+
Table table = new Table();
63+
table.startHeaders();
64+
65+
// Datafeed Info
66+
table.addCell("id", TableColumnAttributeBuilder.builder().setDescription("the datafeed_id").build());
67+
table.addCell("state", TableColumnAttributeBuilder.builder()
68+
.setDescription("the datafeed state")
69+
.setAliases("s")
70+
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
71+
.build());
72+
table.addCell("assignment_explanation",
73+
TableColumnAttributeBuilder.builder("why the datafeed is or is not assigned to a node", false)
74+
.setAliases("ae")
75+
.build());
76+
77+
// Timing stats
78+
table.addCell("bucket.count",
79+
TableColumnAttributeBuilder.builder("bucket count")
80+
.setAliases("bc", "bucketCount")
81+
.build());
82+
table.addCell("search.count",
83+
TableColumnAttributeBuilder.builder("number of searches ran by the datafeed")
84+
.setAliases("sc", "searchCount")
85+
.build());
86+
table.addCell("search.time",
87+
TableColumnAttributeBuilder.builder("the total search time", false)
88+
.setAliases("st", "searchTime")
89+
.build());
90+
table.addCell("search.bucket_avg",
91+
TableColumnAttributeBuilder.builder("the average search time per bucket (millisecond)", false)
92+
.setAliases("sba", "bucketTimeMin")
93+
.build());
94+
table.addCell("search.exp_avg_hour",
95+
TableColumnAttributeBuilder.builder("the exponential average search time per hour (millisecond)", false)
96+
.setAliases("seah", "searchExpAvgHour")
97+
.build());
98+
99+
//Node info
100+
table.addCell("node.id",
101+
TableColumnAttributeBuilder.builder("id of the assigned node", false)
102+
.setAliases("ni", "nodeId")
103+
.build());
104+
table.addCell("node.name",
105+
TableColumnAttributeBuilder.builder("name of the assigned node", false)
106+
.setAliases("nn", "nodeName")
107+
.build());
108+
table.addCell("node.ephemeral_id",
109+
TableColumnAttributeBuilder.builder("ephemeral id of the assigned node", false)
110+
.setAliases("ne", "nodeEphemeralId")
111+
.build());
112+
table.addCell("node.address",
113+
TableColumnAttributeBuilder.builder("network address of the assigned node", false)
114+
.setAliases("na", "nodeAddress")
115+
.build());
116+
117+
table.endHeaders();
118+
return table;
119+
}
120+
121+
private Table buildTable(RestRequest request, GetDatafeedsStatsAction.Response dfStats) {
122+
Table table = getTableWithHeader(request);
123+
dfStats.getResponse().results().forEach(df -> {
124+
table.startRow();
125+
table.addCell(df.getDatafeedId());
126+
table.addCell(df.getDatafeedState().toString());
127+
table.addCell(df.getAssignmentExplanation());
128+
129+
DatafeedTimingStats timingStats = df.getTimingStats();
130+
table.addCell(timingStats == null ? 0 : timingStats.getBucketCount());
131+
table.addCell(timingStats == null ? 0 : timingStats.getSearchCount());
132+
table.addCell(timingStats == null ?
133+
TimeValue.timeValueMillis(0) :
134+
TimeValue.timeValueMillis((long)timingStats.getTotalSearchTimeMs()));
135+
table.addCell(timingStats == null || timingStats.getBucketCount() == 0 ? 0.0 : timingStats.getAvgSearchTimePerBucketMs());
136+
table.addCell(timingStats == null ? 0.0 : timingStats.getExponentialAvgSearchTimePerHourMs());
137+
138+
DiscoveryNode node = df.getNode();
139+
table.addCell(node == null ? null : node.getId());
140+
table.addCell(node == null ? null : node.getName());
141+
table.addCell(node == null ? null : node.getEphemeralId());
142+
table.addCell(node == null ? null : node.getAddress().toString());
143+
144+
table.endRow();
145+
});
146+
return table;
147+
}
148+
}

‎x-pack/plugin/src/test/resources/rest-api-spec/api/cat.ml.datafeeds.json

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"cat.ml.datafeeds":{
3+
"documentation":{
4+
"url":"http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-datafeed-stats.html"
5+
},
6+
"stability":"stable",
7+
"url":{
8+
"paths":[
9+
{
10+
"path":"/_cat/ml/datafeeds",
11+
"methods":[
12+
"GET"
13+
]
14+
},
15+
{
16+
"path":"/_cat/ml/datafeeds/{datafeed_id}",
17+
"methods":[
18+
"GET"
19+
],
20+
"parts":{
21+
"datafeed_id":{
22+
"type":"string",
23+
"description":"The ID of the datafeeds stats to fetch"
24+
}
25+
}
26+
}
27+
]
28+
},
29+
"params":{
30+
"allow_no_datafeeds":{
31+
"type":"boolean",
32+
"required":false,
33+
"description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)"
34+
},
35+
"format":{
36+
"type":"string",
37+
"description":"a short version of the Accept header, e.g. json, yaml"
38+
},
39+
"h":{
40+
"type":"list",
41+
"description":"Comma-separated list of column names to display"
42+
},
43+
"help":{
44+
"type":"boolean",
45+
"description":"Return help information",
46+
"default":false
47+
},
48+
"s":{
49+
"type":"list",
50+
"description":"Comma-separated list of column names or column aliases to sort by"
51+
},
52+
"time":{
53+
"type":"enum",
54+
"description":"The unit in which to display time values",
55+
"options":[
56+
"d (Days)",
57+
"h (Hours)",
58+
"m (Minutes)",
59+
"s (Seconds)",
60+
"ms (Milliseconds)",
61+
"micros (Microseconds)",
62+
"nanos (Nanoseconds)"
63+
]
64+
},
65+
"v":{
66+
"type":"boolean",
67+
"description":"Verbose mode. Display column headers",
68+
"default":false
69+
}
70+
}
71+
}
72+
}

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeed_cat_apis.yml

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
setup:
2+
- skip:
3+
features: headers
4+
- do:
5+
indices.create:
6+
index: airline-data
7+
body:
8+
mappings:
9+
properties:
10+
time:
11+
type: date
12+
airline:
13+
type: keyword
14+
responsetime:
15+
type: float
16+
- do:
17+
headers:
18+
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
19+
ml.put_job:
20+
job_id: job-stats-test
21+
body: >
22+
{
23+
"job_id":"job-stats-test",
24+
"description":"Analysis of response time by airline",
25+
"analysis_config" : {
26+
"bucket_span": "1h",
27+
"detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
28+
},
29+
"analysis_limits" : {
30+
"model_memory_limit": "10mb"
31+
},
32+
"data_description" : {
33+
"time_field":"time",
34+
"time_format":"epoch"
35+
}
36+
}
37+
38+
- do:
39+
headers:
40+
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
41+
ml.put_datafeed:
42+
datafeed_id: datafeed-job-stats-test
43+
body: >
44+
{
45+
"job_id":"job-stats-test",
46+
"indexes":["airline-data"]
47+
}
48+
49+
- do:
50+
headers:
51+
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
52+
ml.put_job:
53+
job_id: jobs-get-stats-datafeed-job
54+
body: >
55+
{
56+
"job_id":"jobs-get-stats-datafeed-job",
57+
"description":"A job with a datafeed",
58+
"analysis_config" : {
59+
"bucket_span": "1h",
60+
"detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
61+
},
62+
"analysis_limits" : {
63+
"model_memory_limit": "10mb"
64+
},
65+
"data_description" : {
66+
"time_field":"time",
67+
"time_format":"yyyy-MM-dd'T'HH:mm:ssX"
68+
}
69+
}
70+
- do:
71+
headers:
72+
Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
73+
ml.put_datafeed:
74+
datafeed_id: datafeed-jobs-get-stats-datafeed-job
75+
body: >
76+
{
77+
"job_id":"jobs-get-stats-datafeed-job",
78+
"indexes":["airline-data"]
79+
}
80+
81+
---
82+
"Test cat datafeeds":
83+
84+
- do:
85+
cat.ml.datafeeds:
86+
datafeed_id: datafeed-job-stats-test
87+
- match:
88+
$body: |
89+
/ #id state bucket.count search.count
90+
^ (datafeed\-job\-stats\-test \s+ \w+ \s+ \d+ \s+ \d+ \n)+ $/
91+
92+
- do:
93+
cat.ml.datafeeds:
94+
v: true
95+
datafeed_id: datafeed-job-stats-test
96+
- match:
97+
$body: |
98+
/^ id \s+ state \s+ bucket\.count \s+ search\.count \n
99+
(datafeed\-job\-stats\-test \s+ \w+ \s+ \d+ \s+ \d+ \n)+ $/
100+
101+
- do:
102+
cat.ml.datafeeds:
103+
h: id,search.count,search.time,search.bucket_avg
104+
v: true
105+
- match:
106+
$body: |
107+
/^ id \s+ search\.count \s+ search\.time \s+ search\.bucket_avg \n
108+
(datafeed\-job\-stats\-test \s+ \d+ \s+ \w+ \s+ .*? \n)+
109+
(datafeed\-jobs\-get\-stats\-datafeed\-job \s+ \d+ \s+ \w+ \s+ .*? \n)+ $/

‎x-pack/plugin/src/test/resources/rest-api-spec/test/ml/job_cat_apis.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ setup:
7070
}
7171
7272
---
73-
"Test get job stats after uploading data prompting the creation of some stats":
73+
"Test cat anomaly detector jobs":
7474

7575
- do:
7676
ml.post_data:

0 commit comments

Comments
 (0)