Skip to content

Commit b909cee

Browse files
imotovelasticmachine
imotov
and
elasticmachine
authored
Expose agg usage in Feature Usage API (#55732)
* Expose agg usage in Feature Usage API Counts usage of the aggs and exposes them on the _nodes/usage/. Closes #53746 * Refactor to include non value sources aggregations * Fix reported values source type for parent and children aggs * Refactor SearchModule constructor * Fix subtype in TTest and IPRanges * Fix more subtypes in aggs that don't register themselves * Fix doc tests * Fix docs * Fix ScriptedMetricAggregatorTests * Fix compilation issues after merge * Fix merge fallout * This gets stale quickly... * Address review comments * Fix tests that were missing proper agg registration in the search module * Fix ScriptedMetricAggregatorTests * Address review comments Co-authored-by: Elastic Machine <[email protected]>
1 parent 785527b commit b909cee

File tree

48 files changed

+539
-103
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+539
-103
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public static TransformConfig randomTransformConfig() {
4545
randomSourceConfig(),
4646
randomDestConfig(),
4747
randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1000, 1000000)),
48-
randomBoolean() ? null : randomSyncConfig(),
48+
randomBoolean() ? null : randomSyncConfig(),
4949
PivotConfigTests.randomPivotConfig(),
5050
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100),
5151
randomBoolean() ? null : Instant.now(),

‎distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java

Lines changed: 100 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,15 @@
1919

2020
package org.elasticsearch.test.rest;
2121

22+
import org.elasticsearch.client.Request;
2223
import org.elasticsearch.client.Response;
2324
import org.elasticsearch.client.ResponseException;
24-
import org.elasticsearch.client.Request;
25+
import org.elasticsearch.common.Strings;
26+
import org.elasticsearch.search.aggregations.AggregationBuilders;
27+
import org.elasticsearch.search.builder.SearchSourceBuilder;
2528

2629
import java.io.IOException;
30+
import java.util.Collections;
2731
import java.util.HashMap;
2832
import java.util.Map;
2933

@@ -41,14 +45,7 @@ public void testWithRestUsage() throws IOException {
4145
Response beforeResponse = client().performRequest(new Request("GET", path));
4246
Map<String, Object> beforeResponseBodyMap = entityAsMap(beforeResponse);
4347
assertThat(beforeResponseBodyMap, notNullValue());
44-
Map<String, Object> before_nodesMap = (Map<String, Object>) beforeResponseBodyMap.get("_nodes");
45-
assertThat(before_nodesMap, notNullValue());
46-
Integer beforeTotal = (Integer) before_nodesMap.get("total");
47-
Integer beforeSuccessful = (Integer) before_nodesMap.get("successful");
48-
Integer beforeFailed = (Integer) before_nodesMap.get("failed");
49-
assertThat(beforeTotal, greaterThan(0));
50-
assertThat(beforeSuccessful, equalTo(beforeTotal));
51-
assertThat(beforeFailed, equalTo(0));
48+
int beforeSuccessful = assertSuccess(beforeResponseBodyMap);
5249

5350
Map<String, Object> beforeNodesMap = (Map<String, Object>) beforeResponseBodyMap.get("nodes");
5451
assertThat(beforeNodesMap, notNullValue());
@@ -98,14 +95,7 @@ public void testWithRestUsage() throws IOException {
9895
Response response = client().performRequest(new Request("GET", "_nodes/usage"));
9996
Map<String, Object> responseBodyMap = entityAsMap(response);
10097
assertThat(responseBodyMap, notNullValue());
101-
Map<String, Object> _nodesMap = (Map<String, Object>) responseBodyMap.get("_nodes");
102-
assertThat(_nodesMap, notNullValue());
103-
Integer total = (Integer) _nodesMap.get("total");
104-
Integer successful = (Integer) _nodesMap.get("successful");
105-
Integer failed = (Integer) _nodesMap.get("failed");
106-
assertThat(total, greaterThan(0));
107-
assertThat(successful, equalTo(total));
108-
assertThat(failed, equalTo(0));
98+
int successful = assertSuccess(responseBodyMap);
10999

110100
Map<String, Object> nodesMap = (Map<String, Object>) responseBodyMap.get("nodes");
111101
assertThat(nodesMap, notNullValue());
@@ -143,4 +133,97 @@ public void testMetricsWithAll() throws IOException {
143133
+ "\"reason\":\"request [_nodes/usage/_all,rest_actions] contains _all and individual metrics [_all,rest_actions]\""));
144134
}
145135

136+
@SuppressWarnings("unchecked")
137+
public void testAggregationUsage() throws IOException {
138+
// First get the current usage figures
139+
String path = randomFrom("_nodes/usage", "_nodes/usage/aggregations", "_nodes/usage/_all");
140+
Response beforeResponse = client().performRequest(new Request("GET", path));
141+
Map<String, Object> beforeResponseBodyMap = entityAsMap(beforeResponse);
142+
assertThat(beforeResponseBodyMap, notNullValue());
143+
int beforeSuccessful = assertSuccess(beforeResponseBodyMap);
144+
145+
Map<String, Object> beforeNodesMap = (Map<String, Object>) beforeResponseBodyMap.get("nodes");
146+
assertThat(beforeNodesMap, notNullValue());
147+
assertThat(beforeNodesMap.size(), equalTo(beforeSuccessful));
148+
149+
Map<String, Map<String, Long>> beforeCombinedAggsUsage = getTotalUsage(beforeNodesMap);
150+
// Do some requests to get some rest usage stats
151+
Request create = new Request("PUT", "/test");
152+
create.setJsonEntity("{\"mappings\": {\"properties\": { \"str\": {\"type\": \"keyword\"}, " +
153+
"\"foo\": {\"type\": \"keyword\"}, \"num\": {\"type\": \"long\"}, \"start\": {\"type\": \"date\"} } }}");
154+
client().performRequest(create);
155+
156+
Request searchRequest = new Request("GET", "/test/_search");
157+
SearchSourceBuilder searchSource = new SearchSourceBuilder()
158+
.aggregation(AggregationBuilders.terms("str_terms").field("str.keyword"))
159+
.aggregation(AggregationBuilders.terms("num_terms").field("num"))
160+
.aggregation(AggregationBuilders.avg("num_avg").field("num"));
161+
searchRequest.setJsonEntity(Strings.toString(searchSource));
162+
searchRequest.setJsonEntity(Strings.toString(searchSource));
163+
client().performRequest(searchRequest);
164+
165+
searchRequest = new Request("GET", "/test/_search");
166+
searchSource = new SearchSourceBuilder()
167+
.aggregation(AggregationBuilders.terms("start").field("start"))
168+
.aggregation(AggregationBuilders.avg("num1").field("num"))
169+
.aggregation(AggregationBuilders.avg("num2").field("num"))
170+
.aggregation(AggregationBuilders.terms("foo").field("foo.keyword"));
171+
String r = Strings.toString(searchSource);
172+
searchRequest.setJsonEntity(Strings.toString(searchSource));
173+
client().performRequest(searchRequest);
174+
175+
Response response = client().performRequest(new Request("GET", "_nodes/usage"));
176+
Map<String, Object> responseBodyMap = entityAsMap(response);
177+
assertThat(responseBodyMap, notNullValue());
178+
int successful = assertSuccess(responseBodyMap);
179+
180+
Map<String, Object> nodesMap = (Map<String, Object>) responseBodyMap.get("nodes");
181+
assertThat(nodesMap, notNullValue());
182+
assertThat(nodesMap.size(), equalTo(successful));
183+
184+
Map<String, Map<String, Long>> afterCombinedAggsUsage = getTotalUsage(nodesMap);
185+
186+
assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "terms", "numeric", 1L);
187+
assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "terms", "date", 1L);
188+
assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "terms", "bytes", 2L);
189+
assertDiff(beforeCombinedAggsUsage, afterCombinedAggsUsage, "avg", "numeric", 3L);
190+
}
191+
192+
private void assertDiff(Map<String, Map<String, Long>> before, Map<String, Map<String, Long>> after, String agg, String vst,
193+
long diff) {
194+
Long valBefore = before.getOrDefault(agg, Collections.emptyMap()).getOrDefault(vst, 0L);
195+
Long valAfter = after.getOrDefault(agg, Collections.emptyMap()).getOrDefault(vst, 0L);
196+
assertThat(agg + "." + vst, valAfter - valBefore, equalTo(diff) );
197+
}
198+
199+
private Map<String, Map<String, Long>> getTotalUsage(Map<String, Object> nodeUsage) {
200+
Map<String, Map<String, Long>> combined = new HashMap<>();
201+
for (Map.Entry<String, Object> nodeEntry : nodeUsage.entrySet()) {
202+
@SuppressWarnings("unchecked")
203+
Map<String, Object> beforeAggsUsage = (Map<String, Object>) ((Map<String, Object>) nodeEntry.getValue()).get("aggregations");
204+
assertThat(beforeAggsUsage, notNullValue());
205+
for (Map.Entry<String, Object> aggEntry : beforeAggsUsage.entrySet()) {
206+
@SuppressWarnings("unchecked") Map<String, Object> aggMap = (Map<String, Object>) aggEntry.getValue();
207+
Map<String, Long> combinedAggMap = combined.computeIfAbsent(aggEntry.getKey(), k -> new HashMap<>());
208+
for (Map.Entry<String, Object> valSourceEntry : aggMap.entrySet()) {
209+
combinedAggMap.put(valSourceEntry.getKey(),
210+
combinedAggMap.getOrDefault(valSourceEntry.getKey(), 0L) + ((Number) valSourceEntry.getValue()).longValue());
211+
}
212+
}
213+
}
214+
return combined;
215+
}
216+
217+
private int assertSuccess(Map<String, Object> responseBodyMap) {
218+
@SuppressWarnings("unchecked") Map<String, Object> nodesResultMap = (Map<String, Object>) responseBodyMap.get("_nodes");
219+
assertThat(nodesResultMap, notNullValue());
220+
Integer total = (Integer) nodesResultMap.get("total");
221+
Integer successful = (Integer) nodesResultMap.get("successful");
222+
Integer failed = (Integer) nodesResultMap.get("failed");
223+
assertThat(total, greaterThan(0));
224+
assertThat(successful, equalTo(total));
225+
assertThat(failed, equalTo(0));
226+
return successful;
227+
}
228+
146229
}

‎docs/reference/cluster/nodes-usage.asciidoc

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,15 @@ of features for each node. All the nodes selective options are explained
3131
==== {api-path-parms-title}
3232

3333
`<metric>`::
34-
(Optional, string) Limits the information returned to the specific metrics.
35-
A comma-separated list of the following options:
34+
(Optional, string) Limits the information returned to the specific metrics.
35+
A comma-separated list of the following options:
3636
+
3737
--
3838
`_all`::
3939
Returns all stats.
40-
40+
4141
`rest_actions`::
42-
Returns the REST actions classname with a count of the number of times
42+
Returns the REST actions classname with a count of the number of times
4343
that action has been called on the node.
4444
--
4545

@@ -79,11 +79,14 @@ The API returns the following response:
7979
"timestamp": 1492553961812, <1>
8080
"since": 1492553906606, <2>
8181
"rest_actions": {
82-
"org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction": 1,
83-
"org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction": 1,
84-
"org.elasticsearch.rest.action.document.RestGetAction": 1,
85-
"org.elasticsearch.rest.action.search.RestSearchAction": 19, <3>
86-
"org.elasticsearch.rest.action.admin.cluster.RestNodesInfoAction": 36
82+
"nodes_usage_action": 1,
83+
"create_index_action": 1,
84+
"document_get_action": 1,
85+
"search_action": 19, <3>
86+
"nodes_info_action": 36
87+
},
88+
"aggregations": {
89+
...
8790
}
8891
}
8992
}
@@ -94,7 +97,9 @@ The API returns the following response:
9497
// TESTRESPONSE[s/1492553961812/$body.$_path/]
9598
// TESTRESPONSE[s/1492553906606/$body.$_path/]
9699
// TESTRESPONSE[s/"rest_actions": [^}]+}/"rest_actions": $body.$_path/]
100+
// TESTRESPONSE[s/"aggregations": [^}]+}/"aggregations": $body.$_path/]
97101
<1> Timestamp for when this nodes usage request was performed.
98102
<2> Timestamp for when the usage information recording was started. This is
99103
equivalent to the time that the node was started.
100104
<3> Search action has been called 19 times for this node.
105+

‎modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,13 @@
3030
import org.apache.lucene.util.NumericUtils;
3131
import org.elasticsearch.index.mapper.MappedFieldType;
3232
import org.elasticsearch.index.mapper.NumberFieldMapper;
33+
import org.elasticsearch.plugins.SearchPlugin;
3334
import org.elasticsearch.search.aggregations.AggregatorTestCase;
35+
import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin;
3436

3537
import java.util.Arrays;
3638
import java.util.Collections;
39+
import java.util.List;
3740

3841
public class MatrixStatsAggregatorTests extends AggregatorTestCase {
3942

@@ -136,4 +139,8 @@ public void testTwoFieldsReduce() throws Exception {
136139
}
137140
}
138141

142+
@Override
143+
protected List<SearchPlugin> getSearchPlugins() {
144+
return Collections.singletonList(new MatrixAggregationPlugin());
145+
}
139146
}

‎modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ public final void testFromXContent() throws Exception {
7373

7474
@Override
7575
protected NamedWriteableRegistry getNamedWriteableRegistry() {
76-
return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
76+
return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()
77+
).getNamedWriteables());
7778
}
7879

7980
@Override

‎modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
import java.io.IOException;
3737
import java.util.Map;
3838

39+
import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE;
40+
3941
public class ChildrenAggregatorFactory extends ValuesSourceAggregatorFactory {
4042

4143
private final Query parentFilter;
@@ -84,4 +86,10 @@ protected Aggregator doCreateInternal(ValuesSource rawValuesSource,
8486
return asMultiBucketAggregator(this, searchContext, parent);
8587
}
8688
}
89+
90+
@Override
91+
public String getStatsSubtype() {
92+
// Child Aggregation is registered in non-standard way, so it might return child's values type
93+
return OTHER_SUBTYPE;
94+
}
8795
}

‎modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
import java.io.IOException;
3737
import java.util.Map;
3838

39+
import static org.elasticsearch.search.aggregations.support.AggregationUsageService.OTHER_SUBTYPE;
40+
3941
public class ParentAggregatorFactory extends ValuesSourceAggregatorFactory {
4042

4143
private final Query parentFilter;
@@ -85,4 +87,10 @@ protected Aggregator doCreateInternal(ValuesSource rawValuesSource,
8587
return asMultiBucketAggregator(this, searchContext, children);
8688
}
8789
}
90+
91+
@Override
92+
public String getStatsSubtype() {
93+
// Parent Aggregation is registered in non-standard way
94+
return OTHER_SUBTYPE;
95+
}
8896
}

‎modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,10 @@
4646
import org.elasticsearch.index.mapper.NumberFieldMapper;
4747
import org.elasticsearch.index.mapper.Uid;
4848
import org.elasticsearch.index.shard.ShardId;
49+
import org.elasticsearch.join.ParentJoinPlugin;
4950
import org.elasticsearch.join.mapper.MetaJoinFieldMapper;
5051
import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
52+
import org.elasticsearch.plugins.SearchPlugin;
5153
import org.elasticsearch.search.aggregations.Aggregation;
5254
import org.elasticsearch.search.aggregations.AggregationBuilder;
5355
import org.elasticsearch.search.aggregations.AggregatorTestCase;
@@ -328,4 +330,9 @@ private void testCaseTermsParentTerms(Query query, IndexSearcher indexSearcher,
328330
LongTerms result = search(indexSearcher, query, aggregationBuilder, fieldType, subFieldType);
329331
verify.accept(result);
330332
}
333+
334+
@Override
335+
protected List<SearchPlugin> getSearchPlugins() {
336+
return Collections.singletonList(new ParentJoinPlugin());
337+
}
331338
}

‎modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,10 @@
4646
import org.elasticsearch.index.mapper.NumberFieldMapper;
4747
import org.elasticsearch.index.mapper.Uid;
4848
import org.elasticsearch.index.shard.ShardId;
49+
import org.elasticsearch.join.ParentJoinPlugin;
4950
import org.elasticsearch.join.mapper.MetaJoinFieldMapper;
5051
import org.elasticsearch.join.mapper.ParentJoinFieldMapper;
52+
import org.elasticsearch.plugins.SearchPlugin;
5153
import org.elasticsearch.search.aggregations.AggregatorTestCase;
5254
import org.elasticsearch.search.aggregations.metrics.InternalMin;
5355
import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
@@ -187,4 +189,9 @@ private void testCase(Query query, IndexSearcher indexSearcher, Consumer<Interna
187189
InternalChildren result = search(indexSearcher, query, aggregationBuilder, fieldType);
188190
verify.accept(result);
189191
}
192+
193+
@Override
194+
protected List<SearchPlugin> getSearchPlugins() {
195+
return Collections.singletonList(new ParentJoinPlugin());
196+
}
190197
}

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ public class RatedRequestsTests extends ESTestCase {
6464
@BeforeClass
6565
public static void init() {
6666
xContentRegistry = new NamedXContentRegistry(
67-
Stream.of(new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents().stream()).flatMap(Function.identity())
68-
.collect(toList()));
67+
Stream.of(new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents().stream())
68+
.flatMap(Function.identity()).collect(toList()));
6969
}
7070

7171
@AfterClass

‎qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,3 +67,9 @@
6767
- match: { hits.total: 1 }
6868
- match: { hits.hits.0._id: q3 }
6969

70+
---
71+
"Verify nodes usage works":
72+
- do:
73+
nodes.usage: {}
74+
- is_true: nodes
75+
- match: { _nodes.failed: 0 }

0 commit comments

Comments
 (0)