Skip to content

Commit 44eecfe

Browse files
prwhelanprwhelan
committed
[Transform] add support for extended_stats
Building off of `stats` and multi-value aggregations, including the limitation: - all values of extended_stats will be mapped to `double` if mapping deduction is used Relates elastic#51925
1 parent 4c1c3b8 commit 44eecfe

File tree

5 files changed

+114
-2
lines changed

5 files changed

+114
-2
lines changed

‎docs/reference/rest-api/common-parms.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -808,6 +808,7 @@ currently supported:
808808
* <<search-aggregations-pipeline-bucket-script-aggregation,Bucket script>>
809809
* <<search-aggregations-pipeline-bucket-selector-aggregation,Bucket selector>>
810810
* <<search-aggregations-metrics-cardinality-aggregation,Cardinality>>
811+
* <<search-aggregations-metrics-extendedstats-aggregation,Extended Stats>>
811812
* <<search-aggregations-bucket-filter-aggregation,Filter>>
812813
* <<search-aggregations-metrics-geobounds-aggregation,Geo bounds>>
813814
* <<search-aggregations-metrics-geocentroid-aggregation,Geo centroid>>

‎server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.io.IOException;
2929
import java.util.Map;
3030
import java.util.Objects;
31+
import java.util.Optional;
3132
import java.util.Set;
3233

3334
public class ExtendedStatsAggregationBuilder extends ValuesSourceAggregationBuilder.MetricsAggregationBuilder<
@@ -87,6 +88,11 @@ public Set<String> metricNames() {
8788
return InternalExtendedStats.METRIC_NAMES;
8889
}
8990

91+
@Override
92+
public Optional<Set<String>> getOutputFieldNames() {
93+
return Optional.of(InternalExtendedStats.METRIC_NAMES);
94+
}
95+
9096
@Override
9197
protected ValuesSourceType defaultValueSourceType() {
9298
return CoreValuesSourceType.NUMERIC;

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2003,6 +2003,74 @@ public void testPivotWithTopMetrics() throws Exception {
20032003
assertEquals("business_3", actual);
20042004
}
20052005

2006+
@SuppressWarnings(value = "unchecked")
2007+
public void testPivotWithExtendedMetrics() throws Exception {
2008+
String transformId = "extended_metrics_transform";
2009+
String transformIndex = "extended_metrics_pivot_reviews";
2010+
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
2011+
2012+
final Request createTransformRequest = createRequestWithAuth(
2013+
"PUT",
2014+
getTransformEndpoint() + transformId,
2015+
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
2016+
);
2017+
2018+
String config = Strings.format("""
2019+
{
2020+
"source": {
2021+
"index": "%s"
2022+
},
2023+
"dest": {
2024+
"index": "%s"
2025+
},
2026+
"pivot": {
2027+
"group_by": {
2028+
"reviewer": {
2029+
"terms": {
2030+
"field": "user_id"
2031+
}
2032+
}
2033+
},
2034+
"aggregations": {
2035+
"stars": {
2036+
"extended_stats": {
2037+
"field": "stars"
2038+
}
2039+
}
2040+
}
2041+
}
2042+
}""", REVIEWS_INDEX_NAME, transformIndex);
2043+
2044+
createTransformRequest.setJsonEntity(config);
2045+
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
2046+
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
2047+
2048+
startAndWaitForTransform(transformId, transformIndex, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS);
2049+
assertTrue(indexExists(transformIndex));
2050+
2051+
Map<String, Object> searchResult = getAsMap(transformIndex + "/_search?q=reviewer:user_4");
2052+
assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult));
2053+
var stdDevMap = (Map<String, Object>) ((List<?>) XContentMapValues.extractValue("hits.hits._source.stars", searchResult)).get(0);
2054+
assertThat(stdDevMap.get("count"), is(equalTo(41.0)));
2055+
assertThat(stdDevMap.get("sum"), is(equalTo(159.0)));
2056+
assertThat(stdDevMap.get("min"), is(equalTo(1.0)));
2057+
assertThat(stdDevMap.get("max"), is(equalTo(5.0)));
2058+
assertThat(stdDevMap.get("avg"), is(equalTo(3.8780487804878048)));
2059+
assertThat(stdDevMap.get("sum_of_squares"), is(equalTo(711.0)));
2060+
assertThat(stdDevMap.get("variance"), is(equalTo(2.3022010707911953)));
2061+
assertThat(stdDevMap.get("variance_population"), is(equalTo(2.3022010707911953)));
2062+
assertThat(stdDevMap.get("variance_sampling"), is(equalTo(2.3597560975609753)));
2063+
assertThat(stdDevMap.get("std_deviation"), is(equalTo(1.5173005868288574)));
2064+
assertThat(stdDevMap.get("std_deviation_population"), is(equalTo(1.5173005868288574)));
2065+
assertThat(stdDevMap.get("std_deviation_sampling"), is(equalTo(1.5361497640402693)));
2066+
assertThat(stdDevMap.get("std_upper"), is(equalTo(6.91264995414552)));
2067+
assertThat(stdDevMap.get("std_lower"), is(equalTo(0.84344760683009)));
2068+
assertThat(stdDevMap.get("std_upper_population"), is(equalTo(6.91264995414552)));
2069+
assertThat(stdDevMap.get("std_lower_population"), is(equalTo(0.84344760683009)));
2070+
assertThat(stdDevMap.get("std_upper_sampling"), is(equalTo(6.950348308568343)));
2071+
assertThat(stdDevMap.get("std_lower_sampling"), is(equalTo(0.8057492524072662)));
2072+
}
2073+
20062074
public void testPivotWithBoxplot() throws Exception {
20072075
String transformId = "boxplot_transform";
20082076
String transformIndex = "boxplot_pivot_reviews";

‎x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregations.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ public final class TransformAggregations {
6060
"date_histogram",
6161
"date_range",
6262
"diversified_sampler",
63-
"extended_stats", // https://github.com/elastic/elasticsearch/issues/51925
6463
"filters",
6564
"geo_distance",
6665
"geohash_grid",
@@ -120,7 +119,8 @@ enum AggregationType {
120119
MISSING("missing", LONG),
121120
TOP_METRICS("top_metrics", SOURCE),
122121
STATS("stats", DOUBLE),
123-
BOXPLOT("boxplot", DOUBLE);
122+
BOXPLOT("boxplot", DOUBLE),
123+
EXTENDED_STATS("extended_stats", DOUBLE);
124124

125125
private final String aggregationType;
126126
private final String targetMapping;

‎x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.search.aggregations.AggregationBuilders;
1919
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
2020
import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
21+
import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
2122
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
2223
import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
2324
import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder;
@@ -31,7 +32,9 @@
3132
import java.util.Map;
3233
import java.util.stream.Collectors;
3334

35+
import static org.hamcrest.Matchers.allOf;
3436
import static org.hamcrest.Matchers.equalTo;
37+
import static org.hamcrest.Matchers.hasEntry;
3538
import static org.hamcrest.Matchers.is;
3639

3740
public class TransformAggregationsTests extends ESTestCase {
@@ -137,6 +140,9 @@ public void testResolveTargetMapping() {
137140
assertEquals("double", TransformAggregations.resolveTargetMapping("stats", null));
138141
assertEquals("double", TransformAggregations.resolveTargetMapping("stats", "int"));
139142

143+
// extended stats
144+
assertEquals("double", TransformAggregations.resolveTargetMapping("extended_stats", "double"));
145+
140146
// boxplot
141147
assertEquals("double", TransformAggregations.resolveTargetMapping("boxplot", "double"));
142148

@@ -220,6 +226,37 @@ public void testGetAggregationOutputTypesStats() {
220226
assertEquals("stats", outputTypes.get("stats.sum"));
221227
}
222228

229+
public void testGetAggregationOutputTypesExtendedStats() {
230+
var extendedStatsAggregationBuilder = new ExtendedStatsAggregationBuilder("extended_stats");
231+
232+
var inputAndOutputTypes = TransformAggregations.getAggregationInputAndOutputTypes(extendedStatsAggregationBuilder);
233+
var outputTypes = inputAndOutputTypes.v2();
234+
assertEquals(18, outputTypes.size());
235+
assertThat(
236+
outputTypes,
237+
allOf(
238+
hasEntry("extended_stats.std_upper_population", "extended_stats"),
239+
hasEntry("extended_stats.variance", "extended_stats"),
240+
hasEntry("extended_stats.avg", "extended_stats"),
241+
hasEntry("extended_stats.min", "extended_stats"),
242+
hasEntry("extended_stats.sum_of_squares", "extended_stats"),
243+
hasEntry("extended_stats.sum", "extended_stats"),
244+
hasEntry("extended_stats.variance_sampling", "extended_stats"),
245+
hasEntry("extended_stats.std_lower_population", "extended_stats"),
246+
hasEntry("extended_stats.std_deviation", "extended_stats"),
247+
hasEntry("extended_stats.std_upper_sampling", "extended_stats"),
248+
hasEntry("extended_stats.std_deviation_population", "extended_stats"),
249+
hasEntry("extended_stats.max", "extended_stats"),
250+
hasEntry("extended_stats.std_lower_sampling", "extended_stats"),
251+
hasEntry("extended_stats.std_deviation_sampling", "extended_stats"),
252+
hasEntry("extended_stats.std_upper", "extended_stats"),
253+
hasEntry("extended_stats.count", "extended_stats"),
254+
hasEntry("extended_stats.variance_population", "extended_stats"),
255+
hasEntry("extended_stats.std_lower", "extended_stats")
256+
)
257+
);
258+
}
259+
223260
public void testGetAggregationOutputTypesRange() {
224261
{
225262
AggregationBuilder rangeAggregationBuilder = new RangeAggregationBuilder("range_agg_name").addUnboundedTo(100)

0 commit comments

Comments
 (0)