diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 44045057e0011..41ba2d18897a1 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -67,6 +67,10 @@ Example response: "available" : true, "enabled" : true }, + "aggregate_metric" : { + "available" : true, + "enabled" : true + }, "analytics" : { "available" : true, "enabled" : true diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 9d6bc57a771a5..cc9693392d409 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -16,7 +16,7 @@ Provides usage information about the installed {xpack} features. === {api-description-title} This API provides information about which features are currently enabled and -available under the current license and some usage statistics. +available under the current license and some usage statistics. [discrete] [[usage-api-query-parms]] @@ -264,6 +264,10 @@ GET /_xpack/usage "analytics" : { "available" : true, "enabled" : true + }, + "aggregate_metric" : { + "available" : true, + "enabled" : true } } ------------------------------------------------------------ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 237f922816321..2a046ee9b55f4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.ccr.CCRInfoTransportAction; import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; +import org.elasticsearch.xpack.core.aggregatemetric.AggregateMetricFeatureSetUsage; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; @@ -490,8 +491,11 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FROZEN_INDICES, FrozenIndicesFeatureSetUsage::new), // Spatial new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SPATIAL, SpatialFeatureSetUsage::new), - // data science - new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new) + // Analytics + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ANALYTICS, AnalyticsFeatureSetUsage::new), + // Aggregate metric field type + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, + XPackField.AGGREGATE_METRIC, AggregateMetricFeatureSetUsage::new) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 3bc1a44e7b820..78d01d423282d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -54,6 +54,8 @@ public final class XPackField { public static final String ANALYTICS = "analytics"; /** Name constant for the enrich plugin. */ public static final String ENRICH = "enrich"; + /** Name constant for the aggregate_metric plugin. */ + public static final String AGGREGATE_METRIC = "aggregate_metric"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java index 0d97119434cc3..e7fcbde6e0096 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java @@ -41,10 +41,11 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction SPATIAL = new XPackInfoFeatureAction(XPackField.SPATIAL); public static final XPackInfoFeatureAction ANALYTICS = new XPackInfoFeatureAction(XPackField.ANALYTICS); public static final XPackInfoFeatureAction ENRICH = new XPackInfoFeatureAction(XPackField.ENRICH); + public static final XPackInfoFeatureAction AGGREGATE_METRIC = new XPackInfoFeatureAction(XPackField.AGGREGATE_METRIC); public static final List ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH + TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH, AGGREGATE_METRIC ); private XPackInfoFeatureAction(String name) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index c696fdeaa3e29..55ef5f4aac50a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -40,10 +40,11 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS + TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, AGGREGATE_METRIC ); private XPackUsageFeatureAction(String name) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/aggregatemetric/AggregateMetricFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/aggregatemetric/AggregateMetricFeatureSetUsage.java new file mode 100644 index 0000000000000..dbbfb365d8e8d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/aggregatemetric/AggregateMetricFeatureSetUsage.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.aggregatemetric; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Objects; + +public class AggregateMetricFeatureSetUsage extends XPackFeatureSet.Usage { + + public AggregateMetricFeatureSetUsage(StreamInput input) throws IOException { + super(input); + } + + public AggregateMetricFeatureSetUsage(boolean available, boolean enabled) { + super(XPackField.AGGREGATE_METRIC, available, enabled); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + AggregateMetricFeatureSetUsage other = (AggregateMetricFeatureSetUsage) obj; + return Objects.equals(available, other.available) && + Objects.equals(enabled, other.enabled); + } + + @Override + public int hashCode() { + return Objects.hash(available, enabled); + } +} diff --git a/x-pack/plugin/mapper-aggregate-metric/build.gradle b/x-pack/plugin/mapper-aggregate-metric/build.gradle new file mode 100644 index 0000000000000..73409b7293c6e --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-aggregate-metric' + description 'Module for the aggregate_metric field type, which allows pre-aggregated fields to be stored a single field.' + classname 'org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-aggregate-metric' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +integTest.enabled = false diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricInfoTransportAction.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricInfoTransportAction.java new file mode 100644 index 0000000000000..47504abaa7e2b --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricInfoTransportAction.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.aggregatemetric; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; + +public class AggregateMetricInfoTransportAction extends XPackInfoFeatureTransportAction { + + @Inject + public AggregateMetricInfoTransportAction( + TransportService transportService, + ActionFilters actionFilters, + Settings settings, + XPackLicenseState licenseState + ) { + super(XPackInfoFeatureAction.AGGREGATE_METRIC.name(), transportService, actionFilters); + } + + @Override + public String name() { + return XPackField.AGGREGATE_METRIC; + } + + @Override + public boolean available() { + return true; + } + + @Override + public boolean enabled() { + return true; + } + +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricMapperPlugin.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricMapperPlugin.java new file mode 100644 index 0000000000000..0279965216442 --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricMapperPlugin.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.aggregatemetric; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonMap; + +public class AggregateMetricMapperPlugin extends Plugin implements MapperPlugin, ActionPlugin { + + @Override + public Map getMappers() { + return singletonMap(AggregateDoubleMetricFieldMapper.CONTENT_TYPE, new AggregateDoubleMetricFieldMapper.TypeParser()); + } + + @Override + public List> getActions() { + return Arrays.asList( + new ActionHandler<>(XPackUsageFeatureAction.AGGREGATE_METRIC, AggregateMetricUsageTransportAction.class), + new ActionHandler<>(XPackInfoFeatureAction.AGGREGATE_METRIC, AggregateMetricInfoTransportAction.class) + ); + } + +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricUsageTransportAction.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricUsageTransportAction.java new file mode 100644 index 0000000000000..973e2bef27435 --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/AggregateMetricUsageTransportAction.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.aggregatemetric; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.aggregatemetric.AggregateMetricFeatureSetUsage; + +public class AggregateMetricUsageTransportAction extends XPackUsageFeatureTransportAction { + + @Inject + public AggregateMetricUsageTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + Settings settings, + XPackLicenseState licenseState + ) { + super( + XPackUsageFeatureAction.AGGREGATE_METRIC.name(), + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver + ); + } + + @Override + protected void masterOperation( + Task task, + XPackUsageRequest request, + ClusterState state, + ActionListener listener + ) { + AggregateMetricFeatureSetUsage usage = new AggregateMetricFeatureSetUsage(true, true); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java new file mode 100644 index 0000000000000..396ff88a2b6d9 --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -0,0 +1,543 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.aggregatemetric.mapper; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentSubParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.SimpleMappedFieldType; +import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +/** A {@link FieldMapper} for a field containing aggregate metrics such as min/max/value_count etc. */ +public class AggregateDoubleMetricFieldMapper extends FieldMapper { + + public static final String CONTENT_TYPE = "aggregate_metric_double"; + + /** + * Mapping field names + */ + public static class Names { + public static final ParseField IGNORE_MALFORMED = new ParseField("ignore_malformed"); + public static final ParseField METRICS = new ParseField("metrics"); + public static final ParseField DEFAULT_METRIC = new ParseField("default_metric"); + } + + /** + * Enum of aggregate metrics supported by this field mapper + */ + enum Metric { + min, + max, + sum, + value_count; + } + + public static class Defaults { + public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); + public static final Explicit> METRICS = new Explicit<>(Collections.emptySet(), false); + public static final Explicit DEFAULT_METRIC = new Explicit<>(Metric.max, false); + public static final AggregateDoubleMetricFieldType FIELD_TYPE = new AggregateDoubleMetricFieldType(); + } + + public static class Builder extends FieldMapper.Builder { + + private Boolean ignoreMalformed; + + /** + * The aggregated metrics supported by the field type + */ + private EnumSet metrics; + + /** + * Set the default metric so that query operations are delegated to it. + */ + private Metric defaultMetric; + + public Builder(String name) { + super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE); + builder = this; + } + + public AggregateDoubleMetricFieldMapper.Builder ignoreMalformed(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + return builder; + } + + protected Explicit ignoreMalformed(BuilderContext context) { + if (ignoreMalformed != null) { + return new Explicit<>(ignoreMalformed, true); + } + if (context.indexSettings() != null) { + return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false); + } + return AggregateDoubleMetricFieldMapper.Defaults.IGNORE_MALFORMED; + } + + public AggregateDoubleMetricFieldMapper.Builder defaultMetric(Metric defaultMetric) { + this.defaultMetric = defaultMetric; + return builder; + } + + protected Explicit defaultMetric(BuilderContext context) { + if (defaultMetric != null) { + if (metrics != null && metrics.contains(defaultMetric) == false) { + // The default_metric is not defined in the "metrics" field + throw new IllegalArgumentException("Metric [" + defaultMetric + "] is not defined in the metrics field."); + } + return new Explicit<>(defaultMetric, true); + } + + // If a single metric is contained, this should be the default + if (metrics != null && metrics.size() == 1) { + return new Explicit<>(metrics.iterator().next(), false); + } + + if (metrics.contains(Defaults.DEFAULT_METRIC.value())) { + return Defaults.DEFAULT_METRIC; + } + throw new IllegalArgumentException( + "Property [" + Names.DEFAULT_METRIC.getPreferredName() + "] must be set for field [" + name() + "]." + ); + } + + public AggregateDoubleMetricFieldMapper.Builder metrics(EnumSet metrics) { + this.metrics = metrics; + return builder; + } + + protected Explicit> metrics(BuilderContext context) { + if (metrics != null) { + return new Explicit<>(metrics, true); + } + return Defaults.METRICS; + } + + @Override + public AggregateDoubleMetricFieldMapper build(BuilderContext context) { + setupFieldType(context); + + if (metrics == null || metrics.isEmpty()) { + throw new IllegalArgumentException( + "Property [" + Names.METRICS.getPreferredName() + "] must be set for field [" + name() + "]." + ); + } + + EnumMap metricMappers = new EnumMap<>(Metric.class); + // Instantiate one NumberFieldMapper instance for each metric + for (Metric m : this.metrics) { + String fieldName = name + "._" + m.name(); + NumberFieldMapper.Builder builder; + + if (m == Metric.value_count) { + // value_count metric can only be an integer and not a double + builder = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.INTEGER); + builder.coerce(false); + } else { + builder = new NumberFieldMapper.Builder(fieldName, NumberFieldMapper.NumberType.DOUBLE); + } + NumberFieldMapper fieldMapper = builder.build(context); + metricMappers.put(m, fieldMapper); + } + + EnumMap metricFields = metricMappers.entrySet() + .stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().fieldType(), + (l, r) -> { throw new IllegalArgumentException("Duplicate keys " + l + "and " + r + "."); }, + () -> new EnumMap<>(Metric.class) + ) + ); + Explicit defaultMetric = defaultMetric(context); + + AggregateDoubleMetricFieldType metricFieldType = (AggregateDoubleMetricFieldType) this.fieldType; + metricFieldType.setMetricFields(metricFields); + metricFieldType.setDefaultMetric(defaultMetric.value()); + + return new AggregateDoubleMetricFieldMapper( + name, + metricFieldType, + defaultFieldType, + context.indexSettings(), + multiFieldsBuilder.build(this, context), + ignoreMalformed(context), + metrics(context), + defaultMetric, + copyTo, + metricMappers + ); + } + } + + public static class TypeParser implements Mapper.TypeParser { + + @Override + public Mapper.Builder parse( + String name, + Map node, + ParserContext parserContext + ) throws MapperParsingException { + AggregateDoubleMetricFieldMapper.Builder builder = new AggregateDoubleMetricFieldMapper.Builder(name); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = entry.getKey(); + Object propNode = entry.getValue(); + if (propName.equals(Names.METRICS.getPreferredName())) { + String metricsStr[] = XContentMapValues.nodeStringArrayValue(propNode); + // Make sure that metrics are supported + EnumSet parsedMetrics = EnumSet.noneOf(Metric.class); + for (int i = 0; i < metricsStr.length; i++) { + try { + Metric m = Metric.valueOf(metricsStr[i]); + parsedMetrics.add(m); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Metric [" + metricsStr[i] + "] is not supported.", e); + } + } + builder.metrics(parsedMetrics); + iterator.remove(); + } else if (propName.equals(Names.DEFAULT_METRIC.getPreferredName())) { + String defaultMetric = XContentMapValues.nodeStringValue( + propNode, + name + "." + Names.DEFAULT_METRIC.getPreferredName() + ); + try { + Metric m = Metric.valueOf(defaultMetric); + builder.defaultMetric(m); + iterator.remove(); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Metric [" + defaultMetric + "] is not supported.", e); + } + } else if (propName.equals(Names.IGNORE_MALFORMED.getPreferredName())) { + builder.ignoreMalformed( + XContentMapValues.nodeBooleanValue(propNode, name + "." + Names.IGNORE_MALFORMED.getPreferredName()) + ); + iterator.remove(); + } else if (TypeParsers.parseMultiField(builder, name, parserContext, propName, propNode)) { + iterator.remove(); + } + } + return builder; + } + } + + public static final class AggregateDoubleMetricFieldType extends SimpleMappedFieldType { + + private EnumMap metricFields; + + private Metric defaultMetric; + + AggregateDoubleMetricFieldType() {} + + AggregateDoubleMetricFieldType(AggregateDoubleMetricFieldType other) { + super(other); + this.metricFields = other.metricFields; + this.defaultMetric = other.defaultMetric; + } + + @Override + public MappedFieldType clone() { + return new AggregateDoubleMetricFieldType(this); + } + + /** + * Return a delegate field type for a given metric sub-field + * @return a field type + */ + private NumberFieldMapper.NumberFieldType delegateFieldType(Metric metric) { + return metricFields.get(metric); + } + + /** + * Return a delegate field type for the default metric sub-field + * @return a field type + */ + private NumberFieldMapper.NumberFieldType delegateFieldType() { + return delegateFieldType(defaultMetric); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + public void setMetricFields(EnumMap metricFields) { + checkIfFrozen(); + this.metricFields = metricFields; + } + + public void setDefaultMetric(Metric defaultMetric) { + checkIfFrozen(); + this.defaultMetric = defaultMetric; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return delegateFieldType().existsQuery(context); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return delegateFieldType().termQuery(value, context); + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + return delegateFieldType().termsQuery(values, context); + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + return delegateFieldType().rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + } + + @Override + public Object valueForDisplay(Object value) { + return delegateFieldType().valueForDisplay(value); + } + + @Override + public DocValueFormat docValueFormat(String format, ZoneId timeZone) { + return delegateFieldType().docValueFormat(format, timeZone); + } + + @Override + public Relation isFieldWithinQuery( + IndexReader reader, + Object from, + Object to, + boolean includeLower, + boolean includeUpper, + ZoneId timeZone, + DateMathParser dateMathParser, + QueryRewriteContext context + ) throws IOException { + return delegateFieldType().isFieldWithinQuery(reader, from, to, includeLower, includeUpper, timeZone, dateMathParser, context); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + return delegateFieldType().fielddataBuilder(fullyQualifiedIndexName); + } + + } + + private final EnumMap metricFieldMappers; + + private Explicit ignoreMalformed; + + /** A set of metrics supported */ + private Explicit> metrics; + + /** The default metric to be when querying this field type */ + protected Explicit defaultMetric; + + private AggregateDoubleMetricFieldMapper( + String simpleName, + MappedFieldType fieldType, + MappedFieldType defaultFieldType, + Settings indexSettings, + MultiFields multiFields, + Explicit ignoreMalformed, + Explicit> metrics, + Explicit defaultMetric, + CopyTo copyTo, + EnumMap metricFieldMappers + ) { + super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); + this.ignoreMalformed = ignoreMalformed; + this.metrics = metrics; + this.defaultMetric = defaultMetric; + this.metricFieldMappers = metricFieldMappers; + } + + @Override + public AggregateDoubleMetricFieldType fieldType() { + return (AggregateDoubleMetricFieldType) super.fieldType(); + } + + @Override + protected String contentType() { + return fieldType.typeName(); + } + + @Override + protected AggregateDoubleMetricFieldMapper clone() { + return (AggregateDoubleMetricFieldMapper) super.clone(); + } + + @Override + public Iterator iterator() { + List mappers = new ArrayList<>(metricFieldMappers.values()); + return mappers.iterator(); + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + if (context.externalValueSet()) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] can't be used in multi-fields"); + } + + context.path().add(simpleName()); + XContentParser.Token token = null; + XContentSubParser subParser = null; + + try { + token = context.parser().currentToken(); + if (token == XContentParser.Token.VALUE_NULL) { + context.path().remove(); + return; + } + + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, context.parser()::getTokenLocation); + subParser = new XContentSubParser(context.parser()); + token = subParser.nextToken(); + while (token != XContentParser.Token.END_OBJECT) { + // should be an object subfield with name a metric name + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, subParser::getTokenLocation); + String fieldName = subParser.currentName(); + Metric metric = Metric.valueOf(fieldName); + + if (metrics.value().contains(metric) == false) { + throw new IllegalArgumentException( + "Aggregate metric [" + metric + "] does not exist in the mapping of field [" + fieldType.name() + "]" + ); + } + + token = subParser.nextToken(); + // Make sure that the value is a number. Probably this will change when + // new aggregate metric types are added (histogram, cardinality etc) + ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, subParser::getTokenLocation); + NumberFieldMapper delegateFieldMapper = metricFieldMappers.get(metric); + + if (context.doc().getField(delegateFieldMapper.fieldType().name()) != null) { + throw new IllegalArgumentException( + "Field [" + + name() + + "] of type [" + + typeName() + + "] does not support indexing multiple values for the same metric in the same field" + ); + } + + delegateFieldMapper.parse(context); + + if (Metric.value_count == metric) { + Number n = context.doc().getField(delegateFieldMapper.fieldType().name()).numericValue(); + if (n.intValue() < 0) { + throw new IllegalArgumentException( + "Aggregate metric [" + metric.name() + "] of field [" + fieldType.name() + "] cannot be a negative number" + ); + } + } + + token = subParser.nextToken(); + } + + for (Metric m : metrics.value()) { + if (context.doc().getField(fieldType().name() + "._" + m.name()) == null) { + throw new IllegalArgumentException( + "Aggregate metric field [" + fieldType.name() + "] must contain all metrics " + metrics.value().toString() + ); + } + } + } catch (Exception e) { + if (ignoreMalformed.value()) { + if (subParser != null) { + // close the subParser so we advance to the end of the object + subParser.close(); + } + context.addIgnoredField(fieldType().name()); + } else { + // Rethrow exception as is. It is going to be caught and nested in a MapperParsingException + // by its FieldMapper.MappedFieldType#parse() + throw e; + } + } + context.path().remove(); + } + + @Override + protected void doMerge(Mapper mergeWith) { + super.doMerge(mergeWith); + AggregateDoubleMetricFieldMapper other = (AggregateDoubleMetricFieldMapper) mergeWith; + if (other.ignoreMalformed.explicit()) { + this.ignoreMalformed = other.ignoreMalformed; + } + + if (other.metrics.explicit()) { + if (this.metrics.value() != null + && metrics.value().isEmpty() == false + && metrics.value().containsAll(other.metrics.value()) == false) { + throw new IllegalArgumentException( + "[" + + fieldType().name() + + "] with field mapper [" + + fieldType().typeName() + + "] " + + "cannot be merged with " + + "[" + + other.fieldType().typeName() + + "] because they contain separate metrics" + ); + } + this.metrics = other.metrics; + } + + if (other.defaultMetric.explicit()) { + this.defaultMetric = other.defaultMetric; + } + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + if (includeDefaults || ignoreMalformed.explicit()) { + builder.field(Names.IGNORE_MALFORMED.getPreferredName(), ignoreMalformed.value()); + } + + if (includeDefaults || metrics.explicit()) { + builder.field(Names.METRICS.getPreferredName(), metrics.value()); + } + + if (includeDefaults || defaultMetric.explicit()) { + builder.field(Names.DEFAULT_METRIC.getPreferredName(), defaultMetric.value()); + } + } +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java new file mode 100644 index 0000000000000..a61b3c1687ef0 --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -0,0 +1,970 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.aggregatemetric.mapper; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.IGNORE_MALFORMED; +import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.METRICS; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.IsInstanceOf.instanceOf; + +public class AggregateDoubleMetricFieldMapperTests extends ESSingleNodeTestCase { + + public static final String METRICS_FIELD = METRICS.getPreferredName(); + public static final String IGNORE_MALFORMED_FIELD = IGNORE_MALFORMED.getPreferredName(); + public static final String CONTENT_TYPE = AggregateDoubleMetricFieldMapper.CONTENT_TYPE; + public static final String DEFAULT_METRIC = AggregateDoubleMetricFieldMapper.Names.DEFAULT_METRIC.getPreferredName(); + + /** + * Test parsing field mapping and adding simple field + */ + public void testParseValue() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max", "sum", "value_count" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Mapper fieldMapper = defaultMapper.mappers().getMapper("metric"); + assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("min", 10.1) + .field("max", 50.0) + .field("sum", 43) + .field("value_count", 14) + .endObject() + .endObject() + ), + XContentType.JSON + ) + ); + + assertThat(doc.rootDoc().getField("metric._min"), notNullValue()); + } + + /** + * Test parsing field mapping and adding simple field + */ + public void testInvalidMapping() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("_doc", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("Property [metrics] must be set for field [metric].")); + } + + /** + * Test parsing an aggregate_metric field that contains no values + */ + public void testParseEmptyValue() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max", "sum" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Exception e = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().startObject("metric").endObject().endObject()), + XContentType.JSON + ) + ) + ); + assertThat(e.getCause().getMessage(), containsString("Aggregate metric field [metric] must contain all metrics")); + } + + /** + * Test parsing an aggregate_metric field that contains no values + * when ignore_malformed = true + */ + public void testParseEmptyValueIgnoreMalformed() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(IGNORE_MALFORMED_FIELD, true) + .field(METRICS_FIELD, new String[] { "min", "max", "sum" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().startObject("metric").endObject().endObject()), + XContentType.JSON + ) + ); + + assertThat(doc.rootDoc().getField("metric"), nullValue()); + } + + /** + * Test adding a metric that other than the supported ones (min, max, sum, value_count) + */ + public void testUnsupportedMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max", "unsupported" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("_doc", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("Metric [unsupported] is not supported.")); + } + + /** + * Test inserting a document containing a metric that has not been defined in the field mapping. + */ + public void testUnmappedMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Exception e = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("min", 10.0) + .field("max", 50.0) + .field("sum", 43) + .endObject() + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat(e.getCause().getMessage(), containsString("Aggregate metric [sum] does not exist in the mapping of field [metric]")); + } + + /** + * Test inserting a document containing a metric that has not been defined in the field mapping. + * Field will be ignored because config ignore_malformed has been set. + */ + public void testUnmappedMetricWithIgnoreMalformed() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max" }) + .field(IGNORE_MALFORMED_FIELD, true) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("min", 10.0) + .field("max", 50.0) + .field("sum", 43) + .endObject() + .endObject() + ), + XContentType.JSON + ) + ); + + assertNull(doc.rootDoc().getField("metric.min")); + } + + /** + * Test inserting a document containing less metrics than those defined in the field mapping. + * An exception will be thrown + */ + public void testMissingMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max", "sum" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Exception e = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("min", 10.0) + .field("max", 50.0) + .endObject() + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat(e.getCause().getMessage(), containsString("Aggregate metric field [metric] must contain all metrics [min, max, sum]")); + } + + /** + * Test inserting a document containing less metrics than those defined in the field mapping. + * Field will be ignored because config ignore_malformed has been set. + */ + public void testMissingMetricWithIgnoreMalformed() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max", "sum" }) + .field(IGNORE_MALFORMED_FIELD, true) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("min", 10.0) + .field("max", 50.0) + .endObject() + .endObject() + ), + XContentType.JSON + ) + ); + + assertNull(doc.rootDoc().getField("metric.min")); + } + + /** + * Test a metric that has an invalid value (string instead of number) + */ + public void testInvalidMetricValue() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Exception e = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("metric").field("min", "10.0").endObject().endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat( + e.getCause().getMessage(), + containsString("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [VALUE_STRING]") + ); + } + + /** + * Test a metric that has an invalid value (string instead of number) + * with ignore_malformed = true + */ + public void testInvalidMetricValueIgnoreMalformed() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min" }) + .field(IGNORE_MALFORMED_FIELD, true) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("metric").field("min", "10.0").endObject().endObject() + ), + XContentType.JSON + ) + ); + assertThat(doc.rootDoc().getField("metric"), nullValue()); + } + + /** + * Test a field that has a negative value for value_count + */ + public void testNegativeValueCount() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Exception e = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("value_count", -55) // value_count cannot be negative value + .endObject() + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat( + e.getCause().getMessage(), + containsString("Aggregate metric [value_count] of field [metric] cannot be a negative number") + ); + } + + /** + * Test a field that has a negative value for value_count with ignore_malformed = true + * No exception will be thrown but the field will be ignored + */ + public void testNegativeValueCountIgnoreMalformed() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(IGNORE_MALFORMED_FIELD, true) + .field(METRICS_FIELD, new String[] { "value_count" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("value_count", -55) // value_count cannot be negative value + .endObject() + .endObject() + ), + XContentType.JSON + ) + ); + + assertThat(doc.rootDoc().getField("metric"), nullValue()); + } + + /** + * Test parsing a value_count metric written as double with zero decimal digits + */ + public void testValueCountDouble() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Mapper fieldMapper = defaultMapper.mappers().getMapper("metric"); + assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("metric").field("value_count", 77.0).endObject().endObject() + ), + XContentType.JSON + ) + ); + + assertThat(doc.rootDoc().getField("metric._value_count"), notNullValue()); + } + + /** + * Test parsing a value_count metric written as double with some decimal digits + */ + public void testInvalidDoubleValueCount() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Exception e = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("metric") + .field("value_count", 45.43) // Only integers can be a value_count + .endObject() + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat( + e.getCause().getMessage(), + containsString( + "failed to parse field [metric._value_count] of type [integer] in document with id '1'." + + " Preview of field's value: '45.43'" + ) + ); + } + + /** + * Test inserting a document containing an array of metrics. An exception must be thrown. + */ + public void testParseArrayValue() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Exception e = expectThrows( + MapperParsingException.class, + () -> defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startArray("metric") + .startObject() + .field("min", 10.0) + .field("max", 50.0) + .endObject() + .startObject() + .field("min", 11.0) + .field("max", 51.0) + .endObject() + .endArray() + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat( + e.getCause().getMessage(), + containsString( + "Field [metric] of type [aggregate_metric_double] does not support " + + "indexing multiple values for the same metric in the same field" + ) + ); + } + + /** + * Test setting the default_metric explicitly + */ + public void testExplicitDefaultMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count", "sum" }) + .field(DEFAULT_METRIC, "sum") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Mapper fieldMapper = defaultMapper.mappers().getMapper("metric"); + assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + assertEquals(AggregateDoubleMetricFieldMapper.Metric.sum, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric.value()); + } + + /** + * Test the default_metric when not set explicitly. When only a single metric is contained, this is set as the default + */ + public void testImplicitDefaultMetricSingleMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Mapper fieldMapper = defaultMapper.mappers().getMapper("metric"); + assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + assertEquals( + AggregateDoubleMetricFieldMapper.Metric.value_count, + ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric.value() + ); + } + + /** + * Test the default_metric when not set explicitly, by default we have set it to be the max. + */ + public void testImplicitDefaultMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count", "sum", "max" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Mapper fieldMapper = defaultMapper.mappers().getMapper("metric"); + assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + assertEquals(AggregateDoubleMetricFieldMapper.Metric.max, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric.value()); + } + + /** + * Test the default_metric when not set explicitly. When more than one metrics are contained + * and max is not one of them, an exception should be thrown. + */ + public void testMissingDefaultMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count", "sum" }) + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("_doc", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("Property [default_metric] must be set for field [metric].")); + } + + /** + * Test setting an invalid value for the default_metric. An exception must be thrown + */ + public void testInvalidDefaultMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count", "sum" }) + .field(DEFAULT_METRIC, "invalid_metric") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("_doc", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("Metric [invalid_metric] is not supported.")); + } + + /** + * Test setting a value for the default_metric that is not contained in the "metrics" field. + * An exception must be thrown + */ + public void testUndefinedDefaultMetric() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "value_count", "sum" }) + .field(DEFAULT_METRIC, "min") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("_doc", new CompressedXContent(mapping)) + ); + assertThat(e.getMessage(), containsString("Metric [min] is not defined in the metrics field.")); + } + + /** + * Test parsing field mapping and adding simple field + */ + public void testParseNestedValue() throws Exception { + ensureGreen(); + + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject("properties") + .startObject("parent") + .startObject("properties") + .startObject("metric") + .field("type", CONTENT_TYPE) + .field(METRICS_FIELD, new String[] { "min", "max", "sum", "value_count" }) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ); + + DocumentMapper defaultMapper = createIndex("test").mapperService() + .documentMapperParser() + .parse("_doc", new CompressedXContent(mapping)); + + Mapper fieldMapper = defaultMapper.mappers().getMapper("parent.metric"); + assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class)); + + ParsedDocument doc = defaultMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("parent") + .startObject("metric") + .field("min", 10.1) + .field("max", 50.0) + .field("sum", 43) + .field("value_count", 14) + .endObject() + .endObject() + .endObject() + ), + XContentType.JSON + ) + ); + + assertThat(doc.rootDoc().getField("parent.metric._min"), notNullValue()); + } + + @Override + protected Collection> getPlugins() { + List> plugins = new ArrayList<>(super.getPlugins()); + plugins.add(AggregateMetricMapperPlugin.class); + plugins.add(LocalStateCompositeXPackPlugin.class); + return plugins; + } + +} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java new file mode 100644 index 0000000000000..f8e98e3333f08 --- /dev/null +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.aggregatemetric.mapper; + +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType; +import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric; + +import java.util.EnumMap; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class AggregateDoubleMetricFieldTypeTests extends FieldTypeTestCase { + + @Override + protected MappedFieldType createDefaultFieldType() { + AggregateDoubleMetricFieldType fieldType = new AggregateDoubleMetricFieldType(); + EnumMap metricFields = new EnumMap<>(Metric.class); + for (Metric m : List.of(Metric.min, Metric.max)) { + String fieldName = "foo" + "._" + m.name(); + NumberFieldMapper.NumberFieldType subfield = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); + subfield.setName(fieldName); + metricFields.put(m, subfield); + } + fieldType.setMetricFields(metricFields); + fieldType.setDefaultMetric(Metric.max); + return fieldType; + } + + public void testTermQuery() { + final MappedFieldType fieldType = createDefaultFieldType(); + Query query = fieldType.termQuery(55.2, null); + assertThat(query, equalTo(DoublePoint.newRangeQuery("foo._max", 55.2, 55.2))); + } + + public void testTermsQuery() { + final MappedFieldType fieldType = createDefaultFieldType(); + Query query = fieldType.termsQuery(asList(55.2, 500.3), null); + assertThat(query, equalTo(DoublePoint.newSetQuery("foo._max", 55.2, 500.3))); + } + + public void testRangeQuery() throws Exception { + final MappedFieldType fieldType = createDefaultFieldType(); + Query query = fieldType.rangeQuery(10.1, 100.1, true, true, null, null, null, null); + assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/aggregate-metrics/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/aggregate-metrics/10_basic.yml new file mode 100644 index 0000000000000..d681efc656ca4 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/aggregate-metrics/10_basic.yml @@ -0,0 +1,209 @@ +--- +"Test exists query on aggregate metric field": + - skip: + version: " - 7.99.99" + reason: "Aggregate metric fields are currently only implemented in 8.0." + + - do: + indices.create: + index: aggregate_metric_test + body: + mappings: + properties: + metric: + type: aggregate_metric_double + metrics: [min, max] + - do: + index: + index: aggregate_metric_test + id: 1 + body: + metric: + min: 18.2 + max: 100 + refresh: true + + - do: + search: + index: aggregate_metric_test + body: + query: + exists: + field: metric + + - match: { hits.total.value: 1 } + + - do: + search: + index: aggregate_metric_test + body: + query: + exists: + field: metric._min + + - match: { hits.total.value: 1 } + + - do: + search: + index: aggregate_metric_test + body: + query: + exists: + field: metric._nonexistent_key + + - match: { hits.total.value: 0 } + +--- +"Test term query on aggregate metric field": + - skip: + version: " - 7.99.99" + reason: "Aggregate metric fields are currently only implemented in 8.0." + + - do: + indices.create: + index: test + body: + mappings: + properties: + metric: + type: aggregate_metric_double + metrics: [min, max] + + - do: + index: + index: test + id: 1 + body: + metric: + min: 18.2 + max: 100 + refresh: true + + - do: + index: + index: test + id: 2 + body: + metric: + min: 50 + max: 1000 + refresh: true + + - do: + search: + index: test + body: + query: + term: + metric: + value: 1000 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "2" } + + - do: + search: + index: test + body: + query: + term: + metric: + value: 100 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "1" } + + + - do: + search: + index: test + body: + query: + term: + metric._min: + value: 50 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "2" } + + +--- +"Test range query on aggregate metric field": + - skip: + version: " - 7.99.99" + reason: "Aggregate metric fields are currently only implemented in 8.0." + + - do: + indices.create: + index: test + body: + mappings: + properties: + metric: + type: aggregate_metric_double + metrics: [min, max] + + - do: + index: + index: test + id: 1 + body: + metric: + min: 18.2 + max: 100 + refresh: true + + - do: + index: + index: test + id: 2 + body: + metric: + min: 50 + max: 1000 + refresh: true + + - do: + search: + index: test + body: + query: + range: + metric: + gte: 900 + lte: 1000 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "2" } + + - do: + search: + index: test + body: + query: + range: + metric: + gte: 90 + lte: 200 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "1" } + + - do: + search: + index: test + body: + query: + range: + metric._min: + gte: 10 + lte: 40 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "1" }