diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java deleted file mode 100644 index 6777071b749fa..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java +++ /dev/null @@ -1,597 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.search.aggregations.metrics; - -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.MockScriptPlugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.bucket.global.Global; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.test.ESIntegTestCase; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static java.util.Collections.emptyMap; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality; -import static org.elasticsearch.search.aggregations.AggregationBuilders.global; -import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.notNullValue; - -@ESIntegTestCase.SuiteScopeTestCase -public class CardinalityIT extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(CustomScriptPlugin.class); - } - - public static class CustomScriptPlugin extends MockScriptPlugin { - - @Override - protected Map, Object>> pluginScripts() { - Map, Object>> scripts = new HashMap<>(); - - scripts.put("_value", vars -> vars.get("_value")); - - scripts.put("doc['str_value'].value", vars -> { - Map doc = (Map) vars.get("doc"); - return doc.get("str_value"); - }); - - scripts.put("doc['str_values']", vars -> { - Map doc = (Map) vars.get("doc"); - ScriptDocValues.Strings strValue = (ScriptDocValues.Strings) doc.get("str_values"); - return strValue; - }); - - scripts.put("doc[' + singleNumericField() + '].value", vars -> { - Map doc = (Map) vars.get("doc"); - return doc.get(singleNumericField()); - }); - - scripts.put("doc[' + multiNumericField(false) + ']", vars -> { - Map doc = (Map) vars.get("doc"); - return (ScriptDocValues) doc.get(multiNumericField(false)); - }); - - return scripts; - } - - @Override - protected Map, Object>> nonDeterministicPluginScripts() { - Map, Object>> scripts = new HashMap<>(); - - scripts.put("Math.random()", vars -> CardinalityIT.randomDouble()); - - return scripts; - } - } - - @Override - public Settings indexSettings() { - return Settings.builder() - .put("index.number_of_shards", numberOfShards()) - .put("index.number_of_replicas", numberOfReplicas()) - .build(); - } - - static long numDocs; - static long precisionThreshold; - - @Override - public void setupSuiteScopeCluster() throws Exception { - - prepareCreate("idx").setMapping( - jsonBuilder().startObject() - .startObject("_doc") - .startObject("properties") - .startObject("str_value") - .field("type", "keyword") - .endObject() - .startObject("str_values") - .field("type", "keyword") - .endObject() - .startObject("l_value") - .field("type", "long") - .endObject() - .startObject("l_values") - .field("type", "long") - .endObject() - .startObject("d_value") - .field("type", "double") - .endObject() - .startObject("d_values") - .field("type", "double") - .endObject() - .endObject() - .endObject() - .endObject() - ).get(); - - numDocs = randomIntBetween(2, 100); - precisionThreshold = randomIntBetween(0, 1 << randomInt(20)); - IndexRequestBuilder[] builders = new IndexRequestBuilder[(int) numDocs]; - for (int i = 0; i < numDocs; ++i) { - builders[i] = client().prepareIndex("idx") - .setSource( - jsonBuilder().startObject() - .field("str_value", "s" + i) - .array("str_values", new String[] { "s" + (i * 2), "s" + (i * 2 + 1) }) - .field("l_value", i) - .array("l_values", new int[] { i * 2, i * 2 + 1 }) - .field("d_value", i) - .array("d_values", new double[] { i * 2, i * 2 + 1 }) - .endObject() - ); - } - indexRandom(true, builders); - createIndex("idx_unmapped"); - - IndexRequestBuilder[] dummyDocsBuilder = new IndexRequestBuilder[10]; - for (int i = 0; i < dummyDocsBuilder.length; i++) { - dummyDocsBuilder[i] = client().prepareIndex("idx").setSource("a_field", "1"); - } - indexRandom(true, dummyDocsBuilder); - - ensureSearchable(); - } - - private void assertCount(Cardinality count, long value) { - if (value <= precisionThreshold) { - // linear counting should be picked, and should be accurate - assertEquals(value, count.getValue()); - } else { - // error is not bound, so let's just make sure it is > 0 - assertThat(count.getValue(), greaterThan(0L)); - } - } - - private static String singleNumericField() { - return randomBoolean() ? "l_value" : "d_value"; - } - - private static String multiNumericField(boolean hash) { - return randomBoolean() ? "l_values" : "d_values"; - } - - public void testUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value")) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, 0); - } - - public void testPartiallyUnmapped() throws Exception { - SearchResponse response = client().prepareSearch("idx", "idx_unmapped") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value")) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testSingleValuedString() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value")) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testSingleValuedNumeric() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField())) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testSingleValuedNumericGetProperty() throws Exception { - SearchResponse searchResponse = client().prepareSearch("idx") - .setQuery(matchAllQuery()) - .addAggregation( - global("global").subAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()) - ) - ) - .get(); - - assertSearchResponse(searchResponse); - - Global global = searchResponse.getAggregations().get("global"); - assertThat(global, notNullValue()); - assertThat(global.getName(), equalTo("global")); - // assertThat(global.getDocCount(), equalTo(numDocs)); - assertThat(global.getAggregations(), notNullValue()); - assertThat(global.getAggregations().asMap().size(), equalTo(1)); - - Cardinality cardinality = global.getAggregations().get("cardinality"); - assertThat(cardinality, notNullValue()); - assertThat(cardinality.getName(), equalTo("cardinality")); - long expectedValue = numDocs; - assertCount(cardinality, expectedValue); - assertThat(((InternalAggregation) global).getProperty("cardinality"), equalTo(cardinality)); - assertThat(((InternalAggregation) global).getProperty("cardinality.value"), equalTo((double) cardinality.getValue())); - assertThat((double) ((InternalAggregation) cardinality).getProperty("value"), equalTo((double) cardinality.getValue())); - } - - public void testSingleValuedNumericHashed() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField())) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testMultiValuedString() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values")) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - - public void testMultiValuedNumeric() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(false))) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - - public void testMultiValuedNumericHashed() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(true))) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - - public void testSingleValuedStringScript() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold) - .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['str_value'].value", emptyMap())) - ) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testMultiValuedStringScript() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold) - .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['str_values']", emptyMap())) - ) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - - public void testSingleValuedNumericScript() throws Exception { - Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc[' + singleNumericField() + '].value", emptyMap()); - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script)) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testMultiValuedNumericScript() throws Exception { - Script script = new Script( - ScriptType.INLINE, - CustomScriptPlugin.NAME, - "doc[' + multiNumericField(false) + ']", - Collections.emptyMap() - ); - SearchResponse response = client().prepareSearch("idx") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script)) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - - public void testSingleValuedStringValueScript() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold) - .field("str_value") - .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())) - ) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testMultiValuedStringValueScript() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold) - .field("str_values") - .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())) - ) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - - public void testSingleValuedNumericValueScript() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold) - .field(singleNumericField()) - .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())) - ) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - - public void testMultiValuedNumericValueScript() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold) - .field(multiNumericField(false)) - .script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())) - ) - .get(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - - public void testAsSubAgg() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .addAggregation( - terms("terms").field("str_value") - .collectMode(randomFrom(SubAggCollectionMode.values())) - .subAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values")) - ) - .get(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - for (Terms.Bucket bucket : terms.getBuckets()) { - Cardinality count = bucket.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, 2); - } - } - - /** - * Make sure that a request using a deterministic script or not using a script get cached. - * Ensure requests using nondeterministic scripts do not get cached. - */ - public void testScriptCaching() throws Exception { - assertAcked( - prepareCreate("cache_test_idx").setMapping("d", "type=long") - .setSettings(Settings.builder().put("requests.cache.enable", true).put("number_of_shards", 1).put("number_of_replicas", 1)) - .get() - ); - indexRandom( - true, - client().prepareIndex("cache_test_idx").setId("1").setSource("s", 1), - client().prepareIndex("cache_test_idx").setId("2").setSource("s", 2) - ); - - // Make sure we are starting with a clear cache - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getHitCount(), - equalTo(0L) - ); - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getMissCount(), - equalTo(0L) - ); - - // Test that a request using a nondeterministic script does not get cached - SearchResponse r = client().prepareSearch("cache_test_idx") - .setSize(0) - .addAggregation( - cardinality("foo").field("d").script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "Math.random()", emptyMap())) - ) - .get(); - assertSearchResponse(r); - - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getHitCount(), - equalTo(0L) - ); - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getMissCount(), - equalTo(0L) - ); - - // Test that a request using a deterministic script gets cached - r = client().prepareSearch("cache_test_idx") - .setSize(0) - .addAggregation( - cardinality("foo").field("d").script(new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "_value", emptyMap())) - ) - .get(); - assertSearchResponse(r); - - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getHitCount(), - equalTo(0L) - ); - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getMissCount(), - equalTo(1L) - ); - - // Ensure that non-scripted requests are cached as normal - r = client().prepareSearch("cache_test_idx").setSize(0).addAggregation(cardinality("foo").field("d")).get(); - assertSearchResponse(r); - - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getHitCount(), - equalTo(0L) - ); - assertThat( - client().admin() - .indices() - .prepareStats("cache_test_idx") - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache() - .getMissCount(), - equalTo(2L) - ); - } -} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java index 26065a72fdf00..b9ef7fe24be4d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java @@ -11,30 +11,126 @@ import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; +import org.elasticsearch.script.MockScriptEngine; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.bucket.global.Global; +import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; -import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; +import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; public class CardinalityAggregatorTests extends AggregatorTestCase { + /** Script to extract the value from any field **/ + public static final String VALUE_SCRIPT = "_value"; + + /** Script to extract the single string value of the 'str_value' field **/ + public static final String STRING_VALUE_SCRIPT = "doc['str_value'].value"; + + /** Script to extract a collection of string values from the 'str_values' field **/ + public static final String STRING_VALUES_SCRIPT = "doc['str_values']"; + + /** Script to extract a single numeric value from the 'number' field **/ + public static final String NUMERIC_VALUE_SCRIPT = "doc['number'].value"; + + /** Script to extract a collection of numeric values from the 'numbers' field **/ + public static final String NUMERIC_VALUES_SCRIPT = "doc['numbers']"; + + public static final int HASHER_DEFAULT_SEED = 17; + + @Override + protected ScriptService getMockScriptService() { + final Map, Object>> scripts = new HashMap<>(); + + scripts.put(VALUE_SCRIPT, vars -> vars.get("_value")); + + scripts.put(STRING_VALUE_SCRIPT, vars -> { + final Map doc = (Map) vars.get("doc"); + return doc.get("str_value"); + }); + + scripts.put(STRING_VALUES_SCRIPT, vars -> { + final Map doc = (Map) vars.get("doc"); + final ScriptDocValues.Strings strValues = (ScriptDocValues.Strings) doc.get("str_values"); + return strValues; + }); + + scripts.put(NUMERIC_VALUE_SCRIPT, vars -> { + final Map doc = (Map) vars.get("doc"); + return doc.get("number"); + }); + + scripts.put(NUMERIC_VALUES_SCRIPT, vars -> { + final Map doc = (Map) vars.get("doc"); + return (ScriptDocValues) doc.get("numbers"); + }); + + MockScriptEngine scriptEngine = new MockScriptEngine( + MockScriptEngine.NAME, + scripts, + Collections.emptyMap(), + Collections.emptyMap() + ); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return CoreValuesSourceType.ALL_CORE; + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new CardinalityAggregationBuilder("cardinality").field(fieldName); + } + public void testNoDocs() throws IOException { testAggregation(new MatchAllDocsQuery(), iw -> { // Intentionally not writing any docs @@ -94,8 +190,8 @@ public void testSomeMatchesNumericDocValues() throws IOException { public void testQueryFiltering() throws IOException { testAggregation(IntPoint.newRangeQuery("number", 0, 5), iw -> { - iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); - iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1))); + iw.addDocument(List.of(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); + iw.addDocument(List.of(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1))); }, card -> { assertEquals(1, card.getValue(), 0); assertTrue(AggregationInspectionHelper.hasValue(card)); @@ -104,14 +200,188 @@ public void testQueryFiltering() throws IOException { public void testQueryFiltersAll() throws IOException { testAggregation(IntPoint.newRangeQuery("number", -1, 0), iw -> { - iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); - iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1))); + iw.addDocument(List.of(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); + iw.addDocument(List.of(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1))); }, card -> { assertEquals(0.0, card.getValue(), 0); assertFalse(AggregationInspectionHelper.hasValue(card)); }); } + public void testSingleValuedString() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("str_value"); + final MappedFieldType mappedFieldTypes = new KeywordFieldMapper.KeywordFieldType("str_value"); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("one")))); + iw.addDocument(singleton(new SortedDocValuesField("unrelatedField", new BytesRef("two")))); + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("three")))); + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("one")))); + }, card -> { + assertEquals(2, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testSingleValuedStringValueScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("str_value") + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, "_value", emptyMap())); + final MappedFieldType mappedFieldTypes = new KeywordFieldMapper.KeywordFieldType("str_value"); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("one")))); + iw.addDocument(singleton(new SortedDocValuesField("unrelatedField", new BytesRef("two")))); + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("three")))); + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("one")))); + }, card -> { + assertEquals(2, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testSingleValuedStringScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").script( + new Script(ScriptType.INLINE, MockScriptEngine.NAME, "doc['str_value'].value", emptyMap()) + ); + final MappedFieldType mappedFieldTypes = new KeywordFieldMapper.KeywordFieldType("str_value"); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("one")))); + iw.addDocument(singleton(new SortedDocValuesField("unrelatedField", new BytesRef("two")))); + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("three")))); + iw.addDocument(singleton(new SortedDocValuesField("str_value", new BytesRef("one")))); + }, card -> { + assertEquals(2, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testMultiValuedStringScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").script( + new Script(ScriptType.INLINE, MockScriptEngine.NAME, "doc['str_values']", emptyMap()) + ); + final MappedFieldType mappedFieldTypes = new KeywordFieldMapper.KeywordFieldType("str_values"); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("two")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("unrelatedField", new BytesRef("two")), + new SortedSetDocValuesField("unrelatedField", new BytesRef("three")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("two")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + }, card -> { + assertEquals(3, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testMultiValuedStringValueScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("str_values") + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, "_value", emptyMap())); + final MappedFieldType mappedFieldTypes = new KeywordFieldMapper.KeywordFieldType("str_values"); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("two")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("unrelatedField", new BytesRef("two")), + new SortedSetDocValuesField("unrelatedField", new BytesRef("three")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("two")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + }, card -> { + assertEquals(3, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testMultiValuedString() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("str_values"); + final MappedFieldType mappedFieldTypes = new KeywordFieldMapper.KeywordFieldType("str_values"); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("two")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("one")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("three")), + new SortedSetDocValuesField("str_values", new BytesRef("two")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("three")), + new SortedSetDocValuesField("str_values", new BytesRef("two")) + ) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("str_values", new BytesRef("two")), + new SortedSetDocValuesField("str_values", new BytesRef("three")) + ) + ); + }, card -> { + assertEquals(3, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + public void testUnmappedMissingString() throws IOException { CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("number").missing("🍌🍌🍌"); @@ -138,6 +408,160 @@ public void testUnmappedMissingNumber() throws IOException { }); } + public void testSingleValuedFieldPartiallyUnmapped() throws IOException { + final Directory directory = newDirectory(); + final RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + final int numDocs = 10; + for (int i = 0; i < numDocs; i++) { + indexWriter.addDocument(singleton(new NumericDocValuesField("number", i + 1))); + } + indexWriter.close(); + + final Directory unmappedDirectory = newDirectory(); + final RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); + unmappedIndexWriter.close(); + + final IndexReader indexReader = DirectoryReader.open(directory); + final IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); + final MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); + final IndexSearcher indexSearcher = newSearcher(multiReader, true, true); + + final MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("number", NumberFieldMapper.NumberType.INTEGER); + final AggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("cardinality").field("number"); + + final CardinalityAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + + final InternalCardinality cardinality = (InternalCardinality) aggregator.buildAggregation(0L); + + assertEquals(10.0, cardinality.getValue(), 0); + assertEquals("cardinality", cardinality.getName()); + assertTrue(AggregationInspectionHelper.hasValue(cardinality)); + + multiReader.close(); + directory.close(); + unmappedDirectory.close(); + } + + public void testSingleValuedNumericValueScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("number") + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, "_value", emptyMap())); + final MappedFieldType mappedFieldTypes = new NumberFieldMapper.NumberFieldType("number", NumberFieldMapper.NumberType.INTEGER); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedNumericDocValuesField("number", 10))); + iw.addDocument(singleton(new SortedNumericDocValuesField("unrelatedField", 11))); + iw.addDocument(singleton(new SortedNumericDocValuesField("number", 12))); + iw.addDocument(singleton(new SortedNumericDocValuesField("number", 12))); + }, card -> { + assertEquals(2, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testSingleValuedNumericScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").script( + new Script(ScriptType.INLINE, MockScriptEngine.NAME, "doc['number'].value", emptyMap()) + ); + final MappedFieldType mappedFieldTypes = new NumberFieldMapper.NumberFieldType("number", NumberFieldMapper.NumberType.INTEGER); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedNumericDocValuesField("number", 10))); + iw.addDocument(singleton(new SortedNumericDocValuesField("unrelatedField", 11))); + iw.addDocument(singleton(new SortedNumericDocValuesField("number", 12))); + iw.addDocument(singleton(new SortedNumericDocValuesField("number", 12))); + }, card -> { + assertEquals(2, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testMultiValuedNumericValueScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("numbers") + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, "_value", emptyMap())); + final MappedFieldType mappedFieldTypes = new NumberFieldMapper.NumberFieldType("numbers", NumberFieldMapper.NumberType.INTEGER); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 10), new SortedNumericDocValuesField("numbers", 12))); + iw.addDocument( + List.of(new SortedNumericDocValuesField("unrelatedField", 11), new SortedNumericDocValuesField("unrelatedField", 12)) + ); + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 11), new SortedNumericDocValuesField("numbers", 12))); + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 12), new SortedNumericDocValuesField("numbers", 13))); + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 12), new SortedNumericDocValuesField("numbers", 13))); + }, card -> { + assertEquals(4, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testMultiValuedNumericScript() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").script( + new Script(ScriptType.INLINE, MockScriptEngine.NAME, "doc['numbers']", emptyMap()) + ); + final MappedFieldType mappedFieldTypes = new NumberFieldMapper.NumberFieldType("numbers", NumberFieldMapper.NumberType.INTEGER); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 10), new SortedNumericDocValuesField("numbers", 12))); + iw.addDocument( + List.of(new SortedNumericDocValuesField("unrelatedField", 11), new SortedNumericDocValuesField("unrelatedField", 12)) + ); + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 11), new SortedNumericDocValuesField("numbers", 12))); + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 12), new SortedNumericDocValuesField("numbers", 13))); + iw.addDocument(List.of(new SortedNumericDocValuesField("numbers", 12), new SortedNumericDocValuesField("numbers", 13))); + }, card -> { + assertEquals(4, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testMultiValuedNumeric() throws IOException { + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("number"); + final MappedFieldType mappedFieldTypes = new NumberFieldMapper.NumberFieldType("number", NumberFieldMapper.NumberType.INTEGER); + + testAggregation(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("number", 7), new SortedNumericDocValuesField("number", 8))); + iw.addDocument(List.of(new SortedNumericDocValuesField("number", 7), new SortedNumericDocValuesField("number", 9))); + iw.addDocument(List.of(new SortedNumericDocValuesField("number", 9), new SortedNumericDocValuesField("number", 8))); + iw.addDocument(List.of(new SortedNumericDocValuesField("number", 8), new SortedNumericDocValuesField("number", 7))); + }, card -> { + assertEquals(3, card.getValue(), 0); + assertTrue(AggregationInspectionHelper.hasValue(card)); + }, mappedFieldTypes); + } + + public void testSingleValuedFieldGlobalAggregation() throws IOException { + final MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("number", NumberFieldMapper.NumberType.LONG); + + final AggregationBuilder aggregationBuilder = AggregationBuilders.global("global") + .subAggregation(AggregationBuilders.cardinality("cardinality").field("number")); + + final int numDocs = 10; + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + for (int i = 0; i < numDocs; i++) { + iw.addDocument(singleton(new NumericDocValuesField("number", (i + 1)))); + iw.addDocument(singleton(new NumericDocValuesField("number", (i + 1)))); + } + }, topLevelAgg -> { + final Global global = (Global) topLevelAgg; + assertNotNull(global); + assertEquals("global", global.getName()); + assertEquals(numDocs * 2, global.getDocCount()); + assertNotNull(global.getAggregations()); + assertEquals(1, global.getAggregations().asMap().size()); + + final Cardinality cardinality = global.getAggregations().get("cardinality"); + assertNotNull(cardinality); + assertEquals("cardinality", cardinality.getName()); + assertEquals(numDocs, cardinality.getValue(), 0); + assertEquals(cardinality, ((InternalAggregation) global).getProperty("cardinality")); + assertEquals(numDocs, (double) ((InternalAggregation) global).getProperty("cardinality.value"), 0); + assertEquals(numDocs, (double) ((InternalAggregation) cardinality).getProperty("value"), 0); + }, fieldType); + } + public void testUnmappedMissingGeoPoint() throws IOException { CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name").field("number") .missing(new GeoPoint(42.39561, -71.13051)); @@ -152,6 +576,88 @@ public void testUnmappedMissingGeoPoint() throws IOException { }); } + public void testAsSubAggregation() throws IOException { + final MappedFieldType mappedFieldTypes[] = { + new KeywordFieldMapper.KeywordFieldType("str_value"), + new NumberFieldMapper.NumberFieldType("number", NumberFieldMapper.NumberType.LONG) }; + + final AggregationBuilder aggregationBuilder = new TermsAggregationBuilder("terms").field("str_value") + .missing("unknown") + .subAggregation(AggregationBuilders.cardinality("cardinality").field("number")); + + testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + final int numDocs = 10; + for (int i = 0; i < numDocs; i++) { + iw.addDocument( + List.of( + new SortedDocValuesField("str_value", new BytesRef((((i + 1) % 2 == 0) ? "even" : "odd"))), + new NumericDocValuesField("number", i + 1) + ) + ); + } + }, topLevelAgg -> { + int expectedTermBucketsCount = 2; // ("even", "odd") + final Terms terms = (StringTerms) topLevelAgg; + assertNotNull(terms); + List buckets = terms.getBuckets(); + assertNotNull(buckets); + assertEquals(expectedTermBucketsCount, buckets.size()); + + for (int i = 0; i < expectedTermBucketsCount; i++) { + final Terms.Bucket bucket = buckets.get(i); + assertNotNull(bucket); + assertEquals(((i + 1) % 2 == 0) ? "odd" : "even", bucket.getKey()); + assertEquals(5L, bucket.getDocCount()); + + final InternalCardinality cardinality = bucket.getAggregations().get("cardinality"); + assertNotNull(cardinality); + assertEquals("cardinality", cardinality.getName()); + assertEquals(5, cardinality.getValue()); + } + }, mappedFieldTypes); + } + + public void testCacheAggregation() throws IOException { + final Directory directory = newDirectory(); + final RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + final int numDocs = 10; + for (int i = 0; i < numDocs; i++) { + indexWriter.addDocument(singleton(new NumericDocValuesField("number", i + 1))); + } + indexWriter.close(); + + final Directory unmappedDirectory = newDirectory(); + final RandomIndexWriter unmappedIndexWriter = new RandomIndexWriter(random(), unmappedDirectory); + unmappedIndexWriter.close(); + + final IndexReader indexReader = DirectoryReader.open(directory); + final IndexReader unamappedIndexReader = DirectoryReader.open(unmappedDirectory); + final MultiReader multiReader = new MultiReader(indexReader, unamappedIndexReader); + final IndexSearcher indexSearcher = newSearcher(multiReader, true, true); + + final MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("number", NumberFieldMapper.NumberType.INTEGER); + final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("cardinality").field("number"); + + final AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); + final CardinalityAggregator aggregator = createAggregator(aggregationBuilder, context); + aggregator.preCollection(); + indexSearcher.search(new MatchAllDocsQuery(), aggregator); + aggregator.postCollection(); + + final InternalCardinality cardinality = (InternalCardinality) aggregator.buildAggregation(0L); + + assertEquals(10.0, cardinality.getValue(), 0); + assertEquals("cardinality", cardinality.getName()); + assertTrue(AggregationInspectionHelper.hasValue(cardinality)); + + // Test that an aggregation not using a script does get cached + assertTrue(context.isCacheable()); + + multiReader.close(); + directory.close(); + unmappedDirectory.close(); + } + private void testAggregation( Query query, CheckedConsumer buildIndex, diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 41fc2ff803634..7b2c69355a497 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -184,7 +184,7 @@ public abstract class AggregatorTestCase extends ESTestCase { ); @Before - public final void initPlugns() { + public final void initPlugins() { List plugins = new ArrayList<>(getSearchPlugins()); plugins.add(new AggCardinalityUpperBoundPlugin()); SearchModule searchModule = new SearchModule(Settings.EMPTY, plugins);