diff --git a/docs/changelog/109034.yaml b/docs/changelog/109034.yaml new file mode 100644 index 0000000000000..cdf1f6fe28d8d --- /dev/null +++ b/docs/changelog/109034.yaml @@ -0,0 +1,5 @@ +pr: 109034 +summary: Fix IOOBE in TTest aggregation when using filters +area: Aggregations +type: bug +issues: [] diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/PairedTTestAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/PairedTTestAggregator.java index 4786f0bd00947..9383726b08a26 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/PairedTTestAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/PairedTTestAggregator.java @@ -48,7 +48,7 @@ protected PairedTTestState getState(long bucket) { @Override protected PairedTTestState getEmptyState() { - return new PairedTTestState(new TTestStats(0, 0, 0), tails); + return new PairedTTestState(TTestStats.EMPTY, tails); } @Override diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestStats.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestStats.java index 4851615ad3c85..832952e957e7a 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestStats.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestStats.java @@ -20,6 +20,8 @@ * Collects basic stats that are needed to perform t-test */ public class TTestStats implements Writeable { + static TTestStats EMPTY = new TTestStats(0, 0, 0); + public final long count; public final double sum; public final double sumOfSqrs; diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/UnpairedTTestAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/UnpairedTTestAggregator.java index 004637a7df7f9..d52a53628799c 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/UnpairedTTestAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/UnpairedTTestAggregator.java @@ -56,12 +56,14 @@ public class UnpairedTTestAggregator extends TTestAggregator @Override protected UnpairedTTestState getState(long bucket) { - return new UnpairedTTestState(a.get(bucket), b.get(bucket), homoscedastic, tails); + final TTestStats aTTestStats = a.getSize() > bucket ? a.get(bucket) : TTestStats.EMPTY; + final TTestStats bTTestStats = b.getSize() > bucket ? b.get(bucket) : TTestStats.EMPTY; + return new UnpairedTTestState(aTTestStats, bTTestStats, homoscedastic, tails); } @Override protected UnpairedTTestState getEmptyState() { - return new UnpairedTTestState(new TTestStats(0, 0, 0), new TTestStats(0, 0, 0), homoscedastic, tails); + return new UnpairedTTestState(TTestStats.EMPTY, TTestStats.EMPTY, homoscedastic, tails); } @Override diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java index da77790e6493c..26c71b8af5102 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregatorTests.java @@ -636,6 +636,77 @@ public void testFiltered() throws IOException { } } + public void testFilteredAsSubAgg() throws IOException { + TTestType tTestType = randomFrom(TTestType.values()); + MappedFieldType fieldType1 = new NumberFieldMapper.NumberFieldType("h", NumberFieldMapper.NumberType.INTEGER); + MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType("a", NumberFieldMapper.NumberType.INTEGER); + MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType("b", NumberFieldMapper.NumberType.INTEGER); + TTestAggregationBuilder ttestAggregationBuilder = new TTestAggregationBuilder("t_test").a( + new MultiValuesSourceFieldConfig.Builder().setFieldName("a").setFilter(QueryBuilders.termQuery("b", 1)).build() + ) + .b(new MultiValuesSourceFieldConfig.Builder().setFieldName("a").setFilter(QueryBuilders.termQuery("b", 2)).build()) + .testType(tTestType); + int tails = randomIntBetween(1, 2); + if (tails == 1 || randomBoolean()) { + ttestAggregationBuilder.tails(tails); + } + HistogramAggregationBuilder aggregationBuilder = new HistogramAggregationBuilder("h").field("h") + .interval(1) + .subAggregation(ttestAggregationBuilder); + int buckets = randomInt(100); + CheckedConsumer buildIndex = iw -> { + for (int i = 0; i < buckets; i++) { + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 102), new IntPoint("b", 1))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 99), new IntPoint("b", 1))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 111), new IntPoint("b", 1))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 97), new IntPoint("b", 1))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 101), new IntPoint("b", 1))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 99), new IntPoint("b", 1))); + + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 89), new IntPoint("b", 2))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 93), new IntPoint("b", 2))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 72), new IntPoint("b", 2))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 98), new IntPoint("b", 2))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 102), new IntPoint("b", 2))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 98), new IntPoint("b", 2))); + + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 189), new IntPoint("b", 3))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 193), new IntPoint("b", 3))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 172), new IntPoint("b", 3))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 198), new IntPoint("b", 3))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 1102), new IntPoint("b", 3))); + iw.addDocument(asList(new NumericDocValuesField("h", i), new NumericDocValuesField("a", 198), new IntPoint("b", 3))); + } + }; + if (tTestType == TTestType.PAIRED) { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> testCase( + buildIndex, + tTest -> fail("Should have thrown exception"), + new AggTestConfig(aggregationBuilder, fieldType1, fieldType2, fieldType3) + ) + ); + assertEquals("Paired t-test doesn't support filters", ex.getMessage()); + } else { + testCase(buildIndex, (Consumer) histogram -> { + if (tTestType == TTestType.HOMOSCEDASTIC) { + assertEquals(buckets, histogram.getBuckets().size()); + for (int i = 0; i < buckets; i++) { + InternalTTest ttest = histogram.getBuckets().get(i).getAggregations().get("t_test"); + assertEquals(0.03928288693 * tails, ttest.getValue(), 0.00001); + } + } else { + assertEquals(buckets, histogram.getBuckets().size()); + for (int i = 0; i < buckets; i++) { + InternalTTest ttest = histogram.getBuckets().get(i).getAggregations().get("t_test"); + assertEquals(0.04538666214 * tails, ttest.getValue(), 0.00001); + } + } + }, new AggTestConfig(aggregationBuilder, fieldType1, fieldType2, fieldType3)); + } + } + public void testFilterByFilterOrScript() throws IOException { boolean fieldInA = randomBoolean(); TTestType tTestType = randomFrom(TTestType.HOMOSCEDASTIC, TTestType.HETEROSCEDASTIC);