10
10
import org .apache .lucene .document .Document ;
11
11
import org .apache .lucene .document .Field ;
12
12
import org .apache .lucene .document .LongPoint ;
13
+ import org .apache .lucene .document .NumericDocValuesField ;
13
14
import org .apache .lucene .document .SortedNumericDocValuesField ;
15
+ import org .apache .lucene .document .SortedSetDocValuesField ;
14
16
import org .apache .lucene .index .DirectoryReader ;
15
17
import org .apache .lucene .index .IndexReader ;
16
18
import org .apache .lucene .index .IndexableField ;
33
35
import org .elasticsearch .index .mapper .DateFieldMapper ;
34
36
import org .elasticsearch .index .mapper .DateFieldMapper .Resolution ;
35
37
import org .elasticsearch .index .mapper .DocCountFieldMapper ;
38
+ import org .elasticsearch .index .mapper .FieldNamesFieldMapper ;
36
39
import org .elasticsearch .index .mapper .KeywordFieldMapper ;
37
40
import org .elasticsearch .index .mapper .KeywordFieldMapper .KeywordFieldType ;
38
41
import org .elasticsearch .index .mapper .MappedFieldType ;
39
42
import org .elasticsearch .index .mapper .NumberFieldMapper ;
40
43
import org .elasticsearch .index .mapper .NumberFieldMapper .NumberType ;
41
44
import org .elasticsearch .index .mapper .ObjectMapper ;
45
+ import org .elasticsearch .index .query .ExistsQueryBuilder ;
42
46
import org .elasticsearch .index .query .MatchAllQueryBuilder ;
43
47
import org .elasticsearch .index .query .MatchQueryBuilder ;
44
48
import org .elasticsearch .index .query .QueryBuilder ;
62
66
import org .elasticsearch .search .aggregations .support .AggregationContext ;
63
67
import org .elasticsearch .search .aggregations .support .AggregationInspectionHelper ;
64
68
import org .elasticsearch .search .internal .ContextIndexSearcherTests .DocumentSubsetDirectoryReader ;
65
- import org .junit .Before ;
66
69
67
70
import java .io .IOException ;
68
71
import java .util .ArrayList ;
73
76
import java .util .Map ;
74
77
import java .util .Set ;
75
78
import java .util .concurrent .TimeUnit ;
79
+ import java .util .function .IntFunction ;
76
80
77
81
import static org .hamcrest .Matchers .both ;
78
82
import static org .hamcrest .Matchers .equalTo ;
86
90
import static org .mockito .Mockito .mock ;
87
91
88
92
public class FiltersAggregatorTests extends AggregatorTestCase {
89
- private MappedFieldType fieldType ;
90
-
91
- @ Before
92
- public void setUpTest () throws Exception {
93
- super .setUp ();
94
- fieldType = new KeywordFieldMapper .KeywordFieldType ("field" );
95
- }
96
-
97
93
public void testEmpty () throws Exception {
98
94
Directory directory = newDirectory ();
99
95
RandomIndexWriter indexWriter = new RandomIndexWriter (random (), directory );
@@ -107,7 +103,12 @@ public void testEmpty() throws Exception {
107
103
}
108
104
FiltersAggregationBuilder builder = new FiltersAggregationBuilder ("test" , filters );
109
105
builder .otherBucketKey ("other" );
110
- InternalFilters response = searchAndReduce (indexSearcher , new MatchAllDocsQuery (), builder , fieldType );
106
+ InternalFilters response = searchAndReduce (
107
+ indexSearcher ,
108
+ new MatchAllDocsQuery (),
109
+ builder ,
110
+ new KeywordFieldMapper .KeywordFieldType ("field" )
111
+ );
111
112
assertEquals (response .getBuckets ().size (), numFilters );
112
113
for (InternalFilters .InternalBucket filter : response .getBuckets ()) {
113
114
assertEquals (filter .getDocCount (), 0 );
@@ -207,7 +208,12 @@ public void testKeyedFilter() throws Exception {
207
208
FiltersAggregationBuilder builder = new FiltersAggregationBuilder ("test" , keys );
208
209
builder .otherBucket (true );
209
210
builder .otherBucketKey ("other" );
210
- final InternalFilters filters = searchAndReduce (indexSearcher , new MatchAllDocsQuery (), builder , fieldType );
211
+ final InternalFilters filters = searchAndReduce (
212
+ indexSearcher ,
213
+ new MatchAllDocsQuery (),
214
+ builder ,
215
+ new KeywordFieldMapper .KeywordFieldType ("field" )
216
+ );
211
217
assertEquals (filters .getBuckets ().size (), 7 );
212
218
assertEquals (filters .getBucketByKey ("foobar" ).getDocCount (), 2 );
213
219
assertEquals (filters .getBucketByKey ("foo" ).getDocCount (), 2 );
@@ -262,7 +268,12 @@ public void testRandom() throws Exception {
262
268
builder .otherBucket (true );
263
269
builder .otherBucketKey ("other" );
264
270
265
- final InternalFilters response = searchAndReduce (indexSearcher , new MatchAllDocsQuery (), builder , fieldType );
271
+ final InternalFilters response = searchAndReduce (
272
+ indexSearcher ,
273
+ new MatchAllDocsQuery (),
274
+ builder ,
275
+ new KeywordFieldMapper .KeywordFieldType ("field" )
276
+ );
266
277
List <InternalFilters .InternalBucket > buckets = response .getBuckets ();
267
278
assertEquals (buckets .size (), filters .length + 1 );
268
279
@@ -822,6 +833,167 @@ public void testSubAggsManyFilters() throws IOException {
822
833
}, dateFt , intFt );
823
834
}
824
835
836
+ public void testDocValuesFieldExistsForDate () throws IOException {
837
+ DateFieldMapper .DateFieldType ft = new DateFieldMapper .DateFieldType ("f" );
838
+ QueryBuilder exists ;
839
+ if (randomBoolean ()) {
840
+ exists = new ExistsQueryBuilder ("f" );
841
+ } else {
842
+ // Range query covering all values in the index is rewritten to exists
843
+ exists = new RangeQueryBuilder ("f" ).gte ("2020-01-01" ).lt ("2020-01-02" );
844
+ }
845
+ long start = DateFieldMapper .DEFAULT_DATE_TIME_FORMATTER .parseMillis ("2020-01-01T00:00:01" );
846
+ docValuesFieldExistsTestCase (exists , ft , true , i -> {
847
+ long date = start + TimeUnit .HOURS .toMillis (i );
848
+ return org .elasticsearch .common .collect .List .of (new LongPoint ("f" , date ), new NumericDocValuesField ("f" , date ));
849
+ });
850
+ }
851
+
852
+ public void testDocValuesFieldExistsForDateWithMultiValuedFields () throws IOException {
853
+ DateFieldMapper .DateFieldType ft = new DateFieldMapper .DateFieldType ("f" );
854
+ long start = DateFieldMapper .DEFAULT_DATE_TIME_FORMATTER .parseMillis ("2020-01-01T00:00:01" );
855
+ docValuesFieldExistsTestCase (new ExistsQueryBuilder ("f" ), ft , true , i -> {
856
+ long date = start + TimeUnit .HOURS .toMillis (i );
857
+ return org .elasticsearch .common .collect .List .of (
858
+ new LongPoint ("f" , date ),
859
+ new LongPoint ("f" , date + 10 ),
860
+ new SortedNumericDocValuesField ("f" , date ),
861
+ new SortedNumericDocValuesField ("f" , date + 10 )
862
+ );
863
+ });
864
+ }
865
+
866
+ public void testDocValuesFieldExistsForDateWithoutData () throws IOException {
867
+ docValuesFieldExistsNoDataTestCase (new DateFieldMapper .DateFieldType ("f" ));
868
+ }
869
+
870
+ public void testDocValuesFieldExistsForNumber () throws IOException {
871
+ NumberFieldMapper .NumberType numberType = randomFrom (NumberFieldMapper .NumberType .values ());
872
+ NumberFieldMapper .NumberFieldType ft = new NumberFieldMapper .NumberFieldType (
873
+ "f" ,
874
+ numberType ,
875
+ true ,
876
+ false ,
877
+ true ,
878
+ true ,
879
+ null ,
880
+ org .elasticsearch .common .collect .Map .of (),
881
+ null
882
+ );
883
+ docValuesFieldExistsTestCase (new ExistsQueryBuilder ("f" ), ft , true , i -> {
884
+ return numberType .createFields ("f" , i , true , true , false );
885
+ });
886
+ }
887
+
888
+ public void testDocValuesFieldExistsForNumberWithoutData () throws IOException {
889
+ docValuesFieldExistsNoDataTestCase (new NumberFieldMapper .NumberFieldType (
890
+ "f" ,
891
+ randomFrom (NumberFieldMapper .NumberType .values ()),
892
+ true ,
893
+ false ,
894
+ true ,
895
+ true ,
896
+ null ,
897
+ org .elasticsearch .common .collect .Map .of (),
898
+ null
899
+ ));
900
+ }
901
+
902
+ public void testDocValuesFieldExistsForKeyword () throws IOException {
903
+ KeywordFieldMapper .KeywordFieldType ft = new KeywordFieldMapper .KeywordFieldType (
904
+ "f" ,
905
+ true ,
906
+ true ,
907
+ org .elasticsearch .common .collect .Map .of ()
908
+ );
909
+ docValuesFieldExistsTestCase (new ExistsQueryBuilder ("f" ), ft , false , i -> {
910
+ BytesRef text = new BytesRef (randomAlphaOfLength (5 ));
911
+ return org .elasticsearch .common .collect .List .of (
912
+ new Field ("f" , text , KeywordFieldMapper .Defaults .FIELD_TYPE ),
913
+ new SortedSetDocValuesField ("f" , text )
914
+ );
915
+ });
916
+ }
917
+
918
+ public void testDocValuesFieldExistsForKeywordWithoutData () throws IOException {
919
+ docValuesFieldExistsNoDataTestCase (
920
+ new KeywordFieldMapper .KeywordFieldType ("f" , true , true , org .elasticsearch .common .collect .Map .of ())
921
+ );
922
+ }
923
+
924
+ private void docValuesFieldExistsTestCase (
925
+ QueryBuilder exists ,
926
+ MappedFieldType fieldType ,
927
+ boolean canUseMetadata ,
928
+ IntFunction <List <? extends IndexableField >> buildDocWithField
929
+ ) throws IOException {
930
+ AggregationBuilder builder = new FiltersAggregationBuilder ("test" , new KeyedFilter ("q1" , exists ));
931
+ CheckedConsumer <RandomIndexWriter , IOException > buildIndex = iw -> {
932
+ for (int i = 0 ; i < 10 ; i ++) {
933
+ iw .addDocument (buildDocWithField .apply (i ));
934
+ }
935
+ for (int i = 0 ; i < 10 ; i ++) {
936
+ iw .addDocument (org .elasticsearch .common .collect .List .of ());
937
+ }
938
+ };
939
+ // Exists queries convert to MatchNone if this isn't defined
940
+ FieldNamesFieldMapper .FieldNamesFieldType fnft = new FieldNamesFieldMapper .FieldNamesFieldType (true );
941
+ debugTestCase (
942
+ builder ,
943
+ new MatchAllDocsQuery (),
944
+ buildIndex ,
945
+ (InternalFilters result , Class <? extends Aggregator > impl , Map <String , Map <String , Object >> debug ) -> {
946
+ assertThat (result .getBuckets (), hasSize (1 ));
947
+ assertThat (result .getBucketByKey ("q1" ).getDocCount (), equalTo (10L ));
948
+
949
+ assertThat (impl , equalTo (FiltersAggregator .FilterByFilter .class ));
950
+ Map <?, ?> filterAggDebug = debug .get ("test" );
951
+ List <?> filtersDebug = (List <?>) filterAggDebug .get ("filters" );
952
+ Map <?, ?> filterDebug = (Map <?, ?>) filtersDebug .get (0 );
953
+ assertThat (filterDebug , hasEntry ("specialized_for" , "docvalues_field_exists" ));
954
+ assertThat ((int ) filterDebug .get ("results_from_metadata" ), canUseMetadata ? greaterThan (0 ) : equalTo (0 ));
955
+ },
956
+ fieldType ,
957
+ fnft
958
+ );
959
+ withAggregator (builder , new MatchAllDocsQuery (), buildIndex , (searcher , aggregator ) -> {
960
+ long estimatedCost = ((FiltersAggregator .FilterByFilter ) aggregator ).estimateCost (Long .MAX_VALUE );
961
+ Map <String , Object > debug = new HashMap <>();
962
+ aggregator .collectDebugInfo (debug ::put );
963
+ List <?> filtersDebug = (List <?>) debug .get ("filters" );
964
+ Map <?, ?> filterDebug = (Map <?, ?>) filtersDebug .get (0 );
965
+ assertThat (estimatedCost , canUseMetadata ? equalTo (0L ) : greaterThan (0L ));
966
+ assertThat ((int ) filterDebug .get ("scorers_prepared_while_estimating_cost" ), canUseMetadata ? equalTo (0 ) : greaterThan (0 ));
967
+ }, fieldType , fnft );
968
+ }
969
+
970
+ private void docValuesFieldExistsNoDataTestCase (
971
+ MappedFieldType fieldType
972
+ ) throws IOException {
973
+ QueryBuilder exists = new ExistsQueryBuilder (fieldType .name ());
974
+ AggregationBuilder builder = new FiltersAggregationBuilder ("test" , new KeyedFilter ("q1" , exists ));
975
+ CheckedConsumer <RandomIndexWriter , IOException > buildIndex = iw -> {
976
+ for (int i = 0 ; i < 10 ; i ++) {
977
+ iw .addDocument (org .elasticsearch .common .collect .List .of ());
978
+ }
979
+ };
980
+ // Exists queries convert to MatchNone if this isn't defined
981
+ FieldNamesFieldMapper .FieldNamesFieldType fnft = new FieldNamesFieldMapper .FieldNamesFieldType (true );
982
+ withAggregator (builder , new MatchAllDocsQuery (), buildIndex , (searcher , aggregator ) -> {
983
+ assertThat (aggregator , instanceOf (FiltersAggregator .FilterByFilter .class ));
984
+ long estimatedCost = ((FiltersAggregator .FilterByFilter ) aggregator ).estimateCost (Long .MAX_VALUE );
985
+ Map <String , Object > debug = collectAndGetFilterDebugInfo (searcher , aggregator );
986
+ assertThat (debug , hasEntry ("specialized_for" , "docvalues_field_exists" ));
987
+ assertThat (estimatedCost , equalTo (0L ));
988
+ assertThat ((int ) debug .get ("results_from_metadata" ), greaterThan (0 ));
989
+ assertThat ((int ) debug .get ("scorers_prepared_while_estimating_cost" ), equalTo (0 ));
990
+ }, fieldType , fnft );
991
+ testCase (builder , new MatchAllDocsQuery (), buildIndex , (InternalFilters result ) -> {
992
+ assertThat (result .getBuckets (), hasSize (1 ));
993
+ assertThat (result .getBucketByKey ("q1" ).getDocCount (), equalTo (0L ));
994
+ }, fieldType , fnft );
995
+ }
996
+
825
997
@ Override
826
998
protected List <ObjectMapper > objectMappers () {
827
999
return MOCK_OBJECT_MAPPERS ;
0 commit comments