Skip to content

Commit 5aeab57

Browse files
mayya-sharipovamayya-sharipova
committed
Handle missing and multiple values in script
Previously in script for numeric fields, there was no way to check if a document is missing a value. Also certain operations on multiple- values fields were missing. This PR adds the following: - return null for doc['field'] if a document is missing a 'field1': Now we can do this: if (doc['field'] == null) {return -1;} return doc['field'].value; or doc['field']?.value ?: -1 - add the following functions for multiple-valued numeric fields: doc['field'].min returns the minumum amoung values doc['field'].max returns the maximum amoung values doc['field'].sum returns the sum of amoung values doc['field'].avg returns the average of values Closes elastic#29286
1 parent e2d770d commit 5aeab57

File tree

11 files changed

+277
-24
lines changed

11 files changed

+277
-24
lines changed

‎docs/painless/painless-getting-started.asciidoc

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ GET hockey/_search
6868
----------------------------------------------------------------
6969
// CONSOLE
7070

71+
7172
Alternatively, you could do the same thing using a script field instead of a function score:
7273

7374
[source,js]
@@ -119,6 +120,21 @@ GET hockey/_search
119120
----------------------------------------------------------------
120121
// CONSOLE
121122

123+
[float]
124+
===== Missing and multiple values
125+
126+
If a document is missing a field `field`, `doc['field']` for this document
127+
will return null.
128+
129+
There is also a number of operations designed for numeric fields,
130+
if a document has multiple values in such a field:
131+
132+
- `doc['field'].min` - gets the minimum value among values
133+
- `doc['field'].max` - gets the maximum value among values
134+
- `doc['field'].sum` - gets the sum of all values
135+
- `doc['field'].avg` - gets the average of all values
136+
137+
122138
[float]
123139
==== Updating Fields with Painless
124140

‎modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Strings {
7373
class org.elasticsearch.index.fielddata.ScriptDocValues$Longs {
7474
Long get(int)
7575
long getValue()
76+
long getMin()
77+
long getMax()
78+
long getSum()
79+
double getAvg()
7680
List getValues()
7781
org.joda.time.ReadableDateTime getDate()
7882
List getDates()
@@ -89,6 +93,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Dates {
8993
class org.elasticsearch.index.fielddata.ScriptDocValues$Doubles {
9094
Double get(int)
9195
double getValue()
96+
double getMin()
97+
double getMax()
98+
double getSum()
99+
double getAvg()
92100
List getValues()
93101
}
94102

‎modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ setup:
9595
script_fields:
9696
bar:
9797
script:
98-
source: "(doc['missing'].value?.length() ?: 0) + params.x;"
98+
source: "(doc['missing']?.value?.length() ?: 0) + params.x;"
9999
params:
100100
x: 5
101101

‎modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_missing_and_multiple_values.yml

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
setup:
2+
- do:
3+
indices.create:
4+
index: test
5+
body:
6+
settings:
7+
number_of_shards: 1
8+
mappings:
9+
_doc:
10+
properties:
11+
dval:
12+
type: double
13+
lval:
14+
type: long
15+
16+
- do:
17+
index:
18+
index: test
19+
type: _doc
20+
id: 1
21+
body: { "dval": 5.5, "lval": 5 }
22+
23+
- do:
24+
index:
25+
index: test
26+
type: _doc
27+
id: 2
28+
body: { "dval": [5.5, 3.5, 4.5] }
29+
30+
31+
- do:
32+
index:
33+
index: test
34+
type: _doc
35+
id: 3
36+
body: { "lval": [5, 3, 4] }
37+
38+
- do:
39+
indices.refresh: {}
40+
41+
---
42+
"check double and long values: missing values and operations on multiple values":
43+
- skip:
44+
version: " - 6.99.99"
45+
reason: Handling missing values and operations on multiple values were added after these versions
46+
47+
- do:
48+
search:
49+
body:
50+
script_fields:
51+
val_dval:
52+
script:
53+
source: "doc['dval']?.value ?: -1.0"
54+
min_dval:
55+
script:
56+
source: "doc['dval']?.min ?: -1.0"
57+
max_dval:
58+
script:
59+
source: "doc['dval']?.max ?: -1.0"
60+
sum_dval:
61+
script:
62+
source: "doc['dval']?.sum ?: -1.0"
63+
avg_dval:
64+
script:
65+
source: "doc['dval']?.avg ?: -1.0"
66+
val_lval:
67+
script:
68+
source: "doc['lval']?.value ?: -1"
69+
min_lval:
70+
script:
71+
source: "doc['lval']?.min ?: -1"
72+
max_lval:
73+
script:
74+
source: "doc['lval']?.max ?: -1"
75+
sum_lval:
76+
script:
77+
source: "doc['lval']?.sum ?: -1"
78+
avg_lval:
79+
script:
80+
source: "doc['lval']?.avg ?: -1"
81+
82+
- match: { hits.hits.0.fields.val_dval.0: 5.5}
83+
- match: { hits.hits.0.fields.min_dval.0: 5.5}
84+
- match: { hits.hits.0.fields.max_dval.0: 5.5}
85+
- match: { hits.hits.0.fields.sum_dval.0: 5.5}
86+
- match: { hits.hits.0.fields.avg_dval.0: 5.5}
87+
88+
- match: { hits.hits.0.fields.val_lval.0: 5}
89+
- match: { hits.hits.0.fields.min_lval.0: 5}
90+
- match: { hits.hits.0.fields.max_lval.0: 5}
91+
- match: { hits.hits.0.fields.sum_lval.0: 5}
92+
- match: { hits.hits.0.fields.avg_lval.0: 5}
93+
94+
- match: { hits.hits.1.fields.val_dval.0: 3.5}
95+
- match: { hits.hits.1.fields.min_dval.0: 3.5}
96+
- match: { hits.hits.1.fields.max_dval.0: 5.5}
97+
- match: { hits.hits.1.fields.sum_dval.0: 13.5}
98+
- match: { hits.hits.1.fields.avg_dval.0: 4.5}
99+
100+
- match: { hits.hits.1.fields.val_lval.0: -1}
101+
- match: { hits.hits.1.fields.min_lval.0: -1}
102+
- match: { hits.hits.1.fields.max_lval.0: -1}
103+
- match: { hits.hits.1.fields.sum_lval.0: -1}
104+
- match: { hits.hits.1.fields.avg_lval.0: -1}
105+
106+
- match: { hits.hits.2.fields.val_dval.0: -1.0}
107+
- match: { hits.hits.2.fields.min_dval.0: -1.0}
108+
- match: { hits.hits.2.fields.max_dval.0: -1.0}
109+
- match: { hits.hits.2.fields.sum_dval.0: -1.0}
110+
- match: { hits.hits.2.fields.avg_dval.0: -1.0}
111+
112+
- match: { hits.hits.2.fields.val_lval.0: 3}
113+
- match: { hits.hits.2.fields.min_lval.0: 3}
114+
- match: { hits.hits.2.fields.max_lval.0: 5}
115+
- match: { hits.hits.2.fields.sum_lval.0: 12}
116+
- match: { hits.hits.2.fields.avg_lval.0: 4}
117+

‎server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java

Lines changed: 88 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
5858
/**
5959
* Set the current doc ID.
6060
*/
61-
public abstract void setNextDocId(int docId) throws IOException;
61+
public abstract boolean setNextDocId(int docId) throws IOException;
6262

6363
/**
6464
* Return a copy of the list of the values for the current document.
@@ -124,9 +124,10 @@ public Longs(SortedNumericDocValues in) {
124124
}
125125

126126
@Override
127-
public void setNextDocId(int docId) throws IOException {
127+
public boolean setNextDocId(int docId) throws IOException {
128128
this.docId = docId;
129-
if (in.advanceExact(docId)) {
129+
boolean docHasValues = in.advanceExact(docId);
130+
if (docHasValues) {
130131
resize(in.docValueCount());
131132
for (int i = 0; i < count; i++) {
132133
values[i] = in.nextValue();
@@ -137,6 +138,7 @@ public void setNextDocId(int docId) throws IOException {
137138
if (dates != null) {
138139
dates.setNextDocId(docId);
139140
}
141+
return docHasValues;
140142
}
141143

142144
/**
@@ -159,6 +161,37 @@ public long getValue() {
159161
return values[0];
160162
}
161163

164+
public long getMin() {
165+
if (count == 0) {
166+
return 0L;
167+
};
168+
return values[0];
169+
}
170+
171+
public long getMax() {
172+
if (count == 0) {
173+
return 0L;
174+
};
175+
return values[count - 1];
176+
}
177+
178+
public long getSum() {
179+
if (count == 0) {
180+
return 0L;
181+
};
182+
long sum = 0L;
183+
for (int i = 0; i < count; i++)
184+
sum += values[i];
185+
return sum;
186+
}
187+
188+
public double getAvg() {
189+
if (count == 0) {
190+
return 0d;
191+
};
192+
return getSum() * 1.0/count;
193+
}
194+
162195
@Deprecated
163196
public ReadableDateTime getDate() throws IOException {
164197
deprecated("getDate on numeric fields is deprecated. Use a date field to get dates.");
@@ -285,13 +318,15 @@ public int size() {
285318
}
286319

287320
@Override
288-
public void setNextDocId(int docId) throws IOException {
289-
if (in.advanceExact(docId)) {
321+
public boolean setNextDocId(int docId) throws IOException {
322+
boolean docHasValues = in.advanceExact(docId);
323+
if (docHasValues) {
290324
count = in.docValueCount();
291325
} else {
292326
count = 0;
293327
}
294328
refreshArray();
329+
return docHasValues;
295330
}
296331

297332
/**
@@ -355,15 +390,17 @@ public Doubles(SortedNumericDoubleValues in) {
355390
}
356391

357392
@Override
358-
public void setNextDocId(int docId) throws IOException {
359-
if (in.advanceExact(docId)) {
393+
public boolean setNextDocId(int docId) throws IOException {
394+
boolean docHasValues = in.advanceExact(docId);
395+
if (docHasValues) {
360396
resize(in.docValueCount());
361397
for (int i = 0; i < count; i++) {
362398
values[i] = in.nextValue();
363399
}
364400
} else {
365401
resize(0);
366402
}
403+
return docHasValues;
367404
}
368405

369406
/**
@@ -386,6 +423,38 @@ public double getValue() {
386423
return values[0];
387424
}
388425

426+
public double getMin() {
427+
if (count == 0) {
428+
return 0d;
429+
};
430+
return values[0];
431+
}
432+
433+
public double getMax() {
434+
if (count == 0) {
435+
return 0d;
436+
};
437+
return values[count - 1];
438+
}
439+
440+
public double getSum() {
441+
if (count == 0) {
442+
return 0d;
443+
};
444+
double sum = 0d;
445+
for (int i = 0; i < count; i++)
446+
sum += values[i];
447+
return sum;
448+
}
449+
450+
public double getAvg() {
451+
if (count == 0) {
452+
return 0d;
453+
};
454+
return getSum() / count;
455+
}
456+
457+
389458
@Override
390459
public Double get(int index) {
391460
return values[index];
@@ -408,8 +477,9 @@ public GeoPoints(MultiGeoPointValues in) {
408477
}
409478

410479
@Override
411-
public void setNextDocId(int docId) throws IOException {
412-
if (in.advanceExact(docId)) {
480+
public boolean setNextDocId(int docId) throws IOException {
481+
boolean docHasValues = in.advanceExact(docId);
482+
if (docHasValues) {
413483
resize(in.docValueCount());
414484
for (int i = 0; i < count; i++) {
415485
GeoPoint point = in.nextValue();
@@ -418,6 +488,7 @@ public void setNextDocId(int docId) throws IOException {
418488
} else {
419489
resize(0);
420490
}
491+
return docHasValues;
421492
}
422493

423494
/**
@@ -528,15 +599,17 @@ public Booleans(SortedNumericDocValues in) {
528599
}
529600

530601
@Override
531-
public void setNextDocId(int docId) throws IOException {
532-
if (in.advanceExact(docId)) {
602+
public boolean setNextDocId(int docId) throws IOException {
603+
boolean docHasValues = in.advanceExact(docId);
604+
if (docHasValues) {
533605
resize(in.docValueCount());
534606
for (int i = 0; i < count; i++) {
535607
values[i] = in.nextValue() == 1;
536608
}
537609
} else {
538610
resize(0);
539611
}
612+
return docHasValues;
540613
}
541614

542615
/**
@@ -584,8 +657,9 @@ abstract static class BinaryScriptDocValues<T> extends ScriptDocValues<T> {
584657
}
585658

586659
@Override
587-
public void setNextDocId(int docId) throws IOException {
588-
if (in.advanceExact(docId)) {
660+
public boolean setNextDocId(int docId) throws IOException {
661+
boolean docHasValues = in.advanceExact(docId);
662+
if (docHasValues) {
589663
resize(in.docValueCount());
590664
for (int i = 0; i < count; i++) {
591665
// We need to make a copy here, because BytesBinaryDVAtomicFieldData's SortedBinaryDocValues
@@ -596,6 +670,7 @@ public void setNextDocId(int docId) throws IOException {
596670
} else {
597671
resize(0);
598672
}
673+
return docHasValues;
599674
}
600675

601676
/**

0 commit comments

Comments
 (0)