Handle missing and multiple values in script

docs/CHANGELOG.asciidoc

@@ -41,6 +41,10 @@ written to by an older Elasticsearch after writing to it with a newer Elasticsea
 
 Do not ignore request analysis/similarity settings on index resize operations when the source index already contains such settings ({pull}30216[#30216])
 
+=== Deprecations
+
+Returning 0 for missing numeric fields in script is deprecated. PR: 
+
 === Regressions
 
 === Known Issues

docs/painless/painless-getting-started.asciidoc

@@ -68,6 +68,7 @@ GET hockey/_search
 ----------------------------------------------------------------
 // CONSOLE
 
+
 Alternatively, you could do the same thing using a script field instead of a function score:
 
 [source,js]
@@ -119,6 +120,31 @@ GET hockey/_search
 ----------------------------------------------------------------
 // CONSOLE
 
+[float]
+===== Missing values
+
+Currently by default, if a document is missing a numeric field `field`,
+`doc['field'].value` returns `0` for this document. This default behaviour
+will be changed in the next major version of elasticsearch. Starting from 7.0,
+if a document is missing a field `field`, `doc['field']` for this document
+will return `null`. From 6.4 version, you can set a system property
+`export ES_JAVA_OPTS="-Des.script.null_for_missing_value=true"' on a node
+to make this node's behaviour compatible with the future major version.
+Otherwise, every time the node starts a deprecation warning will remind you
+about this forthcoming change in 7.x.
+
+
+===== Multiple values
+
+There is a number of operations designed for numeric fields,
+if a document has multiple values in such a field:
+
+- `doc['field'].min` - gets the minimum value among values
+- `doc['field'].max` - gets the maximum value among values
+- `doc['field'].sum` - gets the sum of all values
+- `doc['field'].avg` - gets the average of all values
+
+
 [float]
 ==== Updating Fields with Painless
 

modules/lang-painless/build.gradle

@@ -25,6 +25,7 @@ esplugin {
 }
 
 integTestCluster {
+  systemProperty 'es.script.null_for_missing_value', 'true'
   module project.project(':modules:mapper-extras')
 }
 

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt

@@ -73,6 +73,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Strings {
 class org.elasticsearch.index.fielddata.ScriptDocValues$Longs {
   Long get(int)
   long getValue()
+  long getMin()
+  long getMax()
+  long getSum()
+  double getAvg()
   List getValues()
   org.joda.time.ReadableDateTime getDate()
   List getDates()
@@ -89,6 +93,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Dates {
 class org.elasticsearch.index.fielddata.ScriptDocValues$Doubles {
   Double get(int)
   double getValue()
+  double getMin()
+  double getMax()
+  double getSum()
+  double getAvg()
   List getValues()
 }
 

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml

@@ -95,7 +95,7 @@ setup:
                 script_fields:
                     bar:
                         script:
-                            source: "(doc['missing'].value?.length() ?: 0) + params.x;"
+                            source: "(doc['missing']?.value?.length() ?: 0) + params.x;"
                             params:
                                 x: 5
 

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_missing_and_multiple_values.yml

@@ -0,0 +1,117 @@
+setup:
+    - do:
+        indices.create:
+            index: test
+            body:
+                settings:
+                    number_of_shards: 1
+                mappings:
+                    _doc:
+                        properties:
+                            dval:
+                                type: double
+                            lval:
+                                type: long
+
+    - do:
+        index:
+            index:  test
+            type:   _doc
+            id:     1
+            body:   { "dval": 5.5, "lval": 5 }
+
+    - do:
+        index:
+            index:  test
+            type:   _doc
+            id:     2
+            body:   { "dval": [5.5, 3.5, 4.5] }
+
+
+    - do:
+        index:
+            index:  test
+            type:   _doc
+            id:     3
+            body:   { "lval": [5, 3, 4] }
+
+    - do:
+        indices.refresh: {}
+
+---
+"check double and long values: missing values and operations on multiple values":
+    - skip:
+        version: " - 6.3.99"
+        reason: Handling missing values and operations on multiple values were added from 6.4
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    val_dval:
+                        script:
+                            source: "doc['dval']?.value ?: -1.0"
+                    min_dval:
+                        script:
+                            source: "doc['dval']?.min ?: -1.0"
+                    max_dval:
+                        script:
+                            source: "doc['dval']?.max ?: -1.0"
+                    sum_dval:
+                        script:
+                            source: "doc['dval']?.sum ?: -1.0"
+                    avg_dval:
+                        script:
+                            source: "doc['dval']?.avg ?: -1.0"
+                    val_lval:
+                        script:
+                            source: "doc['lval']?.value ?: -1"
+                    min_lval:
+                        script:
+                            source: "doc['lval']?.min ?: -1"
+                    max_lval:
+                        script:
+                            source: "doc['lval']?.max ?: -1"
+                    sum_lval:
+                        script:
+                            source: "doc['lval']?.sum ?: -1"
+                    avg_lval:
+                        script:
+                            source: "doc['lval']?.avg ?: -1"
+
+    - match: { hits.hits.0.fields.val_dval.0: 5.5}
+    - match: { hits.hits.0.fields.min_dval.0: 5.5}
+    - match: { hits.hits.0.fields.max_dval.0: 5.5}
+    - match: { hits.hits.0.fields.sum_dval.0: 5.5}
+    - match: { hits.hits.0.fields.avg_dval.0: 5.5}
+
+    - match: { hits.hits.0.fields.val_lval.0: 5}
+    - match: { hits.hits.0.fields.min_lval.0: 5}
+    - match: { hits.hits.0.fields.max_lval.0: 5}
+    - match: { hits.hits.0.fields.sum_lval.0: 5}
+    - match: { hits.hits.0.fields.avg_lval.0: 5}
+
+    - match: { hits.hits.1.fields.val_dval.0: 3.5}
+    - match: { hits.hits.1.fields.min_dval.0: 3.5}
+    - match: { hits.hits.1.fields.max_dval.0: 5.5}
+    - match: { hits.hits.1.fields.sum_dval.0: 13.5}
+    - match: { hits.hits.1.fields.avg_dval.0: 4.5}
+
+    - match: { hits.hits.1.fields.val_lval.0: -1}
+    - match: { hits.hits.1.fields.min_lval.0: -1}
+    - match: { hits.hits.1.fields.max_lval.0: -1}
+    - match: { hits.hits.1.fields.sum_lval.0: -1}
+    - match: { hits.hits.1.fields.avg_lval.0: -1}
+
+    - match: { hits.hits.2.fields.val_dval.0: -1.0}
+    - match: { hits.hits.2.fields.min_dval.0: -1.0}
+    - match: { hits.hits.2.fields.max_dval.0: -1.0}
+    - match: { hits.hits.2.fields.sum_dval.0: -1.0}
+    - match: { hits.hits.2.fields.avg_dval.0: -1.0}
+
+    - match: { hits.hits.2.fields.val_lval.0: 3}
+    - match: { hits.hits.2.fields.min_lval.0: 3}
+    - match: { hits.hits.2.fields.max_lval.0: 5}
+    - match: { hits.hits.2.fields.sum_lval.0: 12}
+    - match: { hits.hits.2.fields.avg_lval.0: 4}
+