Skip to content

Commit bcf7669

Browse files
romseygeekromseygeek
committed
Add index-time scripts to geo_point field mapper (#71861)
This commit adds the ability to define an index-time geo_point field with a script parameter, allowing you to calculate points from other values within the indexed document.
1 parent 1776a4f commit bcf7669

File tree

11 files changed

+503
-35
lines changed

11 files changed

+503
-35
lines changed

‎docs/reference/mapping/types/geo-point.asciidoc

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,20 +123,43 @@ The following parameters are accepted by `geo_point` fields:
123123

124124
If `true`, malformed geo-points are ignored. If `false` (default),
125125
malformed geo-points throw an exception and reject the whole document.
126-
A geo-point is considered malformed if its latitude is outside the range
126+
A geo-point is considered malformed if its latitude is outside the range
127127
-90 <= latitude <= 90, or if its longitude is outside the range -180 <= longitude <= 180.
128+
Note that this cannot be set if the `script` parameter is used.
128129

129130
`ignore_z_value`::
130131

131132
If `true` (default) three dimension points will be accepted (stored in source)
132133
but only latitude and longitude values will be indexed; the third dimension is
133134
ignored. If `false`, geo-points containing any more than latitude and longitude
134-
(two dimensions) values throw an exception and reject the whole document.
135+
(two dimensions) values throw an exception and reject the whole document. Note
136+
that this cannot be set if the `script` parameter is used.
135137

136138
<<null-value,`null_value`>>::
137139

138140
Accepts an geopoint value which is substituted for any explicit `null` values.
139-
Defaults to `null`, which means the field is treated as missing.
141+
Defaults to `null`, which means the field is treated as missing. Note that this
142+
cannot be set if the `script` parameter is used.
143+
144+
`on_script_error`::
145+
146+
Defines what to do if the script defined by the `script` parameter
147+
throws an error at indexing time. Accepts `fail` (default), which
148+
will cause the entire document to be rejected, and `continue`, which
149+
will register the field in the document's
150+
<<mapping-ignored-field,`_ignored`>> metadata field and continue
151+
indexing. This parameter can only be set if the `script` field is
152+
also set.
153+
154+
`script`::
155+
156+
If this parameter is set, then the field will index values generated
157+
by this script, rather than reading the values directly from the
158+
source. If a value is set for this field on the input document, then
159+
the document will be rejected with an error.
160+
Scripts are in the same format as their
161+
<<runtime-mapping-fields,runtime equivalent>>, and should emit points
162+
as a pair of (lat, lon) double values.
140163

141164
==== Using geo-points in scripts
142165

‎modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
---
2+
setup:
3+
- do:
4+
indices.create:
5+
index: locations
6+
body:
7+
settings:
8+
number_of_shards: 1
9+
number_of_replicas: 0
10+
mappings:
11+
properties:
12+
location_from_doc_value:
13+
type: geo_point
14+
script:
15+
source: |
16+
emit(doc["location"].lat, doc["location"].lon);
17+
location_from_source:
18+
type: geo_point
19+
script:
20+
source: |
21+
emit(params._source.location.lat, params._source.location.lon);
22+
timestamp:
23+
type: date
24+
location:
25+
type: geo_point
26+
- do:
27+
bulk:
28+
index: locations
29+
refresh: true
30+
body: |
31+
{"index":{}}
32+
{"timestamp": "1998-04-30T14:30:17-05:00", "location" : {"lat": 13.5, "lon" : 34.89}}
33+
{"index":{}}
34+
{"timestamp": "1998-04-30T14:30:53-05:00", "location" : {"lat": -7.9, "lon" : 120.78}}
35+
{"index":{}}
36+
{"timestamp": "1998-04-30T14:31:12-05:00", "location" : {"lat": 45.78, "lon" : -173.45}}
37+
{"index":{}}
38+
{"timestamp": "1998-04-30T14:31:19-05:00", "location" : {"lat": 32.45, "lon" : 45.6}}
39+
{"index":{}}
40+
{"timestamp": "1998-04-30T14:31:22-05:00", "location" : {"lat": -63.24, "lon" : 31.0}}
41+
{"index":{}}
42+
{"timestamp": "1998-04-30T14:31:27-05:00", "location" : {"lat": 0.0, "lon" : 0.0}}
43+
44+
45+
---
46+
"get mapping":
47+
- do:
48+
indices.get_mapping:
49+
index: locations
50+
- match: {locations.mappings.properties.location_from_source.type: geo_point }
51+
- match:
52+
locations.mappings.properties.location_from_source.script.source: |
53+
emit(params._source.location.lat, params._source.location.lon);
54+
- match: {locations.mappings.properties.location_from_source.script.lang: painless }
55+
56+
---
57+
"fetch fields from source":
58+
- do:
59+
search:
60+
index: locations
61+
body:
62+
sort: timestamp
63+
fields: [location, location_from_doc_value, location_from_source]
64+
- match: {hits.total.value: 6}
65+
- match: {hits.hits.0.fields.location.0.type: "Point" }
66+
- match: {hits.hits.0.fields.location.0.coordinates: [34.89, 13.5] }
67+
# calculated from scripts adds annoying extra precision
68+
- match: { hits.hits.0.fields.location_from_doc_value.0.type: "Point" }
69+
- match: { hits.hits.0.fields.location_from_doc_value.0.coordinates: [ 34.889999935403466, 13.499999991618097 ] }
70+
- match: { hits.hits.0.fields.location_from_source.0.type: "Point" }
71+
- match: { hits.hits.0.fields.location_from_source.0.coordinates: [ 34.889999935403466, 13.499999991618097 ] }
72+
73+
---
74+
"exists query":
75+
- do:
76+
search:
77+
index: locations
78+
body:
79+
query:
80+
exists:
81+
field: location_from_source
82+
- match: {hits.total.value: 6}
83+
84+
---
85+
"geo bounding box query":
86+
- do:
87+
search:
88+
index: locations
89+
body:
90+
query:
91+
geo_bounding_box:
92+
location_from_source:
93+
top_left:
94+
lat: 10
95+
lon: -10
96+
bottom_right:
97+
lat: -10
98+
lon: 10
99+
- match: {hits.total.value: 1}
100+
101+
---
102+
"geo shape query":
103+
- do:
104+
search:
105+
index: locations
106+
body:
107+
query:
108+
geo_shape:
109+
location_from_source:
110+
shape:
111+
type: "envelope"
112+
coordinates: [ [ -10, 10 ], [ 10, -10 ] ]
113+
- match: {hits.total.value: 1}
114+
115+
---
116+
"geo distance query":
117+
- do:
118+
search:
119+
index: locations
120+
body:
121+
query:
122+
geo_distance:
123+
distance: "2000km"
124+
location_from_source:
125+
lat: 0
126+
lon: 0
127+
- match: {hits.total.value: 1}
128+
129+
---
130+
"bounds agg":
131+
- do:
132+
search:
133+
index: locations
134+
body:
135+
aggs:
136+
bounds:
137+
geo_bounds:
138+
field: "location"
139+
wrap_longitude: false
140+
bounds_from_doc_value:
141+
geo_bounds:
142+
field: "location_from_doc_value"
143+
wrap_longitude: false
144+
bounds_from_source:
145+
geo_bounds:
146+
field: "location_from_source"
147+
wrap_longitude: false
148+
- match: {hits.total.value: 6}
149+
- match: {aggregations.bounds.bounds.top_left.lat: 45.7799999602139 }
150+
- match: {aggregations.bounds.bounds.top_left.lon: -173.4500000718981 }
151+
- match: {aggregations.bounds.bounds.bottom_right.lat: -63.240000014193356 }
152+
- match: {aggregations.bounds.bounds.bottom_right.lon: 120.77999993227422 }
153+
- match: {aggregations.bounds_from_doc_value.bounds.top_left.lat: 45.7799999602139 }
154+
- match: {aggregations.bounds_from_doc_value.bounds.top_left.lon: -173.4500000718981 }
155+
- match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lat: -63.240000014193356 }
156+
- match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lon: 120.77999993227422 }
157+
- match: {aggregations.bounds_from_source.bounds.top_left.lat: 45.7799999602139 }
158+
- match: {aggregations.bounds_from_source.bounds.top_left.lon: -173.4500000718981 }
159+
- match: {aggregations.bounds_from_source.bounds.bottom_right.lat: -63.240000014193356 }
160+
- match: {aggregations.bounds_from_source.bounds.bottom_right.lon: 120.77999993227422 }
161+
162+
---
163+
"geo_distance sort":
164+
- do:
165+
search:
166+
index: locations
167+
body:
168+
sort:
169+
_geo_distance:
170+
location_from_source:
171+
lat: 0.0
172+
lon: 0.0
173+
- match: {hits.total.value: 6}
174+
- match: {hits.hits.0._source.location.lat: 0.0 }
175+
- match: {hits.hits.0._source.location.lon: 0.0 }
176+
- match: {hits.hits.1._source.location.lat: 13.5 }
177+
- match: {hits.hits.1._source.location.lon: 34.89 }
178+
- match: {hits.hits.2._source.location.lat: 32.45 }
179+
- match: {hits.hits.2._source.location.lon: 45.6 }
180+
- match: {hits.hits.3._source.location.lat: -63.24 }
181+
- match: {hits.hits.3._source.location.lon: 31.0 }
182+
183+
---
184+
"distance_feature query":
185+
- do:
186+
search:
187+
index: locations
188+
body:
189+
query:
190+
bool:
191+
should:
192+
distance_feature:
193+
field: "location"
194+
pivot: "1000km"
195+
origin: [0.0, 0.0]
196+
197+
- match: {hits.total.value: 6}
198+
- match: {hits.hits.0._source.location.lat: 0.0 }
199+
- match: {hits.hits.0._source.location.lon: 0.0 }
200+
- match: {hits.hits.1._source.location.lat: 13.5 }
201+
- match: {hits.hits.1._source.location.lon: 34.89 }
202+
- match: {hits.hits.2._source.location.lat: 32.45 }
203+
- match: {hits.hits.2._source.location.lon: 45.6 }
204+
- match: {hits.hits.3._source.location.lat: -63.24 }
205+
- match: {hits.hits.3._source.location.lon: 31.0 }
206+

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public final Query termQuery(Object value, SearchExecutionContext context) {
8989
}
9090

9191
@Override
92-
public final ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
92+
public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
9393
String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME;
9494

9595
if (parsesArrayValue) {
@@ -116,13 +116,13 @@ protected Object parseSourceValue(Object value) {
116116

117117
private final Explicit<Boolean> ignoreMalformed;
118118
private final Explicit<Boolean> ignoreZValue;
119-
private final Parser<T> parser;
119+
private final Parser<? extends T> parser;
120120

121121
protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType,
122122
Map<String, NamedAnalyzer> indexAnalyzers,
123123
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> ignoreZValue,
124124
MultiFields multiFields, CopyTo copyTo,
125-
Parser<T> parser) {
125+
Parser<? extends T> parser) {
126126
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null);
127127
this.ignoreMalformed = ignoreMalformed;
128128
this.ignoreZValue = ignoreZValue;
@@ -132,10 +132,24 @@ protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedF
132132
protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType,
133133
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> ignoreZValue,
134134
MultiFields multiFields, CopyTo copyTo,
135-
Parser<T> parser) {
135+
Parser<? extends T> parser) {
136136
this(simpleName, mappedFieldType, Collections.emptyMap(), ignoreMalformed, ignoreZValue, multiFields, copyTo, parser);
137137
}
138138

139+
protected AbstractGeometryFieldMapper(
140+
String simpleName,
141+
MappedFieldType mappedFieldType,
142+
MultiFields multiFields,
143+
CopyTo copyTo,
144+
Parser<? extends T> parser,
145+
String onScriptError
146+
) {
147+
super(simpleName, mappedFieldType, Collections.emptyMap(), multiFields, copyTo, true, onScriptError);
148+
this.ignoreMalformed = new Explicit<>(false, true);
149+
this.ignoreZValue = new Explicit<>(false, true);
150+
this.parser = parser;
151+
}
152+
139153
@Override
140154
public AbstractGeometryFieldType fieldType() {
141155
return (AbstractGeometryFieldType) mappedFieldType;
@@ -155,16 +169,19 @@ protected void parseCreateField(ParseContext context) throws IOException {
155169

156170
@Override
157171
public final void parse(ParseContext context) throws IOException {
158-
parser.parse(context.parser(), v -> index(context, v), e -> {
159-
if (ignoreMalformed()) {
160-
context.addIgnoredField(fieldType().name());
161-
} else {
162-
throw new MapperParsingException(
163-
"Failed to parse field [" + fieldType().name() + "] of type [" + contentType() + "]",
164-
e
165-
);
166-
}
167-
});
172+
if (hasScript) {
173+
throw new MapperParsingException("failed to parse field [" + fieldType().name() + "] of type + " + contentType() + "]",
174+
new IllegalArgumentException("Cannot index data directly into a field with a [script] parameter"));
175+
}
176+
parser.parse(context.parser(), v -> index(context, v), e -> {
177+
if (ignoreMalformed()) {
178+
context.addIgnoredField(fieldType().name());
179+
} else {
180+
throw new MapperParsingException(
181+
"failed to parse field [" + fieldType().name() + "] of type [" + contentType() + "]", e
182+
);
183+
}
184+
});
168185
}
169186

170187
public boolean ignoreMalformed() {

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,18 @@ public static Parameter<ParsedPoint> nullValueParam(Function<FieldMapper, Parsed
3939
protected AbstractPointGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType,
4040
MultiFields multiFields, Explicit<Boolean> ignoreMalformed,
4141
Explicit<Boolean> ignoreZValue, ParsedPoint nullValue, CopyTo copyTo,
42-
Parser<T> parser) {
42+
Parser<? extends T> parser) {
4343
super(simpleName, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo, parser);
4444
this.nullValue = nullValue;
4545
}
4646

47+
protected AbstractPointGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType,
48+
MultiFields multiFields, CopyTo copyTo,
49+
Parser<? extends T> parser, String onScriptError) {
50+
super(simpleName, mappedFieldType, multiFields, copyTo, parser, onScriptError);
51+
this.nullValue = null;
52+
}
53+
4754
@Override
4855
public final boolean parsesArrayValue() {
4956
return true;

0 commit comments

Comments
 (0)