Support spatial fields in field retrieval API.

Although we accept a variety of formats during indexing, spatial data is
returned in a single consistent format. This is GeoJSON by default, but
well-known text is also supported by passing the option 'format: wkt'.

Note that points (in addition to shapes) are returned in GeoJSON by default. The
reasoning is that this gives better consistency, and is the most convenient
format for most REST API users.

Author: jtibshirani

docs/reference/mapping/types.asciidoc

@@ -22,6 +22,7 @@ string::         <<text,`text`>>, <<keyword,`keyword`>> and <<wildcard,`wildcard
 <<nested>>::    `nested` for arrays of JSON objects
 
 [float]
+[[_spatial_datatypes]]
 === Spatial data types
 
 <<geo-point>>::     `geo_point` for lat/lon points

docs/reference/search/search-fields.asciidoc

@@ -87,9 +87,13 @@ POST twitter/_search
 
 <1> Both full field names and wildcard patterns are accepted.
 <2> Using object notation, you can pass a `format` parameter to apply a custom
-    format for the field's values. This is currently supported for
-    <<date,`date` fields>> and <<date_nanos, `date_nanos` fields>>, which
-    accept a <<mapping-date-format,date format>>.
+    format for the field's values. The date fields
+    <<date,`date`>> and <<date_nanos, `date_nanos`>> accept a
+    <<mapping-date-format,date format>>. <<_spatial_datatypes, Spatial fields>>
+    accept either `geojson` for http://www.geojson.org[GeoJSON] (the default)
+    or `wkt` for
+    https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry[Well Known Text].
+    Other field types do not support the `format` parameter.
 
 The values are returned as a flat list in the `fields` section in each hit:
 

modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/10_basic.yml

@@ -57,3 +57,26 @@ setup:
               field: location
 
   - match: {hits.total: 1}
+
+---
+"Test retrieve geo_shape field":
+  - do:
+      search:
+        index: test
+        body:
+          fields: [location]
+          _source: false
+
+  - match: { hits.hits.0.fields.location.0.type: "Point" }
+  - match: { hits.hits.0.fields.location.0.coordinates: [1.0, 1.0] }
+
+  - do:
+      search:
+        index: test
+        body:
+          fields:
+            - field: location
+              format: wkt
+          _source: false
+
+  - match: { hits.hits.0.fields.location.0: "POINT (1.0 1.0)" }

server/src/main/java/org/elasticsearch/common/geo/GeoJson.java

@@ -22,15 +22,21 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.parsers.ShapeParser;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentSubParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.geometry.Circle;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.GeometryCollection;
@@ -50,6 +56,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -610,4 +617,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
     }
 
+    public static Map<?, ?> toXContentMap(Geometry geometry) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        GeoJson.toXContent(geometry, builder, ToXContent.EMPTY_PARAMS);
+        StreamInput input = BytesReference.bytes(builder).streamInput();
+
+        try (XContentParser parser = XContentType.JSON.xContent()
+            .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, input)) {
+            return parser.map();
+        }
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -30,16 +30,21 @@
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.SpatialStrategy;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.support.MapXContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -82,6 +87,11 @@ public interface Parser<Parsed> {
         Parsed parse(XContentParser parser, AbstractGeometryFieldMapper mapper) throws IOException, ParseException;
     }
 
+    public interface Formatter<Parsed> {
+        Object formatGeoJson(Parsed value);
+        Object formatWKT(Parsed value);
+    }
+
     public abstract static class Builder<T extends Builder<T, FT>, FT extends AbstractGeometryFieldType>
             extends FieldMapper.Builder<T> {
         protected Boolean ignoreMalformed;
@@ -143,7 +153,31 @@ public Builder ignoreZValue(final boolean ignoreZValue) {
 
     @Override
     protected Object parseSourceValue(Object value, String format) {
-        throw new UnsupportedOperationException();
+        AbstractGeometryFieldType<Parsed, Processed> mappedFieldType = fieldType();
+        Parser<Parsed> geometryParser = mappedFieldType.geometryParser();
+        Formatter<Parsed> geometryFormatter = mappedFieldType.geometryFormatter();
+
+        Parsed geometry;
+        try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE,
+            Collections.singletonMap("dummy_field", value), XContentType.JSON)) {
+            parser.nextToken(); // start object
+            parser.nextToken(); // field name
+            parser.nextToken(); // field value
+            geometry = geometryParser.parse(parser, this);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } catch (ParseException e) {
+            throw new RuntimeException(e);
+        }
+
+        if (format == null || format.equals("geojson")) {
+            return geometryFormatter.formatGeoJson(geometry);
+        } else if (format.equals("wkt")) {
+            return geometryFormatter.formatWKT(geometry);
+        } else {
+            throw new IllegalArgumentException("Encountered an unsupported format [" + format + "] when " +
+                "loading values for the [" + contentType() +"] field named [" + name() + "]");
+        }
     }
 
     public abstract static class TypeParser<T extends Builder> implements Mapper.TypeParser {
@@ -187,6 +221,7 @@ public T parse(String name, Map<String, Object> node, ParserContext parserContex
     public abstract static class AbstractGeometryFieldType<Parsed, Processed> extends MappedFieldType {
         protected Indexer<Parsed, Processed> geometryIndexer;
         protected Parser<Parsed> geometryParser;
+        protected Formatter<Parsed> geometryFormatter;
         protected QueryProcessor geometryQueryBuilder;
 
         protected AbstractGeometryFieldType(String name, boolean indexed, boolean hasDocValues, Map<String, String> meta) {
@@ -213,6 +248,14 @@ protected Parser<Parsed> geometryParser() {
             return geometryParser;
         }
 
+        public void setGeometryFormatter(Formatter<Parsed> geometryFormatter) {
+            this.geometryFormatter = geometryFormatter;
+        }
+
+        protected Formatter<Parsed> geometryFormatter() {
+            return geometryFormatter;
+        }
+
         public QueryProcessor geometryQueryBuilder() {
             return geometryQueryBuilder;
         }

server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java

@@ -22,12 +22,17 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.geo.GeoJson;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.utils.WellKnownText;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -158,6 +163,7 @@ public interface ParsedPoint {
         void validate(String fieldName);
         void normalize(String fieldName);
         void resetCoords(double x, double y);
+        Point asGeometry();
         default boolean isNormalizable(double coord) {
             return Double.isNaN(coord) == false && Double.isInfinite(coord) == false;
         }
@@ -239,4 +245,30 @@ public List<P> parse(XContentParser parser, AbstractGeometryFieldMapper geometry
             }
         }
     }
+
+    public static class PointFormatter<P extends ParsedPoint> implements Formatter<List<P>> {
+        @Override
+        public Object formatGeoJson(List<P> points) {
+            List<Object> result = new ArrayList<>();
+            for (ParsedPoint point : points) {
+                try {
+                    Geometry geometry = point.asGeometry();
+                    result.add(GeoJson.toXContentMap(geometry));
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+            return result;
+        }
+
+        @Override
+        public Object formatWKT(List<P> points) {
+            List<String> result = new ArrayList<>();
+            for (ParsedPoint point : points) {
+                Geometry geometry = point.asGeometry();
+                result.add(WellKnownText.INSTANCE.toWKT(geometry));
+            }
+            return result;
+        }
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -29,10 +29,11 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.geometry.Point;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointIndexFieldData;
-import org.elasticsearch.index.query.VectorGeoPointShapeQueryProcessor;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper.ParsedGeoPoint;
+import org.elasticsearch.index.query.VectorGeoPointShapeQueryProcessor;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 
 import java.io.IOException;
@@ -49,6 +50,7 @@
 public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper<List<ParsedGeoPoint>, List<? extends GeoPoint>> {
     public static final String CONTENT_TYPE = "geo_point";
     public static final FieldType FIELD_TYPE = new FieldType();
+
     static {
         FIELD_TYPE.setStored(false);
         FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
@@ -70,6 +72,7 @@ public GeoPointFieldMapper build(BuilderContext context, String simpleName, Fiel
             GeoPointFieldType ft = new GeoPointFieldType(buildFullName(context), indexed, hasDocValues, meta);
             ft.setGeometryParser(new PointParser<>());
             ft.setGeometryIndexer(new GeoPointIndexer(ft));
+            ft.setGeometryFormatter(new PointFormatter<>());
             ft.setGeometryQueryBuilder(new VectorGeoPointShapeQueryProcessor());
             return new GeoPointFieldMapper(name, fieldType, ft, multiFields, ignoreMalformed, ignoreZValue, nullValue, copyTo);
         }
@@ -218,6 +221,10 @@ public void resetCoords(double x, double y) {
             this.reset(y, x);
         }
 
+        public Point asGeometry() {
+            return new Point(lon(), lat());
+        }
+
         @Override
         public boolean equals(Object other) {
             double oLat;

server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java

@@ -73,6 +73,7 @@ private GeoShapeFieldType buildFieldType(BuilderContext context) {
                 ignoreZValue().value());
             ft.setGeometryParser((parser, mapper) -> geometryParser.parse(parser));
             ft.setGeometryIndexer(new GeoShapeIndexer(orientation().value().getAsBoolean(), buildFullName(context)));
+            ft.setGeometryFormatter(new GeoShapeFormatter());
             ft.setGeometryQueryBuilder(new VectorGeoShapeQueryProcessor());
             ft.setOrientation(orientation == null ? Defaults.ORIENTATION.value() : orientation);
             return ft;

server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFormatter.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.geo.GeoJson;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.utils.WellKnownText;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+public class GeoShapeFormatter implements AbstractGeometryFieldMapper.Formatter<Geometry> {
+    @Override
+    public Object formatGeoJson(Geometry value) {
+        try {
+            return GeoJson.toXContentMap(value);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    @Override
+    public Object formatWKT(Geometry value) {
+        return WellKnownText.INSTANCE.toWKT(value);
+    }
+}

server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.geo.GeoJson;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.geo.SpatialStrategy;
@@ -43,10 +44,13 @@
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.utils.WellKnownText;
 import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor;
 import org.locationtech.spatial4j.shape.Shape;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -256,6 +260,7 @@ private GeoShapeFieldType buildFieldType(BuilderContext context) {
             setupPrefixTrees(ft);
             ft.setGeometryIndexer(new LegacyGeoShapeIndexer(ft));
             ft.setGeometryParser(ShapeParser::parse);
+            ft.setGeometryFormatter(new LegacyGeoShapeFormatter());
             ft.setGeometryQueryBuilder(new LegacyGeoShapeQueryProcessor(ft));
             ft.setOrientation(orientation == null ? Defaults.ORIENTATION.value() : orientation);
             return ft;
@@ -277,6 +282,23 @@ public LegacyGeoShapeFieldMapper build(BuilderContext context) {
         }
     }
 
+    private static class LegacyGeoShapeFormatter implements Formatter<ShapeBuilder<?, ?, ?>> {
+        @Override
+        public Object formatGeoJson(ShapeBuilder<?, ?, ?> value) {
+            try {
+                Geometry geometry = value.buildGeometry();
+                return GeoJson.toXContentMap(geometry);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
+        public String formatWKT(ShapeBuilder<?, ?, ?> value) {
+            return WellKnownText.INSTANCE.toWKT(value.buildGeometry());
+        }
+    }
+
     public static final class GeoShapeFieldType extends AbstractShapeGeometryFieldType<ShapeBuilder<?, ?, ?>, Shape> {
 
         private String tree = DeprecatedParameters.Defaults.TREE;