Skip to content

Rework geo mappers to index value by value #71696

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 19, 2021
Merged

Rework geo mappers to index value by value #71696

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
28552b7
Rework geo mappers to index value by value
romseygeek Apr 14, 2021
6b6ad29
tests; feedback
romseygeek Apr 15, 2021
f2cdb81
unlock test
romseygeek Apr 15, 2021
7415732
Merge remote-tracking branch 'origin/master' into mapper/geo-indexing
romseygeek Apr 15, 2021
f45a756
Merge remote-tracking branch 'origin/master' into mapper/geo-indexing
romseygeek Apr 15, 2021
aae6f69
tests
romseygeek Apr 15, 2021
7239f79
Merge remote-tracking branch 'origin/master' into mapper/geo-indexing
romseygeek Apr 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 31 additions & 80 deletions server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
*/
package org.elasticsearch.index.mapper;

import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.geo.GeoJsonGeometryFormat;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
Expand All @@ -21,17 +21,15 @@

import java.io.IOException;
import java.io.UncheckedIOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;

/**
* Base field mapper class for all spatial field types
*/
public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends FieldMapper {
public abstract class AbstractGeometryFieldMapper<T> extends FieldMapper {

public static Parameter<Explicit<Boolean>> ignoreMalformedParam(Function<FieldMapper, Explicit<Boolean>> initializer,
boolean ignoreMalformedByDefault) {
Expand All @@ -42,52 +40,46 @@ public static Parameter<Explicit<Boolean>> ignoreZValueParam(Function<FieldMappe
return Parameter.explicitBoolParam("ignore_z_value", true, initializer, true);
}

/**
* Interface representing an preprocessor in geometry indexing pipeline
*/
public interface Indexer<Parsed, Processed> {
Processed prepareForIndexing(Parsed geometry);
Class<Processed> processedClass();
List<IndexableField> indexShape(ParseContext context, Processed shape);
}

/**
* Interface representing parser in geometry indexing pipeline.
*/
public abstract static class Parser<Parsed> {
public abstract static class Parser<T> {
/**
* Parse the given xContent value to an object of type {@link Parsed}. The value can be
* Parse the given xContent value to one or more objects of type {@link T}. The value can be
* in any supported format.
*/
public abstract Parsed parse(XContentParser parser) throws IOException, ParseException;
public abstract void parse(
XContentParser parser,
CheckedConsumer<T, IOException> consumer,
Consumer<Exception> onMalformed) throws IOException;

/**
* Given a parsed value and a format string, formats the value into a plain Java object.
*
* Supported formats include 'geojson' and 'wkt'. The different formats are defined
* as subclasses of {@link org.elasticsearch.common.geo.GeometryFormat}.
*/
public abstract Object format(Parsed value, String format);
public abstract Object format(T value, String format);

/**
* Parses the given value, then formats it according to the 'format' string.
*
* Used by value fetchers to validate and format geo objects
*/
public Object parseAndFormatObject(Object value, String format) {
Parsed geometry;
Object[] formatted = new Object[1];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe it'd be more straightforward to use SetOnce instead of an array.

try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE,
Collections.singletonMap("dummy_field", value), XContentType.JSON)) {
parser.nextToken(); // start object
parser.nextToken(); // field name
parser.nextToken(); // field value
geometry = parse(parser);
parse(parser, v -> {
formatted[0] = format(v, format);
}, e -> {} /* ignore malformed */);
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (ParseException e) {
throw new RuntimeException(e);
}
return format(geometry, format);
return formatted[0];
}
}

Expand Down Expand Up @@ -134,26 +126,24 @@ protected Object parseSourceValue(Object value) {

private final Explicit<Boolean> ignoreMalformed;
private final Explicit<Boolean> ignoreZValue;
private final Indexer<Parsed, Processed> indexer;
private final Parser<Parsed> parser;
private final Parser<T> parser;

protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType,
Map<String, NamedAnalyzer> indexAnalyzers,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> ignoreZValue,
MultiFields multiFields, CopyTo copyTo,
Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
Parser<T> parser) {
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null);
this.ignoreMalformed = ignoreMalformed;
this.ignoreZValue = ignoreZValue;
this.indexer = indexer;
this.parser = parser;
}

protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> ignoreZValue,
MultiFields multiFields, CopyTo copyTo,
Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
this(simpleName, mappedFieldType, Collections.emptyMap(), ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser);
Parser<T> parser) {
this(simpleName, mappedFieldType, Collections.emptyMap(), ignoreMalformed, ignoreZValue, multiFields, copyTo, parser);
}

@Override
Expand All @@ -166,60 +156,21 @@ protected void parseCreateField(ParseContext context) throws IOException {
throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called");
}

protected abstract void addStoredFields(ParseContext context, Processed geometry);
protected abstract void addDocValuesFields(String name, Processed geometry, List<IndexableField> fields, ParseContext context);
protected abstract void addMultiFields(ParseContext context, Processed geometry) throws IOException;
protected abstract void index(ParseContext context, T geometry) throws IOException;

/** parsing logic for geometry indexing */
@Override
public void parse(ParseContext context) throws IOException {
MappedFieldType mappedFieldType = fieldType();

try {
Processed shape = context.parseExternalValue(indexer.processedClass());
if (shape == null) {
Parsed geometry = parser.parse(context.parser());
if (geometry == null) {
return;
}
shape = indexer.prepareForIndexing(geometry);
}

List<IndexableField> fields = new ArrayList<>();
if (mappedFieldType.isSearchable() || mappedFieldType.hasDocValues()) {
fields.addAll(indexer.indexShape(context, shape));
}

// indexed:
List<IndexableField> indexedFields = new ArrayList<>();
if (mappedFieldType.isSearchable()) {
indexedFields.addAll(fields);
}
// stored:
if (fieldType().isStored()) {
addStoredFields(context, shape);
}
// docValues:
if (fieldType().hasDocValues()) {
addDocValuesFields(mappedFieldType.name(), shape, fields, context);
} else if (fieldType().isStored() || fieldType().isSearchable()) {
createFieldNamesField(context);
}

// add the indexed fields to the doc:
for (IndexableField field : indexedFields) {
context.doc().add(field);
}

// add multifields (e.g., used for completion suggester)
addMultiFields(context, shape);
} catch (Exception e) {
if (ignoreMalformed.value() == false) {
throw new MapperParsingException("failed to parse field [{}] of type [{}]", e, fieldType().name(),
fieldType().typeName());
}
context.addIgnoredField(mappedFieldType.name());
}
public final void parse(ParseContext context) throws IOException {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should remove the comment on this method as there is not parsing logic and add documentation ton the new index() method.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

++

parser.parse(context.parser(), v -> index(context, v), e -> {
if (ignoreMalformed()) {
context.addIgnoredField(fieldType().name());
} else {
throw new MapperParsingException(
"Failed to parse field [" + fieldType().name() + "] of type [" + contentType() + "]",
e
);
}
});
}

public boolean ignoreMalformed() {
Expand Down
60 changes: 33 additions & 27 deletions server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.CheckedBiFunction;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.TriFunction;
import org.elasticsearch.common.geo.GeoPoint;
Expand All @@ -20,15 +21,12 @@
import org.elasticsearch.index.mapper.Mapper.TypeParser.ParserContext;

import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;

/** Base class for for spatial fields that only support indexing points */
public abstract class AbstractPointGeometryFieldMapper<Parsed, Processed> extends AbstractGeometryFieldMapper<Parsed, Processed> {
public abstract class AbstractPointGeometryFieldMapper<T> extends AbstractGeometryFieldMapper<T> {

public static Parameter<ParsedPoint> nullValueParam(Function<FieldMapper, ParsedPoint> initializer,
TriFunction<String, ParserContext, Object, ParsedPoint> parser,
Expand All @@ -41,8 +39,8 @@ public static Parameter<ParsedPoint> nullValueParam(Function<FieldMapper, Parsed
protected AbstractPointGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType,
MultiFields multiFields, Explicit<Boolean> ignoreMalformed,
Explicit<Boolean> ignoreZValue, ParsedPoint nullValue, CopyTo copyTo,
Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
super(simpleName, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser);
Parser<T> parser) {
super(simpleName, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo, parser);
this.nullValue = nullValue;
}

Expand All @@ -67,7 +65,7 @@ default boolean isNormalizable(double coord) {
}

/** A parser implementation that can parse the various point formats */
public static class PointParser<P extends ParsedPoint> extends Parser<List<P>> {
public static class PointParser<P extends ParsedPoint> extends Parser<P> {
/**
* Note that this parser is only used for formatting values.
*/
Expand All @@ -88,7 +86,7 @@ public PointParser(String field,
this.field = field;
this.pointSupplier = pointSupplier;
this.objectParser = objectParser;
this.nullValue = nullValue;
this.nullValue = nullValue == null ? null : process(nullValue);
this.ignoreZValue = ignoreZValue;
this.ignoreMalformed = ignoreMalformed;
this.geometryParser = new GeometryParser(true, true, true);
Expand All @@ -104,12 +102,14 @@ private P process(P in) {
}

@Override
public List<P> parse(XContentParser parser) throws IOException, ParseException {

public void parse(
XContentParser parser,
CheckedConsumer<P, IOException> consumer,
Consumer<Exception> onMalformed
) throws IOException {
if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
XContentParser.Token token = parser.nextToken();
P point = pointSupplier.get();
ArrayList<P> points = new ArrayList<>();
if (token == XContentParser.Token.VALUE_NUMBER) {
double x = parser.doubleValue();
parser.nextToken();
Expand All @@ -122,35 +122,41 @@ public List<P> parse(XContentParser parser) throws IOException, ParseException {
}

point.resetCoords(x, y);
points.add(process(point));
consumer.accept(process(point));
} else {
while (token != XContentParser.Token.END_ARRAY) {
points.add(process(objectParser.apply(parser, point)));
parseAndConsumeFromObject(parser, point, consumer, onMalformed);
point = pointSupplier.get();
token = parser.nextToken();
}
}
return points;
} else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
if (nullValue == null) {
return null;
} else {
return Collections.singletonList(nullValue);
if (nullValue != null) {
consumer.accept(nullValue);
}
} else {
return Collections.singletonList(process(objectParser.apply(parser, pointSupplier.get())));
parseAndConsumeFromObject(parser, pointSupplier.get(), consumer, onMalformed);
}
}

private void parseAndConsumeFromObject(
XContentParser parser,
P point,
CheckedConsumer<P, IOException> consumer,
Consumer<Exception> onMalformed
) {
try {
point = objectParser.apply(parser, point);
consumer.accept(point);
} catch (Exception e) {
onMalformed.accept(e);
}
}

@Override
public Object format(List<P> points, String format) {
List<Object> result = new ArrayList<>();
public Object format(P point, String format) {
GeometryFormat<Geometry> geometryFormat = geometryParser.geometryFormat(format);
for (ParsedPoint point : points) {
Geometry geometry = point.asGeometry();
result.add(geometryFormat.toXContentAsObject(geometry));
}
return result;
return geometryFormat.toXContentAsObject(point.asGeometry());
}
}
}
10 changes: 5 additions & 5 deletions server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
/**
* Base class for {@link GeoShapeFieldMapper} and {@link LegacyGeoShapeFieldMapper}
*/
public abstract class AbstractShapeGeometryFieldMapper<Parsed, Processed> extends AbstractGeometryFieldMapper<Parsed, Processed> {
public abstract class AbstractShapeGeometryFieldMapper<T> extends AbstractGeometryFieldMapper<T> {

public static Parameter<Explicit<Boolean>> coerceParam(Function<FieldMapper, Explicit<Boolean>> initializer,
boolean coerceByDefault) {
Expand Down Expand Up @@ -56,8 +56,8 @@ protected AbstractShapeGeometryFieldMapper(String simpleName, MappedFieldType ma
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Explicit<Orientation> orientation,
MultiFields multiFields, CopyTo copyTo,
Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
super(simpleName, mappedFieldType, indexAnalyzers, ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser);
Parser<T> parser) {
super(simpleName, mappedFieldType, indexAnalyzers, ignoreMalformed, ignoreZValue, multiFields, copyTo, parser);
this.coerce = coerce;
this.orientation = orientation;
}
Expand All @@ -66,9 +66,9 @@ protected AbstractShapeGeometryFieldMapper(String simpleName, MappedFieldType ma
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Explicit<Orientation> orientation,
MultiFields multiFields, CopyTo copyTo,
Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
Parser<T> parser) {
this(simpleName, mappedFieldType, Collections.emptyMap(),
ignoreMalformed, coerce, ignoreZValue, orientation, multiFields, copyTo, indexer, parser);
ignoreMalformed, coerce, ignoreZValue, orientation, multiFields, copyTo, parser);
}

@Override
Expand Down
Loading