Skip to content

Commit bc49392

Browse files
nik9000nik9000
authored
Support malformed numbers in synthetic _source (#90428)
This adds support for `ignore_malformed` to numeric fields other than `scaled_float` in synthetic `_source`. Their values are saved to a stored field and loaded to render the `_source`.
1 parent ff242a1 commit bc49392

File tree

50 files changed

+344
-247
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+344
-247
lines changed

‎docs/changelog/90428.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 90428
2+
summary: Support malformed numbers in synthetic `_source`
3+
area: TSDB
4+
type: enhancement
5+
issues: []

‎modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -663,7 +663,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
663663
}
664664

665665
@Override
666-
protected SyntheticSourceSupport syntheticSourceSupport() {
666+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
667667
throw new AssumptionViolatedException("not supported");
668668
}
669669

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -709,7 +709,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
709709
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
710710
);
711711
}
712-
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
712+
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed.value()) {
713713
@Override
714714
protected void writeValue(XContentBuilder b, long value) throws IOException {
715715
b.value(decodeForSyntheticSource(value, scalingFactor));

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,8 @@ protected boolean supportsIgnoreMalformed() {
213213
}
214214

215215
@Override
216-
protected SyntheticSourceSupport syntheticSourceSupport() {
216+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
217+
assertFalse("match_only_text doesn't support ignoreMalformed", ignoreMalformed);
217218
return new MatchOnlyTextSyntheticSourceSupport();
218219
}
219220

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
165165
}
166166

167167
@Override
168-
protected SyntheticSourceSupport syntheticSourceSupport() {
168+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
169169
throw new AssumptionViolatedException("not supported");
170170
}
171171

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ protected boolean allowsNullValues() {
184184
}
185185

186186
@Override
187-
protected SyntheticSourceSupport syntheticSourceSupport() {
187+
protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) {
188188
throw new AssumptionViolatedException("not supported");
189189
}
190190

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

Lines changed: 62 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -337,75 +337,76 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
337337
}
338338

339339
@Override
340-
protected SyntheticSourceSupport syntheticSourceSupport() {
341-
return new SyntheticSourceSupport() {
342-
private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);
343-
private final Double nullValue = usually() ? null : round(randomValue());
344-
345-
@Override
346-
public SyntheticSourceExample example(int maxValues) {
347-
if (randomBoolean()) {
348-
Tuple<Double, Double> v = generateValue();
349-
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
350-
}
351-
List<Tuple<Double, Double>> values = randomList(1, maxValues, this::generateValue);
352-
List<Double> in = values.stream().map(Tuple::v1).toList();
353-
List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
354-
Object out = outList.size() == 1 ? outList.get(0) : outList;
355-
return new SyntheticSourceExample(in, out, this::mapping);
356-
}
340+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
341+
assumeFalse("scaled_float doesn't support ignore_malformed with synthetic _source", ignoreMalformed);
342+
return new ScaledFloatSyntheticSourceSupport();
343+
}
357344

358-
private Tuple<Double, Double> generateValue() {
359-
if (nullValue != null && randomBoolean()) {
360-
return Tuple.tuple(null, nullValue);
361-
}
362-
double d = randomValue();
363-
return Tuple.tuple(d, round(d));
345+
private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport {
346+
private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);
347+
private final Double nullValue = usually() ? null : round(randomValue());
348+
349+
@Override
350+
public SyntheticSourceExample example(int maxValues) {
351+
if (randomBoolean()) {
352+
Tuple<Double, Double> v = generateValue();
353+
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
364354
}
355+
List<Tuple<Double, Double>> values = randomList(1, maxValues, this::generateValue);
356+
List<Double> in = values.stream().map(Tuple::v1).toList();
357+
List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
358+
Object out = outList.size() == 1 ? outList.get(0) : outList;
359+
return new SyntheticSourceExample(in, out, this::mapping);
360+
}
365361

366-
private double round(double d) {
367-
long encoded = Math.round(d * scalingFactor);
368-
double decoded = encoded / scalingFactor;
369-
long reencoded = Math.round(decoded * scalingFactor);
370-
if (encoded != reencoded) {
371-
if (encoded > reencoded) {
372-
return decoded + Math.ulp(decoded);
373-
}
374-
return decoded - Math.ulp(decoded);
375-
}
376-
return decoded;
362+
private Tuple<Double, Double> generateValue() {
363+
if (nullValue != null && randomBoolean()) {
364+
return Tuple.tuple(null, nullValue);
377365
}
366+
double d = randomValue();
367+
return Tuple.tuple(d, round(d));
368+
}
378369

379-
private void mapping(XContentBuilder b) throws IOException {
380-
b.field("type", "scaled_float");
381-
b.field("scaling_factor", scalingFactor);
382-
if (nullValue != null) {
383-
b.field("null_value", nullValue);
384-
}
385-
if (rarely()) {
386-
b.field("index", false);
387-
}
388-
if (rarely()) {
389-
b.field("store", false);
370+
private double round(double d) {
371+
long encoded = Math.round(d * scalingFactor);
372+
double decoded = encoded / scalingFactor;
373+
long reencoded = Math.round(decoded * scalingFactor);
374+
if (encoded != reencoded) {
375+
if (encoded > reencoded) {
376+
return decoded + Math.ulp(decoded);
390377
}
378+
return decoded - Math.ulp(decoded);
391379
}
380+
return decoded;
381+
}
392382

393-
@Override
394-
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
395-
return List.of(
396-
new SyntheticSourceInvalidExample(
397-
equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"),
398-
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false)
399-
),
400-
new SyntheticSourceInvalidExample(
401-
equalTo(
402-
"field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"
403-
),
404-
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true)
405-
)
406-
);
383+
private void mapping(XContentBuilder b) throws IOException {
384+
b.field("type", "scaled_float");
385+
b.field("scaling_factor", scalingFactor);
386+
if (nullValue != null) {
387+
b.field("null_value", nullValue);
407388
}
408-
};
389+
if (rarely()) {
390+
b.field("index", false);
391+
}
392+
if (rarely()) {
393+
b.field("store", false);
394+
}
395+
}
396+
397+
@Override
398+
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
399+
return List.of(
400+
new SyntheticSourceInvalidExample(
401+
equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"),
402+
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false)
403+
),
404+
new SyntheticSourceInvalidExample(
405+
equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"),
406+
b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true)
407+
)
408+
);
409+
}
409410
}
410411

411412
@Override
@@ -504,7 +505,7 @@ private double encodeDecode(double value, double scalingFactor) {
504505
return ScaledFloatFieldMapper.decodeForSyntheticSource(ScaledFloatFieldMapper.encode(value, scalingFactor), scalingFactor);
505506
}
506507

507-
private double randomValue() {
508+
private static double randomValue() {
508509
return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat();
509510
}
510511
}

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -801,7 +801,7 @@ protected boolean supportsIgnoreMalformed() {
801801
}
802802

803803
@Override
804-
protected SyntheticSourceSupport syntheticSourceSupport() {
804+
protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) {
805805
throw new AssumptionViolatedException("not supported");
806806
}
807807

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ protected boolean supportsIgnoreMalformed() {
195195
}
196196

197197
@Override
198-
protected SyntheticSourceSupport syntheticSourceSupport() {
198+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
199199
throw new AssumptionViolatedException("not supported");
200200
}
201201

‎plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,7 @@ protected boolean supportsIgnoreMalformed() {
308308
}
309309

310310
@Override
311-
protected SyntheticSourceSupport syntheticSourceSupport() {
311+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
312312
throw new AssumptionViolatedException("not supported");
313313
}
314314

‎plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -595,7 +595,7 @@ protected boolean supportsIgnoreMalformed() {
595595
}
596596

597597
@Override
598-
protected SyntheticSourceSupport syntheticSourceSupport() {
598+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
599599
throw new AssumptionViolatedException("not supported");
600600
}
601601

‎plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ protected boolean supportsIgnoreMalformed() {
7070
}
7171

7272
@Override
73-
protected SyntheticSourceSupport syntheticSourceSupport() {
73+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
7474
throw new AssumptionViolatedException("not supported");
7575
}
7676

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
466466
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
467467
);
468468
}
469-
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
469+
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), false) {
470470
@Override
471471
protected void writeValue(XContentBuilder b, long value) throws IOException {
472472
b.value(value == 1);

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -955,7 +955,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
955955
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
956956
);
957957
}
958-
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
958+
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed) {
959959
@Override
960960
protected void writeValue(XContentBuilder b, long value) throws IOException {
961961
b.value(fieldType().format(value, fieldType().dateTimeFormatter()));

‎server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -497,7 +497,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
497497
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
498498
);
499499
}
500-
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName()) {
500+
return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed()) {
501501
final GeoPoint point = new GeoPoint();
502502

503503
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -399,8 +399,8 @@ private static void validateParsed(float value) {
399399
}
400400

401401
@Override
402-
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
403-
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
402+
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
403+
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
404404
@Override
405405
protected void writeValue(XContentBuilder b, long value) throws IOException {
406406
b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
@@ -549,8 +549,8 @@ private static void validateParsed(float value) {
549549
}
550550

551551
@Override
552-
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
553-
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
552+
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
553+
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
554554
@Override
555555
protected void writeValue(XContentBuilder b, long value) throws IOException {
556556
b.value(NumericUtils.sortableIntToFloat((int) value));
@@ -677,8 +677,8 @@ private static void validateParsed(double value) {
677677
}
678678

679679
@Override
680-
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
681-
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
680+
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
681+
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
682682
@Override
683683
protected void writeValue(XContentBuilder b, long value) throws IOException {
684684
b.value(NumericUtils.sortableLongToDouble(value));
@@ -774,8 +774,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
774774
}
775775

776776
@Override
777-
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
778-
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
777+
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
778+
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
779779
}
780780
},
781781
SHORT("short", NumericType.SHORT) {
@@ -862,8 +862,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
862862
}
863863

864864
@Override
865-
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
866-
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
865+
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
866+
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
867867
}
868868
},
869869
INTEGER("integer", NumericType.INT) {
@@ -1017,8 +1017,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
10171017
}
10181018

10191019
@Override
1020-
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
1021-
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName);
1020+
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
1021+
return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
10221022
}
10231023
},
10241024
LONG("long", NumericType.LONG) {
@@ -1142,8 +1142,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
11421142
}
11431143

11441144
@Override
1145-
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) {
1146-
return syntheticLongFieldLoader(fieldName, fieldSimpleName);
1145+
SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
1146+
return syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
11471147
}
11481148
};
11491149

@@ -1382,10 +1382,14 @@ public double reduceToStoredPrecision(double value) {
13821382
return ((Number) value).doubleValue();
13831383
}
13841384

1385-
abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName);
1385+
abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed);
13861386

1387-
private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(String fieldName, String fieldSimpleName) {
1388-
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName) {
1387+
private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(
1388+
String fieldName,
1389+
String fieldSimpleName,
1390+
boolean ignoreMalformed
1391+
) {
1392+
return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
13891393
@Override
13901394
protected void writeValue(XContentBuilder b, long value) throws IOException {
13911395
b.value(value);
@@ -1670,6 +1674,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
16701674
} catch (IllegalArgumentException e) {
16711675
if (ignoreMalformed.value() && context.parser().currentToken().isValue()) {
16721676
context.addIgnoredField(mappedFieldType.name());
1677+
if (context.isSyntheticSource()) {
1678+
// Save a copy of the field so synthetic source can load it
1679+
context.doc().add(IgnoreMalformedStoredValues.storedField(name(), context.parser()));
1680+
}
16731681
return;
16741682
} else {
16751683
throw e;
@@ -1757,17 +1765,12 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
17571765
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
17581766
);
17591767
}
1760-
if (ignoreMalformed.value()) {
1761-
throw new IllegalArgumentException(
1762-
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers"
1763-
);
1764-
}
17651768
if (copyTo.copyToFields().isEmpty() != true) {
17661769
throw new IllegalArgumentException(
17671770
"field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
17681771
);
17691772
}
1770-
return type.syntheticFieldLoader(name(), simpleName());
1773+
return type.syntheticFieldLoader(name(), simpleName(), ignoreMalformed.value());
17711774
}
17721775

17731776
// For testing only:

0 commit comments

Comments
 (0)