Skip to content

Commit 9d18d52

Browse files
lktslkts
authored
Add block loader from stored field and source for ip field (elastic#126644)
1 parent 5c5a87a commit 9d18d52

File tree

17 files changed

+331
-12
lines changed

17 files changed

+331
-12
lines changed

‎docs/changelog/126644.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126644
2+
summary: Add block loader from stored field and source for ip field
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java

+42
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.index.mapper;
1111

12+
import org.apache.lucene.document.InetAddressPoint;
1213
import org.apache.lucene.index.LeafReaderContext;
1314
import org.apache.lucene.index.PostingsEnum;
1415
import org.apache.lucene.index.SortedSetDocValues;
@@ -20,6 +21,7 @@
2021
import org.elasticsearch.search.fetch.StoredFieldsSpec;
2122

2223
import java.io.IOException;
24+
import java.net.InetAddress;
2325
import java.util.ArrayList;
2426
import java.util.List;
2527
import java.util.Objects;
@@ -381,6 +383,46 @@ public String toString() {
381383
}
382384
}
383385

386+
/**
387+
* Load {@code ip}s from {@code _source}.
388+
*/
389+
public static class IpsBlockLoader extends SourceBlockLoader {
390+
public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
391+
super(fetcher, lookup);
392+
}
393+
394+
@Override
395+
public Builder builder(BlockFactory factory, int expectedCount) {
396+
return factory.bytesRefs(expectedCount);
397+
}
398+
399+
@Override
400+
public RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) {
401+
return new Ips(fetcher, iter);
402+
}
403+
404+
@Override
405+
protected String name() {
406+
return "Ips";
407+
}
408+
}
409+
410+
private static class Ips extends BlockSourceReader {
411+
Ips(ValueFetcher fetcher, DocIdSetIterator iter) {
412+
super(fetcher, iter);
413+
}
414+
415+
@Override
416+
protected void append(BlockLoader.Builder builder, Object v) {
417+
((BlockLoader.BytesRefBuilder) builder).appendBytesRef(new BytesRef(InetAddressPoint.encode((InetAddress) v)));
418+
}
419+
420+
@Override
421+
public String toString() {
422+
return "BlockSourceReader.Ips";
423+
}
424+
}
425+
384426
/**
385427
* Convert a {@link String} into a utf-8 {@link BytesRef}.
386428
*/

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

+81-5
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
4444
import org.elasticsearch.search.lookup.FieldValues;
4545
import org.elasticsearch.search.lookup.SearchLookup;
46+
import org.elasticsearch.xcontent.XContentParser;
4647

4748
import java.io.IOException;
4849
import java.net.InetAddress;
@@ -51,8 +52,10 @@
5152
import java.util.Arrays;
5253
import java.util.Collection;
5354
import java.util.Collections;
55+
import java.util.List;
5456
import java.util.Map;
5557
import java.util.Objects;
58+
import java.util.Set;
5659
import java.util.function.BiFunction;
5760

5861
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
@@ -213,7 +216,8 @@ public IpFieldMapper build(MapperBuilderContext context) {
213216
parseNullValue(),
214217
scriptValues(),
215218
meta.getValue(),
216-
dimension.getValue()
219+
dimension.getValue(),
220+
context.isSourceSynthetic()
217221
),
218222
builderParams(this, context),
219223
context.isSourceSynthetic(),
@@ -234,6 +238,7 @@ public static final class IpFieldType extends SimpleMappedFieldType {
234238
private final InetAddress nullValue;
235239
private final FieldValues<InetAddress> scriptValues;
236240
private final boolean isDimension;
241+
private final boolean isSyntheticSource;
237242

238243
public IpFieldType(
239244
String name,
@@ -243,12 +248,14 @@ public IpFieldType(
243248
InetAddress nullValue,
244249
FieldValues<InetAddress> scriptValues,
245250
Map<String, String> meta,
246-
boolean isDimension
251+
boolean isDimension,
252+
boolean isSyntheticSource
247253
) {
248254
super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
249255
this.nullValue = nullValue;
250256
this.scriptValues = scriptValues;
251257
this.isDimension = isDimension;
258+
this.isSyntheticSource = isSyntheticSource;
252259
}
253260

254261
public IpFieldType(String name) {
@@ -260,7 +267,7 @@ public IpFieldType(String name, boolean isIndexed) {
260267
}
261268

262269
public IpFieldType(String name, boolean isIndexed, boolean hasDocValues) {
263-
this(name, isIndexed, false, hasDocValues, null, null, Collections.emptyMap(), false);
270+
this(name, isIndexed, false, hasDocValues, null, null, Collections.emptyMap(), false, false);
264271
}
265272

266273
@Override
@@ -452,10 +459,79 @@ public static Query rangeQuery(
452459

453460
@Override
454461
public BlockLoader blockLoader(BlockLoaderContext blContext) {
455-
if (hasDocValues()) {
462+
if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) {
456463
return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
457464
}
458-
return null;
465+
466+
if (isStored()) {
467+
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
468+
}
469+
470+
if (isSyntheticSource) {
471+
return blockLoaderFromFallbackSyntheticSource(blContext);
472+
}
473+
474+
// see #indexValue
475+
BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isIndexed()
476+
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
477+
: BlockSourceReader.lookupMatchingAll();
478+
return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
479+
}
480+
481+
private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) {
482+
var reader = new FallbackSyntheticSourceBlockLoader.SingleValueReader<InetAddress>(nullValue) {
483+
@Override
484+
public void convertValue(Object value, List<InetAddress> accumulator) {
485+
if (value instanceof InetAddress ia) {
486+
accumulator.add(ia);
487+
}
488+
489+
try {
490+
var address = InetAddresses.forString(value.toString());
491+
accumulator.add(address);
492+
} catch (Exception e) {
493+
// Malformed value, skip it
494+
}
495+
}
496+
497+
@Override
498+
protected void parseNonNullValue(XContentParser parser, List<InetAddress> accumulator) throws IOException {
499+
// aligned with #parseCreateField()
500+
String value = parser.text();
501+
502+
try {
503+
var address = InetAddresses.forString(value);
504+
accumulator.add(address);
505+
} catch (Exception e) {
506+
// Malformed value, skip it
507+
}
508+
}
509+
510+
@Override
511+
public void writeToBlock(List<InetAddress> values, BlockLoader.Builder blockBuilder) {
512+
var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
513+
514+
for (var value : values) {
515+
bytesRefBuilder.appendBytesRef(new BytesRef(InetAddressPoint.encode(value)));
516+
}
517+
}
518+
};
519+
520+
return new FallbackSyntheticSourceBlockLoader(reader, name()) {
521+
@Override
522+
public Builder builder(BlockFactory factory, int expectedCount) {
523+
return factory.bytesRefs(expectedCount);
524+
}
525+
};
526+
}
527+
528+
private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
529+
return new SourceValueFetcher(sourcePaths, nullValue) {
530+
@Override
531+
public InetAddress parseSourceValue(Object value) {
532+
return parse(value);
533+
}
534+
};
459535
}
460536

461537
@Override

‎server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java

+2
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ public void testTermQuery() {
105105
null,
106106
null,
107107
Collections.emptyMap(),
108+
false,
108109
false
109110
);
110111
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", MOCK_CONTEXT));
@@ -339,6 +340,7 @@ public void testRangeQuery() {
339340
null,
340341
null,
341342
Collections.emptyMap(),
343+
false,
342344
false
343345
);
344346
IllegalArgumentException e = expectThrows(

‎server/src/test/java/org/elasticsearch/index/mapper/blockloader/IpFieldBlockLoaderTests.java

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.blockloader;
11+
12+
import org.apache.lucene.document.InetAddressPoint;
13+
import org.apache.lucene.util.BytesRef;
14+
import org.elasticsearch.common.network.InetAddresses;
15+
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
16+
import org.elasticsearch.index.mapper.MappedFieldType;
17+
import org.elasticsearch.logsdb.datageneration.FieldType;
18+
19+
import java.util.List;
20+
import java.util.Map;
21+
import java.util.Objects;
22+
23+
public class IpFieldBlockLoaderTests extends BlockLoaderTestCase {
24+
public IpFieldBlockLoaderTests(Params params) {
25+
super(FieldType.IP.toString(), params);
26+
}
27+
28+
@Override
29+
@SuppressWarnings("unchecked")
30+
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
31+
var rawNullValue = (String) fieldMapping.get("null_value");
32+
BytesRef nullValue = convert(rawNullValue, null);
33+
34+
if (value == null) {
35+
return convert(null, nullValue);
36+
}
37+
if (value instanceof String s) {
38+
return convert(s, nullValue);
39+
}
40+
41+
boolean hasDocValues = hasDocValues(fieldMapping, true);
42+
boolean useDocValues = params.preference() == MappedFieldType.FieldExtractPreference.NONE
43+
|| params.preference() == MappedFieldType.FieldExtractPreference.DOC_VALUES
44+
|| params.syntheticSource();
45+
if (hasDocValues && useDocValues) {
46+
var resultList = ((List<String>) value).stream()
47+
.map(v -> convert(v, nullValue))
48+
.filter(Objects::nonNull)
49+
.distinct()
50+
.sorted()
51+
.toList();
52+
return maybeFoldList(resultList);
53+
}
54+
55+
// field is stored or using source
56+
var resultList = ((List<String>) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList();
57+
return maybeFoldList(resultList);
58+
}
59+
60+
private static BytesRef convert(Object value, BytesRef nullValue) {
61+
if (value == null) {
62+
return nullValue;
63+
}
64+
65+
if (value instanceof String s) {
66+
try {
67+
var address = InetAddresses.forString(s);
68+
return new BytesRef(InetAddressPoint.encode(address));
69+
} catch (Exception ex) {
70+
// malformed
71+
return null;
72+
}
73+
}
74+
75+
return null;
76+
}
77+
}

‎server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java

+1
Original file line numberDiff line numberDiff line change
@@ -1328,6 +1328,7 @@ public void testIpField() throws Exception {
13281328
null,
13291329
null,
13301330
Collections.emptyMap(),
1331+
false,
13311332
false
13321333
);
13331334
testCase(iw -> {

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.logsdb.datageneration.fields.leaf.GeoPointFieldDataGenerator;
2020
import org.elasticsearch.logsdb.datageneration.fields.leaf.HalfFloatFieldDataGenerator;
2121
import org.elasticsearch.logsdb.datageneration.fields.leaf.IntegerFieldDataGenerator;
22+
import org.elasticsearch.logsdb.datageneration.fields.leaf.IpFieldDataGenerator;
2223
import org.elasticsearch.logsdb.datageneration.fields.leaf.KeywordFieldDataGenerator;
2324
import org.elasticsearch.logsdb.datageneration.fields.leaf.LongFieldDataGenerator;
2425
import org.elasticsearch.logsdb.datageneration.fields.leaf.ScaledFloatFieldDataGenerator;
@@ -44,7 +45,8 @@ public enum FieldType {
4445
BOOLEAN("boolean"),
4546
DATE("date"),
4647
GEO_POINT("geo_point"),
47-
TEXT("text");
48+
TEXT("text"),
49+
IP("ip");
4850

4951
private final String name;
5052

@@ -69,6 +71,7 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
6971
case DATE -> new DateFieldDataGenerator(dataSource);
7072
case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource);
7173
case TEXT -> new TextFieldDataGenerator(dataSource);
74+
case IP -> new IpFieldDataGenerator(dataSource);
7275
};
7376
}
7477

@@ -89,6 +92,7 @@ public static FieldType tryParse(String name) {
8992
case "date" -> FieldType.DATE;
9093
case "geo_point" -> FieldType.GEO_POINT;
9194
case "text" -> FieldType.TEXT;
95+
case "ip" -> FieldType.IP;
9296
default -> null;
9397
};
9498
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java

+4
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ default DataSourceResponse.PointGenerator handle(DataSourceRequest.PointGenerato
7474
return null;
7575
}
7676

77+
default DataSourceResponse.IpGenerator handle(DataSourceRequest.IpGenerator request) {
78+
return null;
79+
}
80+
7781
default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
7882
return null;
7983
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java

+6
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,12 @@ public DataSourceResponse.PointGenerator accept(DataSourceHandler handler) {
120120
}
121121
}
122122

123+
record IpGenerator() implements DataSourceRequest<DataSourceResponse.IpGenerator> {
124+
public DataSourceResponse.IpGenerator accept(DataSourceHandler handler) {
125+
return handler.handle(this);
126+
}
127+
}
128+
123129
record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
124130
public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
125131
return handler.handle(this);

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java

+3
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.elasticsearch.geometry.Geometry;
1313

14+
import java.net.InetAddress;
1415
import java.time.Instant;
1516
import java.util.Map;
1617
import java.util.Optional;
@@ -50,6 +51,8 @@ record PointGenerator(Supplier<Object> generator) implements DataSourceResponse
5051

5152
record GeoPointGenerator(Supplier<Object> generator) implements DataSourceResponse {}
5253

54+
record IpGenerator(Supplier<InetAddress> generator) implements DataSourceResponse {}
55+
5356
record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
5457

5558
record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}

0 commit comments

Comments
 (0)