Skip to content

Commit 8106853

Browse files
talevytalevy
committed
add ignore_missing option to SplitProcessor
Closes elastic#20840.
1 parent e57720e commit 8106853

File tree

8 files changed

+73
-43
lines changed

8 files changed

+73
-43
lines changed

‎core/src/main/java/org/elasticsearch/ingest/IngestDocument.java

+22
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,28 @@ public <T> T getFieldValue(String path, Class<T> clazz) {
116116
return cast(path, context, clazz);
117117
}
118118

119+
/**
120+
* Returns the value contained in the document for the provided path
121+
*
122+
* @param path The path within the document in dot-notation
123+
* @param clazz The expected class of the field value
124+
* @param ignoreMissing The flag to determine whether to throw an exception when `path` is not found in the document.
125+
* @return the value for the provided path if existing, null otherwise.
126+
* @throws IllegalArgumentException only if ignoreMissing is false and the path is null, empty, invalid, if the field doesn't exist
127+
* or if the field that is found at the provided path is not of the expected type.
128+
*/
129+
public <T> T getFieldValue(String path, Class<T> clazz, boolean ignoreMissing) {
130+
try {
131+
return getFieldValue(path, clazz);
132+
} catch (IllegalArgumentException e) {
133+
if (ignoreMissing && hasField(path) != true) {
134+
return null;
135+
} else {
136+
throw e;
137+
}
138+
}
139+
}
140+
119141
/**
120142
* Returns the value contained in the document with the provided templated path
121143
* @param pathTemplate The path within the document in dot-notation

‎docs/reference/ingest/ingest-node.asciidoc

+4-3
Original file line numberDiff line numberDiff line change
@@ -1442,9 +1442,10 @@ Splits a field into an array using a separator character. Only works on string f
14421442
.Split Options
14431443
[options="header"]
14441444
|======
1445-
| Name | Required | Default | Description
1446-
| `field` | yes | - | The field to split
1447-
| `separator` | yes | - | A regex which matches the separator, eg `,` or `\s+`
1445+
| Name | Required | Default | Description
1446+
| `field` | yes | - | The field to split
1447+
| `separator` | yes | - | A regex which matches the separator, eg `,` or `\s+`
1448+
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
14481449
|======
14491450

14501451
[source,js]

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java

+3-11
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
3434
private final String field;
3535
private final boolean ignoreMissing;
3636

37-
protected AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
37+
AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
3838
super(tag);
3939
this.field = field;
4040
this.ignoreMissing = ignoreMissing;
@@ -50,16 +50,8 @@ boolean isIgnoreMissing() {
5050

5151
@Override
5252
public final void execute(IngestDocument document) {
53-
String val;
53+
String val = document.getFieldValue(field, String.class, ignoreMissing);
5454

55-
try {
56-
val = document.getFieldValue(field, String.class);
57-
} catch (IllegalArgumentException e) {
58-
if (ignoreMissing && document.hasField(field) != true) {
59-
return;
60-
}
61-
throw e;
62-
}
6355
if (val == null && ignoreMissing) {
6456
return;
6557
} else if (val == null) {
@@ -72,7 +64,7 @@ public final void execute(IngestDocument document) {
7264
protected abstract String process(String value);
7365

7466
abstract static class Factory implements Processor.Factory {
75-
protected final String processorType;
67+
final String processorType;
7668

7769
protected Factory(String processorType) {
7870
this.processorType = processorType;

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java

+1-10
Original file line numberDiff line numberDiff line change
@@ -142,18 +142,9 @@ boolean isIgnoreMissing() {
142142

143143
@Override
144144
public void execute(IngestDocument document) {
145-
Object oldValue = null;
145+
Object oldValue = document.getFieldValue(field, Object.class, ignoreMissing);
146146
Object newValue;
147147

148-
try {
149-
oldValue = document.getFieldValue(field, Object.class);
150-
} catch (IllegalArgumentException e) {
151-
if (ignoreMissing) {
152-
return;
153-
}
154-
throw e;
155-
}
156-
157148
if (oldValue == null && ignoreMissing) {
158149
return;
159150
} else if (oldValue == null) {

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java

+1-10
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,7 @@ public GrokProcessor(String tag, Map<String, String> patternBank, List<String> m
5252

5353
@Override
5454
public void execute(IngestDocument ingestDocument) throws Exception {
55-
String fieldValue;
56-
57-
try {
58-
fieldValue = ingestDocument.getFieldValue(matchField, String.class);
59-
} catch (IllegalArgumentException e) {
60-
if (ignoreMissing && ingestDocument.hasField(matchField) != true) {
61-
return;
62-
}
63-
throw e;
64-
}
55+
String fieldValue = ingestDocument.getFieldValue(matchField, String.class, ignoreMissing);
6556

6657
if (fieldValue == null && ignoreMissing) {
6758
return;

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java

+16-4
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,13 @@ public final class SplitProcessor extends AbstractProcessor {
4040

4141
private final String field;
4242
private final String separator;
43+
private final boolean ignoreMissing;
4344

44-
SplitProcessor(String tag, String field, String separator) {
45+
SplitProcessor(String tag, String field, String separator, boolean ignoreMissing) {
4546
super(tag);
4647
this.field = field;
4748
this.separator = separator;
49+
this.ignoreMissing = ignoreMissing;
4850
}
4951

5052
String getField() {
@@ -55,12 +57,20 @@ String getSeparator() {
5557
return separator;
5658
}
5759

60+
boolean isIgnoreMissing() {
61+
return ignoreMissing;
62+
}
63+
5864
@Override
5965
public void execute(IngestDocument document) {
60-
String oldVal = document.getFieldValue(field, String.class);
61-
if (oldVal == null) {
66+
String oldVal = document.getFieldValue(field, String.class, ignoreMissing);
67+
68+
if (oldVal == null && ignoreMissing) {
69+
return;
70+
} else if (oldVal == null) {
6271
throw new IllegalArgumentException("field [" + field + "] is null, cannot split.");
6372
}
73+
6474
String[] strings = oldVal.split(separator);
6575
List<String> splitList = new ArrayList<>(strings.length);
6676
Collections.addAll(splitList, strings);
@@ -77,7 +87,9 @@ public static class Factory implements Processor.Factory {
7787
public SplitProcessor create(Map<String, Processor.Factory> registry, String processorTag,
7888
Map<String, Object> config) throws Exception {
7989
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
80-
return new SplitProcessor(processorTag, field, ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"));
90+
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
91+
return new SplitProcessor(processorTag, field,
92+
ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"), ignoreMissing);
8193
}
8294
}
8395
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public void testCreate() throws Exception {
3939
assertThat(splitProcessor.getTag(), equalTo(processorTag));
4040
assertThat(splitProcessor.getField(), equalTo("field1"));
4141
assertThat(splitProcessor.getSeparator(), equalTo("\\."));
42+
assertFalse(splitProcessor.isIgnoreMissing());
4243
}
4344

4445
public void testCreateNoFieldPresent() throws Exception {

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java

+25-5
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.util.List;
3131
import java.util.Map;
3232

33+
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
3334
import static org.hamcrest.Matchers.containsString;
3435
import static org.hamcrest.Matchers.equalTo;
3536

@@ -38,15 +39,15 @@ public class SplitProcessorTests extends ESTestCase {
3839
public void testSplit() throws Exception {
3940
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
4041
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
41-
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
42+
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false);
4243
processor.execute(ingestDocument);
4344
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
4445
}
4546

4647
public void testSplitFieldNotFound() throws Exception {
4748
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
4849
String fieldName = RandomDocumentPicks.randomFieldName(random());
49-
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
50+
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false);
5051
try {
5152
processor.execute(ingestDocument);
5253
fail("split processor should have failed");
@@ -56,8 +57,9 @@ public void testSplitFieldNotFound() throws Exception {
5657
}
5758

5859
public void testSplitNullValue() throws Exception {
59-
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
60-
Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.");
60+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(),
61+
Collections.singletonMap("field", null));
62+
Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.", false);
6163
try {
6264
processor.execute(ingestDocument);
6365
fail("split processor should have failed");
@@ -66,11 +68,29 @@ public void testSplitNullValue() throws Exception {
6668
}
6769
}
6870

71+
public void testSplitNullValueWithIgnoreMissing() throws Exception {
72+
String fieldName = RandomDocumentPicks.randomFieldName(random());
73+
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
74+
Collections.singletonMap(fieldName, null));
75+
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
76+
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", true);
77+
processor.execute(ingestDocument);
78+
assertIngestDocument(originalIngestDocument, ingestDocument);
79+
}
80+
81+
public void testSplitNonExistentWithIgnoreMissing() throws Exception {
82+
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
83+
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
84+
Processor processor = new SplitProcessor(randomAsciiOfLength(10), "field", "\\.", true);
85+
processor.execute(ingestDocument);
86+
assertIngestDocument(originalIngestDocument, ingestDocument);
87+
}
88+
6989
public void testSplitNonStringValue() throws Exception {
7090
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
7191
String fieldName = RandomDocumentPicks.randomFieldName(random());
7292
ingestDocument.setFieldValue(fieldName, randomInt());
73-
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.");
93+
Processor processor = new SplitProcessor(randomAsciiOfLength(10), fieldName, "\\.", false);
7494
try {
7595
processor.execute(ingestDocument);
7696
fail("split processor should have failed");

0 commit comments

Comments
 (0)