Skip to content

Commit 3b1a6f7

Browse files
talevytalevy
committed
add ignore_missing option to relevant processors (#20194)
1 parent b5a8fe6 commit 3b1a6f7

30 files changed

+432
-160
lines changed

‎core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
import java.io.IOException;
2929

30-
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
30+
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
3131
import static org.hamcrest.CoreMatchers.equalTo;
3232
import static org.hamcrest.CoreMatchers.instanceOf;
3333

‎core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import java.util.Collections;
3535
import java.util.Map;
3636

37-
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
37+
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
3838
import static org.hamcrest.Matchers.equalTo;
3939
import static org.hamcrest.Matchers.instanceOf;
4040
import static org.hamcrest.Matchers.not;

‎core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java

-9
Original file line numberDiff line numberDiff line change
@@ -20,22 +20,13 @@
2020
package org.elasticsearch.action.ingest;
2121

2222
import org.elasticsearch.common.bytes.BytesArray;
23-
import org.elasticsearch.common.bytes.BytesReference;
2423
import org.elasticsearch.common.io.stream.BytesStreamOutput;
2524
import org.elasticsearch.common.io.stream.StreamInput;
26-
import org.elasticsearch.ingest.IngestDocument;
27-
import org.elasticsearch.ingest.RandomDocumentPicks;
2825
import org.elasticsearch.test.ESTestCase;
2926

3027
import java.io.IOException;
31-
import java.util.ArrayList;
32-
import java.util.Iterator;
33-
import java.util.List;
3428

35-
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
3629
import static org.hamcrest.CoreMatchers.equalTo;
37-
import static org.hamcrest.CoreMatchers.instanceOf;
38-
import static org.hamcrest.CoreMatchers.nullValue;
3930

4031
public class SimulatePipelineRequestTests extends ESTestCase {
4132

‎core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
import java.util.Iterator;
3131
import java.util.List;
3232

33-
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
33+
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
3434
import static org.hamcrest.CoreMatchers.equalTo;
3535
import static org.hamcrest.CoreMatchers.instanceOf;
3636
import static org.hamcrest.CoreMatchers.nullValue;

‎core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
import java.io.IOException;
2929

30-
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
30+
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
3131
import static org.hamcrest.Matchers.equalTo;
3232
import static org.hamcrest.Matchers.instanceOf;
3333
import static org.hamcrest.Matchers.is;

‎core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import java.util.Map;
3535

3636
import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
37-
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
37+
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
3838
import static org.hamcrest.Matchers.equalTo;
3939
import static org.hamcrest.Matchers.is;
4040
import static org.hamcrest.Matchers.not;

‎core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java

+1-30
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import java.util.Locale;
3434
import java.util.Map;
3535

36+
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
3637
import static org.hamcrest.Matchers.both;
3738
import static org.hamcrest.Matchers.containsString;
3839
import static org.hamcrest.Matchers.endsWith;
@@ -1004,34 +1005,4 @@ public void testSetInvalidSourceField() throws Exception {
10041005
}
10051006
}
10061007

1007-
public static void assertIngestDocument(Object a, Object b) {
1008-
if (a instanceof Map) {
1009-
Map<?, ?> mapA = (Map<?, ?>) a;
1010-
Map<?, ?> mapB = (Map<?, ?>) b;
1011-
for (Map.Entry<?, ?> entry : mapA.entrySet()) {
1012-
if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
1013-
assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
1014-
}
1015-
}
1016-
} else if (a instanceof List) {
1017-
List<?> listA = (List<?>) a;
1018-
List<?> listB = (List<?>) b;
1019-
for (int i = 0; i < listA.size(); i++) {
1020-
Object value = listA.get(i);
1021-
if (value instanceof List || value instanceof Map) {
1022-
assertIngestDocument(value, listB.get(i));
1023-
}
1024-
}
1025-
} else if (a instanceof byte[]) {
1026-
assertArrayEquals((byte[]) a, (byte[])b);
1027-
} else if (a instanceof IngestDocument) {
1028-
IngestDocument docA = (IngestDocument) a;
1029-
IngestDocument docB = (IngestDocument) b;
1030-
assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
1031-
assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
1032-
} else {
1033-
String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
1034-
assertThat(msg, a, equalTo(b));
1035-
}
1036-
}
10371008
}

‎docs/reference/ingest/ingest-node.asciidoc

+19-13
Original file line numberDiff line numberDiff line change
@@ -706,10 +706,11 @@ such a case, `target_field` will still be updated with the unconverted field val
706706
.Convert Options
707707
[options="header"]
708708
|======
709-
| Name | Required | Default | Description
710-
| `field` | yes | - | The field whose value is to be converted
711-
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
712-
| `type` | yes | - | The type to convert the existing value to
709+
| Name | Required | Default | Description
710+
| `field` | yes | - | The field whose value is to be converted
711+
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
712+
| `type` | yes | - | The type to convert the existing value to
713+
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
713714
|======
714715

715716
[source,js]
@@ -1142,6 +1143,7 @@ Grok expression.
11421143
| `patterns` | yes | - | An ordered list of grok expression to match and extract named captures with. Returns on the first expression in the list that matches.
11431144
| `pattern_definitions` | no | - | A map of pattern-name and pattern tuples defining custom patterns to be used by the current processor. Patterns matching existing names will override the pre-existing definition.
11441145
| `trace_match` | no | false | when true, `_ingest._grok_match_index` will be inserted into your matched document's metadata with the index into the pattern found in `patterns` that matched.
1146+
| `ignore_missing` | no | false | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
11451147
|======
11461148

11471149
Here is an example of using the provided patterns to extract out and name structured fields from a string field in
@@ -1278,8 +1280,9 @@ Converts a string to its lowercase equivalent.
12781280
.Lowercase Options
12791281
[options="header"]
12801282
|======
1281-
| Name | Required | Default | Description
1282-
| `field` | yes | - | The field to make lowercase
1283+
| Name | Required | Default | Description
1284+
| `field` | yes | - | The field to make lowercase
1285+
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
12831286
|======
12841287

12851288
[source,js]
@@ -1320,9 +1323,10 @@ Renames an existing field. If the field doesn't exist or the new name is already
13201323
.Rename Options
13211324
[options="header"]
13221325
|======
1323-
| Name | Required | Default | Description
1324-
| `field` | yes | - | The field to be renamed
1325-
| `target_field` | yes | - | The new name of the field
1326+
| Name | Required | Default | Description
1327+
| `field` | yes | - | The field to be renamed
1328+
| `target_field` | yes | - | The new name of the field
1329+
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
13261330
|======
13271331

13281332
[source,js]
@@ -1462,8 +1466,9 @@ NOTE: This only works on leading and trailing whitespace.
14621466
.Trim Options
14631467
[options="header"]
14641468
|======
1465-
| Name | Required | Default | Description
1466-
| `field` | yes | - | The string-valued field to trim whitespace from
1469+
| Name | Required | Default | Description
1470+
| `field` | yes | - | The string-valued field to trim whitespace from
1471+
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
14671472
|======
14681473

14691474
[source,js]
@@ -1483,8 +1488,9 @@ Converts a string to its uppercase equivalent.
14831488
.Uppercase Options
14841489
[options="header"]
14851490
|======
1486-
| Name | Required | Default | Description
1487-
| `field` | yes | - | The field to make uppercase
1491+
| Name | Required | Default | Description
1492+
| `field` | yes | - | The field to make uppercase
1493+
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
14881494
|======
14891495

14901496
[source,js]

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java

+24-5
Original file line numberDiff line numberDiff line change
@@ -32,22 +32,40 @@
3232
*/
3333
abstract class AbstractStringProcessor extends AbstractProcessor {
3434
private final String field;
35+
private final boolean ignoreMissing;
3536

36-
protected AbstractStringProcessor(String tag, String field) {
37+
protected AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
3738
super(tag);
3839
this.field = field;
40+
this.ignoreMissing = ignoreMissing;
3941
}
4042

4143
public String getField() {
4244
return field;
4345
}
4446

47+
boolean isIgnoreMissing() {
48+
return ignoreMissing;
49+
}
50+
4551
@Override
4652
public final void execute(IngestDocument document) {
47-
String val = document.getFieldValue(field, String.class);
48-
if (val == null) {
53+
String val;
54+
55+
try {
56+
val = document.getFieldValue(field, String.class);
57+
} catch (IllegalArgumentException e) {
58+
if (ignoreMissing && document.hasField(field) != true) {
59+
return;
60+
}
61+
throw e;
62+
}
63+
if (val == null && ignoreMissing) {
64+
return;
65+
} else if (val == null) {
4966
throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
5067
}
68+
5169
document.setFieldValue(field, process(val));
5270
}
5371

@@ -64,9 +82,10 @@ protected Factory(String processorType) {
6482
public AbstractStringProcessor create(Map<String, Processor.Factory> registry, String tag,
6583
Map<String, Object> config) throws Exception {
6684
String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
67-
return newProcessor(tag, field);
85+
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);
86+
return newProcessor(tag, field, ignoreMissing);
6887
}
6988

70-
protected abstract AbstractStringProcessor newProcessor(String processorTag, String field);
89+
protected abstract AbstractStringProcessor newProcessor(String processorTag, String field, boolean ignoreMissing);
7190
}
7291
}

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java

+23-4
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,14 @@ public static Type fromString(String processorTag, String propertyName, String t
114114
private final String field;
115115
private final String targetField;
116116
private final Type convertType;
117+
private final boolean ignoreMissing;
117118

118-
ConvertProcessor(String tag, String field, String targetField, Type convertType) {
119+
ConvertProcessor(String tag, String field, String targetField, Type convertType, boolean ignoreMissing) {
119120
super(tag);
120121
this.field = field;
121122
this.targetField = targetField;
122123
this.convertType = convertType;
124+
this.ignoreMissing = ignoreMissing;
123125
}
124126

125127
String getField() {
@@ -134,11 +136,27 @@ Type getConvertType() {
134136
return convertType;
135137
}
136138

139+
boolean isIgnoreMissing() {
140+
return ignoreMissing;
141+
}
142+
137143
@Override
138144
public void execute(IngestDocument document) {
139-
Object oldValue = document.getFieldValue(field, Object.class);
145+
Object oldValue = null;
140146
Object newValue;
141-
if (oldValue == null) {
147+
148+
try {
149+
oldValue = document.getFieldValue(field, Object.class);
150+
} catch (IllegalArgumentException e) {
151+
if (ignoreMissing) {
152+
return;
153+
}
154+
throw e;
155+
}
156+
157+
if (oldValue == null && ignoreMissing) {
158+
return;
159+
} else if (oldValue == null) {
142160
throw new IllegalArgumentException("Field [" + field + "] is null, cannot be converted to type [" + convertType + "]");
143161
}
144162

@@ -168,7 +186,8 @@ public ConvertProcessor create(Map<String, Processor.Factory> registry, String p
168186
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
169187
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
170188
Type convertType = Type.fromString(processorTag, "type", typeProperty);
171-
return new ConvertProcessor(processorTag, field, targetField, convertType);
189+
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
190+
return new ConvertProcessor(processorTag, field, targetField, convertType, ignoreMissing);
172191
}
173192
}
174193
}

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java

+28-8
Original file line numberDiff line numberDiff line change
@@ -39,21 +39,36 @@ public final class GrokProcessor extends AbstractProcessor {
3939
private final String matchField;
4040
private final Grok grok;
4141
private final boolean traceMatch;
42+
private final boolean ignoreMissing;
4243

43-
public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField) {
44-
this(tag, patternBank, matchPatterns, matchField, false);
45-
}
46-
47-
public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField, boolean traceMatch) {
44+
public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField,
45+
boolean traceMatch, boolean ignoreMissing) {
4846
super(tag);
4947
this.matchField = matchField;
5048
this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch));
5149
this.traceMatch = traceMatch;
50+
this.ignoreMissing = ignoreMissing;
5251
}
5352

5453
@Override
5554
public void execute(IngestDocument ingestDocument) throws Exception {
56-
String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
55+
String fieldValue;
56+
57+
try {
58+
fieldValue = ingestDocument.getFieldValue(matchField, String.class);
59+
} catch (IllegalArgumentException e) {
60+
if (ignoreMissing && ingestDocument.hasField(matchField) != true) {
61+
return;
62+
}
63+
throw e;
64+
}
65+
66+
if (fieldValue == null && ignoreMissing) {
67+
return;
68+
} else if (fieldValue == null) {
69+
throw new IllegalArgumentException("field [" + matchField + "] is null, cannot process it.");
70+
}
71+
5772
Map<String, Object> matches = grok.captures(fieldValue);
5873
if (matches == null) {
5974
throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
@@ -77,10 +92,14 @@ public String getType() {
7792
return TYPE;
7893
}
7994

80-
public Grok getGrok() {
95+
Grok getGrok() {
8196
return grok;
8297
}
8398

99+
boolean isIgnoreMissing() {
100+
return ignoreMissing;
101+
}
102+
84103
String getMatchField() {
85104
return matchField;
86105
}
@@ -128,6 +147,7 @@ public GrokProcessor create(Map<String, Processor.Factory> registry, String proc
128147
String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
129148
List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
130149
boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
150+
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
131151

132152
if (matchPatterns.isEmpty()) {
133153
throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
@@ -139,7 +159,7 @@ public GrokProcessor create(Map<String, Processor.Factory> registry, String proc
139159
}
140160

141161
try {
142-
return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch);
162+
return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing);
143163
} catch (Exception e) {
144164
throw newConfigurationException(TYPE, processorTag, "patterns",
145165
"Invalid regex pattern found in: " + matchPatterns + ". " + e.getMessage());

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
3030

3131
public static final String TYPE = "lowercase";
3232

33-
LowercaseProcessor(String processorTag, String field) {
34-
super(processorTag, field);
33+
LowercaseProcessor(String processorTag, String field, boolean ignoreMissing) {
34+
super(processorTag, field, ignoreMissing);
3535
}
3636

3737
@Override
@@ -51,8 +51,8 @@ public Factory() {
5151
}
5252

5353
@Override
54-
protected LowercaseProcessor newProcessor(String tag, String field) {
55-
return new LowercaseProcessor(tag, field);
54+
protected LowercaseProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
55+
return new LowercaseProcessor(tag, field, ignoreMissing);
5656
}
5757
}
5858
}

0 commit comments

Comments
 (0)