Skip to content

add ignore_missing option to convert,trim,lowercase,uppercase,grok,rename #20194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 9, 2016
Merged

add ignore_missing option to convert,trim,lowercase,uppercase,grok,rename #20194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

import java.io.IOException;

import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;

Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import java.util.Collections;
import java.util.Map;

import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.not;
Expand Down
9 changes: 0 additions & 9 deletions core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,13 @@
package org.elasticsearch.action.ingest;

import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.test.ESTestCase;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;

public class SimulatePipelineRequestTests extends ESTestCase {

Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import java.util.Iterator;
import java.util.List;

import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.nullValue;
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

import java.io.IOException;

import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import java.util.Map;

import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
Expand Down
31 changes: 1 addition & 30 deletions core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import java.util.Locale;
import java.util.Map;

import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.both;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.endsWith;
Expand Down Expand Up @@ -1004,34 +1005,4 @@ public void testSetInvalidSourceField() throws Exception {
}
}

public static void assertIngestDocument(Object a, Object b) {
if (a instanceof Map) {
Map<?, ?> mapA = (Map<?, ?>) a;
Map<?, ?> mapB = (Map<?, ?>) b;
for (Map.Entry<?, ?> entry : mapA.entrySet()) {
if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
}
}
} else if (a instanceof List) {
List<?> listA = (List<?>) a;
List<?> listB = (List<?>) b;
for (int i = 0; i < listA.size(); i++) {
Object value = listA.get(i);
if (value instanceof List || value instanceof Map) {
assertIngestDocument(value, listB.get(i));
}
}
} else if (a instanceof byte[]) {
assertArrayEquals((byte[]) a, (byte[])b);
} else if (a instanceof IngestDocument) {
IngestDocument docA = (IngestDocument) a;
IngestDocument docB = (IngestDocument) b;
assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
} else {
String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
assertThat(msg, a, equalTo(b));
}
}
}
32 changes: 19 additions & 13 deletions docs/reference/ingest/ingest-node.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -706,10 +706,11 @@ such a case, `target_field` will still be updated with the unconverted field val
.Convert Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
| `type` | yes | - | The type to convert the existing value to
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
| `type` | yes | - | The type to convert the existing value to
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======

[source,js]
Expand Down Expand Up @@ -1142,6 +1143,7 @@ Grok expression.
| `patterns` | yes | - | An ordered list of grok expression to match and extract named captures with. Returns on the first expression in the list that matches.
| `pattern_definitions` | no | - | A map of pattern-name and pattern tuples defining custom patterns to be used by the current processor. Patterns matching existing names will override the pre-existing definition.
| `trace_match` | no | false | when true, `_ingest._grok_match_index` will be inserted into your matched document's metadata with the index into the pattern found in `patterns` that matched.
| `ignore_missing` | no | false | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======

Here is an example of using the provided patterns to extract out and name structured fields from a string field in
Expand Down Expand Up @@ -1278,8 +1280,9 @@ Converts a string to its lowercase equivalent.
.Lowercase Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to make lowercase
| Name | Required | Default | Description
| `field` | yes | - | The field to make lowercase
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======

[source,js]
Expand Down Expand Up @@ -1320,9 +1323,10 @@ Renames an existing field. If the field doesn't exist or the new name is already
.Rename Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to be renamed
| `target_field` | yes | - | The new name of the field
| Name | Required | Default | Description
| `field` | yes | - | The field to be renamed
| `target_field` | yes | - | The new name of the field
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
|======

[source,js]
Expand Down Expand Up @@ -1462,8 +1466,9 @@ NOTE: This only works on leading and trailing whitespace.
.Trim Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The string-valued field to trim whitespace from
| Name | Required | Default | Description
| `field` | yes | - | The string-valued field to trim whitespace from
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
|======

[source,js]
Expand All @@ -1483,8 +1488,9 @@ Converts a string to its uppercase equivalent.
.Uppercase Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field to make uppercase
| Name | Required | Default | Description
| `field` | yes | - | The field to make uppercase
| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
|======

[source,js]
Expand Down
29 changes: 24 additions & 5 deletions .../ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,40 @@
*/
abstract class AbstractStringProcessor extends AbstractProcessor {
private final String field;
private final boolean ignoreMissing;

protected AbstractStringProcessor(String tag, String field) {
protected AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
super(tag);
this.field = field;
this.ignoreMissing = ignoreMissing;
}

public String getField() {
return field;
}

boolean isIgnoreMissing() {
return ignoreMissing;
}

@Override
public final void execute(IngestDocument document) {
String val = document.getFieldValue(field, String.class);
if (val == null) {
String val;

try {
val = document.getFieldValue(field, String.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing && document.hasField(field) != true) {
return;
}
throw e;
}
if (val == null && ignoreMissing) {
return;
} else if (val == null) {
throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
}

document.setFieldValue(field, process(val));
}

Expand All @@ -64,9 +82,10 @@ protected Factory(String processorType) {
public AbstractStringProcessor create(Map<String, Processor.Factory> registry, String tag,
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
return newProcessor(tag, field);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);
return newProcessor(tag, field, ignoreMissing);
}

protected abstract AbstractStringProcessor newProcessor(String processorTag, String field);
protected abstract AbstractStringProcessor newProcessor(String processorTag, String field, boolean ignoreMissing);
}
}
27 changes: 23 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,14 @@ public static Type fromString(String processorTag, String propertyName, String t
private final String field;
private final String targetField;
private final Type convertType;
private final boolean ignoreMissing;

ConvertProcessor(String tag, String field, String targetField, Type convertType) {
ConvertProcessor(String tag, String field, String targetField, Type convertType, boolean ignoreMissing) {
super(tag);
this.field = field;
this.targetField = targetField;
this.convertType = convertType;
this.ignoreMissing = ignoreMissing;
}

String getField() {
Expand All @@ -134,11 +136,27 @@ Type getConvertType() {
return convertType;
}

boolean isIgnoreMissing() {
return ignoreMissing;
}

@Override
public void execute(IngestDocument document) {
Object oldValue = document.getFieldValue(field, Object.class);
Object oldValue = null;
Object newValue;
if (oldValue == null) {

try {
oldValue = document.getFieldValue(field, Object.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing) {
return;
}
throw e;
}

if (oldValue == null && ignoreMissing) {
return;
} else if (oldValue == null) {
throw new IllegalArgumentException("Field [" + field + "] is null, cannot be converted to type [" + convertType + "]");
}

Expand Down Expand Up @@ -168,7 +186,8 @@ public ConvertProcessor create(Map<String, Processor.Factory> registry, String p
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
Type convertType = Type.fromString(processorTag, "type", typeProperty);
return new ConvertProcessor(processorTag, field, targetField, convertType);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new ConvertProcessor(processorTag, field, targetField, convertType, ignoreMissing);
}
}
}
36 changes: 28 additions & 8 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,36 @@ public final class GrokProcessor extends AbstractProcessor {
private final String matchField;
private final Grok grok;
private final boolean traceMatch;
private final boolean ignoreMissing;

public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField) {
this(tag, patternBank, matchPatterns, matchField, false);
}

public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField, boolean traceMatch) {
public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField,
boolean traceMatch, boolean ignoreMissing) {
super(tag);
this.matchField = matchField;
this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch));
this.traceMatch = traceMatch;
this.ignoreMissing = ignoreMissing;
}

@Override
public void execute(IngestDocument ingestDocument) throws Exception {
String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
String fieldValue;

try {
fieldValue = ingestDocument.getFieldValue(matchField, String.class);
} catch (IllegalArgumentException e) {
if (ignoreMissing && ingestDocument.hasField(matchField) != true) {
return;
}
throw e;
}

if (fieldValue == null && ignoreMissing) {
return;
} else if (fieldValue == null) {
throw new IllegalArgumentException("field [" + matchField + "] is null, cannot process it.");
}

Map<String, Object> matches = grok.captures(fieldValue);
if (matches == null) {
throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
Expand All @@ -77,10 +92,14 @@ public String getType() {
return TYPE;
}

public Grok getGrok() {
Grok getGrok() {
return grok;
}

boolean isIgnoreMissing() {
return ignoreMissing;
}

String getMatchField() {
return matchField;
}
Expand Down Expand Up @@ -128,6 +147,7 @@ public GrokProcessor create(Map<String, Processor.Factory> registry, String proc
String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);

if (matchPatterns.isEmpty()) {
throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
Expand All @@ -139,7 +159,7 @@ public GrokProcessor create(Map<String, Processor.Factory> registry, String proc
}

try {
return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch);
return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing);
} catch (Exception e) {
throw newConfigurationException(TYPE, processorTag, "patterns",
"Invalid regex pattern found in: " + matchPatterns + ". " + e.getMessage());
Expand Down
8 changes: 4 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor {

public static final String TYPE = "lowercase";

LowercaseProcessor(String processorTag, String field) {
super(processorTag, field);
LowercaseProcessor(String processorTag, String field, boolean ignoreMissing) {
super(processorTag, field, ignoreMissing);
}

@Override
Expand All @@ -51,8 +51,8 @@ public Factory() {
}

@Override
protected LowercaseProcessor newProcessor(String tag, String field) {
return new LowercaseProcessor(tag, field);
protected LowercaseProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
return new LowercaseProcessor(tag, field, ignoreMissing);
}
}
}
Loading