Skip to content

Commit 156d580

Browse files
original-brownbearoriginal-brownbear
authored
Ingest: Enable Templated Fieldnames in Rename (#31690) (#31896)
* Ingest: Enable Templated Fieldnames in Rename
1 parent c9573f6 commit 156d580

File tree

5 files changed

+64
-36
lines changed

5 files changed

+64
-36
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
6262
processors.put(DateProcessor.TYPE, new DateProcessor.Factory(parameters.scriptService));
6363
processors.put(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService));
6464
processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService));
65-
processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory());
65+
processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory(parameters.scriptService));
6666
processors.put(RemoveProcessor.TYPE, new RemoveProcessor.Factory(parameters.scriptService));
6767
processors.put(SplitProcessor.TYPE, new SplitProcessor.Factory());
6868
processors.put(JoinProcessor.TYPE, new JoinProcessor.Factory());

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java

+29-14
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import org.elasticsearch.ingest.ConfigurationUtils;
2424
import org.elasticsearch.ingest.IngestDocument;
2525
import org.elasticsearch.ingest.Processor;
26+
import org.elasticsearch.script.ScriptService;
27+
import org.elasticsearch.script.TemplateScript;
2628

2729
import java.util.Map;
2830

@@ -33,22 +35,22 @@ public final class RenameProcessor extends AbstractProcessor {
3335

3436
public static final String TYPE = "rename";
3537

36-
private final String field;
37-
private final String targetField;
38+
private final TemplateScript.Factory field;
39+
private final TemplateScript.Factory targetField;
3840
private final boolean ignoreMissing;
3941

40-
RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) {
42+
RenameProcessor(String tag, TemplateScript.Factory field, TemplateScript.Factory targetField, boolean ignoreMissing) {
4143
super(tag);
4244
this.field = field;
4345
this.targetField = targetField;
4446
this.ignoreMissing = ignoreMissing;
4547
}
4648

47-
String getField() {
49+
TemplateScript.Factory getField() {
4850
return field;
4951
}
5052

51-
String getTargetField() {
53+
TemplateScript.Factory getTargetField() {
5254
return targetField;
5355
}
5456

@@ -58,28 +60,30 @@ boolean isIgnoreMissing() {
5860

5961
@Override
6062
public void execute(IngestDocument document) {
61-
if (document.hasField(field, true) == false) {
63+
String path = document.renderTemplate(field);
64+
if (document.hasField(path, true) == false) {
6265
if (ignoreMissing) {
6366
return;
6467
} else {
65-
throw new IllegalArgumentException("field [" + field + "] doesn't exist");
68+
throw new IllegalArgumentException("field [" + path + "] doesn't exist");
6669
}
6770
}
6871
// We fail here if the target field point to an array slot that is out of range.
6972
// If we didn't do this then we would fail if we set the value in the target_field
7073
// and then on failure processors would not see that value we tried to rename as we already
7174
// removed it.
72-
if (document.hasField(targetField, true)) {
73-
throw new IllegalArgumentException("field [" + targetField + "] already exists");
75+
String target = document.renderTemplate(targetField);
76+
if (document.hasField(target, true)) {
77+
throw new IllegalArgumentException("field [" + target + "] already exists");
7478
}
7579

76-
Object value = document.getFieldValue(field, Object.class);
77-
document.removeField(field);
80+
Object value = document.getFieldValue(path, Object.class);
81+
document.removeField(path);
7882
try {
79-
document.setFieldValue(targetField, value);
83+
document.setFieldValue(target, value);
8084
} catch (Exception e) {
8185
// setting the value back to the original field shouldn't as we just fetched it from that field:
82-
document.setFieldValue(field, value);
86+
document.setFieldValue(path, value);
8387
throw e;
8488
}
8589
}
@@ -90,13 +94,24 @@ public String getType() {
9094
}
9195

9296
public static final class Factory implements Processor.Factory {
97+
98+
private final ScriptService scriptService;
99+
100+
public Factory(ScriptService scriptService) {
101+
this.scriptService = scriptService;
102+
}
103+
93104
@Override
94105
public RenameProcessor create(Map<String, Processor.Factory> registry, String processorTag,
95106
Map<String, Object> config) throws Exception {
96107
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
108+
TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
109+
"field", field, scriptService);
97110
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
111+
TemplateScript.Factory targetFieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
112+
"target_field", targetField, scriptService);
98113
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
99-
return new RenameProcessor(processorTag, field, targetField, ignoreMissing);
114+
return new RenameProcessor(processorTag, fieldTemplate, targetFieldTemplate , ignoreMissing);
100115
}
101116
}
102117
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.elasticsearch.ingest.IngestDocument;
2323
import org.elasticsearch.ingest.Processor;
24+
import org.elasticsearch.ingest.TestTemplateService;
2425
import org.elasticsearch.test.ESTestCase;
2526

2627
import java.util.Collections;
@@ -86,7 +87,8 @@ public void testEscapeFields_valueField() throws Exception {
8687
// so because foo is no branch field but a value field the `foo.bar` field can't be expanded
8788
// into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
8889
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
89-
Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false);
90+
Processor processor = new RenameProcessor("_tag", new TestTemplateService.MockTemplateScript.Factory("foo"),
91+
new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false);
9092
processor.execute(document);
9193
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
9294
processor.execute(document);

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java

+14-8
Original file line numberDiff line numberDiff line change
@@ -20,44 +20,51 @@
2020
package org.elasticsearch.ingest.common;
2121

2222
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.ingest.TestTemplateService;
2324
import org.elasticsearch.test.ESTestCase;
25+
import org.junit.Before;
2426

27+
import java.util.Collections;
2528
import java.util.HashMap;
2629
import java.util.Map;
2730

2831
import static org.hamcrest.CoreMatchers.equalTo;
2932

3033
public class RenameProcessorFactoryTests extends ESTestCase {
3134

35+
private RenameProcessor.Factory factory;
36+
37+
@Before
38+
public void init() {
39+
factory = new RenameProcessor.Factory(TestTemplateService.instance());
40+
}
41+
3242
public void testCreate() throws Exception {
33-
RenameProcessor.Factory factory = new RenameProcessor.Factory();
3443
Map<String, Object> config = new HashMap<>();
3544
config.put("field", "old_field");
3645
config.put("target_field", "new_field");
3746
String processorTag = randomAlphaOfLength(10);
3847
RenameProcessor renameProcessor = factory.create(null, processorTag, config);
3948
assertThat(renameProcessor.getTag(), equalTo(processorTag));
40-
assertThat(renameProcessor.getField(), equalTo("old_field"));
41-
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
49+
assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
50+
assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
4251
assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
4352
}
4453

4554
public void testCreateWithIgnoreMissing() throws Exception {
46-
RenameProcessor.Factory factory = new RenameProcessor.Factory();
4755
Map<String, Object> config = new HashMap<>();
4856
config.put("field", "old_field");
4957
config.put("target_field", "new_field");
5058
config.put("ignore_missing", true);
5159
String processorTag = randomAlphaOfLength(10);
5260
RenameProcessor renameProcessor = factory.create(null, processorTag, config);
5361
assertThat(renameProcessor.getTag(), equalTo(processorTag));
54-
assertThat(renameProcessor.getField(), equalTo("old_field"));
55-
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
62+
assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
63+
assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
5664
assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
5765
}
5866

5967
public void testCreateNoFieldPresent() throws Exception {
60-
RenameProcessor.Factory factory = new RenameProcessor.Factory();
6168
Map<String, Object> config = new HashMap<>();
6269
config.put("target_field", "new_field");
6370
try {
@@ -69,7 +76,6 @@ public void testCreateNoFieldPresent() throws Exception {
6976
}
7077

7178
public void testCreateNoToPresent() throws Exception {
72-
RenameProcessor.Factory factory = new RenameProcessor.Factory();
7379
Map<String, Object> config = new HashMap<>();
7480
config.put("field", "old_field");
7581
try {

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java

+17-12
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.ingest.IngestDocument;
2323
import org.elasticsearch.ingest.Processor;
2424
import org.elasticsearch.ingest.RandomDocumentPicks;
25+
import org.elasticsearch.ingest.TestTemplateService;
2526
import org.elasticsearch.test.ESTestCase;
2627

2728
import java.util.ArrayList;
@@ -45,7 +46,7 @@ public void testRename() throws Exception {
4546
do {
4647
newFieldName = RandomDocumentPicks.randomFieldName(random());
4748
} while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
48-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
49+
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
4950
processor.execute(ingestDocument);
5051
assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
5152
}
@@ -63,7 +64,7 @@ public void testRenameArrayElement() throws Exception {
6364
document.put("one", one);
6465
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
6566

66-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "item", false);
67+
Processor processor = createRenameProcessor("list.0", "item", false);
6768
processor.execute(ingestDocument);
6869
Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
6970
assertThat(actualObject, instanceOf(List.class));
@@ -76,7 +77,7 @@ public void testRenameArrayElement() throws Exception {
7677
assertThat(actualObject, instanceOf(String.class));
7778
assertThat(actualObject, equalTo("item1"));
7879

79-
processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "list.3", false);
80+
processor = createRenameProcessor("list.0", "list.3", false);
8081
try {
8182
processor.execute(ingestDocument);
8283
fail("processor execute should have failed");
@@ -91,7 +92,7 @@ public void testRenameArrayElement() throws Exception {
9192
public void testRenameNonExistingField() throws Exception {
9293
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
9394
String fieldName = RandomDocumentPicks.randomFieldName(random());
94-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
95+
Processor processor = createRenameProcessor(fieldName,
9596
RandomDocumentPicks.randomFieldName(random()), false);
9697
try {
9798
processor.execute(ingestDocument);
@@ -105,7 +106,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
105106
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
106107
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
107108
String fieldName = RandomDocumentPicks.randomFieldName(random());
108-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
109+
Processor processor = createRenameProcessor(fieldName,
109110
RandomDocumentPicks.randomFieldName(random()), true);
110111
processor.execute(ingestDocument);
111112
assertIngestDocument(originalIngestDocument, ingestDocument);
@@ -114,7 +115,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
114115
public void testRenameNewFieldAlreadyExists() throws Exception {
115116
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
116117
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
117-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), RandomDocumentPicks.randomExistingFieldName(
118+
Processor processor = createRenameProcessor(RandomDocumentPicks.randomExistingFieldName(
118119
random(), ingestDocument), fieldName, false);
119120
try {
120121
processor.execute(ingestDocument);
@@ -129,7 +130,7 @@ public void testRenameExistingFieldNullValue() throws Exception {
129130
String fieldName = RandomDocumentPicks.randomFieldName(random());
130131
ingestDocument.setFieldValue(fieldName, null);
131132
String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random()));
132-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
133+
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
133134
processor.execute(ingestDocument);
134135
assertThat(ingestDocument.hasField(fieldName), equalTo(false));
135136
assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
@@ -149,7 +150,7 @@ public Object put(String key, Object value) {
149150
source.put("list", Collections.singletonList("item"));
150151

151152
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
152-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
153+
Processor processor = createRenameProcessor("list", "new_field", false);
153154
try {
154155
processor.execute(ingestDocument);
155156
fail("processor execute should have failed");
@@ -173,7 +174,7 @@ public Object remove(Object key) {
173174
source.put("list", Collections.singletonList("item"));
174175

175176
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
176-
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
177+
Processor processor = createRenameProcessor("list", "new_field", false);
177178
try {
178179
processor.execute(ingestDocument);
179180
fail("processor execute should have failed");
@@ -188,22 +189,26 @@ public void testRenameLeafIntoBranch() throws Exception {
188189
Map<String, Object> source = new HashMap<>();
189190
source.put("foo", "bar");
190191
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
191-
Processor processor1 = new RenameProcessor(randomAlphaOfLength(10), "foo", "foo.bar", false);
192+
Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
192193
processor1.execute(ingestDocument);
193194
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
194195
assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
195196

196-
Processor processor2 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar", "foo.bar.baz", false);
197+
Processor processor2 = createRenameProcessor( "foo.bar", "foo.bar.baz", false);
197198
processor2.execute(ingestDocument);
198199
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar",
199200
Collections.singletonMap("baz", "bar"))));
200201
assertThat(ingestDocument.getFieldValue("foo.bar", Map.class), equalTo(Collections.singletonMap("baz", "bar")));
201202
assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
202203

203204
// for fun lets try to restore it (which don't allow today)
204-
Processor processor3 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar.baz", "foo", false);
205+
Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false);
205206
Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
206207
assertThat(e.getMessage(), equalTo("field [foo] already exists"));
207208
}
208209

210+
private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) {
211+
return new RenameProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(field),
212+
new TestTemplateService.MockTemplateScript.Factory(targetField), ignoreMissing);
213+
}
209214
}

0 commit comments

Comments
 (0)