Skip to content

Commit cc4d705

Browse files
original-brownbearoriginal-brownbear
authored
Ingest: Add conditional per processor (#32398)
* Ingest: Add conditional per processor * closes #21248
1 parent d93b2a2 commit cc4d705

File tree

15 files changed

+788
-38
lines changed

15 files changed

+788
-38
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java

+10-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.List;
2929
import java.util.Map;
3030
import java.util.Set;
31+
import org.elasticsearch.script.ScriptService;
3132

3233
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
3334
import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
@@ -96,6 +97,13 @@ Processor getProcessor() {
9697
}
9798

9899
public static final class Factory implements Processor.Factory {
100+
101+
private final ScriptService scriptService;
102+
103+
Factory(ScriptService scriptService) {
104+
this.scriptService = scriptService;
105+
}
106+
99107
@Override
100108
public ForEachProcessor create(Map<String, Processor.Factory> factories, String tag,
101109
Map<String, Object> config) throws Exception {
@@ -107,7 +115,8 @@ public ForEachProcessor create(Map<String, Processor.Factory> factories, String
107115
throw newConfigurationException(TYPE, tag, "processor", "Must specify exactly one processor type");
108116
}
109117
Map.Entry<String, Map<String, Object>> entry = entries.iterator().next();
110-
Processor processor = ConfigurationUtils.readProcessor(factories, entry.getKey(), entry.getValue());
118+
Processor processor =
119+
ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue());
111120
return new ForEachProcessor(tag, field, processor, ignoreMissing);
112121
}
113122
}

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
7272
processors.put(ConvertProcessor.TYPE, new ConvertProcessor.Factory());
7373
processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory());
7474
processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService));
75-
processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
75+
processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory(parameters.scriptService));
7676
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService));
7777
processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
7878
processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters)));

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java

+10-6
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.ElasticsearchParseException;
2323
import org.elasticsearch.ingest.Processor;
2424
import org.elasticsearch.ingest.TestProcessor;
25+
import org.elasticsearch.script.ScriptService;
2526
import org.elasticsearch.test.ESTestCase;
2627
import org.hamcrest.Matchers;
2728

@@ -30,14 +31,17 @@
3031
import java.util.Map;
3132

3233
import static org.hamcrest.Matchers.equalTo;
34+
import static org.mockito.Mockito.mock;
3335

3436
public class ForEachProcessorFactoryTests extends ESTestCase {
3537

38+
private final ScriptService scriptService = mock(ScriptService.class);
39+
3640
public void testCreate() throws Exception {
3741
Processor processor = new TestProcessor(ingestDocument -> { });
3842
Map<String, Processor.Factory> registry = new HashMap<>();
3943
registry.put("_name", (r, t, c) -> processor);
40-
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
44+
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
4145

4246
Map<String, Object> config = new HashMap<>();
4347
config.put("field", "_field");
@@ -53,7 +57,7 @@ public void testSetIgnoreMissing() throws Exception {
5357
Processor processor = new TestProcessor(ingestDocument -> { });
5458
Map<String, Processor.Factory> registry = new HashMap<>();
5559
registry.put("_name", (r, t, c) -> processor);
56-
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
60+
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
5761

5862
Map<String, Object> config = new HashMap<>();
5963
config.put("field", "_field");
@@ -71,7 +75,7 @@ public void testCreateWithTooManyProcessorTypes() throws Exception {
7175
Map<String, Processor.Factory> registry = new HashMap<>();
7276
registry.put("_first", (r, t, c) -> processor);
7377
registry.put("_second", (r, t, c) -> processor);
74-
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
78+
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
7579

7680
Map<String, Object> config = new HashMap<>();
7781
config.put("field", "_field");
@@ -84,7 +88,7 @@ public void testCreateWithTooManyProcessorTypes() throws Exception {
8488
}
8589

8690
public void testCreateWithNonExistingProcessorType() throws Exception {
87-
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
91+
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
8892
Map<String, Object> config = new HashMap<>();
8993
config.put("field", "_field");
9094
config.put("processor", Collections.singletonMap("_name", Collections.emptyMap()));
@@ -97,15 +101,15 @@ public void testCreateWithMissingField() throws Exception {
97101
Processor processor = new TestProcessor(ingestDocument -> { });
98102
Map<String, Processor.Factory> registry = new HashMap<>();
99103
registry.put("_name", (r, t, c) -> processor);
100-
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
104+
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
101105
Map<String, Object> config = new HashMap<>();
102106
config.put("processor", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap())));
103107
Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, config));
104108
assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
105109
}
106110

107111
public void testCreateWithMissingProcessor() {
108-
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
112+
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
109113
Map<String, Object> config = new HashMap<>();
110114
config.put("field", "_field");
111115
Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Collections.emptyMap(), null, config));

‎modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
---
2+
teardown:
3+
- do:
4+
ingest.delete_pipeline:
5+
id: "my_pipeline"
6+
ignore: 404
7+
8+
---
9+
"Test conditional processor fulfilled condition":
10+
- do:
11+
ingest.put_pipeline:
12+
id: "my_pipeline"
13+
body: >
14+
{
15+
"description": "_description",
16+
"processors": [
17+
{
18+
"bytes" : {
19+
"if" : "ctx.conditional_field == 'bar'",
20+
"field" : "bytes_source_field",
21+
"target_field" : "bytes_target_field"
22+
}
23+
}
24+
]
25+
}
26+
- match: { acknowledged: true }
27+
28+
- do:
29+
index:
30+
index: test
31+
type: test
32+
id: 1
33+
pipeline: "my_pipeline"
34+
body: {bytes_source_field: "1kb", conditional_field: "bar"}
35+
36+
- do:
37+
get:
38+
index: test
39+
type: test
40+
id: 1
41+
- match: { _source.bytes_source_field: "1kb" }
42+
- match: { _source.conditional_field: "bar" }
43+
- match: { _source.bytes_target_field: 1024 }
44+
45+
---
46+
"Test conditional processor unfulfilled condition":
47+
- do:
48+
ingest.put_pipeline:
49+
id: "my_pipeline"
50+
body: >
51+
{
52+
"description": "_description",
53+
"processors": [
54+
{
55+
"bytes" : {
56+
"if" : "ctx.conditional_field == 'foo'",
57+
"field" : "bytes_source_field",
58+
"target_field" : "bytes_target_field"
59+
}
60+
}
61+
]
62+
}
63+
- match: { acknowledged: true }
64+
65+
- do:
66+
index:
67+
index: test
68+
type: test
69+
id: 1
70+
pipeline: "my_pipeline"
71+
body: {bytes_source_field: "1kb", conditional_field: "bar"}
72+
73+
- do:
74+
get:
75+
index: test
76+
type: test
77+
id: 1
78+
- match: { _source.bytes_source_field: "1kb" }
79+
- match: { _source.conditional_field: "bar" }
80+
- is_false: _source.bytes_target_field
81+

‎server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,11 @@ static Parsed parseWithPipelineId(String pipelineId, Map<String, Object> config,
171171
return new Parsed(pipeline, ingestDocumentList, verbose);
172172
}
173173

174-
static Parsed parse(Map<String, Object> config, boolean verbose, IngestService pipelineStore) throws Exception {
174+
static Parsed parse(Map<String, Object> config, boolean verbose, IngestService ingestService) throws Exception {
175175
Map<String, Object> pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE);
176-
Pipeline pipeline = Pipeline.create(SIMULATED_PIPELINE_ID, pipelineConfig, pipelineStore.getProcessorFactories());
176+
Pipeline pipeline = Pipeline.create(
177+
SIMULATED_PIPELINE_ID, pipelineConfig, ingestService.getProcessorFactories(), ingestService.getScriptService()
178+
);
177179
List<IngestDocument> ingestDocumentList = parseDocs(config);
178180
return new Parsed(pipeline, ingestDocumentList, verbose);
179181
}

0 commit comments

Comments
 (0)