Skip to content

Commit eddab39

Browse files
danhermanndanhermann
authored
Configurable MIME type for mustache template encoding on set processor (#65314)
1 parent dc1e460 commit eddab39

File tree

6 files changed

+106
-7
lines changed

6 files changed

+106
-7
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.ingest.IngestDocument;
2525
import org.elasticsearch.ingest.Processor;
2626
import org.elasticsearch.ingest.ValueSource;
27+
import org.elasticsearch.script.Script;
2728
import org.elasticsearch.script.ScriptService;
2829
import org.elasticsearch.script.TemplateScript;
2930

@@ -110,10 +111,11 @@ public SetProcessor create(Map<String, Processor.Factory> registry, String proce
110111
String description, Map<String, Object> config) throws Exception {
111112
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
112113
String copyFrom = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "copy_from");
114+
String mimeType = ConfigurationUtils.readMimeTypeProperty(TYPE, processorTag, config, "mime_type", "application/json");
113115
ValueSource valueSource = null;
114116
if (copyFrom == null) {
115117
Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
116-
valueSource = ValueSource.wrap(value, scriptService);
118+
valueSource = ValueSource.wrap(value, scriptService, Map.of(Script.CONTENT_TYPE_OPTION, mimeType));
117119
} else {
118120
Object value = config.remove("value");
119121
if (value != null) {
@@ -123,8 +125,7 @@ public SetProcessor create(Map<String, Processor.Factory> registry, String proce
123125
}
124126

125127
boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true);
126-
TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
127-
"field", field, scriptService);
128+
TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);
128129
boolean ignoreEmptyValue = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_empty_value", false);
129130

130131
return new SetProcessor(

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@
2121

2222
import org.elasticsearch.ElasticsearchException;
2323
import org.elasticsearch.ElasticsearchParseException;
24+
import org.elasticsearch.ingest.ConfigurationUtils;
2425
import org.elasticsearch.ingest.TestTemplateService;
2526
import org.elasticsearch.test.ESTestCase;
2627
import org.junit.Before;
2728

29+
import java.util.Arrays;
2830
import java.util.Collections;
2931
import java.util.HashMap;
3032
import java.util.Map;
3133

3234
import static org.hamcrest.CoreMatchers.equalTo;
35+
import static org.hamcrest.Matchers.containsString;
3336

3437
public class SetProcessorFactoryTests extends ESTestCase {
3538

@@ -133,4 +136,26 @@ public void testCreateWithCopyFromAndValue() throws Exception {
133136
() -> factory.create(null, processorTag, null, config));
134137
assertThat(exception.getMessage(), equalTo("[copy_from] cannot set both `copy_from` and `value` in the same processor"));
135138
}
139+
140+
public void testMimeType() throws Exception {
141+
// valid mime type
142+
String expectedMimeType = randomFrom(ConfigurationUtils.VALID_MIME_TYPES);
143+
Map<String, Object> config = new HashMap<>();
144+
config.put("field", "field1");
145+
config.put("value", "value1");
146+
config.put("mime_type", expectedMimeType);
147+
String processorTag = randomAlphaOfLength(10);
148+
SetProcessor setProcessor = factory.create(null, processorTag, null, config);
149+
assertThat(setProcessor.getTag(), equalTo(processorTag));
150+
151+
// invalid mime type
152+
expectedMimeType = randomValueOtherThanMany(m -> Arrays.asList(ConfigurationUtils.VALID_MIME_TYPES).contains(m),
153+
() -> randomAlphaOfLengthBetween(5, 9));
154+
final Map<String, Object> config2 = new HashMap<>();
155+
config2.put("field", "field1");
156+
config2.put("value", "value1");
157+
config2.put("mime_type", expectedMimeType);
158+
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2));
159+
assertThat(e.getMessage(), containsString("property does not contain a supported MIME type [" + expectedMimeType + "]"));
160+
}
136161
}

‎qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
package org.elasticsearch.ingest;
2121

22+
import org.elasticsearch.script.Script;
23+
2224
import java.util.Arrays;
2325
import java.util.Collections;
2426
import java.util.HashMap;
@@ -72,4 +74,27 @@ public void testAccessSourceViaTemplate() {
7274
assertThat(ingestDocument.hasField("index"), is(false));
7375
}
7476

77+
public void testWithConfigurableEncoders() {
78+
Map<String, Object> model = new HashMap<>();
79+
model.put("log_line", "10.10.1.1 - - [17/Nov/2020:04:59:43 +0000] \"GET /info HTTP/1.1\" 200 6229 \"-\" \"-\" 2");
80+
81+
// default encoder should be application/json
82+
ValueSource valueSource = ValueSource.wrap("{{log_line}}", scriptService);
83+
Object result = valueSource.copyAndResolve(model);
84+
assertThat(result,
85+
equalTo("10.10.1.1 - - [17/Nov/2020:04:59:43 +0000] \\\"GET /info HTTP/1.1\\\" 200 6229 \\\"-\\\" \\\"-\\\" 2"));
86+
87+
// text/plain encoder
88+
var scriptOptions = Map.of(Script.CONTENT_TYPE_OPTION, "text/plain");
89+
valueSource = ValueSource.wrap("{{log_line}}", scriptService, scriptOptions);
90+
result = valueSource.copyAndResolve(model);
91+
assertThat(result, equalTo("10.10.1.1 - - [17/Nov/2020:04:59:43 +0000] \"GET /info HTTP/1.1\" 200 6229 \"-\" \"-\" 2"));
92+
93+
// application/x-www-form-urlencoded encoder
94+
scriptOptions = Map.of(Script.CONTENT_TYPE_OPTION, "application/x-www-form-urlencoded");
95+
valueSource = ValueSource.wrap("{{log_line}}", scriptService, scriptOptions);
96+
result = valueSource.copyAndResolve(model);
97+
assertThat(result, equalTo("10.10.1.1+-+-+%5B17%2FNov%2F2020%3A04%3A59%3A43+%2B0000%5D+%22GET+%2Finfo+HTTP%2F1.1%22+200" +
98+
"+6229+%22-%22+%22-%22++2"));
99+
}
75100
}

‎server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public final class ConfigurationUtils {
4949

5050
public static final String TAG_KEY = "tag";
5151
public static final String DESCRIPTION_KEY = "description";
52+
public static final String[] VALID_MIME_TYPES = {"application/json", "text/plain", "application/x-www-form-urlencoded"};
5253

5354
private ConfigurationUtils() {
5455
}
@@ -305,6 +306,18 @@ public static Object readObject(String processorType, String processorTag, Map<S
305306
return value;
306307
}
307308

309+
public static String readMimeTypeProperty(String processorType, String processorTag, Map<String, Object> configuration,
310+
String propertyName, String defaultValue) {
311+
String mimeType = readStringProperty(processorType, processorTag, configuration, propertyName, defaultValue);
312+
313+
if (Arrays.asList(VALID_MIME_TYPES).contains(mimeType) == false) {
314+
throw newConfigurationException(processorType, processorTag, propertyName,
315+
"property does not contain a supported MIME type [" + mimeType + "]");
316+
}
317+
318+
return mimeType;
319+
}
320+
308321
public static ElasticsearchException newConfigurationException(String processorType, String processorTag,
309322
String propertyName, String reason) {
310323
String msg;

‎server/src/main/java/org/elasticsearch/ingest/ValueSource.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626

2727
import java.util.ArrayList;
2828
import java.util.Arrays;
29-
import java.util.Collections;
3029
import java.util.HashMap;
3130
import java.util.List;
3231
import java.util.Map;
@@ -50,21 +49,25 @@ public interface ValueSource {
5049
Object copyAndResolve(Map<String, Object> model);
5150

5251
static ValueSource wrap(Object value, ScriptService scriptService) {
52+
return wrap(value, scriptService, Map.of());
53+
}
54+
55+
static ValueSource wrap(Object value, ScriptService scriptService, Map<String, String> scriptOptions) {
5356

5457
if (value instanceof Map) {
5558
@SuppressWarnings("unchecked")
5659
Map<Object, Object> mapValue = (Map) value;
5760
Map<ValueSource, ValueSource> valueTypeMap = new HashMap<>(mapValue.size());
5861
for (Map.Entry<Object, Object> entry : mapValue.entrySet()) {
59-
valueTypeMap.put(wrap(entry.getKey(), scriptService), wrap(entry.getValue(), scriptService));
62+
valueTypeMap.put(wrap(entry.getKey(), scriptService, scriptOptions), wrap(entry.getValue(), scriptService, scriptOptions));
6063
}
6164
return new MapValue(valueTypeMap);
6265
} else if (value instanceof List) {
6366
@SuppressWarnings("unchecked")
6467
List<Object> listValue = (List) value;
6568
List<ValueSource> valueSourceList = new ArrayList<>(listValue.size());
6669
for (Object item : listValue) {
67-
valueSourceList.add(wrap(item, scriptService));
70+
valueSourceList.add(wrap(item, scriptService, scriptOptions));
6871
}
6972
return new ListValue(valueSourceList);
7073
} else if (value == null || value instanceof Number || value instanceof Boolean) {
@@ -76,7 +79,7 @@ static ValueSource wrap(Object value, ScriptService scriptService) {
7679
// installed for use by REST tests. `value` will not be
7780
// modified if templating is not available
7881
if (scriptService.isLangSupported(DEFAULT_TEMPLATE_LANG) && ((String) value).contains("{{")) {
79-
Script script = new Script(ScriptType.INLINE, DEFAULT_TEMPLATE_LANG, (String) value, Collections.emptyMap());
82+
Script script = new Script(ScriptType.INLINE, DEFAULT_TEMPLATE_LANG, (String) value, scriptOptions, Map.of());
8083
return new TemplatedValue(scriptService.compile(script, TemplateScript.CONTEXT));
8184
} else {
8285
return new ObjectValue(value);

‎server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.ingest;
2121

22+
import org.elasticsearch.ElasticsearchException;
2223
import org.elasticsearch.ElasticsearchParseException;
2324
import org.elasticsearch.script.ScriptService;
2425
import org.elasticsearch.script.TemplateScript;
@@ -32,6 +33,7 @@
3233
import java.util.List;
3334
import java.util.Map;
3435

36+
import static org.hamcrest.Matchers.containsString;
3537
import static org.hamcrest.Matchers.equalTo;
3638
import static org.hamcrest.Matchers.instanceOf;
3739
import static org.hamcrest.Matchers.is;
@@ -114,6 +116,36 @@ public void testReadStringOrIntPropertyInvalidType() {
114116
}
115117
}
116118

119+
public void testReadMimeProperty() {
120+
// valid mime type
121+
String expectedMimeType = randomFrom(ConfigurationUtils.VALID_MIME_TYPES);
122+
config.put("mime_type", expectedMimeType);
123+
String readMimeType = ConfigurationUtils.readMimeTypeProperty(null, null, config, "mime_type", "");
124+
assertThat(readMimeType, equalTo(expectedMimeType));
125+
126+
// missing mime type with valid default
127+
expectedMimeType = randomFrom(ConfigurationUtils.VALID_MIME_TYPES);
128+
config.remove("mime_type");
129+
readMimeType = ConfigurationUtils.readMimeTypeProperty(null, null, config, "mime_type", expectedMimeType);
130+
assertThat(readMimeType, equalTo(expectedMimeType));
131+
132+
// invalid mime type
133+
expectedMimeType = randomValueOtherThanMany(m -> Arrays.asList(ConfigurationUtils.VALID_MIME_TYPES).contains(m),
134+
() -> randomAlphaOfLengthBetween(5, 9));
135+
config.put("mime_type", expectedMimeType);
136+
ElasticsearchException e = expectThrows(ElasticsearchException.class,
137+
() -> ConfigurationUtils.readMimeTypeProperty(null, null, config, "mime_type", ""));
138+
assertThat(e.getMessage(), containsString("property does not contain a supported MIME type [" + expectedMimeType + "]"));
139+
140+
// missing mime type with invalid default
141+
final String invalidDefaultMimeType = randomValueOtherThanMany(m -> Arrays.asList(ConfigurationUtils.VALID_MIME_TYPES).contains(m),
142+
() -> randomAlphaOfLengthBetween(5, 9));
143+
config.remove("mime_type");
144+
e = expectThrows(ElasticsearchException.class,
145+
() -> ConfigurationUtils.readMimeTypeProperty(null, null, config, "mime_type", invalidDefaultMimeType));
146+
assertThat(e.getMessage(), containsString("property does not contain a supported MIME type [" + invalidDefaultMimeType + "]"));
147+
}
148+
117149
public void testReadProcessors() throws Exception {
118150
Processor processor = mock(Processor.class);
119151
Map<String, Processor.Factory> registry =

0 commit comments

Comments
 (0)