Skip to content

Commit 86d6b1b

Browse files
jakelandisjakelandis
committed
ingest: date_index_name processor template resolution
This change adds support for template snippet (e.g. {{foo}}) resolution in the date_index_name processor. The following configuration options will now resolve a templated value if so configured: * index_name_prefix (e.g "index_name_prefix": "myindex-{{foo}}-") * date_rounding (e.g. "date_rounding" : "{{bar}}")
1 parent 07470c9 commit 86d6b1b

File tree

5 files changed

+68
-14
lines changed

5 files changed

+68
-14
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
import org.elasticsearch.ingest.ConfigurationUtils;
3333
import org.elasticsearch.ingest.IngestDocument;
3434
import org.elasticsearch.ingest.Processor;
35+
import org.elasticsearch.ingest.ValueSource;
36+
import org.elasticsearch.script.ScriptService;
3537
import org.joda.time.DateTime;
3638
import org.joda.time.DateTimeZone;
3739
import org.joda.time.format.DateTimeFormat;
@@ -47,16 +49,18 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
4749
private final String indexNameFormat;
4850
private final DateTimeZone timezone;
4951
private final List<Function<String, DateTime>> dateFormats;
52+
private final ScriptService scriptService;
5053

5154
DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
52-
String indexNamePrefix, String dateRounding, String indexNameFormat) {
55+
String indexNamePrefix, String dateRounding, String indexNameFormat, ScriptService scriptService) {
5356
super(tag);
5457
this.field = field;
5558
this.timezone = timezone;
5659
this.dateFormats = dateFormats;
5760
this.indexNamePrefix = indexNamePrefix;
5861
this.dateRounding = dateRounding;
5962
this.indexNameFormat = indexNameFormat;
63+
this.scriptService = scriptService;
6064
}
6165

6266
@Override
@@ -94,7 +98,7 @@ public void execute(IngestDocument ingestDocument) throws Exception {
9498
.append('}')
9599
.append('>');
96100
String dynamicIndexName = builder.toString();
97-
ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), dynamicIndexName);
101+
ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), ValueSource.wrap(dynamicIndexName, scriptService));
98102
}
99103

100104
@Override
@@ -128,6 +132,12 @@ List<Function<String, DateTime>> getDateFormats() {
128132

129133
public static final class Factory implements Processor.Factory {
130134

135+
private final ScriptService scriptService;
136+
137+
public Factory(ScriptService scriptService) {
138+
this.scriptService = scriptService;
139+
}
140+
131141
@Override
132142
public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, String tag,
133143
Map<String, Object> config) throws Exception {
@@ -156,7 +166,8 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
156166
String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
157167
String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
158168
String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
159-
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat);
169+
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix,
170+
dateRounding, indexNameFormat, scriptService);
160171
}
161172
}
162173

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
7373
processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory());
7474
processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService));
7575
processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
76-
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
76+
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService));
7777
processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
7878
processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters)));
7979
processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
@@ -97,12 +97,12 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
9797
Supplier<DiscoveryNodes> nodesInCluster) {
9898
return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
9999
}
100-
100+
101101
@Override
102102
public List<Setting<?>> getSettings() {
103103
return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME);
104104
}
105-
105+
106106
private static ThreadWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) {
107107
long intervalMillis = WATCHDOG_INTERVAL.get(parameters.env.settings()).getMillis();
108108
long maxExecutionTimeMillis = WATCHDOG_MAX_EXECUTION_TIME.get(parameters.env.settings()).getMillis();

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.ingest.common;
2121

2222
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.ingest.TestTemplateService;
2324
import org.elasticsearch.test.ESTestCase;
2425
import org.hamcrest.Matchers;
2526
import org.joda.time.DateTimeZone;
@@ -31,7 +32,7 @@
3132
public class DateIndexNameFactoryTests extends ESTestCase {
3233

3334
public void testDefaults() throws Exception {
34-
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
35+
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
3536
Map<String, Object> config = new HashMap<>();
3637
config.put("field", "_field");
3738
config.put("date_rounding", "y");
@@ -46,7 +47,7 @@ public void testDefaults() throws Exception {
4647
}
4748

4849
public void testSpecifyOptionalSettings() throws Exception {
49-
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
50+
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
5051
Map<String, Object> config = new HashMap<>();
5152
config.put("field", "_field");
5253
config.put("index_name_prefix", "_prefix");
@@ -84,7 +85,7 @@ public void testSpecifyOptionalSettings() throws Exception {
8485
}
8586

8687
public void testRequiredFields() throws Exception {
87-
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
88+
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
8889
Map<String, Object> config = new HashMap<>();
8990
config.put("date_rounding", "y");
9091
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
@@ -95,5 +96,4 @@ public void testRequiredFields() throws Exception {
9596
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
9697
assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
9798
}
98-
9999
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,23 +19,34 @@
1919
package org.elasticsearch.ingest.common;
2020

2121
import org.elasticsearch.ingest.IngestDocument;
22+
import org.elasticsearch.ingest.TestTemplateService;
23+
import org.elasticsearch.script.Script;
24+
import org.elasticsearch.script.ScriptContext;
25+
import org.elasticsearch.script.ScriptService;
26+
import org.elasticsearch.script.TemplateScript;
2227
import org.elasticsearch.test.ESTestCase;
2328
import org.joda.time.DateTime;
2429
import org.joda.time.DateTimeZone;
30+
import org.mockito.Matchers;
31+
import org.mockito.Mockito;
2532

2633
import java.util.Collections;
2734
import java.util.Locale;
2835
import java.util.function.Function;
2936

37+
import static org.elasticsearch.script.Script.DEFAULT_TEMPLATE_LANG;
3038
import static org.hamcrest.CoreMatchers.equalTo;
39+
import static org.mockito.Matchers.any;
40+
import static org.mockito.Mockito.mock;
41+
import static org.mockito.Mockito.when;
3142

3243
public class DateIndexNameProcessorTests extends ESTestCase {
3344

3445
public void testJodaPattern() throws Exception {
3546
Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
3647
DateIndexNameProcessor processor = new DateIndexNameProcessor(
3748
"_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC,
38-
"events-", "y", "yyyyMMdd"
49+
"events-", "y", "yyyyMMdd", TestTemplateService.instance()
3950
);
4051

4152
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
@@ -47,7 +58,7 @@ public void testJodaPattern() throws Exception {
4758
public void testTAI64N()throws Exception {
4859
Function<String, DateTime> function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null);
4960
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
50-
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
61+
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
5162
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
5263
Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"));
5364
dateProcessor.execute(document);
@@ -57,7 +68,7 @@ public void testTAI64N()throws Exception {
5768
public void testUnixMs()throws Exception {
5869
Function<String, DateTime> function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null);
5970
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
60-
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
71+
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
6172
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
6273
Collections.singletonMap("_field", "1000500"));
6374
dateProcessor.execute(document);
@@ -72,11 +83,28 @@ public void testUnixMs()throws Exception {
7283
public void testUnix()throws Exception {
7384
Function<String, DateTime> function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
7485
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
75-
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
86+
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
7687
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
7788
Collections.singletonMap("_field", "1000.5"));
7889
dateProcessor.execute(document);
7990
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
8091
}
8192

93+
public void testTemplateSupported() throws Exception {
94+
ScriptService scriptService = mock(ScriptService.class);
95+
TestTemplateService.MockTemplateScript.Factory factory = new TestTemplateService.MockTemplateScript.Factory("script_result");
96+
when(scriptService.compile(any(Script.class), Matchers.<ScriptContext<TemplateScript.Factory>>any())).thenReturn(factory);
97+
when(scriptService.isLangSupported(DEFAULT_TEMPLATE_LANG)).thenReturn(true);
98+
99+
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field",
100+
Collections.singletonList(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null)),
101+
DateTimeZone.UTC, "events-", "m", "yyyyMMdd", scriptService);
102+
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
103+
Collections.singletonMap("_field", "1000.5"));
104+
dateProcessor.execute(document);
105+
106+
// here we only care that the script was compiled and that it returned what we expect.
107+
Mockito.verify(scriptService).compile(any(Script.class), Matchers.<ScriptContext<TemplateScript.Factory>>any());
108+
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("script_result"));
109+
}
82110
}

‎server/src/main/java/org/elasticsearch/ingest/IngestDocument.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,21 @@ public void setFieldValue(String path, Object value) {
406406
setFieldValue(path, value, false);
407407
}
408408

409+
/**
410+
* Sets the provided value to the provided path in the document.
411+
* Any non existing path element will be created.
412+
* If the last item in the path is a list, the value will replace the existing list as a whole.
413+
* Use {@link #appendFieldValue(String, Object)} to append values to lists instead.
414+
* @param path The path within the document in dot-notation
415+
* @param valueSource The value source that will produce the value or values to append to the existing ones
416+
* @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the
417+
* item identified by the provided path.
418+
*/
419+
public void setFieldValue(String path, ValueSource valueSource) {
420+
Map<String, Object> model = valueSource == null ? null : createTemplateModel();
421+
setFieldValue(path, valueSource == null ? null : valueSource.copyAndResolve(model), false);
422+
}
423+
409424
/**
410425
* Sets the provided value to the provided path in the document.
411426
* Any non existing path element will be created. If the last element is a list,

0 commit comments

Comments
 (0)