Skip to content

Commit 51bb27a

Browse files
jakelandisjakelandis
authored
ingest: date_index_name processor template resolution (#31841)
This change adds support for template snippet (e.g. {{foo}}) resolution in the date_index_name processor. The following configuration options will now resolve a templated value if so configured: * index_name_prefix (e.g "index_name_prefix": "myindex-{{foo}}-") * date_rounding (e.g. "date_rounding" : "{{bar}}") * index_name_format (e.g."index_name_format": "{{baz}}")
1 parent d76293f commit 51bb27a

File tree

4 files changed

+83
-34
lines changed

4 files changed

+83
-34
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
import org.elasticsearch.ingest.ConfigurationUtils;
3333
import org.elasticsearch.ingest.IngestDocument;
3434
import org.elasticsearch.ingest.Processor;
35+
import org.elasticsearch.script.ScriptService;
36+
import org.elasticsearch.script.TemplateScript;
3537
import org.joda.time.DateTime;
3638
import org.joda.time.DateTimeZone;
3739
import org.joda.time.format.DateTimeFormat;
@@ -42,21 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
4244
public static final String TYPE = "date_index_name";
4345

4446
private final String field;
45-
private final String indexNamePrefix;
46-
private final String dateRounding;
47-
private final String indexNameFormat;
47+
private final TemplateScript.Factory indexNamePrefixTemplate;
48+
private final TemplateScript.Factory dateRoundingTemplate;
49+
private final TemplateScript.Factory indexNameFormatTemplate;
4850
private final DateTimeZone timezone;
4951
private final List<Function<String, DateTime>> dateFormats;
5052

5153
DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
52-
String indexNamePrefix, String dateRounding, String indexNameFormat) {
54+
TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
55+
TemplateScript.Factory indexNameFormatTemplate) {
5356
super(tag);
5457
this.field = field;
5558
this.timezone = timezone;
5659
this.dateFormats = dateFormats;
57-
this.indexNamePrefix = indexNamePrefix;
58-
this.dateRounding = dateRounding;
59-
this.indexNameFormat = indexNameFormat;
60+
this.indexNamePrefixTemplate = indexNamePrefixTemplate;
61+
this.dateRoundingTemplate = dateRoundingTemplate;
62+
this.indexNameFormatTemplate = indexNameFormatTemplate;
6063
}
6164

6265
@Override
@@ -83,6 +86,9 @@ public void execute(IngestDocument ingestDocument) throws Exception {
8386
if (dateTime == null) {
8487
throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException);
8588
}
89+
String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate);
90+
String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate);
91+
String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate);
8692

8793
DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
8894
StringBuilder builder = new StringBuilder()
@@ -106,16 +112,16 @@ String getField() {
106112
return field;
107113
}
108114

109-
String getIndexNamePrefix() {
110-
return indexNamePrefix;
115+
TemplateScript.Factory getIndexNamePrefixTemplate() {
116+
return indexNamePrefixTemplate;
111117
}
112118

113-
String getDateRounding() {
114-
return dateRounding;
119+
TemplateScript.Factory getDateRoundingTemplate() {
120+
return dateRoundingTemplate;
115121
}
116122

117-
String getIndexNameFormat() {
118-
return indexNameFormat;
123+
TemplateScript.Factory getIndexNameFormatTemplate() {
124+
return indexNameFormatTemplate;
119125
}
120126

121127
DateTimeZone getTimezone() {
@@ -128,6 +134,12 @@ List<Function<String, DateTime>> getDateFormats() {
128134

129135
public static final class Factory implements Processor.Factory {
130136

137+
private final ScriptService scriptService;
138+
139+
public Factory(ScriptService scriptService) {
140+
this.scriptService = scriptService;
141+
}
142+
131143
@Override
132144
public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, String tag,
133145
Map<String, Object> config) throws Exception {
@@ -154,9 +166,16 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
154166

155167
String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
156168
String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
169+
TemplateScript.Factory indexNamePrefixTemplate =
170+
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService);
157171
String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
172+
TemplateScript.Factory dateRoundingTemplate =
173+
ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService);
158174
String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
159-
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat);
175+
TemplateScript.Factory indexNameFormatTemplate =
176+
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService);
177+
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate,
178+
dateRoundingTemplate, indexNameFormatTemplate);
160179
}
161180
}
162181

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
7373
processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory());
7474
processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService));
7575
processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
76-
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
76+
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService));
7777
processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
7878
processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters)));
7979
processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
@@ -97,12 +97,12 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
9797
Supplier<DiscoveryNodes> nodesInCluster) {
9898
return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
9999
}
100-
100+
101101
@Override
102102
public List<Setting<?>> getSettings() {
103103
return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME);
104104
}
105-
105+
106106
private static ThreadWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) {
107107
long intervalMillis = WATCHDOG_INTERVAL.get(parameters.env.settings()).getMillis();
108108
long maxExecutionTimeMillis = WATCHDOG_MAX_EXECUTION_TIME.get(parameters.env.settings()).getMillis();

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,33 +20,35 @@
2020
package org.elasticsearch.ingest.common;
2121

2222
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.ingest.TestTemplateService;
2324
import org.elasticsearch.test.ESTestCase;
2425
import org.hamcrest.Matchers;
2526
import org.joda.time.DateTimeZone;
2627

2728
import java.util.Arrays;
29+
import java.util.Collections;
2830
import java.util.HashMap;
2931
import java.util.Map;
3032

3133
public class DateIndexNameFactoryTests extends ESTestCase {
3234

3335
public void testDefaults() throws Exception {
34-
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
36+
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
3537
Map<String, Object> config = new HashMap<>();
3638
config.put("field", "_field");
3739
config.put("date_rounding", "y");
3840

3941
DateIndexNameProcessor processor = factory.create(null, null, config);
4042
assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
4143
assertThat(processor.getField(), Matchers.equalTo("_field"));
42-
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
43-
assertThat(processor.getDateRounding(), Matchers.equalTo("y"));
44-
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd"));
44+
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo(""));
45+
assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y"));
46+
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd"));
4547
assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC));
4648
}
4749

4850
public void testSpecifyOptionalSettings() throws Exception {
49-
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
51+
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
5052
Map<String, Object> config = new HashMap<>();
5153
config.put("field", "_field");
5254
config.put("index_name_prefix", "_prefix");
@@ -63,7 +65,7 @@ public void testSpecifyOptionalSettings() throws Exception {
6365
config.put("index_name_format", "yyyyMMdd");
6466

6567
processor = factory.create(null, null, config);
66-
assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
68+
assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd"));
6769

6870
config = new HashMap<>();
6971
config.put("field", "_field");
@@ -80,11 +82,11 @@ public void testSpecifyOptionalSettings() throws Exception {
8082
config.put("date_rounding", "y");
8183

8284
processor = factory.create(null, null, config);
83-
assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
85+
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix"));
8486
}
8587

8688
public void testRequiredFields() throws Exception {
87-
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
89+
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
8890
Map<String, Object> config = new HashMap<>();
8991
config.put("date_rounding", "y");
9092
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
@@ -95,5 +97,4 @@ public void testRequiredFields() throws Exception {
9597
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
9698
assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
9799
}
98-
99100
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,14 @@
1919
package org.elasticsearch.ingest.common;
2020

2121
import org.elasticsearch.ingest.IngestDocument;
22+
import org.elasticsearch.ingest.TestTemplateService;
2223
import org.elasticsearch.test.ESTestCase;
2324
import org.joda.time.DateTime;
2425
import org.joda.time.DateTimeZone;
26+
import org.joda.time.format.DateTimeFormat;
2527

2628
import java.util.Collections;
29+
import java.util.List;
2730
import java.util.Locale;
2831
import java.util.function.Function;
2932

@@ -33,11 +36,8 @@ public class DateIndexNameProcessorTests extends ESTestCase {
3336

3437
public void testJodaPattern() throws Exception {
3538
Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
36-
DateIndexNameProcessor processor = new DateIndexNameProcessor(
37-
"_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC,
38-
"events-", "y", "yyyyMMdd"
39-
);
40-
39+
DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
40+
DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
4141
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
4242
Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z"));
4343
processor.execute(document);
@@ -46,7 +46,7 @@ public void testJodaPattern() throws Exception {
4646

4747
public void testTAI64N()throws Exception {
4848
Function<String, DateTime> function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null);
49-
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
49+
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
5050
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
5151
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
5252
Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"));
@@ -56,7 +56,7 @@ public void testTAI64N()throws Exception {
5656

5757
public void testUnixMs()throws Exception {
5858
Function<String, DateTime> function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null);
59-
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
59+
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
6060
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
6161
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
6262
Collections.singletonMap("_field", "1000500"));
@@ -71,12 +71,41 @@ public void testUnixMs()throws Exception {
7171

7272
public void testUnix()throws Exception {
7373
Function<String, DateTime> function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
74-
DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
74+
DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
7575
DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
7676
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
7777
Collections.singletonMap("_field", "1000.5"));
7878
dateProcessor.execute(document);
7979
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
8080
}
8181

82+
public void testTemplatedFields() throws Exception {
83+
String indexNamePrefix = randomAlphaOfLength(10);
84+
String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s");
85+
String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy");
86+
String date = Integer.toString(randomInt());
87+
Function<String, DateTime> dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
88+
89+
DateIndexNameProcessor dateProcessor = createProcessor("_field",
90+
Collections.singletonList(dateTimeFunction), DateTimeZone.UTC, indexNamePrefix,
91+
dateRounding, indexNameFormat);
92+
93+
IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
94+
Collections.singletonMap("_field", date));
95+
dateProcessor.execute(document);
96+
97+
assertThat(document.getSourceAndMetadata().get("_index"),
98+
equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat)
99+
.print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>"));
100+
}
101+
102+
private DateIndexNameProcessor createProcessor(String field, List<Function<String, DateTime>> dateFormats,
103+
DateTimeZone timezone, String indexNamePrefix, String dateRounding,
104+
String indexNameFormat) {
105+
return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone,
106+
new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix),
107+
new TestTemplateService.MockTemplateScript.Factory(dateRounding),
108+
new TestTemplateService.MockTemplateScript.Factory(indexNameFormat)
109+
);
110+
}
82111
}

0 commit comments

Comments
 (0)