Skip to content

Commit 6416d74

Browse files
talevytalevy
authored
Pre-compile inline scripts in Ingest Script processors (#57960)
This commit introduces an optimization for inline scripts. It keeps the compiled ingest script that the ScriptProcessor.Factory has been creating for validation purposes. Previously, the Script Service's cache was leveraged because it was the best way to handle caching of both stored and inline scripts. Since inline scripts are so widely used in Ingest Node, it is probably best to ensure we are using the pre-compiled version from the beginning.
1 parent bf910e9 commit 6416d74

File tree

3 files changed

+117
-21
lines changed

3 files changed

+117
-21
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.ingest.common;
2121

22+
import org.elasticsearch.common.Nullable;
2223
import org.elasticsearch.common.bytes.BytesReference;
2324
import org.elasticsearch.common.util.CollectionUtils;
2425
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -34,6 +35,7 @@
3435
import org.elasticsearch.script.Script;
3536
import org.elasticsearch.script.ScriptException;
3637
import org.elasticsearch.script.ScriptService;
38+
import org.elasticsearch.script.ScriptType;
3739

3840
import java.io.InputStream;
3941
import java.util.Arrays;
@@ -50,17 +52,19 @@ public final class ScriptProcessor extends AbstractProcessor {
5052

5153
private final Script script;
5254
private final ScriptService scriptService;
55+
private final IngestScript precompiledIngestScript;
5356

5457
/**
5558
* Processor that evaluates a script with an ingest document in its context
56-
*
57-
* @param tag The processor's tag.
59+
* @param tag The processor's tag.
5860
* @param script The {@link Script} to execute.
61+
* @param precompiledIngestScript The {@link Script} precompiled
5962
* @param scriptService The {@link ScriptService} used to execute the script.
6063
*/
61-
ScriptProcessor(String tag, Script script, ScriptService scriptService) {
64+
ScriptProcessor(String tag, Script script, @Nullable IngestScript precompiledIngestScript, ScriptService scriptService) {
6265
super(tag);
6366
this.script = script;
67+
this.precompiledIngestScript = precompiledIngestScript;
6468
this.scriptService = scriptService;
6569
}
6670

@@ -71,8 +75,14 @@ public final class ScriptProcessor extends AbstractProcessor {
7175
*/
7276
@Override
7377
public IngestDocument execute(IngestDocument document) {
74-
IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
75-
factory.newInstance(script.getParams()).execute(document.getSourceAndMetadata());
78+
final IngestScript ingestScript;
79+
if (precompiledIngestScript == null) {
80+
IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
81+
ingestScript = factory.newInstance(script.getParams());
82+
} else {
83+
ingestScript = precompiledIngestScript;
84+
}
85+
ingestScript.execute(document.getSourceAndMetadata());
7686
CollectionUtils.ensureNoSelfReferences(document.getSourceAndMetadata(), "ingest script");
7787
return document;
7888
}
@@ -86,6 +96,10 @@ Script getScript() {
8696
return script;
8797
}
8898

99+
IngestScript getPrecompiledIngestScript() {
100+
return precompiledIngestScript;
101+
}
102+
89103
public static final class Factory implements Processor.Factory {
90104
private final ScriptService scriptService;
91105

@@ -105,13 +119,16 @@ public ScriptProcessor create(Map<String, Processor.Factory> registry, String pr
105119
Arrays.asList("id", "source", "inline", "lang", "params", "options").forEach(config::remove);
106120

107121
// verify script is able to be compiled before successfully creating processor.
122+
IngestScript ingestScript = null;
108123
try {
109-
scriptService.compile(script, IngestScript.CONTEXT);
124+
final IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
125+
if (ScriptType.INLINE.equals(script.getType())) {
126+
ingestScript = factory.newInstance(script.getParams());
127+
}
110128
} catch (ScriptException e) {
111129
throw newConfigurationException(TYPE, processorTag, null, e);
112130
}
113-
114-
return new ScriptProcessor(processorTag, script, scriptService);
131+
return new ScriptProcessor(processorTag, script, ingestScript, scriptService);
115132
}
116133
}
117134
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,15 @@
2020
package org.elasticsearch.ingest.common;
2121

2222
import org.elasticsearch.ElasticsearchException;
23+
import org.elasticsearch.common.settings.Settings;
2324
import org.elasticsearch.common.xcontent.XContentParseException;
25+
import org.elasticsearch.script.IngestScript;
26+
import org.elasticsearch.script.MockScriptEngine;
2427
import org.elasticsearch.script.Script;
2528
import org.elasticsearch.script.ScriptException;
29+
import org.elasticsearch.script.ScriptModule;
2630
import org.elasticsearch.script.ScriptService;
31+
import org.elasticsearch.script.ScriptType;
2732
import org.elasticsearch.test.ESTestCase;
2833
import org.junit.Before;
2934

@@ -51,6 +56,10 @@ public void init() {
5156
}
5257

5358
public void testFactoryValidationWithDefaultLang() throws Exception {
59+
ScriptService mockedScriptService = mock(ScriptService.class);
60+
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
61+
factory = new ScriptProcessor.Factory(mockedScriptService);
62+
5463
Map<String, Object> configMap = new HashMap<>();
5564
String randomType = randomFrom("id", "source");
5665
configMap.put(randomType, "foo");
@@ -61,6 +70,10 @@ public void testFactoryValidationWithDefaultLang() throws Exception {
6170
}
6271

6372
public void testFactoryValidationWithParams() throws Exception {
73+
ScriptService mockedScriptService = mock(ScriptService.class);
74+
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
75+
factory = new ScriptProcessor.Factory(mockedScriptService);
76+
6477
Map<String, Object> configMap = new HashMap<>();
6578
String randomType = randomFrom("id", "source");
6679
Map<String, Object> randomParams = Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10));
@@ -94,6 +107,10 @@ public void testFactoryValidationAtLeastOneScriptingType() throws Exception {
94107
}
95108

96109
public void testInlineBackcompat() throws Exception {
110+
ScriptService mockedScriptService = mock(ScriptService.class);
111+
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
112+
factory = new ScriptProcessor.Factory(mockedScriptService);
113+
97114
Map<String, Object> configMap = new HashMap<>();
98115
configMap.put("inline", "code");
99116

@@ -117,4 +134,44 @@ public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception {
117134

118135
assertThat(exception.getMessage(), is("compile-time exception"));
119136
}
137+
138+
public void testInlineIsCompiled() throws Exception {
139+
String scriptName = "foo";
140+
ScriptService scriptService = new ScriptService(Settings.builder().build(),
141+
Collections.singletonMap(
142+
Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
143+
Script.DEFAULT_SCRIPT_LANG,
144+
Collections.singletonMap(scriptName, ctx -> {
145+
ctx.put("foo", "bar");
146+
return null;
147+
}),
148+
Collections.emptyMap()
149+
)
150+
), new HashMap<>(ScriptModule.CORE_CONTEXTS));
151+
factory = new ScriptProcessor.Factory(scriptService);
152+
153+
Map<String, Object> configMap = new HashMap<>();
154+
configMap.put("source", scriptName);
155+
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap);
156+
assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
157+
assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE));
158+
assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
159+
assertNotNull(processor.getPrecompiledIngestScript());
160+
Map<String, Object> ctx = new HashMap<>();
161+
processor.getPrecompiledIngestScript().execute(ctx);
162+
assertThat(ctx.get("foo"), equalTo("bar"));
163+
}
164+
165+
public void testStoredIsNotCompiled() throws Exception {
166+
ScriptService mockedScriptService = mock(ScriptService.class);
167+
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
168+
factory = new ScriptProcessor.Factory(mockedScriptService);
169+
Map<String, Object> configMap = new HashMap<>();
170+
configMap.put("id", "script_name");
171+
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap);
172+
assertNull(processor.getScript().getLang());
173+
assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED));
174+
assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
175+
assertNull(processor.getPrecompiledIngestScript());
176+
}
120177
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222
import org.elasticsearch.common.settings.Settings;
2323
import org.elasticsearch.ingest.IngestDocument;
2424
import org.elasticsearch.ingest.RandomDocumentPicks;
25+
import org.elasticsearch.script.IngestScript;
2526
import org.elasticsearch.script.MockScriptEngine;
2627
import org.elasticsearch.script.Script;
2728
import org.elasticsearch.script.ScriptModule;
2829
import org.elasticsearch.script.ScriptService;
2930
import org.elasticsearch.script.ScriptType;
3031
import org.elasticsearch.test.ESTestCase;
32+
import org.junit.Before;
3133

3234
import java.util.Collections;
3335
import java.util.HashMap;
@@ -38,18 +40,22 @@
3840

3941
public class ScriptProcessorTests extends ESTestCase {
4042

41-
public void testScripting() throws Exception {
42-
int randomBytesIn = randomInt();
43-
int randomBytesOut = randomInt();
44-
int randomBytesTotal = randomBytesIn + randomBytesOut;
43+
private ScriptService scriptService;
44+
private Script script;
45+
private IngestScript ingestScript;
46+
47+
@Before
48+
public void setupScripting() {
4549
String scriptName = "script";
46-
ScriptService scriptService = new ScriptService(Settings.builder().build(),
50+
scriptService = new ScriptService(Settings.builder().build(),
4751
Collections.singletonMap(
4852
Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
4953
Script.DEFAULT_SCRIPT_LANG,
5054
Collections.singletonMap(
5155
scriptName, ctx -> {
52-
ctx.put("bytes_total", randomBytesTotal);
56+
Integer bytesIn = (Integer) ctx.get("bytes_in");
57+
Integer bytesOut = (Integer) ctx.get("bytes_out");
58+
ctx.put("bytes_total", bytesIn + bytesOut);
5359
return null;
5460
}
5561
),
@@ -58,20 +64,36 @@ Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
5864
),
5965
new HashMap<>(ScriptModule.CORE_CONTEXTS)
6066
);
61-
Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
67+
script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
68+
ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams());
69+
}
70+
71+
public void testScriptingWithoutPrecompiledScriptFactory() throws Exception {
72+
ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, null, scriptService);
73+
IngestDocument ingestDocument = randomDocument();
74+
processor.execute(ingestDocument);
75+
assertIngestDocument(ingestDocument);
76+
}
77+
78+
public void testScriptingWithPrecompiledIngestScript() {
79+
ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, ingestScript, scriptService);
80+
IngestDocument ingestDocument = randomDocument();
81+
processor.execute(ingestDocument);
82+
assertIngestDocument(ingestDocument);
83+
}
6284

85+
private IngestDocument randomDocument() {
6386
Map<String, Object> document = new HashMap<>();
6487
document.put("bytes_in", randomInt());
6588
document.put("bytes_out", randomInt());
66-
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
67-
68-
ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, scriptService);
69-
70-
processor.execute(ingestDocument);
89+
return RandomDocumentPicks.randomIngestDocument(random(), document);
90+
}
7191

92+
private void assertIngestDocument(IngestDocument ingestDocument) {
7293
assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_in"));
7394
assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_out"));
7495
assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_total"));
75-
assertThat(ingestDocument.getSourceAndMetadata().get("bytes_total"), is(randomBytesTotal));
96+
int bytesTotal = ingestDocument.getFieldValue("bytes_in", Integer.class) + ingestDocument.getFieldValue("bytes_out", Integer.class);
97+
assertThat(ingestDocument.getSourceAndMetadata().get("bytes_total"), is(bytesTotal));
7698
}
7799
}

0 commit comments

Comments
 (0)