Skip to content

Pre-compile inline scripts in Ingest Script processors #57960

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 15, 2020
Merged

Pre-compile inline scripts in Ingest Script processors #57960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca324d7
Pre-compile inline scripts in Ingest Script processors
talevy Jun 10, 2020
434b47f
Merge remote-tracking branch 'elastic/master' into precompileinline
talevy Jun 12, 2020
39be478
Merge remote-tracking branch 'elastic/master' into precompileinline
talevy Jun 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 25 additions & 8 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.ingest.common;

import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
Expand All @@ -34,6 +35,7 @@
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptException;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;

import java.io.InputStream;
import java.util.Arrays;
Expand All @@ -50,17 +52,19 @@ public final class ScriptProcessor extends AbstractProcessor {

private final Script script;
private final ScriptService scriptService;
private final IngestScript precompiledIngestScript;

/**
* Processor that evaluates a script with an ingest document in its context
*
* @param tag The processor's tag.
* @param tag The processor's tag.
* @param script The {@link Script} to execute.
* @param precompiledIngestScript The {@link Script} precompiled
* @param scriptService The {@link ScriptService} used to execute the script.
*/
ScriptProcessor(String tag, Script script, ScriptService scriptService) {
ScriptProcessor(String tag, Script script, @Nullable IngestScript precompiledIngestScript, ScriptService scriptService) {
super(tag);
this.script = script;
this.precompiledIngestScript = precompiledIngestScript;
this.scriptService = scriptService;
}

Expand All @@ -71,8 +75,14 @@ public final class ScriptProcessor extends AbstractProcessor {
*/
@Override
public IngestDocument execute(IngestDocument document) {
IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
factory.newInstance(script.getParams()).execute(document.getSourceAndMetadata());
final IngestScript ingestScript;
if (precompiledIngestScript == null) {
IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
ingestScript = factory.newInstance(script.getParams());
} else {
ingestScript = precompiledIngestScript;
}
ingestScript.execute(document.getSourceAndMetadata());
CollectionUtils.ensureNoSelfReferences(document.getSourceAndMetadata(), "ingest script");
return document;
}
Expand All @@ -86,6 +96,10 @@ Script getScript() {
return script;
}

IngestScript getPrecompiledIngestScript() {
return precompiledIngestScript;
}

public static final class Factory implements Processor.Factory {
private final ScriptService scriptService;

Expand All @@ -105,13 +119,16 @@ public ScriptProcessor create(Map<String, Processor.Factory> registry, String pr
Arrays.asList("id", "source", "inline", "lang", "params", "options").forEach(config::remove);

// verify script is able to be compiled before successfully creating processor.
IngestScript ingestScript = null;
try {
scriptService.compile(script, IngestScript.CONTEXT);
final IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
if (ScriptType.INLINE.equals(script.getType())) {
ingestScript = factory.newInstance(script.getParams());
}
} catch (ScriptException e) {
throw newConfigurationException(TYPE, processorTag, null, e);
}

return new ScriptProcessor(processorTag, script, scriptService);
return new ScriptProcessor(processorTag, script, ingestScript, scriptService);
}
}
}
Expand Down
57 changes: 57 additions & 0 deletions ...est-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,15 @@
package org.elasticsearch.ingest.common;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParseException;
import org.elasticsearch.script.IngestScript;
import org.elasticsearch.script.MockScriptEngine;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptException;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;

Expand Down Expand Up @@ -51,6 +56,10 @@ public void init() {
}

public void testFactoryValidationWithDefaultLang() throws Exception {
ScriptService mockedScriptService = mock(ScriptService.class);
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
factory = new ScriptProcessor.Factory(mockedScriptService);

Map<String, Object> configMap = new HashMap<>();
String randomType = randomFrom("id", "source");
configMap.put(randomType, "foo");
Expand All @@ -61,6 +70,10 @@ public void testFactoryValidationWithDefaultLang() throws Exception {
}

public void testFactoryValidationWithParams() throws Exception {
ScriptService mockedScriptService = mock(ScriptService.class);
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
factory = new ScriptProcessor.Factory(mockedScriptService);

Map<String, Object> configMap = new HashMap<>();
String randomType = randomFrom("id", "source");
Map<String, Object> randomParams = Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10));
Expand Down Expand Up @@ -94,6 +107,10 @@ public void testFactoryValidationAtLeastOneScriptingType() throws Exception {
}

public void testInlineBackcompat() throws Exception {
ScriptService mockedScriptService = mock(ScriptService.class);
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
factory = new ScriptProcessor.Factory(mockedScriptService);

Map<String, Object> configMap = new HashMap<>();
configMap.put("inline", "code");

Expand All @@ -117,4 +134,44 @@ public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception {

assertThat(exception.getMessage(), is("compile-time exception"));
}

public void testInlineIsCompiled() throws Exception {
String scriptName = "foo";
ScriptService scriptService = new ScriptService(Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
Script.DEFAULT_SCRIPT_LANG,
Collections.singletonMap(scriptName, ctx -> {
ctx.put("foo", "bar");
return null;
}),
Collections.emptyMap()
)
), new HashMap<>(ScriptModule.CORE_CONTEXTS));
factory = new ScriptProcessor.Factory(scriptService);

Map<String, Object> configMap = new HashMap<>();
configMap.put("source", scriptName);
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap);
assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE));
assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
assertNotNull(processor.getPrecompiledIngestScript());
Map<String, Object> ctx = new HashMap<>();
processor.getPrecompiledIngestScript().execute(ctx);
assertThat(ctx.get("foo"), equalTo("bar"));
}

public void testStoredIsNotCompiled() throws Exception {
ScriptService mockedScriptService = mock(ScriptService.class);
when(mockedScriptService.compile(any(), any())).thenReturn(mock(IngestScript.Factory.class));
factory = new ScriptProcessor.Factory(mockedScriptService);
Map<String, Object> configMap = new HashMap<>();
configMap.put("id", "script_name");
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), configMap);
assertNull(processor.getScript().getLang());
assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED));
assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
assertNull(processor.getPrecompiledIngestScript());
}
}
48 changes: 35 additions & 13 deletions ...les/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.script.IngestScript;
import org.elasticsearch.script.MockScriptEngine;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptModule;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;

import java.util.Collections;
import java.util.HashMap;
Expand All @@ -38,18 +40,22 @@

public class ScriptProcessorTests extends ESTestCase {

public void testScripting() throws Exception {
int randomBytesIn = randomInt();
int randomBytesOut = randomInt();
int randomBytesTotal = randomBytesIn + randomBytesOut;
private ScriptService scriptService;
private Script script;
private IngestScript ingestScript;

@Before
public void setupScripting() {
String scriptName = "script";
ScriptService scriptService = new ScriptService(Settings.builder().build(),
scriptService = new ScriptService(Settings.builder().build(),
Collections.singletonMap(
Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
Script.DEFAULT_SCRIPT_LANG,
Collections.singletonMap(
scriptName, ctx -> {
ctx.put("bytes_total", randomBytesTotal);
Integer bytesIn = (Integer) ctx.get("bytes_in");
Integer bytesOut = (Integer) ctx.get("bytes_out");
ctx.put("bytes_total", bytesIn + bytesOut);
return null;
}
),
Expand All @@ -58,20 +64,36 @@ Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(
),
new HashMap<>(ScriptModule.CORE_CONTEXTS)
);
Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams());
}

public void testScriptingWithoutPrecompiledScriptFactory() throws Exception {
ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, null, scriptService);
IngestDocument ingestDocument = randomDocument();
processor.execute(ingestDocument);
assertIngestDocument(ingestDocument);
}

public void testScriptingWithPrecompiledIngestScript() {
ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, ingestScript, scriptService);
IngestDocument ingestDocument = randomDocument();
processor.execute(ingestDocument);
assertIngestDocument(ingestDocument);
}

private IngestDocument randomDocument() {
Map<String, Object> document = new HashMap<>();
document.put("bytes_in", randomInt());
document.put("bytes_out", randomInt());
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);

ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), script, scriptService);

processor.execute(ingestDocument);
return RandomDocumentPicks.randomIngestDocument(random(), document);
}

private void assertIngestDocument(IngestDocument ingestDocument) {
assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_in"));
assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_out"));
assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_total"));
assertThat(ingestDocument.getSourceAndMetadata().get("bytes_total"), is(randomBytesTotal));
int bytesTotal = ingestDocument.getFieldValue("bytes_in", Integer.class) + ingestDocument.getFieldValue("bytes_out", Integer.class);
assertThat(ingestDocument.getSourceAndMetadata().get("bytes_total"), is(bytesTotal));
}
}