From 5b5e61752939bb2d762b0592a0965a862178186c Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Fri, 15 Nov 2019 16:32:36 -0700 Subject: [PATCH 1/3] Scripting: add available languages & contexts API Adds `GET /_script_language` to support Kibana dynamic scripting language selection. Response contains whether `inline` and/or `stored` scripts are enabled as determined by the `script.allowed_types` settings. For each scripting language registered, such as `painless`, `expression`, `mustache` or custom, available contexts for the language are included as determined by the `script.allowed_contexts` setting. Response format: ``` { "types_allowed": [ "inline", "stored" ], "language_contexts": [ { "language": "expression", "contexts": [ "aggregation_selector", "aggs" ... ] }, { "language": "painless", "contexts": [ "aggregation_selector", "aggs", "aggs_combine", ... ] } ... ] } ``` Fixes: #49463 --- .../expression/ExpressionScriptEngine.java | 88 ++++++---- .../script/mustache/MustacheScriptEngine.java | 6 + .../painless/PainlessScriptEngine.java | 5 + .../expertscript/ExpertScriptPlugin.java | 6 + .../elasticsearch/action/ActionModule.java | 5 + .../GetScriptLanguageAction.java | 31 ++++ .../GetScriptLanguageRequest.java | 42 +++++ .../GetScriptLanguageResponse.java | 78 +++++++++ .../TransportGetScriptLanguageAction.java | 43 +++++ .../cluster/RestGetScriptLanguageAction.java | 51 ++++++ .../elasticsearch/script/ScriptEngine.java | 6 + .../script/ScriptLanguagesInfo.java | 164 ++++++++++++++++++ .../elasticsearch/script/ScriptService.java | 22 +++ .../GetScriptLanguageResponseTests.java | 136 +++++++++++++++ .../script/ScriptLanguagesInfoTests.java | 76 ++++++++ .../functionscore/ExplainableScriptIT.java | 5 + .../search/suggest/SuggestSearchIT.java | 5 + .../script/MockScriptEngine.java | 29 ++++ 18 files changed, 761 insertions(+), 37 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetScriptLanguageAction.java create mode 100644 server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java create mode 100644 server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index afe507279027d..7d4ac042efc1d 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -55,6 +55,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; /** * Provides the infrastructure for Lucene expressions as a scripting language for Elasticsearch. @@ -65,6 +66,41 @@ public class ExpressionScriptEngine implements ScriptEngine { public static final String NAME = "expression"; + private static Map, Function> contexts = Map.of( + BucketAggregationScript.CONTEXT, + ExpressionScriptEngine::newBucketAggregationScriptFactory, + + BucketAggregationSelectorScript.CONTEXT, + (Expression expr) -> { + BucketAggregationScript.Factory factory = newBucketAggregationScriptFactory(expr); + BucketAggregationSelectorScript.Factory wrappedFactory = parameters -> new BucketAggregationSelectorScript(parameters) { + @Override + public boolean execute() { + return factory.newInstance(getParams()).execute().doubleValue() == 1.0; + } + }; + return wrappedFactory; + }, + + FilterScript.CONTEXT, + (Expression expr) -> (FilterScript.Factory) (p, lookup) -> newFilterScript(expr, lookup, p), + + ScoreScript.CONTEXT, + (Expression expr) -> (ScoreScript.Factory) (p, lookup) -> newScoreScript(expr, lookup, p), + + TermsSetQueryScript.CONTEXT, + (Expression expr) -> (TermsSetQueryScript.Factory) (p, lookup) -> newTermsSetQueryScript(expr, lookup, p), + + AggregationScript.CONTEXT, + (Expression expr) -> (AggregationScript.Factory) (p, lookup) -> newAggregationScript(expr, lookup, p), + + NumberSortScript.CONTEXT, + (Expression expr) -> (NumberSortScript.Factory) (p, lookup) -> newSortScript(expr, lookup, p), + + FieldScript.CONTEXT, + (Expression expr) -> (FieldScript.Factory) (p, lookup) -> newFieldScript(expr, lookup, p) + ); + @Override public String getType() { return NAME; @@ -102,37 +138,15 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE } } }); - if (context.instanceClazz.equals(BucketAggregationScript.class)) { - return context.factoryClazz.cast(newBucketAggregationScriptFactory(expr)); - } else if (context.instanceClazz.equals(BucketAggregationSelectorScript.class)) { - BucketAggregationScript.Factory factory = newBucketAggregationScriptFactory(expr); - BucketAggregationSelectorScript.Factory wrappedFactory = parameters -> new BucketAggregationSelectorScript(parameters) { - @Override - public boolean execute() { - return factory.newInstance(getParams()).execute().doubleValue() == 1.0; - } - }; - return context.factoryClazz.cast(wrappedFactory); - } else if (context.instanceClazz.equals(FilterScript.class)) { - FilterScript.Factory factory = (p, lookup) -> newFilterScript(expr, lookup, p); - return context.factoryClazz.cast(factory); - } else if (context.instanceClazz.equals(ScoreScript.class)) { - ScoreScript.Factory factory = (p, lookup) -> newScoreScript(expr, lookup, p); - return context.factoryClazz.cast(factory); - } else if (context.instanceClazz.equals(TermsSetQueryScript.class)) { - TermsSetQueryScript.Factory factory = (p, lookup) -> newTermsSetQueryScript(expr, lookup, p); - return context.factoryClazz.cast(factory); - } else if (context.instanceClazz.equals(AggregationScript.class)) { - AggregationScript.Factory factory = (p, lookup) -> newAggregationScript(expr, lookup, p); - return context.factoryClazz.cast(factory); - } else if (context.instanceClazz.equals(NumberSortScript.class)) { - NumberSortScript.Factory factory = (p, lookup) -> newSortScript(expr, lookup, p); - return context.factoryClazz.cast(factory); - } else if (context.instanceClazz.equals(FieldScript.class)) { - FieldScript.Factory factory = (p, lookup) -> newFieldScript(expr, lookup, p); - return context.factoryClazz.cast(factory); + if (contexts.containsKey(context) == false) { + throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]"); } - throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]"); + return context.factoryClazz.cast(contexts.get(context).apply(expr)); + } + + @Override + public List> getSupportedContexts() { + return new ArrayList<>(contexts.keySet()); } private static BucketAggregationScript.Factory newBucketAggregationScriptFactory(Expression expr) { @@ -166,7 +180,7 @@ public Double execute() { }; } - private NumberSortScript.LeafFactory newSortScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { + private static NumberSortScript.LeafFactory newSortScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings, // instead of complicating SimpleBindings (which should stay simple) SimpleBindings bindings = new SimpleBindings(); @@ -193,7 +207,7 @@ private NumberSortScript.LeafFactory newSortScript(Expression expr, SearchLookup return new ExpressionNumberSortScript(expr, bindings, needsScores); } - private TermsSetQueryScript.LeafFactory newTermsSetQueryScript(Expression expr, SearchLookup lookup, + private static TermsSetQueryScript.LeafFactory newTermsSetQueryScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings, // instead of complicating SimpleBindings (which should stay simple) @@ -216,7 +230,7 @@ private TermsSetQueryScript.LeafFactory newTermsSetQueryScript(Expression expr, return new ExpressionTermSetQueryScript(expr, bindings); } - private AggregationScript.LeafFactory newAggregationScript(Expression expr, SearchLookup lookup, + private static AggregationScript.LeafFactory newAggregationScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings, // instead of complicating SimpleBindings (which should stay simple) @@ -252,7 +266,7 @@ private AggregationScript.LeafFactory newAggregationScript(Expression expr, Sear return new ExpressionAggregationScript(expr, bindings, needsScores, specialValue); } - private FieldScript.LeafFactory newFieldScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { + private static FieldScript.LeafFactory newFieldScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { SimpleBindings bindings = new SimpleBindings(); for (String variable : expr.variables) { try { @@ -273,7 +287,7 @@ private FieldScript.LeafFactory newFieldScript(Expression expr, SearchLookup loo * This is a hack for filter scripts, which must return booleans instead of doubles as expression do. * See https://github.com/elastic/elasticsearch/issues/26429. */ - private FilterScript.LeafFactory newFilterScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { + private static FilterScript.LeafFactory newFilterScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { ScoreScript.LeafFactory searchLeafFactory = newScoreScript(expr, lookup, vars); return ctx -> { ScoreScript script = searchLeafFactory.newInstance(ctx); @@ -290,7 +304,7 @@ public void setDocument(int docid) { }; } - private ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { + private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLookup lookup, @Nullable Map vars) { // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings, // instead of complicating SimpleBindings (which should stay simple) SimpleBindings bindings = new SimpleBindings(); @@ -327,7 +341,7 @@ private ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLookup loo /** * converts a ParseException at compile-time or link-time to a ScriptException */ - private ScriptException convertToScriptException(String message, String source, String portion, Throwable cause) { + private static ScriptException convertToScriptException(String message, String source, String portion, Throwable cause) { List stack = new ArrayList<>(); stack.add(portion); StringBuilder pointer = new StringBuilder(); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java index ca28d12a7bda5..5b96864f1006c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java @@ -40,6 +40,7 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Collections; +import java.util.List; import java.util.Map; /** @@ -79,6 +80,11 @@ public T compile(String templateName, String templateSource, ScriptContext> getSupportedContexts() { + return List.of(TemplateScript.CONTEXT); + } + private CustomMustacheFactory createMustacheFactory(Map options) { if (options == null || options.isEmpty() || options.containsKey(Script.CONTENT_TYPE_OPTION) == false) { return new CustomMustacheFactory(); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index 11bfbe3b40fc6..3a6afd06cf75a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -146,6 +146,11 @@ public Loader run() { } } + @Override + public List> getSupportedContexts() { + return new ArrayList<>(contextsToCompilers.keySet()); + } + /** * Generates a stateful factory class that will return script instances. Acts as a middle man between * the {@link ScriptContext#factoryClazz} and the {@link ScriptContext#instanceClazz} when used so that diff --git a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java index 0b65084dee466..d04cba86e80f8 100644 --- a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java +++ b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collection; +import java.util.List; import java.util.Map; /** @@ -76,6 +77,11 @@ public void close() { // optionally close resources } + @Override + public List> getSupportedContexts() { + return List.of(ScoreScript.CONTEXT); + } + private static class PureDfLeafFactory implements LeafFactory { private final Map params; private final SearchLookup lookup; diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 10dcf6943f867..3d60a1fb698d2 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -80,10 +80,12 @@ import org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.GetScriptContextAction; +import org.elasticsearch.action.admin.cluster.storedscripts.GetScriptLanguageAction; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportGetScriptContextAction; +import org.elasticsearch.action.admin.cluster.storedscripts.TransportGetScriptLanguageAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportGetStoredScriptAction; import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction; import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction; @@ -249,6 +251,7 @@ import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction; import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction; +import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction; import org.elasticsearch.rest.action.admin.cluster.RestGetSnapshotsAction; import org.elasticsearch.rest.action.admin.cluster.RestGetStoredScriptAction; import org.elasticsearch.rest.action.admin.cluster.RestGetTaskAction; @@ -522,6 +525,7 @@ public void reg actions.register(GetStoredScriptAction.INSTANCE, TransportGetStoredScriptAction.class); actions.register(DeleteStoredScriptAction.INSTANCE, TransportDeleteStoredScriptAction.class); actions.register(GetScriptContextAction.INSTANCE, TransportGetScriptContextAction.class); + actions.register(GetScriptLanguageAction.INSTANCE, TransportGetScriptLanguageAction.class); actions.register(FieldCapabilitiesAction.INSTANCE, TransportFieldCapabilitiesAction.class); actions.register(TransportFieldCapabilitiesIndexAction.TYPE, TransportFieldCapabilitiesIndexAction.class); @@ -662,6 +666,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestPutStoredScriptAction(restController)); registerHandler.accept(new RestDeleteStoredScriptAction(restController)); registerHandler.accept(new RestGetScriptContextAction(restController)); + registerHandler.accept(new RestGetScriptLanguageAction(restController)); registerHandler.accept(new RestFieldCapabilitiesAction(restController)); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java new file mode 100644 index 0000000000000..d4c6ae2de052c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageAction.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.storedscripts; + +import org.elasticsearch.action.ActionType; + +public class GetScriptLanguageAction extends ActionType { + public static final GetScriptLanguageAction INSTANCE = new GetScriptLanguageAction(); + public static final String NAME = "cluster:admin/script_language/get"; + + private GetScriptLanguageAction() { + super(NAME, GetScriptLanguageResponse::new); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java new file mode 100644 index 0000000000000..c5433be2febfa --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageRequest.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.storedscripts; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +public class GetScriptLanguageRequest extends ActionRequest { + public GetScriptLanguageRequest() { + super(); + } + + GetScriptLanguageRequest(StreamInput in) throws IOException { + super(in); + } + + @Override + public ActionRequestValidationException validate() { return null; } + + @Override + public String toString() { return "get script languages"; } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java new file mode 100644 index 0000000000000..7d8ea7654c4d4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponse.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.storedscripts; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.ScriptLanguagesInfo; + +import java.io.IOException; +import java.util.Objects; + +public class GetScriptLanguageResponse extends ActionResponse implements StatusToXContentObject, Writeable { + public final ScriptLanguagesInfo info; + + GetScriptLanguageResponse(ScriptLanguagesInfo info) { + this.info = info; + } + + GetScriptLanguageResponse(StreamInput in) throws IOException { + super(in); + info = new ScriptLanguagesInfo(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + info.writeTo(out); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + public static GetScriptLanguageResponse fromXContent(XContentParser parser) throws IOException { + return new GetScriptLanguageResponse(ScriptLanguagesInfo.fromXContent(parser)); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + GetScriptLanguageResponse that = (GetScriptLanguageResponse) o; + return info.equals(that.info); + } + + @Override + public int hashCode() { return Objects.hash(info); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return info.toXContent(builder, params); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java new file mode 100644 index 0000000000000..96f07de533c25 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetScriptLanguageAction.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.storedscripts; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; + +public class TransportGetScriptLanguageAction extends HandledTransportAction { + private final ScriptService scriptService; + + @Inject + public TransportGetScriptLanguageAction(TransportService transportService, ActionFilters actionFilters, ScriptService scriptService) { + super(GetScriptLanguageAction.NAME, transportService, actionFilters, GetScriptLanguageRequest::new); + this.scriptService = scriptService; + } + + @Override + protected void doExecute(Task task, GetScriptLanguageRequest request, ActionListener listener) { + listener.onResponse(new GetScriptLanguageResponse(scriptService.getScriptLanguages())); + } +} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetScriptLanguageAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetScriptLanguageAction.java new file mode 100644 index 0000000000000..c9246b910cf4f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetScriptLanguageAction.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.cluster; + +import org.elasticsearch.action.admin.cluster.storedscripts.GetScriptLanguageAction; +import org.elasticsearch.action.admin.cluster.storedscripts.GetScriptLanguageRequest; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestGetScriptLanguageAction extends BaseRestHandler { + @Inject + public RestGetScriptLanguageAction(RestController controller) { + controller.registerHandler(GET, "/_script_language", this); + } + + @Override public String getName() { + return "script_language_action"; + } + + @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + return channel -> client.execute(GetScriptLanguageAction.INSTANCE, + new GetScriptLanguageRequest(), + new RestToXContentListener<>(channel)); + } + +} diff --git a/server/src/main/java/org/elasticsearch/script/ScriptEngine.java b/server/src/main/java/org/elasticsearch/script/ScriptEngine.java index bd32cce0b3781..2374befa3f585 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptEngine.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptEngine.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.List; import java.util.Map; /** @@ -45,4 +46,9 @@ public interface ScriptEngine extends Closeable { @Override default void close() throws IOException {} + + /** + * Script contexts supported by this engine. + */ + List> getSupportedContexts(); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java b/server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java new file mode 100644 index 0000000000000..fe56d93147196 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class ScriptLanguagesInfo implements ToXContentObject, Writeable { + private static final ParseField TYPES_ALLOWED = new ParseField("types_allowed"); + private static final ParseField LANGUAGE_CONTEXTS = new ParseField("language_contexts"); + private static final ParseField LANGUAGE = new ParseField("language"); + private static final ParseField CONTEXTS = new ParseField("contexts"); + + public final Set typesAllowed; + public final Map> languageContexts; + + public ScriptLanguagesInfo(Set typesAllowed, Map> languageContexts) { + this.typesAllowed = typesAllowed != null ? Set.copyOf(typesAllowed): Collections.emptySet(); + this.languageContexts = languageContexts != null ? Map.copyOf(languageContexts): Collections.emptyMap(); + } + + public ScriptLanguagesInfo(StreamInput in) throws IOException { + typesAllowed = readStringSet(in); + languageContexts = readStringMapSet(in); + } + + @SuppressWarnings("unchecked") + public static ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("script_languages_info", true, + (a) -> new ScriptLanguagesInfo( + new HashSet<>((List)a[0]), + ((List>>)a[1]).stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2)) + ) + ); + + @SuppressWarnings("unchecked") + private static ConstructingObjectParser>,Void> LANGUAGE_CONTEXT_PARSER = + new ConstructingObjectParser<>("language_contexts", true, + (m, name) -> new Tuple<>((String)m[0], Set.copyOf((List)m[1])) + ); + + static { + PARSER.declareStringArray(constructorArg(), TYPES_ALLOWED); + PARSER.declareObjectArray(constructorArg(), LANGUAGE_CONTEXT_PARSER, LANGUAGE_CONTEXTS); + LANGUAGE_CONTEXT_PARSER.declareString(constructorArg(), LANGUAGE); + LANGUAGE_CONTEXT_PARSER.declareStringArray(constructorArg(), CONTEXTS); + } + + public static ScriptLanguagesInfo fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeStringSet(out, typesAllowed); + writeStringMapSet(out, languageContexts); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ScriptLanguagesInfo that = (ScriptLanguagesInfo) o; + return Objects.equals(typesAllowed, that.typesAllowed) && + Objects.equals(languageContexts, that.languageContexts); + } + + @Override + public int hashCode() { + return Objects.hash(typesAllowed, languageContexts); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject().startArray(TYPES_ALLOWED.getPreferredName()); + for (String type: typesAllowed.stream().sorted().collect(Collectors.toList())) { + builder.value(type); + } + + builder.endArray().startArray(LANGUAGE_CONTEXTS.getPreferredName()); + List>> languagesByName = languageContexts.entrySet().stream().sorted( + Map.Entry.comparingByKey() + ).collect(Collectors.toList()); + + for (Map.Entry> languageContext: languagesByName) { + builder.startObject().field(LANGUAGE.getPreferredName(), languageContext.getKey()).startArray(CONTEXTS.getPreferredName()); + for (String context: languageContext.getValue().stream().sorted().collect(Collectors.toList())) { + builder.value(context); + } + builder.endArray().endObject(); + } + + return builder.endArray().endObject(); + } + + private static Map> readStringMapSet(StreamInput in) throws IOException { + Map> values = new HashMap<>(); + for (int i = in.readInt(); i > 0; i--) { + values.put(in.readString(), readStringSet(in)); + } + return values; + } + + private static void writeStringMapSet(StreamOutput out, Map> values) throws IOException { + out.writeInt(values.size()); + for (Map.Entry> value: values.entrySet()) { + out.writeString(value.getKey()); + writeStringSet(out, value.getValue()); + } + } + + private static Set readStringSet(StreamInput in) throws IOException { + Set values = new HashSet<>(); + for (int i = in.readInt(); i > 0; i--) { + values.add(in.readString()); + } + return values; + } + + private static void writeStringSet(StreamOutput out, Set values) throws IOException { + out.writeInt(values.size()); + for (String value: values) { + out.writeString(value); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index 799515256d302..a1788ee74a163 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -52,12 +52,14 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; public class ScriptService implements Closeable, ClusterStateApplier { @@ -546,6 +548,26 @@ public Set getContextInfos() { return infos; } + public ScriptLanguagesInfo getScriptLanguages() { + Set types = typesAllowed; + if (types == null) { + types = new HashSet<>(); + for (ScriptType type: ScriptType.values()) { + types.add(type.getName()); + } + } + + final Set contexts = contextsAllowed != null ? contextsAllowed : this.contexts.keySet(); + Map> languageContexts = new HashMap<>(); + engines.forEach( + (key, value) -> languageContexts.put( + key, + value.getSupportedContexts().stream().map(c -> c.name).filter(contexts::contains).collect(Collectors.toSet()) + ) + ); + return new ScriptLanguagesInfo(types, languageContexts); + } + public ScriptStats stats() { return scriptMetrics.stats(); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java new file mode 100644 index 0000000000000..f330cb36b7296 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.storedscripts; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.script.ScriptLanguagesInfo; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class GetScriptLanguageResponseTests extends AbstractSerializingTestCase { + private static int MAX_VALUES = 4; + private static final int MIN_LENGTH = 1; + private static final int MAX_LENGTH = 16; + + @Override + protected GetScriptLanguageResponse createTestInstance() { + if (randomBoolean()) { + return new GetScriptLanguageResponse( + new ScriptLanguagesInfo(Collections.emptySet(), Collections.emptyMap()) + ); + } + return new GetScriptLanguageResponse(randomInstance()); + } + + @Override + protected GetScriptLanguageResponse doParseInstance(XContentParser parser) throws IOException { + return GetScriptLanguageResponse.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { return GetScriptLanguageResponse::new; } + + @Override + protected GetScriptLanguageResponse mutateInstance(GetScriptLanguageResponse instance) throws IOException { + switch (randomInt(2)) { + case 0: + // mutate typesAllowed + return new GetScriptLanguageResponse( + new ScriptLanguagesInfo(mutateStringSet(instance.info.typesAllowed), instance.info.languageContexts) + ); + case 1: + // Add language + String language = randomValueOtherThanMany( + instance.info.languageContexts::containsKey, + () -> randomAlphaOfLengthBetween(MIN_LENGTH, MAX_LENGTH) + ); + Map> languageContexts = new HashMap<>(); + instance.info.languageContexts.forEach(languageContexts::put); + languageContexts.put(language, randomStringSet(randomIntBetween(1, MAX_VALUES))); + return new GetScriptLanguageResponse(new ScriptLanguagesInfo(instance.info.typesAllowed, languageContexts)); + default: + // Mutate languageContexts + Map> lc = new HashMap<>(); + if (instance.info.languageContexts.size() == 0) { + lc.put(randomAlphaOfLengthBetween(MIN_LENGTH, MAX_LENGTH), randomStringSet(randomIntBetween(1, MAX_VALUES))); + } else { + int toModify = randomInt(instance.info.languageContexts.size()-1); + List keys = new ArrayList<>(instance.info.languageContexts.keySet()); + for (int i=0; i value = instance.info.languageContexts.get(keys.get(i)); + if (i == toModify) { + value = mutateStringSet(instance.info.languageContexts.get(keys.get(i))); + } + lc.put(key, value); + } + } + return new GetScriptLanguageResponse(new ScriptLanguagesInfo(instance.info.typesAllowed, lc)); + } + } + + private static ScriptLanguagesInfo randomInstance() { + Map> contexts = new HashMap<>(); + for (String context: randomStringSet(randomIntBetween(1, MAX_VALUES))) { + contexts.put(context, randomStringSet(randomIntBetween(1, MAX_VALUES))); + } + return new ScriptLanguagesInfo(randomStringSet(randomInt(MAX_VALUES)), contexts); + } + + private static Set randomStringSet(int numInstances) { + Set rand = new HashSet<>(numInstances); + for (int i = 0; i < numInstances; i++) { + rand.add(randomValueOtherThanMany(rand::contains, () -> randomAlphaOfLengthBetween(MIN_LENGTH, MAX_LENGTH))); + } + return rand; + } + + private static Set mutateStringSet(Set strings) { + if (strings.isEmpty()) { + return Set.of(randomAlphaOfLengthBetween(MIN_LENGTH, MAX_LENGTH)); + } + + if (randomBoolean()) { + Set updated = new HashSet<>(strings); + updated.add(randomValueOtherThanMany(updated::contains, () -> randomAlphaOfLengthBetween(MIN_LENGTH, MAX_LENGTH))); + return updated; + } else { + List sorted = strings.stream().sorted().collect(Collectors.toList()); + int toRemove = randomInt(sorted.size() - 1); + Set updated = new HashSet<>(); + for (int i = 0; i < sorted.size(); i++) { + if (i != toRemove) { + updated.add(sorted.get(i)); + } + } + return updated; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java new file mode 100644 index 0000000000000..611d9c1f56dc4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ScriptLanguagesInfoTests extends ESTestCase { + // Test empty types allowed, INLINE only, STORED only, INLINE and STORED - DONE + // Test empty contexts allowed with different engines, search only, update only, search and update + // Test expression, mustache, painless and empty engines + + public void testEmptyTypesAllowedReturnsAllTypes() { + ScriptService ss = getMockScriptService(Settings.EMPTY); + ScriptLanguagesInfo info = ss.getScriptLanguages(); + ScriptType[] types = ScriptType.values(); + assertEquals(types.length, info.typesAllowed.size()); + for(ScriptType type: types) { + assertTrue("[" + type.getName() + "] is allowed", info.typesAllowed.contains(type.getName())); + } + } + + public void testSingleTypesAllowedReturnsThatType() { + for (ScriptType type: ScriptType.values()) { + ScriptService ss = getMockScriptService( + Settings.builder().put("script.allowed_types", type.getName()).build() + ); + ScriptLanguagesInfo info = ss.getScriptLanguages(); + assertEquals(1, info.typesAllowed.size()); + assertTrue("[" + type.getName() + "] is allowed", info.typesAllowed.contains(type.getName())); + } + } + + public void testBothTypesAllowedReturnsBothTypes() { + List types = Arrays.stream(ScriptType.values()).map(ScriptType::getName).collect(Collectors.toList()); + Settings.Builder settings = Settings.builder().putList("script.allowed_types", types); + ScriptService ss = getMockScriptService(settings.build()); + ScriptLanguagesInfo info = ss.getScriptLanguages(); + assertEquals(types.size(), info.typesAllowed.size()); + for(String type: types) { + assertTrue("[" + type + "] is allowed", info.typesAllowed.contains(type)); + } + } + + private ScriptService getMockScriptService(Settings settings) { + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, + Collections.singletonMap("test_script", script -> 1), + Collections.emptyMap()); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + + return new ScriptService(settings, engines, ScriptModule.CORE_CONTEXTS); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index 7bcc3e58cc2aa..a183780050b09 100644 --- a/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -90,6 +90,11 @@ public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { }; return context.factoryClazz.cast(factory); } + + @Override + public List> getSupportedContexts() { + return List.of(ScoreScript.CONTEXT); + } }; } } diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java index f242eb8c18506..50685ddbe06a4 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java @@ -1155,6 +1155,11 @@ public String execute() { }; return context.factoryClazz.cast(factory); } + + @Override + public List> getSupportedContexts() { + return List.of(TemplateScript.CONTEXT); + } } public void testPhraseSuggesterCollate() throws InterruptedException, ExecutionException, IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index e16060a0c6786..5296b02d3b28d 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -304,6 +304,35 @@ public double execute(Map params1, double[] values) { throw new IllegalArgumentException("mock script engine does not know how to handle context [" + context.name + "]"); } + @Override + public List> getSupportedContexts() { + // TODO(stu): make part of `compile()` + return List.of( + FieldScript.CONTEXT, + TermsSetQueryScript.CONTEXT, + NumberSortScript.CONTEXT, + StringSortScript.CONTEXT, + IngestScript.CONTEXT, + AggregationScript.CONTEXT, + IngestConditionalScript.CONTEXT, + UpdateScript.CONTEXT, + BucketAggregationScript.CONTEXT, + BucketAggregationSelectorScript.CONTEXT, + SignificantTermsHeuristicScoreScript.CONTEXT, + TemplateScript.CONTEXT, + FilterScript.CONTEXT, + SimilarityScript.CONTEXT, + SimilarityWeightScript.CONTEXT, + MovingFunctionScript.CONTEXT, + ScoreScript.CONTEXT, + ScriptedMetricAggContexts.InitScript.CONTEXT, + ScriptedMetricAggContexts.MapScript.CONTEXT, + ScriptedMetricAggContexts.CombineScript.CONTEXT, + ScriptedMetricAggContexts.ReduceScript.CONTEXT, + IntervalFilterScript.CONTEXT + ); + } + private Map createVars(Map params) { Map vars = new HashMap<>(); vars.put("params", params); From 178d597589dd98a90a5fb2ddc5ac53607bc4d567 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 3 Dec 2019 16:21:35 -0700 Subject: [PATCH 2/3] Test script context filter, add yml test --- .../client/RestHighLevelClientTests.java | 1 + .../api/get_script_languages.json | 19 ++++++ .../test/scripts/25_get_script_languages.yml | 9 +++ .../script/ScriptLanguagesInfoTests.java | 66 +++++++++++++++++-- 4 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/scripts/25_get_script_languages.yml diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 598ccce0f33e0..3135239530199 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -765,6 +765,7 @@ public void testApiNamingConventions() throws Exception { "cluster.remote_info", "create", "get_script_context", + "get_script_languages", "get_source", "indices.exists_type", "indices.get_upgrade", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json new file mode 100644 index 0000000000000..5a45392d9ee11 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json @@ -0,0 +1,19 @@ +{ + "get_script_languages":{ + "documentation":{ + "description":"Returns available script types, languages and contexts" + }, + "stability":"experimental", + "url":{ + "paths":[ + { + "path":"/_script_language", + "methods":[ + "GET" + ] + } + ] + }, + "params":{} + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scripts/25_get_script_languages.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scripts/25_get_script_languages.yml new file mode 100644 index 0000000000000..f4d764324e2dd --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scripts/25_get_script_languages.yml @@ -0,0 +1,9 @@ +"Action to get script languages": + - skip: + version: " - 7.6.0" + reason: "get_script_languages introduced in 7.6.0" + - do: + get_script_languages: {} + + - match: { types_allowed.0: "inline" } + - match: { types_allowed.1: "stored" } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java index 611d9c1f56dc4..38139103ed2ab 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptLanguagesInfoTests.java @@ -22,17 +22,17 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; public class ScriptLanguagesInfoTests extends ESTestCase { - // Test empty types allowed, INLINE only, STORED only, INLINE and STORED - DONE - // Test empty contexts allowed with different engines, search only, update only, search and update - // Test expression, mustache, painless and empty engines - public void testEmptyTypesAllowedReturnsAllTypes() { ScriptService ss = getMockScriptService(Settings.EMPTY); ScriptLanguagesInfo info = ss.getScriptLanguages(); @@ -73,4 +73,62 @@ private ScriptService getMockScriptService(Settings settings) { return new ScriptService(settings, engines, ScriptModule.CORE_CONTEXTS); } + + + public interface MiscContext { + void execute(); + Object newInstance(); + } + + public void testOnlyScriptEngineContextsReturned() { + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, + Collections.singletonMap("test_script", script -> 1), + Collections.emptyMap()); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + + Map> mockContexts = scriptEngine.getSupportedContexts().stream().collect(Collectors.toMap( + c -> c.name, + Function.identity() + )); + String miscContext = "misc_context"; + assertFalse(mockContexts.containsKey(miscContext)); + + Map> mockAndMiscContexts = new HashMap<>(mockContexts); + mockAndMiscContexts.put(miscContext, new ScriptContext<>(miscContext, MiscContext.class)); + + ScriptService ss = new ScriptService(Settings.EMPTY, engines, mockAndMiscContexts); + ScriptLanguagesInfo info = ss.getScriptLanguages(); + + assertTrue(info.languageContexts.containsKey(MockScriptEngine.NAME)); + assertEquals(1, info.languageContexts.size()); + assertEquals(mockContexts.keySet(), info.languageContexts.get(MockScriptEngine.NAME)); + } + + public void testContextsAllowedSettingRespected() { + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, + Collections.singletonMap("test_script", script -> 1), + Collections.emptyMap()); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + Map> mockContexts = scriptEngine.getSupportedContexts().stream().collect(Collectors.toMap( + c -> c.name, + Function.identity() + )); + + List allContexts = new ArrayList<>(mockContexts.keySet()); + List allowed = allContexts.subList(0, allContexts.size()/2); + String miscContext = "misc_context"; + allowed.add(miscContext); + // check that allowing more than available doesn't pollute the returned contexts + Settings.Builder settings = Settings.builder().putList("script.allowed_contexts", allowed); + + Map> mockAndMiscContexts = new HashMap<>(mockContexts); + mockAndMiscContexts.put(miscContext, new ScriptContext<>(miscContext, MiscContext.class)); + + ScriptService ss = new ScriptService(settings.build(), engines, mockAndMiscContexts); + ScriptLanguagesInfo info = ss.getScriptLanguages(); + + assertTrue(info.languageContexts.containsKey(MockScriptEngine.NAME)); + assertEquals(1, info.languageContexts.size()); + assertEquals(new HashSet<>(allContexts.subList(0, allContexts.size()/2)), info.languageContexts.get(MockScriptEngine.NAME)); + } } From 17791579e4e4bbcc31bccd102f12b618a442fed0 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 3 Dec 2019 18:08:59 -0700 Subject: [PATCH 3/3] ScriptLanguagesInfo resp javadoc, getSupportedContexts() returns Set, use readMap(StreamInput::readString --- .../expression/ExpressionScriptEngine.java | 5 +- .../script/mustache/MustacheScriptEngine.java | 6 +- .../painless/PainlessScriptEngine.java | 4 +- .../expertscript/ExpertScriptPlugin.java | 6 +- .../elasticsearch/script/ScriptEngine.java | 4 +- .../script/ScriptLanguagesInfo.java | 78 ++++++++++--------- .../functionscore/ExplainableScriptIT.java | 5 +- .../search/suggest/SuggestSearchIT.java | 5 +- .../script/MockScriptEngine.java | 5 +- 9 files changed, 64 insertions(+), 54 deletions(-) diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index 7d4ac042efc1d..3df42b53cbb34 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -55,6 +55,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; /** @@ -145,8 +146,8 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE } @Override - public List> getSupportedContexts() { - return new ArrayList<>(contexts.keySet()); + public Set> getSupportedContexts() { + return contexts.keySet(); } private static BucketAggregationScript.Factory newBucketAggregationScriptFactory(Expression expr) { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java index 5b96864f1006c..f453905089fdd 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java @@ -40,8 +40,8 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Collections; -import java.util.List; import java.util.Map; +import java.util.Set; /** * Main entry point handling template registration, compilation and @@ -81,8 +81,8 @@ public T compile(String templateName, String templateSource, ScriptContext> getSupportedContexts() { - return List.of(TemplateScript.CONTEXT); + public Set> getSupportedContexts() { + return Set.of(TemplateScript.CONTEXT); } private CustomMustacheFactory createMustacheFactory(Map options) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index 3a6afd06cf75a..7f64e992bc122 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -147,8 +147,8 @@ public Loader run() { } @Override - public List> getSupportedContexts() { - return new ArrayList<>(contextsToCompilers.keySet()); + public Set> getSupportedContexts() { + return contextsToCompilers.keySet(); } /** diff --git a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java index d04cba86e80f8..5259d32a2837b 100644 --- a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java +++ b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java @@ -34,8 +34,8 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collection; -import java.util.List; import java.util.Map; +import java.util.Set; /** * An example script plugin that adds a {@link ScriptEngine} implementing expert scoring. @@ -78,8 +78,8 @@ public void close() { } @Override - public List> getSupportedContexts() { - return List.of(ScoreScript.CONTEXT); + public Set> getSupportedContexts() { + return Set.of(ScoreScript.CONTEXT); } private static class PureDfLeafFactory implements LeafFactory { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptEngine.java b/server/src/main/java/org/elasticsearch/script/ScriptEngine.java index 2374befa3f585..9ace06d701d14 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptEngine.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptEngine.java @@ -21,8 +21,8 @@ import java.io.Closeable; import java.io.IOException; -import java.util.List; import java.util.Map; +import java.util.Set; /** * A script language implementation. @@ -50,5 +50,5 @@ default void close() throws IOException {} /** * Script contexts supported by this engine. */ - List> getSupportedContexts(); + Set> getSupportedContexts(); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java b/server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java index fe56d93147196..d8bfb4f499fe5 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptLanguagesInfo.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -41,6 +40,44 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +/** + * The allowable types, languages and their corresponding contexts. When serialized there is a top level types_allowed list, + * meant to reflect the setting script.allowed_types with the allowed types (eg inline, stored). + * + * The top-level language_contexts list of objects have the language (eg. painless, + * mustache) and a list of contexts available for the language. It is the responsibility of the caller to ensure + * these contexts are filtered by the script.allowed_contexts setting. + * + * The json serialization of the object has the form: + * + * { + * "types_allowed": [ + * "inline", + * "stored" + * ], + * "language_contexts": [ + * { + * "language": "expression", + * "contexts": [ + * "aggregation_selector", + * "aggs" + * ... + * ] + * }, + * { + * "language": "painless", + * "contexts": [ + * "aggregation_selector", + * "aggs", + * "aggs_combine", + * ... + * ] + * } + * ... + * ] + * } + * + */ public class ScriptLanguagesInfo implements ToXContentObject, Writeable { private static final ParseField TYPES_ALLOWED = new ParseField("types_allowed"); private static final ParseField LANGUAGE_CONTEXTS = new ParseField("language_contexts"); @@ -56,8 +93,8 @@ public ScriptLanguagesInfo(Set typesAllowed, Map> lan } public ScriptLanguagesInfo(StreamInput in) throws IOException { - typesAllowed = readStringSet(in); - languageContexts = readStringMapSet(in); + typesAllowed = in.readSet(StreamInput::readString); + languageContexts = in.readMap(StreamInput::readString, sin -> sin.readSet(StreamInput::readString)); } @SuppressWarnings("unchecked") @@ -88,8 +125,8 @@ public static ScriptLanguagesInfo fromXContent(XContentParser parser) throws IOE @Override public void writeTo(StreamOutput out) throws IOException { - writeStringSet(out, typesAllowed); - writeStringMapSet(out, languageContexts); + out.writeStringCollection(typesAllowed); + out.writeMap(languageContexts, StreamOutput::writeString, StreamOutput::writeStringCollection); } @Override @@ -130,35 +167,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endArray().endObject(); } - - private static Map> readStringMapSet(StreamInput in) throws IOException { - Map> values = new HashMap<>(); - for (int i = in.readInt(); i > 0; i--) { - values.put(in.readString(), readStringSet(in)); - } - return values; - } - - private static void writeStringMapSet(StreamOutput out, Map> values) throws IOException { - out.writeInt(values.size()); - for (Map.Entry> value: values.entrySet()) { - out.writeString(value.getKey()); - writeStringSet(out, value.getValue()); - } - } - - private static Set readStringSet(StreamInput in) throws IOException { - Set values = new HashSet<>(); - for (int i = in.readInt(); i > 0; i--) { - values.add(in.readString()); - } - return values; - } - - private static void writeStringSet(StreamOutput out, Set values) throws IOException { - out.writeInt(values.size()); - for (String value: values) { - out.writeString(value); - } - } } diff --git a/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index a183780050b09..e6bbcf5dd57d0 100644 --- a/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import static org.elasticsearch.client.Requests.searchRequest; @@ -92,8 +93,8 @@ public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { } @Override - public List> getSupportedContexts() { - return List.of(ScoreScript.CONTEXT); + public Set> getSupportedContexts() { + return Set.of(ScoreScript.CONTEXT); } }; } diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java index 50685ddbe06a4..9495bc444265e 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java @@ -55,6 +55,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.ExecutionException; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; @@ -1157,8 +1158,8 @@ public String execute() { } @Override - public List> getSupportedContexts() { - return List.of(TemplateScript.CONTEXT); + public Set> getSupportedContexts() { + return Set.of(TemplateScript.CONTEXT); } } diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 5296b02d3b28d..e278fbad85aff 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -35,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static java.util.Collections.emptyMap; @@ -305,9 +306,9 @@ public double execute(Map params1, double[] values) { } @Override - public List> getSupportedContexts() { + public Set> getSupportedContexts() { // TODO(stu): make part of `compile()` - return List.of( + return Set.of( FieldScript.CONTEXT, TermsSetQueryScript.CONTEXT, NumberSortScript.CONTEXT,