Skip to content

Commit 0866031

Browse files
authored
Scripting: split out compile limits and caching (#52498)
Phase 1 of adding compilation limits per context. * Refactor rate limiting and caching into separate class, `ScriptCache`, which will be used per context. * Disable compilation limit for certain tests. Refs: #50152
1 parent 3bce3ec commit 0866031

File tree

12 files changed

+415
-221
lines changed

12 files changed

+415
-221
lines changed

server/src/main/java/org/elasticsearch/node/Node.java

+19-9
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@
141141
import org.elasticsearch.repositories.RepositoriesModule;
142142
import org.elasticsearch.repositories.RepositoriesService;
143143
import org.elasticsearch.rest.RestController;
144+
import org.elasticsearch.script.ScriptContext;
145+
import org.elasticsearch.script.ScriptEngine;
144146
import org.elasticsearch.script.ScriptModule;
145147
import org.elasticsearch.script.ScriptService;
146148
import org.elasticsearch.search.SearchModule;
@@ -346,6 +348,7 @@ protected Node(
346348
client = new NodeClient(settings, threadPool);
347349
final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
348350
final ScriptModule scriptModule = new ScriptModule(settings, pluginsService.filterPlugins(ScriptPlugin.class));
351+
final ScriptService scriptService = newScriptService(settings, scriptModule.engines, scriptModule.contexts);
349352
AnalysisModule analysisModule = new AnalysisModule(this.environment, pluginsService.filterPlugins(AnalysisPlugin.class));
350353
// this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
351354
// so we might be late here already
@@ -358,20 +361,20 @@ protected Node(
358361

359362
final SettingsModule settingsModule =
360363
new SettingsModule(settings, additionalSettings, additionalSettingsFilter, settingsUpgraders);
361-
scriptModule.registerClusterSettingsListeners(settingsModule.getClusterSettings());
364+
scriptModule.registerClusterSettingsListeners(scriptService, settingsModule.getClusterSettings());
362365
resourcesToClose.add(resourceWatcherService);
363366
final NetworkService networkService = new NetworkService(
364367
getCustomNameResolvers(pluginsService.filterPlugins(DiscoveryPlugin.class)));
365368

366369
List<ClusterPlugin> clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class);
367370
final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
368-
clusterService.addStateApplier(scriptModule.getScriptService());
371+
clusterService.addStateApplier(scriptService);
369372
resourcesToClose.add(clusterService);
370373
clusterService.addLocalNodeMasterListener(
371374
new ConsistentSettingsService(settings, clusterService, settingsModule.getConsistentSettings())
372375
.newHashPublisher());
373376
final IngestService ingestService = new IngestService(clusterService, threadPool, this.environment,
374-
scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(),
377+
scriptService, analysisModule.getAnalysisRegistry(),
375378
pluginsService.filterPlugins(IngestPlugin.class), client);
376379
final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client);
377380
final UsageService usageService = new UsageService();
@@ -446,7 +449,7 @@ protected Node(
446449
final IndicesService indicesService =
447450
new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry, analysisModule.getAnalysisRegistry(),
448451
clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry,
449-
threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays, scriptModule.getScriptService(),
452+
threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays, scriptService,
450453
clusterService, client, metaStateService, engineFactoryProviders, indexStoreFactories);
451454

452455
final AliasValidator aliasValidator = new AliasValidator();
@@ -466,7 +469,7 @@ protected Node(
466469

467470
Collection<Object> pluginComponents = pluginsService.filterPlugins(Plugin.class).stream()
468471
.flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
469-
scriptModule.getScriptService(), xContentRegistry, environment, nodeEnvironment,
472+
scriptService, xContentRegistry, environment, nodeEnvironment,
470473
namedWriteableRegistry, clusterModule.getIndexNameExpressionResolver()).stream())
471474
.collect(Collectors.toList());
472475

@@ -523,12 +526,12 @@ protected Node(
523526
clusterService.getClusterSettings(), pluginsService.filterPlugins(DiscoveryPlugin.class),
524527
clusterModule.getAllocationService(), environment.configFile(), gatewayMetaState, rerouteService);
525528
this.nodeService = new NodeService(settings, threadPool, monitorService, discoveryModule.getDiscovery(),
526-
transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(),
529+
transportService, indicesService, pluginsService, circuitBreakerService, scriptService,
527530
httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter(), responseCollectorService,
528531
searchTransportService);
529532

530533
final SearchService searchService = newSearchService(clusterService, indicesService,
531-
threadPool, scriptModule.getScriptService(), bigArrays, searchModule.getFetchPhase(),
534+
threadPool, scriptService, bigArrays, searchModule.getFetchPhase(),
532535
responseCollectorService, circuitBreakerService);
533536

534537
final List<PersistentTasksExecutor<?>> tasksExecutors = pluginsService
@@ -558,7 +561,7 @@ protected Node(
558561
b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
559562
b.bind(BigArrays.class).toInstance(bigArrays);
560563
b.bind(PageCacheRecycler.class).toInstance(pageCacheRecycler);
561-
b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
564+
b.bind(ScriptService.class).toInstance(scriptService);
562565
b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry());
563566
b.bind(IngestService.class).toInstance(ingestService);
564567
b.bind(UsageService.class).toInstance(usageService);
@@ -575,7 +578,7 @@ protected Node(
575578
b.bind(Transport.class).toInstance(transport);
576579
b.bind(TransportService.class).toInstance(transportService);
577580
b.bind(NetworkService.class).toInstance(networkService);
578-
b.bind(UpdateHelper.class).toInstance(new UpdateHelper(scriptModule.getScriptService()));
581+
b.bind(UpdateHelper.class).toInstance(new UpdateHelper(scriptService));
579582
b.bind(MetaDataIndexUpgradeService.class).toInstance(metaDataIndexUpgradeService);
580583
b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
581584
b.bind(GatewayMetaState.class).toInstance(gatewayMetaState);
@@ -1032,6 +1035,13 @@ protected SearchService newSearchService(ClusterService clusterService, IndicesS
10321035
scriptService, bigArrays, fetchPhase, responseCollectorService, circuitBreakerService);
10331036
}
10341037

1038+
/**
1039+
* Creates a new the ScriptService. This method can be overwritten by tests to inject mock implementations.
1040+
*/
1041+
protected ScriptService newScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<String, ScriptContext<?>> contexts) {
1042+
return new ScriptService(settings, engines, contexts);
1043+
}
1044+
10351045
/**
10361046
* Get Custom Name Resolvers list based on a Discovery Plugins list
10371047
* @param discoveryPlugins Discovery plugins list
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.script;
21+
22+
import org.apache.logging.log4j.LogManager;
23+
import org.apache.logging.log4j.Logger;
24+
import org.elasticsearch.common.breaker.CircuitBreaker;
25+
import org.elasticsearch.common.breaker.CircuitBreakingException;
26+
import org.elasticsearch.common.cache.Cache;
27+
import org.elasticsearch.common.cache.CacheBuilder;
28+
import org.elasticsearch.common.cache.RemovalListener;
29+
import org.elasticsearch.common.cache.RemovalNotification;
30+
import org.elasticsearch.common.collect.Tuple;
31+
import org.elasticsearch.common.unit.TimeValue;
32+
33+
import java.util.Map;
34+
import java.util.Objects;
35+
36+
/**
37+
* Script cache and compilation rate limiter.
38+
*/
39+
public class ScriptCache {
40+
41+
private static final Logger logger = LogManager.getLogger(ScriptService.class);
42+
43+
private Cache<CacheKey, Object> cache;
44+
private final ScriptMetrics scriptMetrics = new ScriptMetrics();
45+
46+
private final Object lock = new Object();
47+
48+
private Tuple<Integer, TimeValue> rate;
49+
private long lastInlineCompileTime;
50+
private double scriptsPerTimeWindow;
51+
private double compilesAllowedPerNano;
52+
53+
// Cache settings
54+
private int cacheSize;
55+
private TimeValue cacheExpire;
56+
57+
public ScriptCache(
58+
int cacheMaxSize,
59+
TimeValue cacheExpire,
60+
Tuple<Integer, TimeValue> maxCompilationRate
61+
) {
62+
CacheBuilder<CacheKey, Object> cacheBuilder = CacheBuilder.builder();
63+
if (cacheMaxSize >= 0) {
64+
cacheBuilder.setMaximumWeight(cacheMaxSize);
65+
}
66+
67+
if (cacheExpire.getNanos() != 0) {
68+
cacheBuilder.setExpireAfterAccess(cacheExpire);
69+
}
70+
71+
logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, cacheExpire);
72+
this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build();
73+
74+
this.lastInlineCompileTime = System.nanoTime();
75+
76+
this.cacheSize = cacheMaxSize;
77+
this.cacheExpire = cacheExpire;
78+
this.setMaxCompilationRate(maxCompilationRate);
79+
}
80+
81+
private Cache<CacheKey,Object> buildCache() {
82+
CacheBuilder<CacheKey, Object> cacheBuilder = CacheBuilder.builder();
83+
if (cacheSize >= 0) {
84+
cacheBuilder.setMaximumWeight(cacheSize);
85+
}
86+
if (cacheExpire.getNanos() != 0) {
87+
cacheBuilder.setExpireAfterAccess(cacheExpire);
88+
}
89+
return cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build();
90+
}
91+
92+
<FactoryType> FactoryType compile(
93+
ScriptContext<FactoryType> context,
94+
ScriptEngine scriptEngine,
95+
String id,
96+
String idOrCode,
97+
ScriptType type,
98+
Map<String, String> options
99+
) {
100+
String lang = scriptEngine.getType();
101+
CacheKey cacheKey = new CacheKey(lang, idOrCode, context.name, options);
102+
Object compiledScript = cache.get(cacheKey);
103+
104+
if (compiledScript != null) {
105+
return context.factoryClazz.cast(compiledScript);
106+
}
107+
108+
// Synchronize so we don't compile scripts many times during multiple shards all compiling a script
109+
synchronized (lock) {
110+
// Retrieve it again in case it has been put by a different thread
111+
compiledScript = cache.get(cacheKey);
112+
113+
if (compiledScript == null) {
114+
try {
115+
// Either an un-cached inline script or indexed script
116+
// If the script type is inline the name will be the same as the code for identification in exceptions
117+
// but give the script engine the chance to be better, give it separate name + source code
118+
// for the inline case, then its anonymous: null.
119+
if (logger.isTraceEnabled()) {
120+
logger.trace("context [{}]: compiling script, type: [{}], lang: [{}], options: [{}]", context.name, type,
121+
lang, options);
122+
}
123+
// Check whether too many compilations have happened
124+
checkCompilationLimit();
125+
compiledScript = scriptEngine.compile(id, idOrCode, context, options);
126+
} catch (ScriptException good) {
127+
// TODO: remove this try-catch completely, when all script engines have good exceptions!
128+
throw good; // its already good
129+
} catch (Exception exception) {
130+
throw new GeneralScriptException("Failed to compile " + type + " script [" + id + "] using lang [" + lang + "]",
131+
exception);
132+
}
133+
134+
// Since the cache key is the script content itself we don't need to
135+
// invalidate/check the cache if an indexed script changes.
136+
scriptMetrics.onCompilation();
137+
cache.put(cacheKey, compiledScript);
138+
}
139+
140+
}
141+
142+
return context.factoryClazz.cast(compiledScript);
143+
}
144+
145+
public ScriptStats stats() {
146+
return scriptMetrics.stats();
147+
}
148+
149+
/**
150+
* Check whether there have been too many compilations within the last minute, throwing a circuit breaking exception if so.
151+
* This is a variant of the token bucket algorithm: https://en.wikipedia.org/wiki/Token_bucket
152+
*
153+
* It can be thought of as a bucket with water, every time the bucket is checked, water is added proportional to the amount of time that
154+
* elapsed since the last time it was checked. If there is enough water, some is removed and the request is allowed. If there is not
155+
* enough water the request is denied. Just like a normal bucket, if water is added that overflows the bucket, the extra water/capacity
156+
* is discarded - there can never be more water in the bucket than the size of the bucket.
157+
*/
158+
void checkCompilationLimit() {
159+
if (rate.v1() == 0 && rate.v2().getNanos() == 0) {
160+
// unlimited
161+
return;
162+
}
163+
164+
long now = System.nanoTime();
165+
long timePassed = now - lastInlineCompileTime;
166+
lastInlineCompileTime = now;
167+
168+
scriptsPerTimeWindow += (timePassed) * compilesAllowedPerNano;
169+
170+
// It's been over the time limit anyway, readjust the bucket to be level
171+
if (scriptsPerTimeWindow > rate.v1()) {
172+
scriptsPerTimeWindow = rate.v1();
173+
}
174+
175+
// If there is enough tokens in the bucket, allow the request and decrease the tokens by 1
176+
if (scriptsPerTimeWindow >= 1) {
177+
scriptsPerTimeWindow -= 1.0;
178+
} else {
179+
scriptMetrics.onCompilationLimit();
180+
// Otherwise reject the request
181+
throw new CircuitBreakingException("[script] Too many dynamic script compilations within, max: [" +
182+
rate.v1() + "/" + rate.v2() +"]; please use indexed, or scripts with parameters instead; " +
183+
"this limit can be changed by the [script.max_compilations_rate] setting",
184+
CircuitBreaker.Durability.TRANSIENT);
185+
}
186+
}
187+
188+
/**
189+
* This configures the maximum script compilations per five minute window.
190+
*
191+
* @param newRate the new expected maximum number of compilations per five minute window
192+
*/
193+
void setMaxCompilationRate(Tuple<Integer, TimeValue> newRate) {
194+
synchronized (lock) {
195+
this.rate = newRate;
196+
// Reset the counter to allow new compilations
197+
this.scriptsPerTimeWindow = rate.v1();
198+
this.compilesAllowedPerNano = ((double) rate.v1()) / newRate.v2().nanos();
199+
200+
this.cache = buildCache();
201+
}
202+
}
203+
204+
/**
205+
* A small listener for the script cache that calls each
206+
* {@code ScriptEngine}'s {@code scriptRemoved} method when the
207+
* script has been removed from the cache
208+
*/
209+
private class ScriptCacheRemovalListener implements RemovalListener<CacheKey, Object> {
210+
@Override
211+
public void onRemoval(RemovalNotification<CacheKey, Object> notification) {
212+
if (logger.isDebugEnabled()) {
213+
logger.debug(
214+
"removed [{}] from cache, reason: [{}]",
215+
notification.getValue(),
216+
notification.getRemovalReason()
217+
);
218+
}
219+
scriptMetrics.onCacheEviction();
220+
}
221+
}
222+
223+
private static final class CacheKey {
224+
final String lang;
225+
final String idOrCode;
226+
final String context;
227+
final Map<String, String> options;
228+
229+
private CacheKey(String lang, String idOrCode, String context, Map<String, String> options) {
230+
this.lang = lang;
231+
this.idOrCode = idOrCode;
232+
this.context = context;
233+
this.options = options;
234+
}
235+
236+
@Override
237+
public boolean equals(Object o) {
238+
if (this == o) return true;
239+
if (o == null || getClass() != o.getClass()) return false;
240+
CacheKey cacheKey = (CacheKey) o;
241+
return Objects.equals(lang, cacheKey.lang) &&
242+
Objects.equals(idOrCode, cacheKey.idOrCode) &&
243+
Objects.equals(context, cacheKey.context) &&
244+
Objects.equals(options, cacheKey.options);
245+
}
246+
247+
@Override
248+
public int hashCode() {
249+
return Objects.hash(lang, idOrCode, context, options);
250+
}
251+
}
252+
}

server/src/main/java/org/elasticsearch/script/ScriptModule.java

+5-10
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ public class ScriptModule {
6666
).collect(Collectors.toMap(c -> c.name, Function.identity()));
6767
}
6868

69-
private final ScriptService scriptService;
69+
public final Map<String, ScriptEngine> engines;
70+
public final Map<String, ScriptContext<?>> contexts;
7071

7172
public ScriptModule(Settings settings, List<ScriptPlugin> scriptPlugins) {
7273
Map<String, ScriptEngine> engines = new HashMap<>();
@@ -89,20 +90,14 @@ public ScriptModule(Settings settings, List<ScriptPlugin> scriptPlugins) {
8990
}
9091
}
9192
}
92-
scriptService = new ScriptService(settings, Collections.unmodifiableMap(engines), Collections.unmodifiableMap(contexts));
93-
}
94-
95-
/**
96-
* Service responsible for managing scripts.
97-
*/
98-
public ScriptService getScriptService() {
99-
return scriptService;
93+
this.engines = Collections.unmodifiableMap(engines);
94+
this.contexts = Collections.unmodifiableMap(contexts);
10095
}
10196

10297
/**
10398
* Allow the script service to register any settings update handlers on the cluster settings
10499
*/
105-
public void registerClusterSettingsListeners(ClusterSettings clusterSettings) {
100+
public void registerClusterSettingsListeners(ScriptService scriptService, ClusterSettings clusterSettings) {
106101
scriptService.registerClusterSettingsListeners(clusterSettings);
107102
}
108103
}

0 commit comments

Comments
 (0)