Scripting: Per-context script cache, default off

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -359,10 +359,13 @@ public void apply(Settings value, Settings current, Settings previous) {
             NetworkService.TCP_RECEIVE_BUFFER_SIZE,
             IndexSettings.QUERY_STRING_ANALYZE_WILDCARD,
             IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD,
+            ScriptService.SCRIPT_CACHE_SIZE_SETTING_DEPRECATED,
+            ScriptService.SCRIPT_CACHE_EXPIRE_SETTING_DEPRECATED,
+            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED,
             ScriptService.SCRIPT_CACHE_SIZE_SETTING,
             ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
-            ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
             ScriptService.SCRIPT_MAX_COMPILATIONS_RATE,
+            ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
             ScriptService.TYPES_ALLOWED_SETTING,
             ScriptService.CONTEXTS_ALLOWED_SETTING,
             IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING,

server/src/main/java/org/elasticsearch/script/ScriptCache.java

@@ -184,6 +184,29 @@ void checkCompilationLimit() {
                 CircuitBreaker.Durability.TRANSIENT);
         }
     }
+    /**
+     * This sets the cache size.  Replaces existing cache.
+     * @param newSize the new size
+     */
+    void setScriptCacheSize(Integer newSize) {
+        synchronized (lock) {
+            this.cacheSize = newSize.intValue();
+
+            this.cache = buildCache();
+        }
+    }
+
+    /**
+     * This sets the cache expiration time.  Replaces existing cache.
+     * @param newExpire the new expiration
+     */
+    void setScriptCacheExpire(TimeValue newExpire) {
+        synchronized (lock) {
+            this.cacheExpire = newExpire;
+
+            this.cache = buildCache();
+        }
+    }
 
     /**
      * This configures the maximum script compilations per five minute window.

server/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -45,6 +45,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -93,15 +94,52 @@ public class ScriptService implements Closeable, ClusterStateApplier {
                 }
             };
 
-    public static final Setting<Integer> SCRIPT_CACHE_SIZE_SETTING =
+    public static final Setting<Integer> SCRIPT_CACHE_SIZE_SETTING_DEPRECATED =
         Setting.intSetting("script.cache.max_size", 100, 0, Property.NodeScope);
-    public static final Setting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING =
+    public static final Setting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING_DEPRECATED =
         Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope);
     public static final Setting<Integer> SCRIPT_MAX_SIZE_IN_BYTES =
         Setting.intSetting("script.max_size_in_bytes", 65535, 0, Property.Dynamic, Property.NodeScope);
-    public static final Setting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE =
+    public static final Setting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED =
             new Setting<>("script.max_compilations_rate", "75/5m", MAX_COMPILATION_RATE_FUNCTION, Property.Dynamic, Property.NodeScope);
 
+    // Per-context settings
+    static final String contextPrefix = "script.context.";
+
+    // script.context.<context-name>.{cache_max_size, cache_expire, max_compilations_rate}
+    public static final Setting.AffixSetting<Integer> SCRIPT_CACHE_SIZE_SETTING =
+        Setting.affixKeySetting(
+            contextPrefix,
+            "cache_max_size",
+            key -> Setting.intSetting(
+                key,
+                100,
+                0,
+                Property.NodeScope,
+                Property.Dynamic)
+        );
+    public static final Setting.AffixSetting<TimeValue> SCRIPT_CACHE_EXPIRE_SETTING =
+        Setting.affixKeySetting(
+            contextPrefix,
+            "cache_expire",
+            key -> Setting.positiveTimeSetting(
+                key,
+                TimeValue.timeValueMillis(0),
+                Property.NodeScope,
+                Property.Dynamic)
+        );
+    public static final Setting.AffixSetting<Tuple<Integer, TimeValue>> SCRIPT_MAX_COMPILATIONS_RATE =
+        Setting.affixKeySetting(
+            contextPrefix,
+            "max_compilations_rate",
+            key -> new Setting<>(
+                key,
+                "75/5m",
+                MAX_COMPILATION_RATE_FUNCTION,
+                Property.NodeScope,
+                Property.Dynamic)
+        );
+
     public static final String ALLOW_NONE = "none";
 
     public static final Setting<List<String>> TYPES_ALLOWED_SETTING =
@@ -113,7 +151,8 @@ public class ScriptService implements Closeable, ClusterStateApplier {
     private final Set<String> typesAllowed;
     private final Set<String> contextsAllowed;
 
-    final ScriptCache compiler;
+    final ScriptCache generalCache;
+    final Map<String, ScriptCache> contextCache = new HashMap<>();
 
     private final Map<String, ScriptEngine> engines;
     private final Map<String, ScriptContext<?>> contexts;
@@ -201,13 +240,138 @@ public ScriptService(Settings settings, Map<String, ScriptEngine> engines, Map<S
         }
 
         this.setMaxSizeInBytes(SCRIPT_MAX_SIZE_IN_BYTES.get(settings));
-        compiler = new ScriptCache(
-                    SCRIPT_CACHE_SIZE_SETTING.get(settings),
-                    SCRIPT_CACHE_EXPIRE_SETTING.get(settings),
+        generalCache = initCache(settings, contextCache, contexts.keySet());
+    }
+
+    /**
+     * initialize contextCache with per context settings or return a general script cache
+     */
+    ScriptCache initCache(Settings settings, Map<String, ScriptCache> contextCache, Set<String> contextNames) {
+        if (useGeneralCacheSettings(settings)) {
+            return new ScriptCache(
+                SCRIPT_CACHE_SIZE_SETTING_DEPRECATED.get(settings),
+                SCRIPT_CACHE_EXPIRE_SETTING_DEPRECATED.get(settings),
+                compilationLimitsEnabled() ?
+                    SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED.get(settings):
+                    new Tuple<>(0, TimeValue.ZERO)
+            );
+        }
+
+        Map<String, Integer> sizePerContext = SCRIPT_CACHE_SIZE_SETTING.getAsMap(settings);
+        Map<String, TimeValue> expirePerContext = SCRIPT_CACHE_EXPIRE_SETTING.getAsMap(settings);
+        Map<String, Tuple<Integer, TimeValue>> ratePerContext = SCRIPT_MAX_COMPILATIONS_RATE.getAsMap(settings);
+
+        List<String> badContexts = invalidContextSettings(contextNames, sizePerContext.keySet(), expirePerContext.keySet(),
+                                    ratePerContext.keySet());
+        if (badContexts.size() > 0) {
+            throw new IllegalArgumentException("Invalid contexts for script cache settings [" + String.join(", ", badContexts) + "]");
+        }
+
+        for (String contextName: contexts.keySet()) {
+            contextCache.put(contextName,
+                new ScriptCache(
+                    sizePerContext.getOrDefault(contextName, SCRIPT_CACHE_SIZE_SETTING.getDefault(settings)),
+                    expirePerContext.getOrDefault(contextName, SCRIPT_CACHE_EXPIRE_SETTING.getDefault(settings)),
                     compilationLimitsEnabled() ?
-                        SCRIPT_MAX_COMPILATIONS_RATE.get(settings):
+                        ratePerContext.getOrDefault(contextName, SCRIPT_MAX_COMPILATIONS_RATE.getDefault(settings)) :
                         new Tuple<>(0, TimeValue.ZERO)
-                    );
+                )
+            );
+        }
+
+        return null;
+    }
+
+    /**
+     * Return all setting values that do not correspond to the set of contexts.  Visible for testing.
+     */
+    static List<String> invalidContextSettings(Set<String> contextNames, Set<String> size, Set<String> expire, Set<String> rate) {
+        Set<String> badSize = new HashSet<>(size);
+        Set<String> badExpire = new HashSet<>(expire);
+        Set<String> badRate = new HashSet<>(rate);
+        for (String context: contextNames) {
+            badSize.remove(context);
+            badExpire.remove(context);
+            badRate.remove(context);
+        }
+
+        List<String> badSettings = new ArrayList<>();
+        for (String sizeContext: badSize) {
+            badSettings.add(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(sizeContext).getKey());
+        }
+        for (String expireContext: badExpire) {
+            badSettings.add(SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(expireContext).getKey());
+        }
+        for (String rateContext: badRate) {
+            badSettings.add(SCRIPT_MAX_COMPILATIONS_RATE.getConcreteSettingForNamespace(rateContext).getKey());
+        }
+
+        Collections.sort(badSettings);
+        return badSettings;
+    }
+
+    /**
+     * Ensure that either the general cache settings are given or the per context settings are given.  If neither are set, the default is to
+     * using general cache settings.
+     * Visible for testing
+     * TODO(stu): change default to per context when context defaults exist
+     */
+    static boolean useGeneralCacheSettings(Settings settings) {
+        Set<String> generalKeys = new HashSet<>();
+        Set<String> contextKeys = new HashSet<>();
+        for (String key: settings.keySet()) {
+            if (SCRIPT_CACHE_SIZE_SETTING_DEPRECATED.match(key) ||
+                SCRIPT_CACHE_EXPIRE_SETTING_DEPRECATED.match(key) ||
+                SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED.match(key)) {
+
+                generalKeys.add(key);
+
+            } else if (
+                SCRIPT_CACHE_SIZE_SETTING.getRawKey().match(key) ||
+                SCRIPT_CACHE_EXPIRE_SETTING.getRawKey().match(key) ||
+                SCRIPT_MAX_COMPILATIONS_RATE.getRawKey().match(key)) {
+
+                contextKeys.add(key);
+
+            }
+        }
+
+        if (generalKeys.size() > 0) {
+            if (contextKeys.size() > 0) {
+                throw new IllegalArgumentException("Cannot combine deprecated general script cache settings [" +
+                    generalKeys.stream().sorted().collect(Collectors.joining(", ")) +
+                    "] with context specific script cache settings [" +
+                    contextKeys.stream().sorted().collect(Collectors.joining(", ")) + ']'
+                );
+            }
+
+            return true;
+        }
+
+        // TODO(stu): switch to false when per context defaults exist
+        return contextKeys.size() == 0;
+    }
+
+    private void contextExists(String context, Object o) {
+        if (generalCache != null) {
+            throw new IllegalArgumentException("Cannot update context script cache settings when general script cache is in use");
+        }
+        if (contextCache.containsKey(context) == false) {
+            throw new IllegalArgumentException("Context [" + context + "] does not exist");
+        }
+    }
+
+
+    private void setScriptCacheSize(String context, Integer newSize) {
+        contextCache.get(context).setScriptCacheSize(newSize);
+    }
+
+    private void setScriptCacheExpire(String context, TimeValue newExpire) {
+        contextCache.get(context).setScriptCacheExpire(newExpire);
+    }
+
+    private void setMaxCompilationRate(String context, Tuple<Integer, TimeValue> newRate) {
+        contextCache.get(context).setMaxCompilationRate(newRate);
     }
 
     /**
@@ -219,7 +383,29 @@ boolean compilationLimitsEnabled() {
 
     void registerClusterSettingsListeners(ClusterSettings clusterSettings) {
         clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_SIZE_IN_BYTES, this::setMaxSizeInBytes);
-        clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_COMPILATIONS_RATE, compiler::setMaxCompilationRate);
+        clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED,
+                // Don't deref potentially null generalCache
+                r -> {
+                    if (generalCache != null) {
+                        generalCache.setMaxCompilationRate(r);
+                    }
+                },
+                s -> {
+                    if (generalCache == null) {
+                        // This validator is run on every settings update.
+                        // Here we try to detect if it's a spurious update to the default.
+                        // This _will_ allow through bad updates to the default.
+                        if (SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED.getDefault(Settings.EMPTY).equals(s) &&
+                            SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED.exists(settings) == false) {
+                            return;
+                        }
+                        throw new IllegalArgumentException("Using context script caches rather than general script cache");
+                    }
+                }
+            );
+        clusterSettings.addAffixUpdateConsumer(SCRIPT_CACHE_SIZE_SETTING, this::setScriptCacheSize, this::contextExists);
+        clusterSettings.addAffixUpdateConsumer(SCRIPT_CACHE_EXPIRE_SETTING, this::setScriptCacheExpire, this::contextExists);
+        clusterSettings.addAffixUpdateConsumer(SCRIPT_MAX_COMPILATIONS_RATE, this::setMaxCompilationRate, this::contextExists);
     }
 
     @Override
@@ -304,7 +490,15 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
             logger.trace("compiling lang: [{}] type: [{}] script: {}", lang, type, idOrCode);
         }
 
-        return compiler.compile(context, scriptEngine, id, idOrCode, type, options);
+        if (generalCache != null) {
+            return generalCache.compile(context, scriptEngine, id, idOrCode, type, options);
+        }
+
+        if (contextCache.containsKey(context.name) == false) {
+            throw new IllegalArgumentException("script context [" + context.name + "] has no script cache");
+        }
+
+        return contextCache.get(context.name).compile(context, scriptEngine, id, idOrCode, type, options);
     }
 
     public boolean isLangSupported(String lang) {
@@ -468,7 +662,11 @@ public ScriptLanguagesInfo getScriptLanguages() {
     }
 
     public ScriptStats stats() {
-        return compiler.stats();
+        if (generalCache != null) {
+            return generalCache.stats();
+        }
+
+        return ScriptStats.sum(contextCache.values().stream().map(ScriptCache::stats)::iterator);
     }
 
     @Override

server/src/main/java/org/elasticsearch/script/ScriptStats.java

@@ -79,4 +79,20 @@ static final class Fields {
         static final String CACHE_EVICTIONS = "cache_evictions";
         static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered";
     }
+
+    public static ScriptStats sum(Iterable<ScriptStats> stats) {
+        long compilations = 0;
+        long cacheEvictions = 0;
+        long compilationLimitTriggered = 0;
+        for (ScriptStats stat: stats) {
+            compilations += stat.compilations;
+            cacheEvictions += stat.cacheEvictions;
+            compilationLimitTriggered += stat.compilationLimitTriggered;
+        }
+        return new ScriptStats(
+            compilations,
+            cacheEvictions,
+            compilationLimitTriggered
+        );
+    }
 }

server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java

@@ -69,7 +69,7 @@ private Node startNode() throws NodeValidationException {
             .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo"))
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
             .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
-            .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "1000/1m")
+            .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED.getKey(), "1000/1m")
             .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put("transport.type", getTestTransportType())
             .put(Node.NODE_DATA_SETTING.getKey(), true)

server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java

@@ -29,9 +29,9 @@ public class ScriptCacheTests extends ESTestCase {
     // simply by multiplying by five, so even setting it to one, requires five compilations to break
     public void testCompilationCircuitBreaking() throws Exception {
         ScriptCache cache = new ScriptCache(
-            ScriptService.SCRIPT_CACHE_SIZE_SETTING.get(Settings.EMPTY),
-            ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.get(Settings.EMPTY),
-            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.get(Settings.EMPTY)
+            ScriptService.SCRIPT_CACHE_SIZE_SETTING_DEPRECATED.get(Settings.EMPTY),
+            ScriptService.SCRIPT_CACHE_EXPIRE_SETTING_DEPRECATED.get(Settings.EMPTY),
+            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_DEPRECATED.get(Settings.EMPTY)
         );
         cache.setMaxCompilationRate(Tuple.tuple(1, TimeValue.timeValueMinutes(1)));
         cache.checkCompilationLimit(); // should pass