Scripting: Per-context script cache, default off

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -364,10 +364,13 @@ public void apply(Settings value, Settings current, Settings previous) {
             NetworkService.TCP_RECEIVE_BUFFER_SIZE,
             IndexSettings.QUERY_STRING_ANALYZE_WILDCARD,
             IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD,
+            ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING,
+            ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
+            ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE,
             ScriptService.SCRIPT_CACHE_SIZE_SETTING,
             ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
-            ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
             ScriptService.SCRIPT_MAX_COMPILATIONS_RATE,
+            ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
             ScriptService.TYPES_ALLOWED_SETTING,
             ScriptService.CONTEXTS_ALLOWED_SETTING,
             IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING,

server/src/main/java/org/elasticsearch/script/ScriptCache.java

@@ -45,14 +45,14 @@ public class ScriptCache {
 
     private final Object lock = new Object();
 
-    private Tuple<Integer, TimeValue> rate;
+    Tuple<Integer, TimeValue> rate;
     private long lastInlineCompileTime;
     private double scriptsPerTimeWindow;
     private double compilesAllowedPerNano;
 
     // Cache settings
-    private int cacheSize;
-    private TimeValue cacheExpire;
+    int cacheSize;
+    TimeValue cacheExpire;
 
     public ScriptCache(
             int cacheMaxSize,
@@ -184,6 +184,29 @@ void checkCompilationLimit() {
                 CircuitBreaker.Durability.TRANSIENT);
         }
     }
+    /**
+     * This sets the cache size.  Replaces existing cache.
+     * @param newSize the new size
+     */
+    void setScriptCacheSize(Integer newSize) {
+        synchronized (lock) {
+            this.cacheSize = newSize.intValue();
+
+            this.cache = buildCache();
+        }
+    }
+
+    /**
+     * This sets the cache expiration time.  Replaces existing cache.
+     * @param newExpire the new expiration
+     */
+    void setScriptCacheExpire(TimeValue newExpire) {
+        synchronized (lock) {
+            this.cacheExpire = newExpire;
+
+            this.cache = buildCache();
+        }
+    }
 
     /**
      * This configures the maximum script compilations per five minute window.

server/src/main/java/org/elasticsearch/script/ScriptStats.java

@@ -79,4 +79,20 @@ static final class Fields {
         static final String CACHE_EVICTIONS = "cache_evictions";
         static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered";
     }
+
+    public static ScriptStats sum(Iterable<ScriptStats> stats) {
+        long compilations = 0;
+        long cacheEvictions = 0;
+        long compilationLimitTriggered = 0;
+        for (ScriptStats stat: stats) {
+            compilations += stat.compilations;
+            cacheEvictions += stat.cacheEvictions;
+            compilationLimitTriggered += stat.compilationLimitTriggered;
+        }
+        return new ScriptStats(
+            compilations,
+            cacheEvictions,
+            compilationLimitTriggered
+        );
+    }
 }

server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java

@@ -69,7 +69,7 @@ private Node startNode() throws NodeValidationException {
             .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo"))
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
             .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
-            .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "1000/1m")
+            .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE.getKey(), "1000/1m")
             .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put("transport.type", getTestTransportType())
             .put(Node.NODE_DATA_SETTING.getKey(), true)

server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java

@@ -29,9 +29,9 @@ public class ScriptCacheTests extends ESTestCase {
     // simply by multiplying by five, so even setting it to one, requires five compilations to break
     public void testCompilationCircuitBreaking() throws Exception {
         ScriptCache cache = new ScriptCache(
-            ScriptService.SCRIPT_CACHE_SIZE_SETTING.get(Settings.EMPTY),
-            ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.get(Settings.EMPTY),
-            ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.get(Settings.EMPTY)
+            ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY),
+            ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY),
+            ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE.get(Settings.EMPTY)
         );
         cache.setMaxCompilationRate(Tuple.tuple(1, TimeValue.timeValueMinutes(1)));
         cache.checkCompilationLimit(); // should pass

server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java

@@ -38,10 +38,20 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 
+import static org.elasticsearch.script.ScriptService.Cache;
+import static org.elasticsearch.script.ScriptService.CacheSettings;
 import static org.elasticsearch.script.ScriptService.MAX_COMPILATION_RATE_FUNCTION;
+import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING;
+import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING;
+import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE;
+import static org.elasticsearch.script.ScriptService.SCRIPT_MAX_COMPILATIONS_RATE;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -222,7 +232,7 @@ public void testMultipleCompilationsCountedInCompilationStats() throws IOExcepti
 
     public void testCompilationStatsOnCacheHit() throws IOException {
         Settings.Builder builder = Settings.builder();
-        builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
         buildScriptService(builder.build());
         Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap());
         ScriptContext<?> context = randomFrom(contexts.values());
@@ -239,7 +249,7 @@ public void testIndexedScriptCountedInCompilationStats() throws IOException {
 
     public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
         Settings.Builder builder = Settings.builder();
-        builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1);
         buildScriptService(builder.build());
         scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values()));
         scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(contexts.values()));
@@ -309,6 +319,176 @@ public void testMaxSizeLimit() throws Exception {
                 iae.getMessage());
     }
 
+    public void testGeneralCacheConstructor() {
+        CacheSettings general = randomCacheSettings();
+        Set<String> contexts = randomContexts();
+        Cache cache = new Cache(general, contexts);
+        assertNotNull(cache.general);
+        assertEquals(cache.contextCache.keySet(), contexts);
+        for (ScriptCache cachePerContext: cache.contextCache.values()) {
+            assertSame(cache.general, cachePerContext);
+        }
+    }
+
+    public void testContextCacheConstructor() {
+        Map<String, CacheSettings> contextSettings = new HashMap<>();
+        List<String> contexts = List.of("abc", "def", "hij", "lmn", "opq", "rst");
+        for (String context: contexts) {
+            contextSettings.put(context, randomCacheSettings());
+        }
+        CacheSettings generalSettings = randomCacheSettings();
+        Set<String> contextSizeSet = new HashSet<>(contexts.subList(0, 2));
+        Set<String> contextExpireSet = new HashSet<>(contexts.subList(2, 4));
+
+        // General unset: always use context settings
+        Cache cache = new Cache(contextSettings, generalSettings, false, contextSizeSet, false, contextExpireSet);
+        assertNull(cache.general);
+        for (Map.Entry<String, ScriptCache> entry: cache.contextCache.entrySet()) {
+            ScriptCache contextCache = entry.getValue();
+            CacheSettings contextSetting = contextSettings.get(entry.getKey());
+            assertEquals(contextSetting.size.intValue(), contextCache.cacheSize);
+            assertEquals(contextSetting.expire, contextCache.cacheExpire);
+            assertEquals(contextSetting.compileRate, contextCache.rate);
+        }
+
+        // General set: use context settings if set, otherwise use general
+        cache = new Cache(contextSettings, generalSettings, true, contextSizeSet, true, contextExpireSet);
+        // size not set, use general
+        for (String context: List.of("hij", "lmn", "opq", "rst")) {
+            assertEquals(generalSettings.size.intValue(), cache.contextCache.get(context).cacheSize);
+        }
+        // size is set, use context size
+        for (String context: List.of("abc", "def")) {
+            assertEquals(contextSettings.get(context).size.intValue(), cache.contextCache.get(context).cacheSize);
+        }
+        // expire not set, use general
+        for (String context: List.of("abc", "def", "opq", "rst")) {
+            assertEquals(generalSettings.expire, cache.contextCache.get(context).cacheExpire);
+        }
+        // expire is set, use context expire
+        for (String context: List.of("hij", "lmn")) {
+            assertEquals(contextSettings.get(context).expire, cache.contextCache.get(context).cacheExpire);
+        }
+    }
+
+    private Set<String> randomContexts() {
+        return new HashSet<>(randomList(10, () -> randomAlphaOfLengthBetween(1, 20)));
+    }
+
+    private TimeValue randomTimeValueObject() {
+        return TimeValue.timeValueSeconds(randomIntBetween(0, 3600));
+    }
+
+    private CacheSettings randomCacheSettings() {
+        return new CacheSettings(
+            randomIntBetween(0, 1000),
+            randomTimeValueObject(),
+            new Tuple<>(randomIntBetween(10, 1000), randomTimeValueObject())
+        );
+    }
+
+    public void testGeneralSettingsCacheUpdater() {
+        Settings settings = Settings.builder()
+            .put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 101)
+            .put(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), TimeValue.timeValueMinutes(60))
+            .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE.getKey(), "80/2m")
+            .build();
+        List<ScriptContext<?>> contextList = List.of(FieldScript.CONTEXT, AggregationScript.CONTEXT, UpdateScript.CONTEXT,
+                                                        IngestScript.CONTEXT);
+        Map<String, ScriptContext<?>> contexts = new HashMap<>();
+        for (ScriptContext<?> context: contextList) {
+            contexts.put(context.name, context);
+        }
+
+        AtomicReference<Cache> cacheRef = new AtomicReference<>();
+        ScriptService.CacheUpdater updater = new ScriptService.CacheUpdater(settings, contexts, true, cacheRef);
+
+        // SCRIPT_GENERAL_MAX_COMPILATIONS_RATE is set, should get general settings
+        assertFalse(updater.isContextCacheEnabled());
+        assertNotNull(cacheRef.get().general);
+        assertEquals(0, updater.contextSizeSet.size());
+        assertEquals(0, updater.contextExpireSet.size());
+
+        // Test flipping to the "use-context" rate
+        updater.setMaxCompilationRate(ScriptService.USE_CONTEXT_RATE_VALUE);
+        assertTrue(updater.isContextCacheEnabled());
+        assertNull(cacheRef.get().general);
+
+        HashMap<String, CacheSettings> contextSettings = new HashMap<>();
+        for (String context: contexts.keySet()) {
+            CacheSettings cacheSetting = randomCacheSettings();
+            contextSettings.put(context, cacheSetting);
+            updater.setMaxCompilationRate(context, cacheSetting.compileRate);
+            updater.setScriptCacheExpire(context, cacheSetting.expire);
+            updater.setScriptCacheSize(context, cacheSetting.size);
+        }
+
+        Map<String, ScriptCache> contextCaches = cacheRef.get().contextCache;
+        for (String context: contexts.keySet()) {
+            ScriptCache cache = contextCaches.get(context);
+            assertNotNull(cache);
+            CacheSettings contextSetting = contextSettings.get(context);
+            assertEquals(cache.cacheExpire, contextSetting.expire);
+            assertEquals(cache.cacheSize, contextSetting.size.intValue());
+            assertEquals(cache.rate, contextSetting.compileRate);
+        }
+
+        // Test flipping back from the "use-context" rate
+        updater.setMaxCompilationRate(new Tuple<>(80, TimeValue.timeValueMinutes(3)));
+        assertFalse(updater.isContextCacheEnabled());
+        assertNotNull(cacheRef.get().general);
+    }
+
+    public void testContextSettingsCacheUpdater() {
+        List<ScriptContext<?>> contextList = List.of(FieldScript.CONTEXT, AggregationScript.CONTEXT, UpdateScript.CONTEXT,
+                                                    IngestScript.CONTEXT);
+        Map<String, ScriptContext<?>> contexts = new HashMap<>();
+        for (ScriptContext<?> context : contextList) {
+            contexts.put(context.name, context);
+        }
+
+        Map<String, CacheSettings> contextSettings = new HashMap<>();
+        Settings.Builder settingBuilder = Settings.builder();
+        for (ScriptContext<?> context: contextList.subList(0, 2)) {
+            CacheSettings cs = randomCacheSettings();
+            contextSettings.put(context.name, cs);
+            settingBuilder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context.name).getKey(),
+                               cs.size.toString());
+            settingBuilder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context.name).getKey(),
+                               cs.expire.seconds() + "s");
+            settingBuilder.put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getConcreteSettingForNamespace(context.name).getKey(),
+                               cs.compileRate.v1() + "/" +  cs.compileRate.v2().getMillis() + "ms"
+            );
+        }
+
+        CacheSettings generalSettings = randomCacheSettings();
+        settingBuilder.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), generalSettings.size.toString());
+        settingBuilder.put(SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(), generalSettings.expire.seconds() + "s");
+        settingBuilder.put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE.getKey(), ScriptService.USE_CONTEXT_RATE_KEY);
+
+        AtomicReference<Cache> cacheRef = new AtomicReference<>();
+        ScriptService.CacheUpdater updater = new ScriptService.CacheUpdater(settingBuilder.build(), contexts, true, cacheRef);
+        assertTrue(updater.isContextCacheEnabled());
+        Cache cache = cacheRef.get();
+        assertNull(cache.general);
+
+        for (ScriptContext<?> context: contextList) {
+            String name = context.name;
+            ScriptCache sc = cache.contextCache.get(name);
+            assertNotNull(sc);
+            if (contextSettings.containsKey(name)) {
+                CacheSettings cs = contextSettings.get(name);
+                assertEquals(sc.cacheExpire, cs.expire);
+                assertEquals(sc.cacheSize, cs.size.intValue());
+                assertEquals(sc.rate, cs.compileRate);
+            } else {
+                assertEquals(sc.cacheExpire, generalSettings.expire);
+                assertEquals(sc.cacheSize, generalSettings.size.intValue());
+                assertEquals(sc.rate, SCRIPT_MAX_COMPILATIONS_RATE.getDefault(Settings.EMPTY));
+            }
+        }
+    }
+
     private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         try {
             scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext);

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -495,10 +495,10 @@ private static Settings getRandomNodeSettings(long seed) {
         }
 
         if (random.nextBoolean()) {
-            builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 0, 2000));
+            builder.put(ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 0, 2000));
         }
         if (random.nextBoolean()) {
-            builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getKey(),
+            builder.put(ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.getKey(),
                     timeValueMillis(RandomNumbers.randomIntBetween(random, 750, 10000000)).getStringRep());
         }
 

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java

@@ -215,7 +215,7 @@ private NodeConfigurationSource createNodeConfigurationSource(final String leade
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b");
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b");
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "1b");
-        builder.put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE.getKey(), "2048/1m");
+        builder.put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE.getKey(), "2048/1m");
         // wait short time for other active shards before actually deleting, default 30s not needed in tests
         builder.put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
         builder.putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()); // empty list disables a port scan for other nodes