Scripting: enable regular expressions by default (#63029) (#63272)

* Setting `script.painless.regex.enabled` has a new option,
  `use-factor`, the default.  This defaults to using regular
  expressions but limiting the complexity of the regular
  expressions.

  In addition to `use-factor`, the setting can be `true`, as
  before, which enables regular expressions without limiting them.

  `false` totally disables regular expressions, which was the
  old default.

* New setting `script.painless.regex.limit-factor`.  This limits
  regular expression complexity by limiting the number characters
  a regular expression can consider based on input length.

  The default is `6`, so a regular expression can consider
  `6` * input length number of characters.  With input
  `foobarbaz` (length `9`), for example, the regular expression
  can consider `54` (`6 * 9`) characters.

  This reduces the impact of exponential backtracking in Java's
  regular expression engine.

* add `@inject_constant` annotation to whitelist.

  This annotation signals that a compiler settings will
  be injected at the beginning of a whitelisted method.

  The format is `argnum=settingname`:
  `1=foo_setting 2=bar_setting`.

  Argument numbers must start at one and must be sequential.

* Augment
  `Pattern.split(CharSequence)`
  `Pattern.split(CharSequence, int)`,
  `Pattern.splitAsStream(CharSequence)`
  `Pattern.matcher(CharSequence)`
  to take the value of `script.painless.regex.limit-factor` as a
  an injected parameter, limiting as explained above when this
  setting is in use.

Fixes: #49873
Backport of: 93f29a4

Author: stu-elastic

modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotation.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.spi.annotation;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Inject compiler setting constants.
+ * Format: {@code inject_constant["1=foo_compiler_setting", 2="bar_compiler_setting"]} injects "foo_compiler_setting and
+ * "bar_compiler_setting" as the first two arguments (other than receiver reference for instance methods) to the annotated method.
+ */
+public class InjectConstantAnnotation {
+    public static final String NAME = "inject_constant";
+    public final List<String> injects;
+    public InjectConstantAnnotation(List<String> injects) {
+        this.injects = Collections.unmodifiableList(injects);
+    }
+}

modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/InjectConstantAnnotationParser.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.spi.annotation;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+public class InjectConstantAnnotationParser implements WhitelistAnnotationParser {
+
+    public static final InjectConstantAnnotationParser INSTANCE = new InjectConstantAnnotationParser();
+
+    private InjectConstantAnnotationParser() {}
+
+    @Override
+    public Object parse(Map<String, String> arguments) {
+        if (arguments.isEmpty()) {
+            throw new IllegalArgumentException("[@inject_constant] requires at least one name to inject");
+        }
+        ArrayList<String> argList = new ArrayList<>(arguments.size());
+        for (int i = 1; i <= arguments.size(); i++) {
+            String argNum = Integer.toString(i);
+            if (arguments.containsKey(argNum) == false) {
+                throw new IllegalArgumentException("[@inject_constant] missing argument number [" + argNum + "]");
+            }
+            argList.add(arguments.get(argNum));
+        }
+
+        return new InjectConstantAnnotation(argList);
+    }
+}

modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java

@@ -35,7 +35,8 @@ public interface WhitelistAnnotationParser {
             Stream.of(
                     new AbstractMap.SimpleEntry<>(NoImportAnnotation.NAME, NoImportAnnotationParser.INSTANCE),
                     new AbstractMap.SimpleEntry<>(DeprecatedAnnotation.NAME, DeprecatedAnnotationParser.INSTANCE),
-                    new AbstractMap.SimpleEntry<>(NonDeterministicAnnotation.NAME, NonDeterministicAnnotationParser.INSTANCE)
+                    new AbstractMap.SimpleEntry<>(NonDeterministicAnnotation.NAME, NonDeterministicAnnotationParser.INSTANCE),
+                    new AbstractMap.SimpleEntry<>(InjectConstantAnnotation.NAME, InjectConstantAnnotationParser.INSTANCE)
             ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
     );
 

modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java

@@ -220,7 +220,7 @@ ScriptScope compile(Loader loader, String name, String source, CompilerSettings
         ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1);
         new PainlessSemanticHeaderPhase().visitClass(root, scriptScope);
         new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope);
-        // TODO(stu): Make this phase optional #60156
+        // TODO: Make this phase optional #60156
         new DocFieldsPhase().visitClass(root, scriptScope);
         new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope);
         ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode();
@@ -255,7 +255,7 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de
         ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1);
         new PainlessSemanticHeaderPhase().visitClass(root, scriptScope);
         new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope);
-        // TODO(stu): Make this phase optional #60156
+        // TODO: Make this phase optional #60156
         new DocFieldsPhase().visitClass(root, scriptScope);
         new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope);
         ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode();

modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java

@@ -21,16 +21,28 @@
 
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.painless.api.Augmentation;
+
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * Settings to use when compiling a script.
  */
 public final class CompilerSettings {
     /**
-     * Are regexes enabled? This is a node level setting because regexes break out of painless's lovely sandbox and can cause stack
-     * overflows and we can't analyze the regex to be sure it won't.
+     * Are regexes enabled? If {@code true}, regexes are enabled and unlimited by the limit factor.  If {@code false}, they are completely
+     * disabled. If {@code use-limit}, the default, regexes are enabled but limited in complexity according to the
+     * {@code script.painless.regex.limit-factor} setting.
+     */
+    public static final Setting<RegexEnabled> REGEX_ENABLED =
+        new Setting<>("script.painless.regex.enabled", RegexEnabled.LIMITED.value, RegexEnabled::parse, Property.NodeScope);
+
+    /**
+     * How complex can a regex be?  This is the number of characters that can be considered expressed as a multiple of string length.
      */
-    public static final Setting<Boolean> REGEX_ENABLED = Setting.boolSetting("script.painless.regex.enabled", false, Property.NodeScope);
+    public static final Setting<Integer> REGEX_LIMIT_FACTOR =
+        Setting.intSetting("script.painless.regex.limit-factor", 6, 1, Property.NodeScope);
 
     /**
      * Constant to be used when specifying the maximum loop counter when compiling a script.
@@ -65,12 +77,20 @@ public final class CompilerSettings {
      * For testing. Do not use.
      */
     private int initialCallSiteDepth = 0;
+    private int testInject0 = 2;
+    private int testInject1 = 4;
+    private int testInject2 = 6;
 
     /**
-     * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple
-     * <strong>looking</strong> regexes can cause stack overflows.
+     * Are regexes enabled? Defaults to using the factor setting.
      */
-    private boolean regexesEnabled = false;
+    private RegexEnabled regexesEnabled = RegexEnabled.LIMITED;
+
+
+    /**
+     * How complex can regexes be?  Expressed as a multiple of the input string.
+     */
+    private int regexLimitFactor = 0;
 
     /**
      * Returns the value for the cumulative total number of statements that can be made in all loops
@@ -123,18 +143,82 @@ public void setInitialCallSiteDepth(int depth) {
     }
 
     /**
-     * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple
-     * <strong>looking</strong> regexes can cause stack overflows.
+     * Are regexes enabled?
      */
-    public boolean areRegexesEnabled() {
+    public RegexEnabled areRegexesEnabled() {
         return regexesEnabled;
     }
 
     /**
-     * Are regexes enabled? They are currently disabled by default because they break out of the loop counter and even fairly simple
-     * <strong>looking</strong> regexes can cause stack overflows.
+     * Are regexes enabled or limited?
      */
-    public void setRegexesEnabled(boolean regexesEnabled) {
+    public void setRegexesEnabled(RegexEnabled regexesEnabled) {
         this.regexesEnabled = regexesEnabled;
     }
+
+    /**
+     * What is the limitation on regex complexity?  How many multiples of input length can a regular expression consider?
+     */
+    public void setRegexLimitFactor(int regexLimitFactor) {
+        this.regexLimitFactor = regexLimitFactor;
+    }
+
+    /**
+     * What is the limit factor for regexes?
+     */
+    public int getRegexLimitFactor() {
+        return regexLimitFactor;
+    }
+
+    /**
+     * Get compiler settings as a map.  This is used to inject compiler settings into augmented methods with the {@code @inject_constant}
+     * annotation.
+     */
+    public Map<String, Object> asMap() {
+        int regexLimitFactor = this.regexLimitFactor;
+        if (regexesEnabled == RegexEnabled.TRUE) {
+            regexLimitFactor = Augmentation.UNLIMITED_PATTERN_FACTOR;
+        } else if (regexesEnabled == RegexEnabled.FALSE) {
+            regexLimitFactor = Augmentation.DISABLED_PATTERN_FACTOR;
+        }
+        Map<String, Object> map = new HashMap<>();
+        map.put("regex_limit_factor", regexLimitFactor);
+
+        // for testing only
+        map.put("testInject0", testInject0);
+        map.put("testInject1", testInject1);
+        map.put("testInject2", testInject2);
+
+        return map;
+    }
+
+    /**
+     * Options for {@code script.painless.regex.enabled} setting.
+     */
+    public enum RegexEnabled {
+        TRUE("true"),
+        FALSE("false"),
+        LIMITED("limited");
+        final String value;
+
+        RegexEnabled(String value) {
+            this.value = value;
+        }
+
+        /**
+         * Parse string value, necessary because `valueOf` would require strings to be upper case.
+         */
+        public static RegexEnabled parse(String value) {
+            if (TRUE.value.equals(value)) {
+                return TRUE;
+            } else if (FALSE.value.equals(value)) {
+                return FALSE;
+            } else if (LIMITED.value.equals(value)) {
+                return LIMITED;
+            }
+            throw new IllegalArgumentException(
+                "invalid value [" + value + "] must be one of [" + TRUE.value + "," + FALSE.value + "," + LIMITED.value + "]"
+            );
+        }
+    }
 }

modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java

@@ -182,6 +182,8 @@ static MethodHandle arrayLengthGetter(Class<?> arrayType) {
      * Otherwise it returns a handle to the matching method.
      * <p>
      * @param painlessLookup the whitelist
+     * @param functions user defined functions and lambdas
+     * @param constants available constants to be used if the method has the {@code InjectConstantAnnotation}
      * @param methodHandlesLookup caller's lookup
      * @param callSiteType callsite's type
      * @param receiverClass Class of the object to invoke the method on.
@@ -191,8 +193,8 @@ static MethodHandle arrayLengthGetter(Class<?> arrayType) {
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      * @throws Throwable if a method reference cannot be converted to an functional interface
      */
-    static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable functions,
-            MethodHandles.Lookup methodHandlesLookup, MethodType callSiteType, Class<?> receiverClass, String name, Object args[])
+    static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable functions, Map<String, Object> constants,
+            MethodHandles.Lookup methodHandlesLookup, MethodType callSiteType, Class<?> receiverClass, String name, Object[] args)
             throws Throwable {
 
          String recipeString = (String) args[0];
@@ -206,7 +208,15 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu
                          "[" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + (numArguments - 1) + "] not found");
              }
 
-             return painlessMethod.methodHandle;
+             MethodHandle handle = painlessMethod.methodHandle;
+             Object[] injections = PainlessLookupUtility.buildInjections(painlessMethod, constants);
+
+             if (injections.length > 0) {
+                 // method handle contains the "this" pointer so start injections at 1
+                 handle = MethodHandles.insertArguments(handle, 1, injections);
+             }
+
+             return handle;
          }
 
          // convert recipe string to a bitset for convenience (the code below should be refactored...)
@@ -236,7 +246,13 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu
                     "dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found");
         }
 
-         MethodHandle handle = method.methodHandle;
+        MethodHandle handle = method.methodHandle;
+        Object[] injections = PainlessLookupUtility.buildInjections(method, constants);
+
+        if (injections.length > 0) {
+            // method handle contains the "this" pointer so start injections at 1
+            handle = MethodHandles.insertArguments(handle, 1, injections);
+        }
 
          int replaced = 0;
          upTo = 1;
@@ -257,22 +273,25 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu
                      // we have everything.
                      filter = lookupReferenceInternal(painlessLookup,
                                                       functions,
+                                                      constants,
                                                       methodHandlesLookup,
                                                       interfaceType,
                                                       type,
                                                       call,
-                                                      numCaptures);
+                                                      numCaptures
+                     );
                  } else if (signature.charAt(0) == 'D') {
                      // the interface type is now known, but we need to get the implementation.
                      // this is dynamically based on the receiver type (and cached separately, underneath
                      // this cache). It won't blow up since we never nest here (just references)
-                     Class<?> captures[] = new Class<?>[numCaptures];
+                     Class<?>[] captures = new Class<?>[numCaptures];
                      for (int capture = 0; capture < captures.length; capture++) {
                          captures[capture] = callSiteType.parameterType(i + 1 + capture);
                      }
                      MethodType nestedType = MethodType.methodType(interfaceType, captures);
                      CallSite nested = DefBootstrap.bootstrap(painlessLookup,
                                                               functions,
+                                                              constants,
                                                               methodHandlesLookup,
                                                               call,
                                                               nestedType,
@@ -300,8 +319,10 @@ static MethodHandle lookupMethod(PainlessLookup painlessLookup, FunctionTable fu
       * This is just like LambdaMetaFactory, only with a dynamic type. The interface type is known,
       * so we simply need to lookup the matching implementation method based on receiver type.
       */
-    static MethodHandle lookupReference(PainlessLookup painlessLookup, FunctionTable functions,
-            MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class<?> receiverClass, String name) throws Throwable {
+    static MethodHandle lookupReference(PainlessLookup painlessLookup, FunctionTable functions, Map<String, Object> constants,
+            MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class<?> receiverClass, String name)
+            throws Throwable {
+
         Class<?> interfaceType = painlessLookup.canonicalTypeNameToType(interfaceClass);
         if (interfaceType == null) {
             throw new IllegalArgumentException("type [" + interfaceClass + "] not found");
@@ -317,25 +338,30 @@ static MethodHandle lookupReference(PainlessLookup painlessLookup, FunctionTable
                     "dynamic method [" + typeToCanonicalTypeName(receiverClass) + ", " + name + "/" + arity + "] not found");
         }
 
-        return lookupReferenceInternal(painlessLookup, functions, methodHandlesLookup,
-            interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass),
-            implMethod.javaMethod.getName(), 1);
+        return lookupReferenceInternal(painlessLookup, functions, constants,
+                methodHandlesLookup, interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass),
+                implMethod.javaMethod.getName(), 1);
      }
 
      /** Returns a method handle to an implementation of clazz, given method reference signature. */
-    private static MethodHandle lookupReferenceInternal(PainlessLookup painlessLookup, FunctionTable functions,
-            MethodHandles.Lookup methodHandlesLookup, Class<?> clazz, String type, String call, int captures) throws Throwable {
-        final FunctionRef ref = FunctionRef.create(painlessLookup, functions, null, clazz, type, call, captures);
+    private static MethodHandle lookupReferenceInternal(
+            PainlessLookup painlessLookup, FunctionTable functions, Map<String, Object> constants,
+            MethodHandles.Lookup methodHandlesLookup, Class<?> clazz, String type, String call, int captures
+            ) throws Throwable {
+
+        final FunctionRef ref = FunctionRef.create(painlessLookup, functions, null, clazz, type, call, captures, constants);
         final CallSite callSite = LambdaBootstrap.lambdaBootstrap(
-            methodHandlesLookup,
-            ref.interfaceMethodName,
-            ref.factoryMethodType,
-            ref.interfaceMethodType,
-            ref.delegateClassName,
-            ref.delegateInvokeType,
-            ref.delegateMethodName,
-            ref.delegateMethodType,
-            ref.isDelegateInterface ? 1 : 0
+                methodHandlesLookup,
+                ref.interfaceMethodName,
+                ref.factoryMethodType,
+                ref.interfaceMethodType,
+                ref.delegateClassName,
+                ref.delegateInvokeType,
+                ref.delegateMethodName,
+                ref.delegateMethodType,
+                ref.isDelegateInterface ? 1 : 0,
+                ref.isDelegateAugmented ? 1 : 0,
+                ref.delegateInjections
         );
         return callSite.dynamicInvoker().asType(MethodType.methodType(clazz, ref.factoryMethodType.parameterArray()));
      }