Add grok and dissect methods to runtime fields (#68088)

This adds a `grok` and a `dissect` method to runtime fields which
returns a `Matcher` style object you can use to get the matched
patterns. A fairly simple script to extract the "verb" from an apache
log line with `grok` would look like this:
```
String verb = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.verb;
if (verb != null) {
  emit(verb);
}
```

And `dissect` would look like:
```
String verb = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{status} %{size}').extract(doc["message"].value)?.verb;
if (verb != null) {
  emit(verb);
}
```

We'll work later to get it down to a clean looking one liner, but for
now, this'll do.

The `grok` and `dissect` methods are special in that they only run at
script compile time. You can't pass non-constants to them. They'll
produce compile errors if you send in a bad pattern. This is nice
because they can be expensive to "compile" and there are many other
optimizations we can make when the patterns are available up front.

Closes #67825

Author: nik9000

libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java

@@ -102,7 +102,7 @@ public final class DissectKey {
         }
 
         if (name == null || (name.isEmpty() && !skip)) {
-            throw new DissectException.KeyParse(key, "The key name could be determined");
+            throw new DissectException.KeyParse(key, "The key name could not be determined");
         }
     }
 

libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java

@@ -34,7 +34,8 @@
 import java.util.stream.Collectors;
 
 /**
- * <p>Splits (dissects) a string into its parts based on a pattern.</p><p>A dissect pattern is composed of a set of keys and delimiters.
+ * Splits (dissects) a string into its parts based on a pattern.
+ * <p>A dissect pattern is composed of a set of keys and delimiters.
  * For example the dissect pattern: <pre>%{a} %{b},%{c}</pre> has 3 keys (a,b,c) and two delimiters (space and comma). This pattern will
  * match a string of the form: <pre>foo bar,baz</pre> and will result a key/value pairing of <pre>a=foo, b=bar, and c=baz.</pre>
  * <p>Matches are all or nothing. For example, the same pattern will NOT match <pre>foo bar baz</pre> since all of the delimiters did not
@@ -280,7 +281,19 @@ public Map<String, String> parse(String inputString) {
         }
         Map<String, String> results = dissectMatch.getResults();
 
-        if (dissectMatch.isValid(results) == false) {
+        return dissectMatch.isValid(results) ? results : null;
+    }
+
+    /**
+     * <p>Entry point to dissect a string into it's parts.</p>
+     *
+     * @param inputString The string to dissect
+     * @return the key/value Map of the results
+     * @throws DissectException if unable to dissect a pair into it's parts.
+     */
+    public Map<String, String> forceParse(String inputString) {
+        Map<String, String> results = parse(inputString);
+        if (results == null) {
             throw new DissectException.FindMatch(pattern, inputString);
         }
         return results;

libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java

@@ -349,11 +349,12 @@ public void testJsonSpecification() throws Exception {
         }
     }
 
-    private DissectException assertFail(String pattern, String input){
-        return expectThrows(DissectException.class, () -> new DissectParser(pattern, null).parse(input));
+    private DissectException assertFail(String pattern, String input) {
+        return expectThrows(DissectException.class, () -> new DissectParser(pattern, null).forceParse(input));
     }
 
     private void assertMiss(String pattern, String input) {
+        assertNull(new DissectParser(pattern, null).parse(input)); 
         DissectException e = assertFail(pattern, input);
         assertThat(e.getMessage(), CoreMatchers.containsString("Unable to find match for dissect pattern"));
         assertThat(e.getMessage(), CoreMatchers.containsString(pattern));

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java

@@ -54,7 +54,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
         } else if (input == null) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
         }
-        dissectParser.parse(input).forEach(ingestDocument::setFieldValue);
+        dissectParser.forceParse(input).forEach(ingestDocument::setFieldValue);
         return ingestDocument;
     }
 

modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.spi.annotation;
+
+/**
+ * Methods annotated with this must be run at compile time so their arguments
+ * must all be constant and they produce a constant.
+ */
+public class CompileTimeOnlyAnnotation {
+    public static final String NAME = "compile_time_only";
+
+    public static final CompileTimeOnlyAnnotation INSTANCE = new CompileTimeOnlyAnnotation();
+
+    private CompileTimeOnlyAnnotation() {}
+}

modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.spi.annotation;
+
+import java.util.Map;
+
+/**
+ * Methods annotated with {@link CompileTimeOnlyAnnotation} must be run at
+ * compile time so their arguments must all be constant and they produce a
+ * constant.
+ */
+public class CompileTimeOnlyAnnotationParser implements WhitelistAnnotationParser {
+
+    public static final CompileTimeOnlyAnnotationParser INSTANCE = new CompileTimeOnlyAnnotationParser();
+
+    private CompileTimeOnlyAnnotationParser() {}
+
+    @Override
+    public Object parse(Map<String, String> arguments) {
+        if (arguments.isEmpty() == false) {
+            throw new IllegalArgumentException(
+                "unexpected parameters for [@" + CompileTimeOnlyAnnotation.NAME + "] annotation, found " + arguments
+            );
+        }
+
+        return CompileTimeOnlyAnnotation.INSTANCE;
+    }
+}

modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java

@@ -36,7 +36,8 @@ public interface WhitelistAnnotationParser {
                     new AbstractMap.SimpleEntry<>(NoImportAnnotation.NAME, NoImportAnnotationParser.INSTANCE),
                     new AbstractMap.SimpleEntry<>(DeprecatedAnnotation.NAME, DeprecatedAnnotationParser.INSTANCE),
                     new AbstractMap.SimpleEntry<>(NonDeterministicAnnotation.NAME, NonDeterministicAnnotationParser.INSTANCE),
-                    new AbstractMap.SimpleEntry<>(InjectConstantAnnotation.NAME, InjectConstantAnnotationParser.INSTANCE)
+                    new AbstractMap.SimpleEntry<>(InjectConstantAnnotation.NAME, InjectConstantAnnotationParser.INSTANCE),
+                    new AbstractMap.SimpleEntry<>(CompileTimeOnlyAnnotation.NAME, CompileTimeOnlyAnnotationParser.INSTANCE)
             ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
     );
 

modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.painless.node.SClass;
 import org.elasticsearch.painless.phase.DefaultConstantFoldingOptimizationPhase;
 import org.elasticsearch.painless.phase.DefaultIRTreeToASMBytesPhase;
+import org.elasticsearch.painless.phase.DefaultStaticConstantExtractionPhase;
 import org.elasticsearch.painless.phase.DefaultStringConcatenationOptimizationPhase;
 import org.elasticsearch.painless.phase.DocFieldsPhase;
 import org.elasticsearch.painless.phase.PainlessSemanticAnalysisPhase;
@@ -227,6 +228,7 @@ ScriptScope compile(Loader loader, String name, String source, CompilerSettings
         ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode();
         new DefaultStringConcatenationOptimizationPhase().visitClass(classNode, null);
         new DefaultConstantFoldingOptimizationPhase().visitClass(classNode, null);
+        new DefaultStaticConstantExtractionPhase().visitClass(classNode, scriptScope);
         new DefaultIRTreeToASMBytesPhase().visitScript(classNode);
         byte[] bytes = classNode.getBytes();
 
@@ -263,6 +265,7 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de
         ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode();
         new DefaultStringConcatenationOptimizationPhase().visitClass(classNode, null);
         new DefaultConstantFoldingOptimizationPhase().visitClass(classNode, null);
+        new DefaultStaticConstantExtractionPhase().visitClass(classNode, scriptScope);
         classNode.setDebugStream(debugStream);
         new DefaultIRTreeToASMBytesPhase().visitScript(classNode);
 

modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java

@@ -21,6 +21,7 @@
 
 import java.lang.reflect.Method;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 public class PainlessInstanceBinding {
@@ -30,13 +31,21 @@ public class PainlessInstanceBinding {
 
     public final Class<?> returnType;
     public final List<Class<?>> typeParameters;
+    public final Map<Class<?>, Object> annotations;
 
-    PainlessInstanceBinding(Object targetInstance, Method javaMethod, Class<?> returnType, List<Class<?>> typeParameters) {
+    PainlessInstanceBinding(
+        Object targetInstance,
+        Method javaMethod,
+        Class<?> returnType,
+        List<Class<?>> typeParameters,
+        Map<Class<?>, Object> annotations
+    ) {
         this.targetInstance = targetInstance;
         this.javaMethod = javaMethod;
 
         this.returnType = returnType;
         this.typeParameters = typeParameters;
+        this.annotations = annotations;
     }
 
     @Override
@@ -54,7 +63,8 @@ public boolean equals(Object object) {
         return targetInstance == that.targetInstance &&
                 Objects.equals(javaMethod, that.javaMethod) &&
                 Objects.equals(returnType, that.returnType) &&
-                Objects.equals(typeParameters, that.typeParameters);
+                Objects.equals(typeParameters, that.typeParameters) &&
+                Objects.equals(annotations, that.annotations);
     }
 
     @Override

modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.painless.spi.WhitelistField;
 import org.elasticsearch.painless.spi.WhitelistInstanceBinding;
 import org.elasticsearch.painless.spi.WhitelistMethod;
+import org.elasticsearch.painless.spi.annotation.CompileTimeOnlyAnnotation;
 import org.elasticsearch.painless.spi.annotation.InjectConstantAnnotation;
 import org.elasticsearch.painless.spi.annotation.NoImportAnnotation;
 import org.objectweb.asm.ClassWriter;
@@ -174,7 +175,8 @@ public static PainlessLookup buildFromWhitelists(List<Whitelist> whitelists) {
                     origin = whitelistInstanceBinding.origin;
                     painlessLookupBuilder.addPainlessInstanceBinding(
                             whitelistInstanceBinding.targetInstance, whitelistInstanceBinding.methodName,
-                            whitelistInstanceBinding.returnCanonicalTypeName, whitelistInstanceBinding.canonicalTypeNameParameters);
+                            whitelistInstanceBinding.returnCanonicalTypeName, whitelistInstanceBinding.canonicalTypeNameParameters,
+                            whitelistInstanceBinding.painlessAnnotations);
                 }
             }
         } catch (Exception exception) {
@@ -393,6 +395,10 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
                     "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae);
         }
 
+        if (annotations.containsKey(CompileTimeOnlyAnnotation.class)) {
+            throw new IllegalArgumentException("constructors can't have @" + CompileTimeOnlyAnnotation.NAME);
+        }
+
         MethodType methodType = methodHandle.type();
 
         String painlessConstructorKey = buildPainlessConstructorKey(typeParametersSize);
@@ -574,6 +580,10 @@ public void addPainlessMethod(Class<?> targetClass, Class<?> augmentedClass,
             }
         }
 
+        if (annotations.containsKey(CompileTimeOnlyAnnotation.class)) {
+            throw new IllegalArgumentException("regular methods can't have @" + CompileTimeOnlyAnnotation.NAME);
+        }
+
         MethodType methodType = methodHandle.type();
         boolean isStatic = augmentedClass == null && Modifier.isStatic(javaMethod.getModifiers());
         String painlessMethodKey = buildPainlessMethodKey(methodName, typeParametersSize);
@@ -989,6 +999,10 @@ public void addPainlessClassBinding(Class<?> targetClass, String methodName, Cla
                     "invalid method name [" + methodName + "] for class binding [" + targetCanonicalClassName + "].");
         }
 
+        if (annotations.containsKey(CompileTimeOnlyAnnotation.class)) {
+            throw new IllegalArgumentException("class bindings can't have @" + CompileTimeOnlyAnnotation.NAME);
+        }
+
         Method[] javaMethods = targetClass.getMethods();
         Method javaMethod = null;
 
@@ -1079,7 +1093,8 @@ public void addPainlessClassBinding(Class<?> targetClass, String methodName, Cla
     }
 
     public void addPainlessInstanceBinding(Object targetInstance,
-            String methodName, String returnCanonicalTypeName, List<String> canonicalTypeNameParameters) {
+            String methodName, String returnCanonicalTypeName, List<String> canonicalTypeNameParameters,
+            Map<Class<?>, Object> painlessAnnotations) {
 
         Objects.requireNonNull(targetInstance);
         Objects.requireNonNull(methodName);
@@ -1108,10 +1123,16 @@ public void addPainlessInstanceBinding(Object targetInstance,
                     "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]");
         }
 
-        addPainlessInstanceBinding(targetInstance, methodName, returnType, typeParameters);
+        addPainlessInstanceBinding(targetInstance, methodName, returnType, typeParameters, painlessAnnotations);
     }
 
-    public void addPainlessInstanceBinding(Object targetInstance, String methodName, Class<?> returnType, List<Class<?>> typeParameters) {
+    public void addPainlessInstanceBinding(
+        Object targetInstance,
+        String methodName,
+        Class<?> returnType,
+        List<Class<?>> typeParameters,
+        Map<Class<?>, Object> painlessAnnotations
+    ) {
         Objects.requireNonNull(targetInstance);
         Objects.requireNonNull(methodName);
         Objects.requireNonNull(returnType);
@@ -1189,7 +1210,7 @@ public void addPainlessInstanceBinding(Object targetInstance, String methodName,
 
         PainlessInstanceBinding existingPainlessInstanceBinding = painlessMethodKeysToPainlessInstanceBindings.get(painlessMethodKey);
         PainlessInstanceBinding newPainlessInstanceBinding =
-                new PainlessInstanceBinding(targetInstance, javaMethod, returnType, typeParameters);
+                new PainlessInstanceBinding(targetInstance, javaMethod, returnType, typeParameters, painlessAnnotations);
 
         if (existingPainlessInstanceBinding == null) {
             newPainlessInstanceBinding = painlessInstanceBindingCache.computeIfAbsent(newPainlessInstanceBinding, key -> key);
@@ -1200,11 +1221,13 @@ public void addPainlessInstanceBinding(Object targetInstance, String methodName,
                     "[[" + targetCanonicalClassName + "], " +
                     "[" + methodName + "], " +
                     "[" + typeToCanonicalTypeName(returnType) + "], " +
-                    typesToCanonicalTypeNames(typeParameters) + "] and " +
+                    typesToCanonicalTypeNames(typeParameters) + "], " +
+                    painlessAnnotations + " and " +
                     "[[" + targetCanonicalClassName + "], " +
                     "[" + methodName + "], " +
                     "[" + typeToCanonicalTypeName(existingPainlessInstanceBinding.returnType) + "], " +
-                    typesToCanonicalTypeNames(existingPainlessInstanceBinding.typeParameters) + "]");
+                    typesToCanonicalTypeNames(existingPainlessInstanceBinding.typeParameters) + "], " +
+                    existingPainlessInstanceBinding.annotations);
         }
     }