EQL: Add string function (#54470)

* EQL: Add string() function
* EQL: Reorder queryfolder_tests
* EQL: Add test queries
* EQL: Fix InternalEqlScriptUtils.string and test case
* EQL: Fix testStringFunctionWithText error message
* EQL: Flatten ToStringFunctionPipe.equals
* EQL: Reorder painless whitelist
* EQL: Address feedback and remove string(null) handling
* EQL: Move string(pid) test over
* EQL: Rename source -> value

Author: rw-access

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml

@@ -13,3 +13,7 @@ expected_event_ids = [95]
 query = '''
 file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
 '''
+
+[[queries]]
+query = 'process where string(serial_event_id) = "1"'
+expected_event_ids  = [1]

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -12,8 +12,9 @@
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith;
-import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.StringContains;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
 import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
@@ -37,6 +38,7 @@ private static FunctionDefinition[][] functions() {
                 def(IndexOf.class, IndexOf::new, "indexof"),
                 def(Length.class, Length::new, "length"),
                 def(StartsWith.class, StartsWith::new, "startswith"),
+                def(ToString.class, ToString::new, "string"),
                 def(StringContains.class, StringContains::new, "stringcontains"),
                 def(Substring.class, Substring::new, "substring"),
                 def(Wildcard.class, Wildcard::new, "wildcard"),

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ToString.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.ToStringFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * EQL specific string function that wraps object.toString.
+ */
+public class ToString extends ScalarFunction {
+
+    private final Expression value;
+
+    public ToString(Source source, Expression src) {
+        super(source, Collections.singletonList(src));
+        this.value = src;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        return isExact(value, sourceText(), ParamOrdinal.DEFAULT);
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new ToStringFunctionPipe(source(), this, Expressions.pipe(value));
+    }
+
+    @Override
+    public boolean foldable() {
+        return value.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(value.fold());
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, ToString::new, value);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(value);
+
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s)"),
+                "string",
+                sourceScript.template()),
+                paramsBuilder()
+                    .script(sourceScript.params())
+                    .build(), dataType());
+    }
+
+    @Override
+    public ScriptTemplate scriptWithField(FieldAttribute field) {
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
+                paramsBuilder().variable(field.exactAttribute().name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 1) {
+            throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
+        }
+
+        return new ToString(source(), newChildren.get(0));
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ToStringFunctionPipe.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class ToStringFunctionPipe extends Pipe {
+
+    private final Pipe source;
+
+    public ToStringFunctionPipe(Source source, Expression expression, Pipe src) {
+        super(source, expression, Collections.singletonList(src));
+        this.source = src;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        if (newChildren.size() != 1) {
+            throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
+        }
+        return new ToStringFunctionPipe(source(), expression(), newChildren.get(0));
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        Pipe newSource = source.resolveAttributes(resolver);
+        if (newSource == source) {
+            return this;
+        }
+        return replaceChildren(Collections.singletonList(newSource));
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved();
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<ToStringFunctionPipe> info() {
+        return NodeInfo.create(this, ToStringFunctionPipe::new, expression(), source);
+    }
+
+    @Override
+    public ToStringFunctionProcessor asProcessor() {
+        return new ToStringFunctionProcessor(source.asProcessor());
+    }
+
+    public Pipe src() {
+        return source;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        return Objects.equals(source, ((ToStringFunctionPipe) obj).source);
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ToStringFunctionProcessor.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ToStringFunctionProcessor implements Processor {
+
+    public static final String NAME = "sstr";
+
+    private final Processor source;
+
+    public ToStringFunctionProcessor(Processor source) {
+        this.source = source;
+    }
+
+    public ToStringFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source.process(input));
+    }
+
+    public static Object doProcess(Object source) {
+        return source == null ? null : source.toString();
+    }
+
+    protected Processor source() {
+        return source;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        ToStringFunctionProcessor other = (ToStringFunctionProcessor) obj;
+        return Objects.equals(source(), other.source());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source());
+    }
+
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java

@@ -11,8 +11,9 @@
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor;
-import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.StringContainsFunctionProcessor;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToStringFunctionProcessor;
 import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
 
 /*
@@ -44,6 +45,10 @@ public static Boolean startsWith(String s, String pattern) {
         return (Boolean) StartsWithFunctionProcessor.doProcess(s, pattern);
     }
 
+    public static String string(Object s) {
+        return (String) ToStringFunctionProcessor.doProcess(s);
+    }
+
     public static Boolean stringContains(String string, String substring) {
         return (Boolean) StringContainsFunctionProcessor.doProcess(string, substring);
     }

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -12,7 +12,7 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl
 
 #
 # Utilities
-# 
+#
   def docValue(java.util.Map, String)
   boolean nullSafeFilter(Boolean)
   double nullSafeSortNumeric(Number)
@@ -54,12 +54,13 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
 
 #
 # ASCII Functions
-# 
+#
   String  between(String, String, String, Boolean, Boolean)
   Boolean endsWith(String, String)
   Integer indexOf(String, String, Number)
   Integer length(String)
   Boolean startsWith(String, String)
+  String  string(Object)
   Boolean stringContains(String, String)
   String  substring(String, Number, Number)
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -337,4 +337,12 @@ public void testMultiField() {
         accept(idxr, "foo where multi_field_nested.end_date == ''");
         accept(idxr, "foo where multi_field_nested.start_date == 'bar'");
     }
+
+    public void testStringFunctionWithText() {
+        final IndexResolution idxr = loadIndexResolution("mapping-multi-field.json");
+        assertEquals("1:15: [string(multi_field.english)] cannot operate on field " +
+                "of data type [text]: No keyword/multi-field defined exact matches for [english]; " +
+                "define one or use MATCH/QUERY instead",
+            error(idxr, "process where string(multi_field.english) == 'foo'"));
+    }
 }

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -113,6 +113,13 @@ InternalQlScriptUtils.docValue(doc,params.v0),params.v1))"
 "params":{"v0":"process_name","v1":"foo"}
 ;
 
+stringFunction
+process where string(pid) == "123";
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalEqlScriptUtils.string(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))",
+"params":{"v0":"pid","v1":"123"}
+;
+
 indexOfFunction
 process where indexOf(user_name, 'A', 2) > 0
 ;