EQL: Add concat function

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml

@@ -14,6 +14,39 @@ query = '''
 file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
 '''
 
+[[queries]]
+description = "test string concatenation. update test to avoid case-sensitivity issues"
+query = '''
+process where concat(serial_event_id, '::', process_name, '::', opcode) == '5::wininit.exe::3'
+'''
+expected_event_ids  = [5]
+
+
+[[queries]]
+query = 'process where concat(serial_event_id) = "1"'
+expected_event_ids  = [1]
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) != null'
+expected_event_ids  = [2, 3]
+
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) == null'
+expected_event_ids  = [1, 4]
+
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(process_name, null, null) == null'
+expected_event_ids  = [1, 2, 3, 4]
+
+
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null'
+expected_event_ids  = [1, 2, 3, 4]
+
+
 [[queries]]
 query = 'process where string(serial_event_id) = "1"'
 expected_event_ids  = [1]

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -8,6 +8,7 @@
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
@@ -40,6 +41,7 @@ private static FunctionDefinition[][] functions() {
             new FunctionDefinition[] {
                 def(Between.class, Between::new, 2, "between"),
                 def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"),
+                def(Concat.class, Concat::new, "concat"),
                 def(EndsWith.class, EndsWith::new, "endswith"),
                 def(IndexOf.class, IndexOf::new, "indexof"),
                 def(Length.class, Length::new, "length"),

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Concat.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * EQL specific concat function to build a string of all input arguments concatenated.
+ */
+public class Concat extends ScalarFunction {
+
+    private final List<Expression> values;
+
+    public Concat(Source source, List<Expression> values) {
+        super(source, values);
+        this.values = values;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = TypeResolution.TYPE_RESOLVED;
+        for (Expression value : values) {
+            resolution = isExact(value, sourceText(), Expressions.ParamOrdinal.DEFAULT);
+
+            if (resolution.unresolved()) {
+                return resolution;
+            }
+        }
+
+        return resolution;
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new ConcatFunctionPipe(source(), this, Expressions.pipe(values));
+    }
+
+    @Override
+    public boolean foldable() {
+        return Expressions.foldable(values);
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(Expressions.fold(values));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Concat::new, values);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        List<ScriptTemplate> templates = new ArrayList<>();
+        for (Expression ex : children()) {
+            templates.add(asScript(ex));
+        }
+
+        StringJoiner template = new StringJoiner(",", "{eql}.concat([", "])");
+        ParamsBuilder params = paramsBuilder();
+
+        for (ScriptTemplate scriptTemplate : templates) {
+            template.add(scriptTemplate.template());
+            params.script(scriptTemplate.params());
+        }
+
+        return new ScriptTemplate(formatTemplate(template.toString()), params.build(), dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Concat(source(), newChildren);
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ConcatFunctionPipe.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class ConcatFunctionPipe extends Pipe {
+
+    private final List<Pipe> values;
+
+    public ConcatFunctionPipe(Source source, Expression expression, List<Pipe> values) {
+        super(source, expression, values);
+        this.values = values;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        return new ConcatFunctionPipe(source(), expression(), newChildren);
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        List<Pipe> newValues = new ArrayList<>(values.size());
+        for (Pipe v : values) {
+            newValues.add(v.resolveAttributes(resolver));
+        }
+
+        if (newValues == values) {
+            return this;
+        }
+
+        return replaceChildren(newValues);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        for (Pipe p : values) {
+            if (p.supportedByAggsOnlyQuery() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public boolean resolved() {
+        for (Pipe p : values) {
+            if (p.resolved() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        for (Pipe v : values) {
+            v.collectFields(sourceBuilder);
+        }
+    }
+
+    @Override
+    protected NodeInfo<ConcatFunctionPipe> info() {
+        return NodeInfo.create(this, ConcatFunctionPipe::new, expression(), values);
+    }
+
+    @Override
+    public ConcatFunctionProcessor asProcessor() {
+        List<Processor> processors = new ArrayList<>(values.size());
+        for (Pipe p: values) {
+            processors.add(p.asProcessor());
+        }
+        return new ConcatFunctionProcessor(processors);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(values);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        return Objects.equals(values, ((ConcatFunctionPipe) obj).values);
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ConcatFunctionProcessor.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class ConcatFunctionProcessor implements Processor {
+
+    public static final String NAME = "scon";
+
+    private final List<Processor> values;
+
+    public ConcatFunctionProcessor(List<Processor> values) {
+        this.values = values;
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        for (Processor v: values) {
+            out.writeNamedWriteable(v);
+        }
+    }
+
+    @Override
+    public Object process(Object input) {
+        List<Object> processed = new ArrayList<>(values.size());
+        for (Processor v: values) {
+            processed.add(v.process(input));
+        }
+        return doProcess(processed);
+    }
+
+    public static Object doProcess(List<Object> inputs) {
+        if (inputs == null) {
+            return null;
+        }
+
+        StringBuilder str = new StringBuilder();
+
+        for (Object input: inputs) {
+            if (input == null) {
+                return null;
+            }
+
+            str.append(input.toString());
+        }
+
+        return str.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        return Objects.equals(values, ((ConcatFunctionProcessor) obj).values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(values);
+    }
+
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
@@ -16,6 +17,8 @@
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToStringFunctionProcessor;
 import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
 
+import java.util.List;
+
 /*
  * Whitelisted class for EQL scripts.
  * Acts as a registry of the various static methods used <b>internally</b> by the scalar functions
@@ -29,6 +32,10 @@ public static String between(String s, String left, String right, Boolean greedy
         return (String) BetweenFunctionProcessor.doProcess(s, left, right, greedy, caseSensitive);
     }
 
+    public static String concat(List<Object> values) {
+        return (String) ConcatFunctionProcessor.doProcess(values);
+    }
+
     public static Boolean endsWith(String s, String pattern) {
         return (Boolean) EndsWithFunctionProcessor.doProcess(s, pattern);
     }

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -61,6 +61,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
 # ASCII Functions
 #
   String  between(String, String, String, Boolean, Boolean)
+  String  concat(java.util.List)
   Boolean endsWith(String, String)
   Integer indexOf(String, String, Number)
   Integer length(String)

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -119,8 +119,6 @@ public void testFunctionParsingUnknown() {
     public void testFunctionVerificationUnknown() {
         assertEquals("1:34: Unknown function [number]",
                 error("process where serial_event_id == number('5')"));
-        assertEquals("1:15: Unknown function [concat]",
-                error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'"));
     }
 
     // Test unsupported array indexes