Skip to content

Commit 3890820

Browse files
rw-accessrw-access
committed
EQL: Add concat function (#55193)
* EQL: Add concat function * EQL: for loop spacing for concat * EQL: return unresolved arguments to concat early * EQL: Add concat integration tests * EQL: Fix concat query fail test * EQL: Add class for concat function testing * EQL: Add concat integration tests * EQL: Update concat() null behavior
1 parent a7968a1 commit 3890820

File tree

12 files changed

+425
-24
lines changed

12 files changed

+425
-24
lines changed

‎x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,39 @@ query = '''
1414
file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
1515
'''
1616

17+
[[queries]]
18+
description = "test string concatenation. update test to avoid case-sensitivity issues"
19+
query = '''
20+
process where concat(serial_event_id, '::', process_name, '::', opcode) == '5::wininit.exe::3'
21+
'''
22+
expected_event_ids = [5]
23+
24+
25+
[[queries]]
26+
query = 'process where concat(serial_event_id) = "1"'
27+
expected_event_ids = [1]
28+
29+
[[queries]]
30+
query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) != null'
31+
expected_event_ids = [2, 3]
32+
33+
34+
[[queries]]
35+
query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) == null'
36+
expected_event_ids = [1, 4]
37+
38+
39+
[[queries]]
40+
query = 'process where serial_event_id < 5 and concat(process_name, null, null) == null'
41+
expected_event_ids = [1, 2, 3, 4]
42+
43+
44+
45+
[[queries]]
46+
query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null'
47+
expected_event_ids = [1, 2, 3, 4]
48+
49+
1750
[[queries]]
1851
query = 'process where string(serial_event_id) = "1"'
1952
expected_event_ids = [1]

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
1010
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
11+
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Concat;
1112
import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
1213
import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
1314
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
@@ -40,6 +41,7 @@ private static FunctionDefinition[][] functions() {
4041
new FunctionDefinition[] {
4142
def(Between.class, Between::new, 2, "between"),
4243
def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"),
44+
def(Concat.class, Concat::new, "concat"),
4345
def(EndsWith.class, EndsWith::new, "endswith"),
4446
def(IndexOf.class, IndexOf::new, "indexof"),
4547
def(Length.class, Length::new, "length"),

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Concat.java

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
8+
9+
import org.elasticsearch.xpack.ql.expression.Expression;
10+
import org.elasticsearch.xpack.ql.expression.Expressions;
11+
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
12+
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
13+
import org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder;
14+
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
15+
import org.elasticsearch.xpack.ql.tree.NodeInfo;
16+
import org.elasticsearch.xpack.ql.tree.Source;
17+
import org.elasticsearch.xpack.ql.type.DataType;
18+
import org.elasticsearch.xpack.ql.type.DataTypes;
19+
20+
import java.util.ArrayList;
21+
import java.util.List;
22+
import java.util.StringJoiner;
23+
24+
import static org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor.doProcess;
25+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isExact;
26+
import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
27+
28+
/**
29+
* EQL specific concat function to build a string of all input arguments concatenated.
30+
*/
31+
public class Concat extends ScalarFunction {
32+
33+
private final List<Expression> values;
34+
35+
public Concat(Source source, List<Expression> values) {
36+
super(source, values);
37+
this.values = values;
38+
}
39+
40+
@Override
41+
protected TypeResolution resolveType() {
42+
if (!childrenResolved()) {
43+
return new TypeResolution("Unresolved children");
44+
}
45+
46+
TypeResolution resolution = TypeResolution.TYPE_RESOLVED;
47+
for (Expression value : values) {
48+
resolution = isExact(value, sourceText(), Expressions.ParamOrdinal.DEFAULT);
49+
50+
if (resolution.unresolved()) {
51+
return resolution;
52+
}
53+
}
54+
55+
return resolution;
56+
}
57+
58+
@Override
59+
protected Pipe makePipe() {
60+
return new ConcatFunctionPipe(source(), this, Expressions.pipe(values));
61+
}
62+
63+
@Override
64+
public boolean foldable() {
65+
return Expressions.foldable(values);
66+
}
67+
68+
@Override
69+
public Object fold() {
70+
return doProcess(Expressions.fold(values));
71+
}
72+
73+
@Override
74+
protected NodeInfo<? extends Expression> info() {
75+
return NodeInfo.create(this, Concat::new, values);
76+
}
77+
78+
@Override
79+
public ScriptTemplate asScript() {
80+
List<ScriptTemplate> templates = new ArrayList<>();
81+
for (Expression ex : children()) {
82+
templates.add(asScript(ex));
83+
}
84+
85+
StringJoiner template = new StringJoiner(",", "{eql}.concat([", "])");
86+
ParamsBuilder params = paramsBuilder();
87+
88+
for (ScriptTemplate scriptTemplate : templates) {
89+
template.add(scriptTemplate.template());
90+
params.script(scriptTemplate.params());
91+
}
92+
93+
return new ScriptTemplate(formatTemplate(template.toString()), params.build(), dataType());
94+
}
95+
96+
@Override
97+
public DataType dataType() {
98+
return DataTypes.KEYWORD;
99+
}
100+
101+
@Override
102+
public Expression replaceChildren(List<Expression> newChildren) {
103+
return new Concat(source(), newChildren);
104+
}
105+
106+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ConcatFunctionPipe.java

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
7+
8+
import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
9+
import org.elasticsearch.xpack.ql.expression.Expression;
10+
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
11+
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
12+
import org.elasticsearch.xpack.ql.tree.NodeInfo;
13+
import org.elasticsearch.xpack.ql.tree.Source;
14+
15+
import java.util.ArrayList;
16+
import java.util.List;
17+
import java.util.Objects;
18+
19+
public class ConcatFunctionPipe extends Pipe {
20+
21+
private final List<Pipe> values;
22+
23+
public ConcatFunctionPipe(Source source, Expression expression, List<Pipe> values) {
24+
super(source, expression, values);
25+
this.values = values;
26+
}
27+
28+
@Override
29+
public final Pipe replaceChildren(List<Pipe> newChildren) {
30+
return new ConcatFunctionPipe(source(), expression(), newChildren);
31+
}
32+
33+
@Override
34+
public final Pipe resolveAttributes(AttributeResolver resolver) {
35+
List<Pipe> newValues = new ArrayList<>(values.size());
36+
for (Pipe v : values) {
37+
newValues.add(v.resolveAttributes(resolver));
38+
}
39+
40+
if (newValues == values) {
41+
return this;
42+
}
43+
44+
return replaceChildren(newValues);
45+
}
46+
47+
@Override
48+
public boolean supportedByAggsOnlyQuery() {
49+
for (Pipe p : values) {
50+
if (p.supportedByAggsOnlyQuery() == false) {
51+
return false;
52+
}
53+
}
54+
return true;
55+
}
56+
57+
@Override
58+
public boolean resolved() {
59+
for (Pipe p : values) {
60+
if (p.resolved() == false) {
61+
return false;
62+
}
63+
}
64+
return true;
65+
}
66+
67+
@Override
68+
public final void collectFields(QlSourceBuilder sourceBuilder) {
69+
for (Pipe v : values) {
70+
v.collectFields(sourceBuilder);
71+
}
72+
}
73+
74+
@Override
75+
protected NodeInfo<ConcatFunctionPipe> info() {
76+
return NodeInfo.create(this, ConcatFunctionPipe::new, expression(), values);
77+
}
78+
79+
@Override
80+
public ConcatFunctionProcessor asProcessor() {
81+
List<Processor> processors = new ArrayList<>(values.size());
82+
for (Pipe p: values) {
83+
processors.add(p.asProcessor());
84+
}
85+
return new ConcatFunctionProcessor(processors);
86+
}
87+
88+
@Override
89+
public int hashCode() {
90+
return Objects.hash(values);
91+
}
92+
93+
@Override
94+
public boolean equals(Object obj) {
95+
if (this == obj) {
96+
return true;
97+
}
98+
99+
if (obj == null || getClass() != obj.getClass()) {
100+
return false;
101+
}
102+
103+
return Objects.equals(values, ((ConcatFunctionPipe) obj).values);
104+
}
105+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ConcatFunctionProcessor.java

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
7+
8+
import org.elasticsearch.common.io.stream.StreamOutput;
9+
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
10+
11+
import java.io.IOException;
12+
import java.util.ArrayList;
13+
import java.util.List;
14+
import java.util.Objects;
15+
16+
public class ConcatFunctionProcessor implements Processor {
17+
18+
public static final String NAME = "scon";
19+
20+
private final List<Processor> values;
21+
22+
public ConcatFunctionProcessor(List<Processor> values) {
23+
this.values = values;
24+
}
25+
26+
@Override
27+
public final void writeTo(StreamOutput out) throws IOException {
28+
for (Processor v: values) {
29+
out.writeNamedWriteable(v);
30+
}
31+
}
32+
33+
@Override
34+
public Object process(Object input) {
35+
List<Object> processed = new ArrayList<>(values.size());
36+
for (Processor v: values) {
37+
processed.add(v.process(input));
38+
}
39+
return doProcess(processed);
40+
}
41+
42+
public static Object doProcess(List<Object> inputs) {
43+
if (inputs == null) {
44+
return null;
45+
}
46+
47+
StringBuilder str = new StringBuilder();
48+
49+
for (Object input: inputs) {
50+
if (input == null) {
51+
return null;
52+
}
53+
54+
str.append(input.toString());
55+
}
56+
57+
return str.toString();
58+
}
59+
60+
@Override
61+
public boolean equals(Object obj) {
62+
if (this == obj) {
63+
return true;
64+
}
65+
66+
if (obj == null || getClass() != obj.getClass()) {
67+
return false;
68+
}
69+
70+
return Objects.equals(values, ((ConcatFunctionProcessor) obj).values);
71+
}
72+
73+
@Override
74+
public int hashCode() {
75+
return Objects.hash(values);
76+
}
77+
78+
79+
@Override
80+
public String getWriteableName() {
81+
return NAME;
82+
}
83+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
88

99
import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor;
10+
import org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor;
1011
import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor;
1112
import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor;
1213
import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
@@ -16,6 +17,8 @@
1617
import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToStringFunctionProcessor;
1718
import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
1819

20+
import java.util.List;
21+
1922
/*
2023
* Whitelisted class for EQL scripts.
2124
* Acts as a registry of the various static methods used <b>internally</b> by the scalar functions
@@ -29,6 +32,10 @@ public static String between(String s, String left, String right, Boolean greedy
2932
return (String) BetweenFunctionProcessor.doProcess(s, left, right, greedy, caseSensitive);
3033
}
3134

35+
public static String concat(List<Object> values) {
36+
return (String) ConcatFunctionProcessor.doProcess(values);
37+
}
38+
3239
public static Boolean endsWith(String s, String pattern) {
3340
return (Boolean) EndsWithFunctionProcessor.doProcess(s, pattern);
3441
}

‎x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
6161
# ASCII Functions
6262
#
6363
String between(String, String, String, Boolean, Boolean)
64+
String concat(java.util.List)
6465
Boolean endsWith(String, String)
6566
Integer indexOf(String, String, Number)
6667
Integer length(String)

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,6 @@ public void testFunctionParsingUnknown() {
119119
public void testFunctionVerificationUnknown() {
120120
assertEquals("1:34: Unknown function [number]",
121121
error("process where serial_event_id == number('5')"));
122-
assertEquals("1:15: Unknown function [concat]",
123-
error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'"));
124122
}
125123

126124
// Test unsupported array indexes

0 commit comments

Comments
 (0)