Skip to content

Commit d02f774

Browse files
aleksmausaleksmaus
authored
EQL: implement cidrMatch function (elastic#54186) (elastic#54928)
Related to elastic#54132
1 parent 6571374 commit d02f774

File tree

9 files changed

+282
-38
lines changed

9 files changed

+282
-38
lines changed

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616
import org.elasticsearch.client.RestHighLevelClient;
1717
import org.elasticsearch.client.eql.EqlSearchRequest;
1818
import org.elasticsearch.client.eql.EqlSearchResponse;
19+
import org.elasticsearch.client.indices.CreateIndexRequest;
1920
import org.elasticsearch.common.Strings;
21+
import org.elasticsearch.common.io.Streams;
2022
import org.elasticsearch.common.xcontent.XContentParser;
2123
import org.elasticsearch.common.xcontent.XContentType;
2224
import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -56,6 +58,12 @@ private static void setupData(CommonEqlActionTestCase tc) throws Exception {
5658
return;
5759
}
5860

61+
CreateIndexRequest request = new CreateIndexRequest(testIndexName)
62+
.mapping(Streams.readFully(CommonEqlActionTestCase.class.getResourceAsStream("/mapping-default.json")),
63+
XContentType.JSON);
64+
65+
tc.highLevelClient().indices().create(request, RequestOptions.DEFAULT);
66+
5967
BulkRequest bulk = new BulkRequest();
6068
bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
6169

‎x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
{
2+
"properties" : {
3+
"command_line" : {
4+
"type" : "keyword"
5+
},
6+
"event" : {
7+
"properties" : {
8+
"category" : {
9+
"type" : "keyword"
10+
}
11+
}
12+
},
13+
"md5" : {
14+
"type" : "keyword"
15+
},
16+
"parent_process_name": {
17+
"type" : "keyword"
18+
},
19+
"parent_process_path": {
20+
"type" : "keyword"
21+
},
22+
"pid" : {
23+
"type" : "long"
24+
},
25+
"ppid" : {
26+
"type" : "long"
27+
},
28+
"process_name": {
29+
"type" : "keyword"
30+
},
31+
"process_path": {
32+
"type" : "keyword"
33+
},
34+
"subtype" : {
35+
"type" : "keyword"
36+
},
37+
"@timestamp" : {
38+
"type" : "date"
39+
},
40+
"user" : {
41+
"type" : "keyword"
42+
},
43+
"user_name" : {
44+
"type" : "keyword"
45+
},
46+
"user_domain": {
47+
"type" : "keyword"
48+
},
49+
"hostname" : {
50+
"type" : "text",
51+
"fields" : {
52+
"keyword" : {
53+
"type" : "keyword",
54+
"ignore_above" : 256
55+
}
56+
}
57+
},
58+
"opcode" : {
59+
"type" : "long"
60+
},
61+
"file_name" : {
62+
"type" : "text",
63+
"fields" : {
64+
"keyword" : {
65+
"type" : "keyword",
66+
"ignore_above" : 256
67+
}
68+
}
69+
},
70+
"serial_event_id" : {
71+
"type" : "long"
72+
},
73+
"source_address" : {
74+
"type" : "ip"
75+
},
76+
"exit_code" : {
77+
"type" : "long"
78+
}
79+
}
80+
}

‎x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,30 +1057,6 @@ query = '''
10571057
file where between(file_path, "dev", ".json", true) == "\\testlogs\\something"
10581058
'''
10591059

1060-
[[queries]]
1061-
expected_event_ids = [75304, 75305]
1062-
query = '''
1063-
network where cidrMatch(source_address, "10.6.48.157/8")
1064-
'''
1065-
1066-
[[queries]]
1067-
expected_event_ids = []
1068-
query = '''
1069-
network where cidrMatch(source_address, "192.168.0.0/16")
1070-
'''
1071-
1072-
[[queries]]
1073-
expected_event_ids = [75304, 75305]
1074-
query = '''
1075-
network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8")
1076-
1077-
'''
1078-
[[queries]]
1079-
expected_event_ids = [75304, 75305]
1080-
query = '''
1081-
network where cidrMatch(source_address, "0.0.0.0/0")
1082-
'''
1083-
10841060
[[queries]]
10851061
expected_event_ids = [7, 14, 22, 29, 44]
10861062
query = '''

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
package org.elasticsearch.xpack.eql.expression.function;
88

9+
import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
910
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
1011
import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
1112
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
@@ -30,6 +31,7 @@ private static FunctionDefinition[][] functions() {
3031
// String
3132
new FunctionDefinition[] {
3233
def(Between.class, Between::new, 2, "between"),
34+
def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"),
3335
def(EndsWith.class, EndsWith::new, "endswith"),
3436
def(Length.class, Length::new, "length"),
3537
def(StartsWith.class, StartsWith::new, "startswith"),

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
8+
9+
import org.elasticsearch.xpack.ql.expression.Expression;
10+
import org.elasticsearch.xpack.ql.expression.Expressions;
11+
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
12+
import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction;
13+
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
14+
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
15+
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
16+
import org.elasticsearch.xpack.ql.tree.NodeInfo;
17+
import org.elasticsearch.xpack.ql.tree.Source;
18+
import org.elasticsearch.xpack.ql.type.DataType;
19+
import org.elasticsearch.xpack.ql.type.DataTypes;
20+
import org.elasticsearch.xpack.ql.util.CollectionUtils;
21+
22+
import java.util.List;
23+
24+
import static java.util.Collections.singletonList;
25+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable;
26+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact;
27+
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
28+
29+
/**
30+
* EQL specific cidrMatch function
31+
* Returns true if the source address matches any of the provided CIDR blocks.
32+
* Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch
33+
*/
34+
public class CIDRMatch extends BaseSurrogateFunction {
35+
36+
private final Expression field;
37+
private final List<Expression> addresses;
38+
39+
public CIDRMatch(Source source, Expression field, List<Expression> addresses) {
40+
super(source, CollectionUtils.combine(singletonList(field), addresses));
41+
this.field = field;
42+
this.addresses = addresses;
43+
}
44+
45+
@Override
46+
protected NodeInfo<? extends Expression> info() {
47+
return NodeInfo.create(this, CIDRMatch::new, field, addresses);
48+
}
49+
50+
@Override
51+
public Expression replaceChildren(List<Expression> newChildren) {
52+
if (newChildren.size() < 2) {
53+
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
54+
}
55+
return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
56+
}
57+
58+
@Override
59+
public DataType dataType() {
60+
return DataTypes.BOOLEAN;
61+
}
62+
63+
@Override
64+
protected TypeResolution resolveType() {
65+
if (!childrenResolved()) {
66+
return new TypeResolution("Unresolved children");
67+
}
68+
69+
TypeResolution resolution = isIPAndExact(field, sourceText(), Expressions.ParamOrdinal.FIRST);
70+
if (resolution.unresolved()) {
71+
return resolution;
72+
}
73+
74+
for (Expression addr : addresses) {
75+
// Currently we have limited enum for ordinal numbers
76+
// So just using default here for error messaging
77+
resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT);
78+
if (resolution.unresolved()) {
79+
return resolution;
80+
}
81+
}
82+
83+
int index = 1;
84+
85+
for (Expression addr : addresses) {
86+
87+
resolution = isFoldable(addr, sourceText(), ParamOrdinal.fromIndex(index));
88+
if (resolution.unresolved()) {
89+
break;
90+
}
91+
92+
resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.fromIndex(index));
93+
if (resolution.unresolved()) {
94+
break;
95+
}
96+
97+
index++;
98+
}
99+
100+
return resolution;
101+
}
102+
103+
@Override
104+
public ScalarFunction makeSubstitute() {
105+
ScalarFunction func = null;
106+
107+
for (Expression address : addresses) {
108+
final Equals eq = new Equals(source(), field, address);
109+
func = (func == null) ? eq : new Or(source(), func, eq);
110+
}
111+
112+
return func;
113+
}
114+
}

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,6 @@ public void testFunctionVerificationUnknown() {
133133
error("process where serial_event_id == number('5')"));
134134
assertEquals("1:15: Unknown function [concat]",
135135
error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'"));
136-
assertEquals("1:15: Unknown function [cidrMatch]",
137-
error("network where cidrMatch(source_address, \"192.168.0.0/16\", \"10.6.48.157/8\")"));
138136
}
139137

140138
// Test unsupported array indexes

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -62,27 +62,35 @@ public void testBetweenWrongTypeParams() {
6262
error("process where between(process_name, \"s\", \"e\", false, 2)"));
6363
}
6464

65-
public void testPropertyEquationFilterUnsupported() {
66-
QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
67-
() -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)"));
65+
public void testCIDRMatchNonIPField() {
66+
VerificationException e = expectThrows(VerificationException.class,
67+
() -> plan("process where cidrMatch(hostname, \"10.0.0.0/8\")"));
6868
String msg = e.getMessage();
69-
assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg);
69+
assertEquals("Found 1 problem\n" +
70+
"line 1:15: first argument of [cidrMatch(hostname, \"10.0.0.0/8\")] must be [ip], found value [hostname] type [text]", msg);
7071
}
7172

72-
public void testPropertyEquationInClauseFilterUnsupported() {
73+
public void testCIDRMatchMissingValue() {
74+
ParsingException e = expectThrows(ParsingException.class,
75+
() -> plan("process where cidrMatch(source_address)"));
76+
String msg = e.getMessage();
77+
assertEquals("line 1:16: error building [cidrmatch]: expects at least two arguments", msg);
78+
}
79+
80+
public void testCIDRMatchAgainstField() {
7381
VerificationException e = expectThrows(VerificationException.class,
74-
() -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")"));
82+
() -> plan("process where cidrMatch(source_address, hostname)"));
7583
String msg = e.getMessage();
76-
assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " +
77-
"offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
84+
assertEquals("Found 1 problem\n" +
85+
"line 1:15: second argument of [cidrMatch(source_address, hostname)] must be a constant, received [hostname]", msg);
7886
}
7987

80-
public void testLengthFunctionWithInexact() {
88+
public void testCIDRMatchNonString() {
8189
VerificationException e = expectThrows(VerificationException.class,
82-
() -> plan("process where length(plain_text) > 0"));
90+
() -> plan("process where cidrMatch(source_address, 12345)"));
8391
String msg = e.getMessage();
84-
assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
85-
+ "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
92+
assertEquals("Found 1 problem\n" +
93+
"line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg);
8694
}
8795

8896
public void testEndsWithFunctionWithInexact() {
@@ -93,6 +101,29 @@ public void testEndsWithFunctionWithInexact() {
93101
+ "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
94102
}
95103

104+
public void testLengthFunctionWithInexact() {
105+
VerificationException e = expectThrows(VerificationException.class,
106+
() -> plan("process where length(plain_text) > 0"));
107+
String msg = e.getMessage();
108+
assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
109+
+ "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
110+
}
111+
112+
public void testPropertyEquationFilterUnsupported() {
113+
QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
114+
() -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)"));
115+
String msg = e.getMessage();
116+
assertEquals("Line 1:74: Comparisons against variables are not (currently) supported; offender [pid] in [==]", msg);
117+
}
118+
119+
public void testPropertyEquationInClauseFilterUnsupported() {
120+
VerificationException e = expectThrows(VerificationException.class,
121+
() -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")"));
122+
String msg = e.getMessage();
123+
assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " +
124+
"offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
125+
}
126+
96127
public void testStartsWithFunctionWithInexact() {
97128
VerificationException e = expectThrows(VerificationException.class,
98129
() -> plan("process where startsWith(plain_text, \"foo\") == true"));

‎x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,27 @@ InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),par
129129
"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"}
130130
;
131131

132+
cidrMatchFunctionOne
133+
process where cidrMatch(source_address, "10.0.0.0/8")
134+
;
135+
"term":{"source_address":{"value":"10.0.0.0/8"
136+
;
137+
138+
cidrMatchFunctionTwo
139+
process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16")
140+
;
141+
"term":{"source_address":{"value":"10.0.0.0/8"
142+
"term":{"source_address":{"value":"192.168.0.0/16"
143+
;
144+
145+
cidrMatchFunctionThree
146+
process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32")
147+
;
148+
"term":{"source_address":{"value":"10.0.0.0/8"
149+
"term":{"source_address":{"value":"192.168.0.0/16"
150+
"term":{"source_address":{"value":"2001:db8::/32"
151+
;
152+
132153
wildcardFunctionSingleArgument
133154
process where wildcard(process_path, "*\\red_ttp\\wininit.*")
134155
;

0 commit comments

Comments
 (0)