Skip to content

Commit 0e600f7

Browse files
astefanastefan
authored
EQL: Add optional fields and limit joining keys on non-null values only (#79677) (#79807)
Add optional fields, with usage inside queries' filters and as join keys. Optional fields have a question mark in front of their name (`?some_field`) and can be used in standalone event queries and sequences. If the field exists in at least one index that's getting queried, that field will actually be used in queries. If the field doesn't exist in any of the indices, all its mentions in query filters will be replaced with `null`. For sequences, optional fields as keys can have `null` as a value, whereas non-optional fields will be restricted to non-null values only. For example, a query like ``` sequence by ?process.entity_id, process.pid [process where transID == 2] [file where transID == 0] with runs=2 ``` can return a sequence with a join key `[null,123]`. If the sequence will use `process.pid` as an optional field (`sequence by ?process.entity_id, ?process.pid`), as well, the sequence can now return join keys as `[null,123]`, `[null,null]`, `[512,null]`.
1 parent 7e9b8cd commit 0e600f7

40 files changed

+1406
-374
lines changed

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestC
4545
private final String query;
4646
private final String name;
4747
private final long[] eventIds;
48+
/**
49+
* Join keys can be of multiple types, but toml is very restrictive and doesn't allow mixed types values in the same array of values
50+
* For now, every value will be converted to a String.
51+
*/
52+
private final String[] joinKeys;
4853

4954
@Before
5055
public void setup() throws Exception {
@@ -79,18 +84,19 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
7984
name = "" + (counter);
8085
}
8186

82-
results.add(new Object[] { spec.query(), name, spec.expectedEventIds() });
87+
results.add(new Object[] { spec.query(), name, spec.expectedEventIds(), spec.joinKeys() });
8388
}
8489

8590
return results;
8691
}
8792

88-
BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds) {
93+
BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
8994
this.index = index;
9095

9196
this.query = query;
9297
this.name = name;
9398
this.eventIds = eventIds;
99+
this.joinKeys = joinKeys;
94100
}
95101

96102
public void test() throws Exception {
@@ -192,6 +198,44 @@ protected void assertSequences(List<Sequence> sequences) {
192198
.flatMap(s -> s.events().stream())
193199
.collect(toList());
194200
assertEvents(events);
201+
List<Object> keys = sequences.stream()
202+
.flatMap(s -> s.joinKeys().stream())
203+
.collect(toList());
204+
assertEvents(events);
205+
assertJoinKeys(keys);
206+
}
207+
208+
private void assertJoinKeys(List<Object> keys) {
209+
logger.debug("Join keys {}", new Object() {
210+
public String toString() {
211+
return keysToString(keys);
212+
}
213+
});
214+
215+
if (joinKeys == null || joinKeys.length == 0) {
216+
return;
217+
}
218+
String[] actual = new String[keys.size()];
219+
int i = 0;
220+
for (Object key : keys) {
221+
if (key == null) {
222+
actual[i] = "null";
223+
} else {
224+
actual[i] = key.toString();
225+
}
226+
i++;
227+
}
228+
assertArrayEquals(LoggerMessageFormat.format(null, "unexpected result for spec[{}] [{}] -> {} vs {}", name, query,
229+
Arrays.toString(joinKeys), Arrays.toString(actual)), joinKeys, actual);
230+
}
231+
232+
private String keysToString(List<Object> keys) {
233+
StringJoiner sj = new StringJoiner(",", "[", "]");
234+
for (Object key : keys) {
235+
sj.add(key.toString());
236+
sj.add("\n");
237+
}
238+
return sj.toString();
195239
}
196240

197241
@Override

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@ public static List<Object[]> readTestSpecs() throws Exception {
2121
}
2222

2323
// constructor for "local" rest tests
24-
public EqlDateNanosSpecTestCase(String query, String name, long[] eventIds) {
25-
this(TEST_NANOS_INDEX, query, name, eventIds);
24+
public EqlDateNanosSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
25+
this(TEST_NANOS_INDEX, query, name, eventIds, joinKeys);
2626
}
2727

2828
// constructor for multi-cluster tests
29-
public EqlDateNanosSpecTestCase(String index, String query, String name, long[] eventIds) {
30-
super(index, query, name, eventIds);
29+
public EqlDateNanosSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
30+
super(index, query, name, eventIds, joinKeys);
3131
}
3232

3333
@Override

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@ public static List<Object[]> readTestSpecs() throws Exception {
2121
}
2222

2323
// constructor for "local" rest tests
24-
public EqlExtraSpecTestCase(String query, String name, long[] eventIds) {
25-
this(TEST_EXTRA_INDEX, query, name, eventIds);
24+
public EqlExtraSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
25+
this(TEST_EXTRA_INDEX, query, name, eventIds, joinKeys);
2626
}
2727

2828
// constructor for multi-cluster tests
29-
public EqlExtraSpecTestCase(String index, String query, String name, long[] eventIds) {
30-
super(index, query, name, eventIds);
29+
public EqlExtraSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
30+
super(index, query, name, eventIds, joinKeys);
3131
}
3232

3333
@Override

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ public class EqlSpec {
1919
private String[] tags;
2020
private String query;
2121
private long[] expectedEventIds;
22+
private String[] joinKeys;
2223

2324
public String name() {
2425
return name;
@@ -68,6 +69,14 @@ public void expectedEventIds(long[] expectedEventIds) {
6869
this.expectedEventIds = expectedEventIds;
6970
}
7071

72+
public String[] joinKeys() {
73+
return joinKeys;
74+
}
75+
76+
public void joinKeys(String[] joinKeys) {
77+
this.joinKeys = joinKeys;
78+
}
79+
7180
@Override
7281
public String toString() {
7382
String str = "";
@@ -83,6 +92,10 @@ public String toString() {
8392
if (expectedEventIds != null) {
8493
str = appendWithComma(str, "expected_event_ids", Arrays.toString(expectedEventIds));
8594
}
95+
96+
if (joinKeys != null) {
97+
str = appendWithComma(str, "join_keys", Arrays.toString(joinKeys));
98+
}
8699
return str;
87100
}
88101

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ private static List<EqlSpec> readFromStream(InputStream is, Set<String> uniqueTe
101101
}
102102
spec.expectedEventIds(expectedEventIds);
103103
}
104+
105+
arr = table.getList("join_keys");
106+
spec.joinKeys(arr != null ? arr.toArray(new String[0]) : new String[0]);
104107
validateAndAddSpec(testSpecs, spec, uniqueTestNames);
105108
}
106109

‎x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,12 @@ protected String tiebreaker() {
2828
}
2929

3030
// constructor for "local" rest tests
31-
public EqlSpecTestCase(String query, String name, long[] eventIds) {
32-
this(TEST_INDEX, query, name, eventIds);
31+
public EqlSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
32+
this(TEST_INDEX, query, name, eventIds, joinKeys);
3333
}
3434

3535
// constructor for multi-cluster tests
36-
public EqlSpecTestCase(String index, String query, String name, long[] eventIds) {
37-
super(index, query, name, eventIds);
36+
public EqlSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
37+
super(index, query, name, eventIds, joinKeys);
3838
}
3939
}

‎x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,3 +444,63 @@ query = '''
444444
process where substring(command_line, 5) regex (".*?net[1]? localgroup.*?", ".*? myappserver.py .*?")
445445
'''
446446

447+
[[queries]]
448+
description = "This query has an important meaning when considering its pair - sequenceWithRequiredUserDomain - it shows that when using '?' for a key field, this one becomes null-aware, meaning a null value can be used as a join key"
449+
name = "sequenceWithOptionalUserDomain"
450+
expected_event_ids = [1, 57]
451+
join_keys = ["null"]
452+
query = '''
453+
sequence by ?user_domain [process where true] [registry where true]
454+
'''
455+
456+
[[queries]]
457+
name = "sequenceWithRequiredUserDomain"
458+
expected_event_ids = []
459+
join_keys = []
460+
query = '''
461+
sequence by user_domain [process where true] [registry where true]
462+
'''
463+
464+
[[queries]]
465+
description = "An equivalent (without optional keys) query can be found in test_queries.toml - twoSequencesWithTwoKeys"
466+
name = "twoSequencesWithTwoKeys_AndOptionals"
467+
query = '''
468+
sequence by ?x
469+
[process where true] by unique_pid, process_path, ?z
470+
[process where opcode == 1] by unique_ppid, parent_process_path, ?w
471+
'''
472+
expected_event_ids = [48, 53,
473+
53, 54,
474+
54, 56,
475+
97, 98]
476+
join_keys = ["null", "48", "C:\\Python27\\python.exe", "null",
477+
"null", "53", "C:\\Windows\\System32\\cmd.exe", "null",
478+
"null", "54", "C:\\Python27\\python.exe", "null",
479+
"null", "750058", "C:\\Windows\\System32\\net.exe", "null"]
480+
481+
[[queries]]
482+
name = "sequenceOnOneNullKey"
483+
query = '''
484+
sequence
485+
[process where parent_process_path == null] by parent_process_path
486+
[any where true] by parent_process_path
487+
'''
488+
expected_event_ids = []
489+
490+
[[queries]]
491+
name = "sequenceOnTwoNullKeys"
492+
query = '''
493+
sequence by ppid
494+
[process where parent_process_path == null] by parent_process_path
495+
[any where true] by parent_process_path
496+
'''
497+
expected_event_ids = []
498+
499+
[[queries]]
500+
name = "sequenceOnImplicitNullKeys"
501+
query = '''
502+
sequence by ppid, parent_process_path
503+
[process where parent_process_path == null]
504+
[any where true]
505+
'''
506+
expected_event_ids = []

‎x-pack/plugin/eql/qa/common/src/main/resources/data/extra.data

Lines changed: 122 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,137 @@
2121
"@timestamp": "10",
2222
"event_type": "REQUEST",
2323
"transID": 1235,
24-
"sequence": 1
24+
"sequence": 4
2525
},
2626
{
2727
"@timestamp": "11",
2828
"event_type": "ERROR",
2929
"transID": 1235,
30-
"sequence": 2
30+
"sequence": 5
3131
},
3232
{
3333
"@timestamp": "11",
3434
"event_type": "STAT",
3535
"transID": 1235,
36-
"sequence": 3
36+
"sequence": 6
37+
},
38+
{
39+
"@timestamp": "100",
40+
"event_type": "OPTIONAL",
41+
"optional_field_default_null": null,
42+
"sequence": 7
43+
},
44+
{
45+
"@timestamp": "101",
46+
"event_type": "OPTIONAL",
47+
"sequence": 8,
48+
"transID": 1235
49+
},
50+
{
51+
"@timestamp": "102",
52+
"event_type": "OPTIONAL",
53+
"optional_field_default_null": null,
54+
"sequence": 9
55+
},
56+
{
57+
"@timestamp": "1",
58+
"event_type": "process",
59+
"transID": 1,
60+
"process.pid": 123,
61+
"sequence": 10
62+
},
63+
{
64+
"@timestamp": "1",
65+
"event_type": "process",
66+
"transID": 2,
67+
"sequence": 11
68+
},
69+
{
70+
"@timestamp": "2",
71+
"event_type": "process",
72+
"transID": 2,
73+
"process.pid": 123,
74+
"sequence": 12
75+
},
76+
{
77+
"@timestamp": "3",
78+
"event_type": "file",
79+
"transID": 0,
80+
"process.pid": 123,
81+
"sequence": 13
82+
},
83+
{
84+
"@timestamp": "4",
85+
"event_type": "file",
86+
"transID": 0,
87+
"process.pid": 123,
88+
"sequence": 14
89+
},
90+
{
91+
"@timestamp": "5",
92+
"event_type": "file",
93+
"transID": 0,
94+
"process.pid": 123,
95+
"sequence": 15
96+
},
97+
{
98+
"@timestamp": "6",
99+
"event_type": "file",
100+
"transID": 0,
101+
"sequence": 16
102+
},
103+
{
104+
"@timestamp": "6",
105+
"event_type": "file",
106+
"transID": 0,
107+
"sequence": 17
108+
},
109+
{
110+
"@timestamp": "7",
111+
"event_type": "process",
112+
"transID": 2,
113+
"process.entity_id": 512,
114+
"process.pid": 123,
115+
"sequence": 18
116+
},
117+
{
118+
"@timestamp": "8",
119+
"event_type": "file",
120+
"transID": 0,
121+
"process.entity_id": 512,
122+
"process.pid": 123,
123+
"sequence": 19
124+
},
125+
{
126+
"@timestamp": "9",
127+
"event_type": "file",
128+
"transID": 0,
129+
"process.entity_id": 512,
130+
"process.pid": 123,
131+
"sequence": 20
132+
},
133+
{
134+
"@timestamp": "10",
135+
"event_type": "file",
136+
"transID": 0,
137+
"process.entity_id": 512,
138+
"process.pid": 123,
139+
"sequence": 21
140+
},
141+
{
142+
"@timestamp": "11",
143+
"event_type": "file",
144+
"transID": 0,
145+
"process.entity_id": 512,
146+
"process.pid": 123,
147+
"sequence": 22
148+
},
149+
{
150+
"@timestamp": "12",
151+
"event_type": "file",
152+
"transID": 0,
153+
"process.entity_id": 512,
154+
"process.pid": 123,
155+
"sequence": 23
37156
}
38157
]

‎x-pack/plugin/eql/qa/common/src/main/resources/data/extra.mapping

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@
2020
"path": "sequence"
2121
}
2222
}
23+
},
24+
"optional_field_mapping_only": {
25+
"type": "keyword"
26+
},
27+
"optional_field_default_null": {
28+
"type": "keyword",
29+
"null_value": "NULL"
2330
}
2431
}
2532
}

0 commit comments

Comments
 (0)