diff --git a/docs/reference/eql/limitations.asciidoc b/docs/reference/eql/limitations.asciidoc index dfd0d1ee65b09..ac302f8ae1679 100644 --- a/docs/reference/eql/limitations.asciidoc +++ b/docs/reference/eql/limitations.asciidoc @@ -41,5 +41,3 @@ queries that contain: ** {eql-ref}/pipes.html#sort[`sort`] ** {eql-ref}/pipes.html#unique[`unique`] ** {eql-ref}/pipes.html#unique-count[`unique_count`] - -* The `until` {eql-ref}/sequences.html[sequence keyword] \ No newline at end of file diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc index 8fdf1a8ff5a30..a1da93992cf1d 100644 --- a/docs/reference/eql/search.asciidoc +++ b/docs/reference/eql/search.asciidoc @@ -31,6 +31,8 @@ PUT /sec_logs/_bulk?refresh { "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } {"index":{"_index" : "sec_logs", "_id" : "5"}} { "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } } +{"index":{"_index" : "sec_logs", "_id" : "6"}} +{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } } ---- // TESTSETUP @@ -99,7 +101,7 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } - }, + }, "sort": [ 1607252645000 ] @@ -389,6 +391,27 @@ contains the shared `agent.id` value for each matching event. } ---- // TESTRESPONSE[s/"took": 60/"took": $body.took/] + +You can use the <> to specify an expiration +event for sequences. Matching sequences must end before this event. + +The following request adds +`until [ process where event.type == "termination" ]` to the previous EQL query. +This ensures matching sequences end before a process termination event. + +[source,console] +---- +GET /sec_logs/_eql/search +{ + "query": """ + sequence by agent.id with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + until [ process where event.type == "termination" ] + """ +} +---- +// TEST[s/search/search\?filter_path\=\-\*\.sequences\.\*events\.\*fields/] ==== [discrete] @@ -547,7 +570,7 @@ tiebreaker for events with the same timestamp. } ---- // TESTRESPONSE[s/"took": 34/"took": $body.took/] -<1> The event's <>, converted to +<1> The event's <>, converted to milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch] <2> The event's `event.id` value. diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc index b826318062788..85de1258c15f4 100644 --- a/docs/reference/eql/syntax.asciidoc +++ b/docs/reference/eql/syntax.asciidoc @@ -485,7 +485,7 @@ sequence by user.name ---- ==== -You can combine the `sequence by` and `with maxspan` keywords to constrain a +You can combine the `sequence by` and `with maxspan` keywords to constrain a sequence by both field values and a timespan. [source,eql] @@ -513,6 +513,89 @@ sequence by user.name with maxspan=15m ---- ==== +[discrete] +[[eql-until-keyword]] +==== `until` keyword + +You can use the `until` keyword to specify an expiration event for sequences. +Matching sequences must end before this event, which is not included the +results. If this event occurs within a sequence, the sequence is not considered +a match. + +[source,eql] +---- +sequence + [ event_category_1 where condition_1 ] + [ event_category_2 where condition_2 ] + ... +until [ event_category_2 where condition_2 ] +---- + +.*Example* +[%collapsible] +==== +The following EQL sequence query uses the `until` keyword to end sequences +before a process termination event. Process termination events have an event +category of `process` and `event.type` value of `termination`. + +[source,eql] +---- +sequence + [ file where file.extension == "exe" ] + [ process where true ] +until [ process where event.type == "termination" ] +---- +==== + +[TIP] +==== +The `until` keyword can be helpful when searching for process sequences in +Windows event logs, such as those ingested using +{winlogbeat-ref}/index.html[Winlogbeat]. + +In Windows, a process ID (PID) is unique only while a process is running. After +a process terminates, its PID can be reused. + +You can search for a sequence of events with the same PID value using the `by` +and `sequence by` keywords. + +.*Example* +[%collapsible] +===== +The following EQL query uses the `sequence by` keyword to match a sequence of +events that share the same `process.pid` value. + +[source,eql] +---- +sequence by process.pid + [ process where process.name == "cmd.exe" ] + [ process where process.name == "whoami.exe" ] +---- +===== + +However, due to PID reuse, this can result in a matching sequence that +contains events across unrelated processes. To prevent false positives, you can +use the `until` keyword to end matching sequences before a process termination +event. + +.*Example* +[%collapsible] +===== +The following EQL query uses the `until` keyword to end sequences before +`process` events with an `event.type` of `termination`. These events indicate a +process has been terminated. + +[source,eql] +---- +sequence by process.pid + [ process where process.name == "cmd.exe" ] + [ process where process.name == "whoami.exe" ] +until [ process where event.type == "termination" ] +---- +===== + +==== + [discrete] [[eql-functions]] === Functions