From a579ff84d7868376e335780dba9ed4b6d8322e28 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Thu, 8 Apr 2021 09:25:21 -0400 Subject: [PATCH 1/3] [DOCS] EQL/SQL: Document `runtime_fields` parameter --- docs/reference/eql/eql-search-api.asciidoc | 2 + docs/reference/eql/eql.asciidoc | 71 +++++++++++++++++++++- docs/reference/search/search.asciidoc | 27 +++++--- docs/reference/sql/endpoints/rest.asciidoc | 45 +++++++++++++- 4 files changed, 135 insertions(+), 10 deletions(-) diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 00267fdd830b6..380ba18ec4306 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -294,6 +294,8 @@ command]. NOTE: This parameter may change the set of returned hits. However, it does not change the sort order of hits in the response. +include::{es-repo-dir}/search/search.asciidoc[tag=runtime-mappings-def] + `size`:: (Optional, integer or float) For <>, the maximum number of matching events to diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc index 232e2bf4adf9c..f67b4993edb79 100644 --- a/docs/reference/eql/eql.asciidoc +++ b/docs/reference/eql/eql.asciidoc @@ -442,7 +442,7 @@ GET /my-data-stream/_eql/search?filter_path=-hits.events._source "process.*", <1> { "field": "@timestamp", <2> - "format": "epoch_millis" + "format": "epoch_millis" } ] } @@ -532,6 +532,75 @@ The values are returned as a flat list in the `fields` section of each hit: // TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] // TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] +[discrete] +[[eql-use-runtime-fields]] +=== Use runtime fields + +Use the `runtime_mappings` parameter to extract and create <> during a search. Use the `fields` parameter to include runtime fields +in the response. + +The following search create a `day_of_week` runtime field from the `@timestamp` +and returns it in the response. + +[source,console] +---- +GET /my-data-stream/_eql/search?filter_path=-hits.events._source +{ + "runtime_mappings": { + "day_of_week": { + "type": "keyword", + "script": "emit(doc['@timestamp'].value.dayOfWeekEnum.toString())" + } + }, + "query": """ + process where process.name == "regsvr32.exe" + """, + "fields": [ + "@timestamp", + "day_of_week" + ] +} +---- +// TEST[setup:sec_logs] + +The API returns: + +[source,console-result] +---- +{ + "is_partial": false, + "is_running": false, + "took": 60, + "timed_out": false, + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "events": [ + { + "_index": ".ds-my-data-stream-2099.12.07-000001", + "_id": "OQmfCaduce8zoHT93o4H", + "fields": { + "@timestamp": [ + "2099-12-07T11:07:09.000Z" + ], + "day_of_week": [ + "MONDAY" + ] + } + }, + ... + ] + } +} +---- +// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/\.\.\./$body.hits.events.1/] + [discrete] [[specify-a-timestamp-or-event-category-field]] === Specify a timestamp or event category field diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 1e916ba9ce1e0..081c27e3a0319 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -428,29 +428,37 @@ Period of time used to extend the life of the PIT. <>. [[search-api-body-runtime]] +// tag::runtime-mappings-def[] `runtime_mappings`:: -(Optional, object) -Defines a <> in the search request that -exist only as part of the query. Fields defined in the search request take -precedence over fields defined with the same name in the index mappings. +(Optional, object of objects) +Defines one or more <> in the search +request. These fields take precedence over mapped fields with the same name. + .Properties of `runtime_mappings` objects [%collapsible%open] ==== + +``:: +(Required, object) +Configuration for the runtime field. The key is the field name. ++ +.Properties of `` +[%collapsible%open] +===== + `type`:: (Required, string) -<> of the object, which can be any of the following: +<>, which can be any of the following: + include::{es-ref-dir}/mapping/runtime.asciidoc[tag=runtime-data-types] `script`:: (Optional, string) -<> that is executed at query time. The +<> executed at query time. The script has access to the entire context of a document, including the original `_source` and any mapped fields plus their values. + -Your script must include `emit` to return calculated values. For -example: +This script must include `emit` to return calculated values. For example: + [source,js] ---- @@ -459,7 +467,10 @@ example: } ---- // NOTCONSOLE + +===== ==== +// end::runtime-mappings-def[] [[request-body-search-seq-no-primary-term]] `seq_no_primary_term`:: diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc index 0d95b45889211..065df80775560 100644 --- a/docs/reference/sql/endpoints/rest.asciidoc +++ b/docs/reference/sql/endpoints/rest.asciidoc @@ -522,6 +522,44 @@ POST /_sql?format=txt [IMPORTANT] The recommended way of passing values to a query is with question mark placeholders, to avoid any attempts of hacking or SQL injection. +[[sql-runtime-fields]] +=== Use runtime fields + +Use the `runtime_mappings` parameter to extract and create <>, or columns, from existing ones during a search. + +The following search create a `release_day_of_week` runtime field from +`release_date` and returns it in the response. + +[source,console] +---- +POST _sql?format=txt +{ + "runtime_mappings": { + "release_day_of_week": { + "type": "keyword", + "script": """ + emit(doc['release_date'].value.dayOfWeekEnum.toString()) + """ + } + }, + "query": """ + SELECT * FROM library WHERE page_count > 300 AND author = 'Frank Herbert' + """ +} +---- +// TEST[setup:library] + +The API returns: + +[source,txt] +---- + author | name | page_count | release_date |release_day_of_week +---------------+---------------+---------------+------------------------+------------------- +Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z|TUESDAY +---- +// TESTRESPONSE[non_json] + [[sql-rest-fields]] === Supported REST parameters @@ -530,7 +568,7 @@ the request time-outs or localization information (such as timezone). The table below lists the supported parameters: -[cols="^m,^m,^5"] +[cols="> in the search +request. These fields take precedence over mapped fields with the same name. + |=== Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <> chapter. From 3b74f247f7f5b7ae77c40e697deeca26cb99a5e0 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Thu, 8 Apr 2021 16:08:10 -0400 Subject: [PATCH 2/3] fix typos --- docs/reference/eql/eql.asciidoc | 2 +- docs/reference/sql/endpoints/rest.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc index f67b4993edb79..3f3473b6e404c 100644 --- a/docs/reference/eql/eql.asciidoc +++ b/docs/reference/eql/eql.asciidoc @@ -540,7 +540,7 @@ Use the `runtime_mappings` parameter to extract and create <> during a search. Use the `fields` parameter to include runtime fields in the response. -The following search create a `day_of_week` runtime field from the `@timestamp` +The following search creates a `day_of_week` runtime field from the `@timestamp` and returns it in the response. [source,console] diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc index 065df80775560..74e628f108cd0 100644 --- a/docs/reference/sql/endpoints/rest.asciidoc +++ b/docs/reference/sql/endpoints/rest.asciidoc @@ -528,7 +528,7 @@ The recommended way of passing values to a query is with question mark placehold Use the `runtime_mappings` parameter to extract and create <>, or columns, from existing ones during a search. -The following search create a `release_day_of_week` runtime field from +The following search creates a `release_day_of_week` runtime field from `release_date` and returns it in the response. [source,console] From 28511cc73fb861b7e28f9ee0c895fe0f7c86ee56 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Mon, 19 Apr 2021 08:57:58 -0400 Subject: [PATCH 3/3] Add xref --- docs/reference/sql/endpoints/rest.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc index 74e628f108cd0..77d9be3d29051 100644 --- a/docs/reference/sql/endpoints/rest.asciidoc +++ b/docs/reference/sql/endpoints/rest.asciidoc @@ -9,6 +9,7 @@ * <> * <> * <> +* <> * <> [[sql-rest-overview]]