Skip to content

Commit e6c4c08

Browse files
astefanastefan
authored
Adds runtime_mappings to EQL and SQL requests (#71356) (#71432)
* Adds `runtime_mappings` to EQL and SQL requests allowing users to define search time runtime fields which will be used in queries (cherry picked from commit b8266e5)
1 parent 9d2f67c commit e6c4c08

File tree

60 files changed

+1086
-446
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1086
-446
lines changed

‎client/rest-high-level/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ dependencies {
5151
exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations'
5252
}
5353
testImplementation(project(':x-pack:plugin:eql'))
54+
testImplementation(project(':x-pack:plugin:ql:test-fixtures'))
5455
}
5556

5657
tasks.named('forbiddenApisMain').configure {

‎client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java

+21-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,11 @@
2020
import java.io.IOException;
2121
import java.util.Arrays;
2222
import java.util.List;
23+
import java.util.Map;
2324
import java.util.Objects;
2425

26+
import static java.util.Collections.emptyMap;
27+
2528
public class EqlSearchRequest implements Validatable, ToXContentObject {
2629

2730
private String[] indices;
@@ -32,6 +35,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
3235
private String eventCategoryField = "event.category";
3336
private String resultPosition = "tail";
3437
private List<FieldAndFormat> fetchFields;
38+
private Map<String, Object> runtimeMappings = emptyMap();
3539

3640
private int size = 10;
3741
private int fetchSize = 1000;
@@ -55,6 +59,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
5559
static final String KEY_KEEP_ALIVE = "keep_alive";
5660
static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion";
5761
static final String KEY_FETCH_FIELDS = "fields";
62+
static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings";
5863

5964
public EqlSearchRequest(String indices, String query) {
6065
indices(indices);
@@ -87,6 +92,9 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
8792
if (fetchFields != null) {
8893
builder.field(KEY_FETCH_FIELDS, fetchFields);
8994
}
95+
if (runtimeMappings != null && runtimeMappings.isEmpty() == false) {
96+
builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings);
97+
}
9098
builder.endObject();
9199
return builder;
92100
}
@@ -161,6 +169,15 @@ public EqlSearchRequest fetchFields(List<FieldAndFormat> fetchFields) {
161169
return this;
162170
}
163171

172+
public Map<String, Object> runtimeMappings() {
173+
return runtimeMappings;
174+
}
175+
176+
public EqlSearchRequest runtimeMappings(Map<String, Object> runtimeMappings) {
177+
this.runtimeMappings = runtimeMappings;
178+
return this;
179+
}
180+
164181
public int size() {
165182
return this.size;
166183
}
@@ -243,7 +260,8 @@ public boolean equals(Object o) {
243260
Objects.equals(keepAlive, that.keepAlive) &&
244261
Objects.equals(keepOnCompletion, that.keepOnCompletion) &&
245262
Objects.equals(resultPosition, that.resultPosition) &&
246-
Objects.equals(fetchFields, that.fetchFields);
263+
Objects.equals(fetchFields, that.fetchFields) &&
264+
Objects.equals(runtimeMappings, that.runtimeMappings);
247265
}
248266

249267
@Override
@@ -262,7 +280,8 @@ public int hashCode() {
262280
keepAlive,
263281
keepOnCompletion,
264282
resultPosition,
265-
fetchFields);
283+
fetchFields,
284+
runtimeMappings);
266285
}
267286

268287
public String[] indices() {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java

+5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import java.io.IOException;
1919
import java.util.Collections;
2020

21+
import static org.elasticsearch.xpack.ql.TestUtils.randomRuntimeMappings;
2122
import static org.hamcrest.Matchers.equalTo;
2223

2324
public class EqlSearchRequestTests extends AbstractRequestTestCase<EqlSearchRequest, org.elasticsearch.xpack.eql.action.EqlSearchRequest> {
@@ -50,6 +51,9 @@ protected EqlSearchRequest createClientTestInstance() {
5051
eqlSearchRequest.filter(QueryBuilders.termQuery(randomAlphaOfLength(10), randomInt(100)));
5152
}
5253
}
54+
if (randomBoolean()) {
55+
eqlSearchRequest.runtimeMappings(randomRuntimeMappings());
56+
}
5357
return eqlSearchRequest;
5458
}
5559

@@ -70,6 +74,7 @@ protected void assertInstances(org.elasticsearch.xpack.eql.action.EqlSearchReque
7074
assertThat(serverInstance.indices(), equalTo(clientTestInstance.indices()));
7175
assertThat(serverInstance.fetchSize(), equalTo(clientTestInstance.fetchSize()));
7276
assertThat(serverInstance.size(), equalTo(clientTestInstance.size()));
77+
assertThat(serverInstance.runtimeMappings(), equalTo(clientTestInstance.runtimeMappings()));
7378
}
7479

7580
@Override

‎x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml

+192
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
---
2+
setup:
3+
- do:
4+
indices.create:
5+
index: eql_test
6+
body:
7+
mappings:
8+
runtime:
9+
day_of_week:
10+
type: keyword
11+
script:
12+
source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
13+
- do:
14+
bulk:
15+
refresh: true
16+
body:
17+
- index:
18+
_index: eql_test
19+
_id: 1
20+
- event:
21+
- category: process
22+
"@timestamp": 2020-02-03T12:34:56Z
23+
user: SYSTEM
24+
id: 123
25+
valid: false
26+
raw_message: "199.72.81.55 - - [01/Jul/1995:00:00:01 -0400] GET /history/apollo/ HTTP/1.0 200 6245"
27+
- index:
28+
_index: eql_test
29+
_id: 2
30+
- event:
31+
- category: process
32+
"@timestamp": 2020-02-04T12:34:56Z
33+
user: SYSTEM
34+
id: 123
35+
valid: true
36+
raw_message: "199.72.81.123 - - [01/Jul/1995:00:00:02 -0400] GET /history/apollo/a HTTP/1.0 200 500"
37+
- index:
38+
_index: eql_test
39+
_id: 3
40+
- event:
41+
- category: process
42+
"@timestamp": 2020-02-05T12:34:56Z
43+
user: SYSTEM
44+
id: 123
45+
valid: true
46+
raw_message: "199.72.81.34 - - [01/Jul/1995:00:00:03 -0400] GET /history/apollo/b HTTP/1.0 200 1500"
47+
- index:
48+
_index: eql_test
49+
_id: 4
50+
- event:
51+
- category: process
52+
"@timestamp": 2020-02-05T12:34:57Z
53+
user: SYSTEM
54+
id: 123
55+
56+
57+
---
58+
"Execute EQL events query with search time keyword runtime field":
59+
- do:
60+
eql.search:
61+
index: eql_test
62+
body:
63+
query: 'process where is_valid=="YES"'
64+
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","is_valid"]
65+
runtime_mappings: {"is_valid": {"type":"keyword","script":"if (doc['valid'].size()==0 || doc['valid'].value == false) emit('NO'); else emit('YES')"}}
66+
67+
- match: {timed_out: false}
68+
- match: {hits.total.value: 2}
69+
- match: {hits.total.relation: "eq"}
70+
- match: {hits.events.0._id: "2"}
71+
- match: {hits.events.0.fields.@timestamp: ["1580819696000"]}
72+
- match: {hits.events.0.fields.id: [123]}
73+
- match: {hits.events.0.fields.valid: [true]}
74+
- match: {hits.events.0.fields.is_valid: ["YES"]}
75+
- match: {hits.events.1._id: "3"}
76+
- match: {hits.events.1.fields.@timestamp: ["1580906096000"]}
77+
- match: {hits.events.1.fields.id: [123]}
78+
- match: {hits.events.1.fields.valid: [true]}
79+
- match: {hits.events.1.fields.is_valid: ["YES"]}
80+
81+
---
82+
"Execute EQL events query with search time ip runtime field":
83+
- do:
84+
eql.search:
85+
index: eql_test
86+
filter_path: "hits.events._source.raw_message,hits.events.fields.address,hits.events._id"
87+
body:
88+
query: 'process where true'
89+
fields: ["address"]
90+
runtime_mappings: {"address": {"type": "ip","script": "if (doc[\"raw_message.keyword\"].size() == 0) return; else {Matcher m = /\\d+\\.\\d+\\.\\d+\\.\\d+/.matcher(doc[\"raw_message.keyword\"].value);if (m.find()) emit(m.group());}"}}
91+
92+
- match: {hits.events.0._id: "1"}
93+
- match: {hits.events.0.fields.address: ["199.72.81.55"]}
94+
- match: {hits.events.0._source.raw_message: "199.72.81.55 - - [01/Jul/1995:00:00:01 -0400] GET /history/apollo/ HTTP/1.0 200 6245"}
95+
- match: {hits.events.1._id: "2"}
96+
- match: {hits.events.1.fields.address: ["199.72.81.123"]}
97+
- match: {hits.events.1._source.raw_message: "199.72.81.123 - - [01/Jul/1995:00:00:02 -0400] GET /history/apollo/a HTTP/1.0 200 500"}
98+
- match: {hits.events.2._id: "3"}
99+
- match: {hits.events.2.fields.address: ["199.72.81.34"]}
100+
- match: {hits.events.2._source.raw_message: "199.72.81.34 - - [01/Jul/1995:00:00:03 -0400] GET /history/apollo/b HTTP/1.0 200 1500"}
101+
- match: {hits.events.3._id: "4"}
102+
- match: {hits.events.3.fields.address: null}
103+
- match: {hits.events.3._source.raw_message: null}
104+
105+
---
106+
"Execute EQL events query with search time runtime field overriding mapping level runtime field":
107+
- do:
108+
eql.search:
109+
index: eql_test
110+
body:
111+
query: 'process where user == "SYSTEM"'
112+
fields: ["id","day_of_week"]
113+
runtime_mappings: {"day_of_week": {"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
114+
115+
- match: {timed_out: false}
116+
- match: {hits.total.value: 4}
117+
- match: {hits.total.relation: "eq"}
118+
- match: {hits.events.0._id: "1"}
119+
- match: {hits.events.0._source.user: "SYSTEM"}
120+
- match: {hits.events.0._source.valid: false}
121+
- match: {hits.events.0.fields.id: [123]}
122+
- match: {hits.events.0.fields.day_of_week: [1]}
123+
- match: {hits.events.1._id: "2"}
124+
- match: {hits.events.1._source.valid: true}
125+
- match: {hits.events.1.fields.id: [123]}
126+
- match: {hits.events.1.fields.day_of_week: [2]}
127+
- match: {hits.events.2._id: "3"}
128+
- match: {hits.events.2._source.valid: true}
129+
- match: {hits.events.2.fields.id: [123]}
130+
- match: {hits.events.2.fields.day_of_week: [3]}
131+
- match: {hits.events.3._id: "4"}
132+
- match: {hits.events.3.fields.id: [123]}
133+
- match: {hits.events.3.fields.day_of_week: [3]}
134+
135+
---
136+
"Execute EQL sequence with search time runtime fields overriding mapping level runtime field":
137+
- do:
138+
eql.search:
139+
index: eql_test
140+
body:
141+
query: 'sequence by user [process where user == "SYSTEM"] [process where true] [process where day_of_week == 3]'
142+
fields: ["day_of_week"]
143+
runtime_mappings: {"day_of_week": {"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
144+
145+
- match: {timed_out: false}
146+
- match: {hits.total.value: 2}
147+
- match: {hits.total.relation: "eq"}
148+
- match: {hits.sequences.0.join_keys.0: "SYSTEM"}
149+
- match: {hits.sequences.0.events.0._id: "1"}
150+
- match: {hits.sequences.0.events.0._source.@timestamp: "2020-02-03T12:34:56Z"}
151+
- match: {hits.sequences.0.events.0.fields.day_of_week: [1]}
152+
- match: {hits.sequences.0.events.1._id: "2"}
153+
- match: {hits.sequences.0.events.1._source.@timestamp: "2020-02-04T12:34:56Z"}
154+
- match: {hits.sequences.0.events.1.fields.day_of_week: [2]}
155+
- match: {hits.sequences.0.events.2._id: "3"}
156+
- match: {hits.sequences.0.events.2._source.@timestamp: "2020-02-05T12:34:56Z"}
157+
- match: {hits.sequences.0.events.2.fields.day_of_week: [3]}
158+
- match: {hits.sequences.1.join_keys.0: "SYSTEM"}
159+
- match: {hits.sequences.1.events.0._id: "2"}
160+
- match: {hits.sequences.1.events.0._source.@timestamp: "2020-02-04T12:34:56Z"}
161+
- match: {hits.sequences.1.events.0.fields.day_of_week: [2]}
162+
- match: {hits.sequences.1.events.1._id: "3"}
163+
- match: {hits.sequences.1.events.1._source.@timestamp: "2020-02-05T12:34:56Z"}
164+
- match: {hits.sequences.1.events.1.fields.day_of_week: [3]}
165+
- match: {hits.sequences.1.events.2._id: "4"}
166+
- match: {hits.sequences.1.events.2._source.@timestamp: "2020-02-05T12:34:57Z"}
167+
- match: {hits.sequences.1.events.2.fields.day_of_week: [3]}
168+
169+
---
170+
"Validate valid runtime mappings request":
171+
- do:
172+
eql.search:
173+
index: eql_test
174+
body:
175+
query: 'process where user == "SYSTEM"'
176+
fields: ["id","day_of_week"]
177+
runtime_mappings: {"day_of_week": {"script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
178+
catch: bad_request
179+
- match: { error.root_cause.0.type: "action_request_validation_exception" }
180+
- match: { error.root_cause.0.reason: "Validation Failed: 1: No type specified for runtime field [day_of_week];" }
181+
182+
183+
- do:
184+
eql.search:
185+
index: eql_test
186+
body:
187+
query: 'process where user == "SYSTEM"'
188+
fields: ["id","day_of_week"]
189+
runtime_mappings: {"day_of_week": [{"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}]}
190+
catch: bad_request
191+
- match: { error.root_cause.0.type: "action_request_validation_exception" }
192+
- match: { error.root_cause.0.reason: "Validation Failed: 1: Expected map for runtime field [day_of_week] definition but got [String];" }

0 commit comments

Comments
 (0)