Adds runtime_mappings to EQL and SQL requests

client/rest-high-level/build.gradle

@@ -54,6 +54,7 @@ dependencies {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations'
   }
   testImplementation(project(':x-pack:plugin:eql'))
+  testImplementation(project(':x-pack:plugin:ql:test-fixtures'))
 }
 
 tasks.named('forbiddenApisMain').configure {

client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java

@@ -20,8 +20,11 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Collections.emptyMap;
+
 public class EqlSearchRequest implements Validatable, ToXContentObject {
 
     private String[] indices;
@@ -32,6 +35,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
     private String eventCategoryField = "event.category";
     private String resultPosition = "tail";
     private List<FieldAndFormat> fetchFields;
+    private Map<String, Object> runtimeMappings = emptyMap();
 
     private int size = 10;
     private int fetchSize = 1000;
@@ -55,6 +59,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
     static final String KEY_KEEP_ALIVE = "keep_alive";
     static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion";
     static final String KEY_FETCH_FIELDS = "fields";
+    static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings";
 
     public EqlSearchRequest(String indices, String query) {
         indices(indices);
@@ -87,6 +92,9 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         if (fetchFields != null) {
             builder.field(KEY_FETCH_FIELDS, fetchFields);
         }
+        if (runtimeMappings != null && runtimeMappings.isEmpty() == false) {
+            builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings);
+        }
         builder.endObject();
         return builder;
     }
@@ -161,6 +169,15 @@ public EqlSearchRequest fetchFields(List<FieldAndFormat> fetchFields) {
         return this;
     }
 
+    public Map<String, Object> runtimeMappings() {
+        return runtimeMappings;
+    }
+
+    public EqlSearchRequest runtimeMappings(Map<String, Object> runtimeMappings) {
+        this.runtimeMappings = runtimeMappings;
+        return this;
+    }
+
     public int size() {
         return this.size;
     }
@@ -243,7 +260,8 @@ public boolean equals(Object o) {
             Objects.equals(keepAlive, that.keepAlive) &&
             Objects.equals(keepOnCompletion, that.keepOnCompletion) &&
             Objects.equals(resultPosition, that.resultPosition) &&
-            Objects.equals(fetchFields, that.fetchFields);
+            Objects.equals(fetchFields, that.fetchFields) &&
+            Objects.equals(runtimeMappings, that.runtimeMappings);
     }
 
     @Override
@@ -262,7 +280,8 @@ public int hashCode() {
             keepAlive,
             keepOnCompletion,
             resultPosition,
-            fetchFields);
+            fetchFields,
+            runtimeMappings);
     }
 
     public String[] indices() {

client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java

@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.xpack.ql.TestUtils.randomRuntimeMappings;
 import static org.hamcrest.Matchers.equalTo;
 
 public class EqlSearchRequestTests extends AbstractRequestTestCase<EqlSearchRequest, org.elasticsearch.xpack.eql.action.EqlSearchRequest> {
@@ -50,6 +51,9 @@ protected EqlSearchRequest createClientTestInstance() {
                 eqlSearchRequest.filter(QueryBuilders.termQuery(randomAlphaOfLength(10), randomInt(100)));
             }
         }
+        if (randomBoolean()) {
+            eqlSearchRequest.runtimeMappings(randomRuntimeMappings());
+        }
         return eqlSearchRequest;
     }
 
@@ -70,6 +74,7 @@ protected void assertInstances(org.elasticsearch.xpack.eql.action.EqlSearchReque
         assertThat(serverInstance.indices(), equalTo(clientTestInstance.indices()));
         assertThat(serverInstance.fetchSize(), equalTo(clientTestInstance.fetchSize()));
         assertThat(serverInstance.size(), equalTo(clientTestInstance.size()));
+        assertThat(serverInstance.runtimeMappings(), equalTo(clientTestInstance.runtimeMappings()));
     }
 
     @Override

x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml

@@ -0,0 +1,192 @@
+---
+setup:
+  - do:
+      indices.create:
+          index:  eql_test
+          body:
+            mappings:
+              runtime:
+                day_of_week:
+                  type: keyword
+                  script:
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: eql_test
+              _id:    1
+          - event:
+              - category: process
+            "@timestamp": 2020-02-03T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: false
+            raw_message: "199.72.81.55 - - [01/Jul/1995:00:00:01 -0400] GET /history/apollo/ HTTP/1.0 200 6245"
+          - index:
+              _index: eql_test
+              _id:    2
+          - event:
+              - category: process
+            "@timestamp": 2020-02-04T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
+            raw_message: "199.72.81.123 - - [01/Jul/1995:00:00:02 -0400] GET /history/apollo/a HTTP/1.0 200 500"
+          - index:
+              _index: eql_test
+              _id:    3
+          - event:
+              - category: process
+            "@timestamp": 2020-02-05T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
+            raw_message: "199.72.81.34 - - [01/Jul/1995:00:00:03 -0400] GET /history/apollo/b HTTP/1.0 200 1500"
+          - index:
+              _index: eql_test
+              _id:    4
+          - event:
+              - category: process
+            "@timestamp": 2020-02-05T12:34:57Z
+            user: SYSTEM
+            id: 123
+
+
+---
+"Execute EQL events query with search time keyword runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where is_valid=="YES"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","is_valid"]
+          runtime_mappings: {"is_valid": {"type":"keyword","script":"if (doc['valid'].size()==0 || doc['valid'].value == false) emit('NO'); else emit('YES')"}}
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._id: "2"}
+  - match: {hits.events.0.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.valid: [true]}
+  - match: {hits.events.0.fields.is_valid: ["YES"]}
+  - match: {hits.events.1._id: "3"}
+  - match: {hits.events.1.fields.@timestamp: ["1580906096000"]}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.valid: [true]}
+  - match: {hits.events.1.fields.is_valid: ["YES"]}
+
+---
+"Execute EQL events query with search time ip runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        filter_path: "hits.events._source.raw_message,hits.events.fields.address,hits.events._id"
+        body:
+          query: 'process where true'
+          fields: ["address"]
+          runtime_mappings: {"address": {"type": "ip","script": "if (doc[\"raw_message.keyword\"].size() == 0) return; else {Matcher m = /\\d+\\.\\d+\\.\\d+\\.\\d+/.matcher(doc[\"raw_message.keyword\"].value);if (m.find()) emit(m.group());}"}}
+
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields.address: ["199.72.81.55"]}
+  - match: {hits.events.0._source.raw_message: "199.72.81.55 - - [01/Jul/1995:00:00:01 -0400] GET /history/apollo/ HTTP/1.0 200 6245"}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1.fields.address: ["199.72.81.123"]}
+  - match: {hits.events.1._source.raw_message: "199.72.81.123 - - [01/Jul/1995:00:00:02 -0400] GET /history/apollo/a HTTP/1.0 200 500"}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2.fields.address: ["199.72.81.34"]}
+  - match: {hits.events.2._source.raw_message: "199.72.81.34 - - [01/Jul/1995:00:00:03 -0400] GET /history/apollo/b HTTP/1.0 200 1500"}
+  - match: {hits.events.3._id: "4"}
+  - match: {hits.events.3.fields.address: null}
+  - match: {hits.events.3._source.raw_message: null}
+
+---
+"Execute EQL events query with search time runtime field overriding mapping level runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: ["id","day_of_week"]
+          runtime_mappings: {"day_of_week": {"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 4}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._source.valid: false}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.day_of_week: [1]}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1._source.valid: true}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.day_of_week: [2]}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2._source.valid: true}
+  - match: {hits.events.2.fields.id: [123]}
+  - match: {hits.events.2.fields.day_of_week: [3]}
+  - match: {hits.events.3._id: "4"}
+  - match: {hits.events.3.fields.id: [123]}
+  - match: {hits.events.3.fields.day_of_week: [3]}
+
+---
+"Execute EQL sequence with search time runtime fields overriding mapping level runtime field":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true] [process where day_of_week == 3]'
+          fields: ["day_of_week"]
+          runtime_mappings: {"day_of_week": {"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0._source.@timestamp: "2020-02-03T12:34:56Z"}
+  - match: {hits.sequences.0.events.0.fields.day_of_week: [1]}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1._source.@timestamp: "2020-02-04T12:34:56Z"}
+  - match: {hits.sequences.0.events.1.fields.day_of_week: [2]}
+  - match: {hits.sequences.0.events.2._id: "3"}
+  - match: {hits.sequences.0.events.2._source.@timestamp: "2020-02-05T12:34:56Z"}
+  - match: {hits.sequences.0.events.2.fields.day_of_week: [3]}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0._source.@timestamp: "2020-02-04T12:34:56Z"}
+  - match: {hits.sequences.1.events.0.fields.day_of_week: [2]}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1._source.@timestamp: "2020-02-05T12:34:56Z"}
+  - match: {hits.sequences.1.events.1.fields.day_of_week: [3]}
+  - match: {hits.sequences.1.events.2._id: "4"}
+  - match: {hits.sequences.1.events.2._source.@timestamp: "2020-02-05T12:34:57Z"}
+  - match: {hits.sequences.1.events.2.fields.day_of_week: [3]}
+
+---
+"Validate valid runtime mappings request":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: ["id","day_of_week"]
+          runtime_mappings: {"day_of_week": {"script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}}
+      catch: bad_request
+  - match: { error.root_cause.0.type: "action_request_validation_exception" }
+  - match: { error.root_cause.0.reason: "Validation Failed: 1: No type specified for runtime field [day_of_week];" }
+
+
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: ["id","day_of_week"]
+          runtime_mappings: {"day_of_week": [{"type":"long","script":"emit(doc['@timestamp'].value.dayOfWeekEnum.getValue())"}]}
+      catch: bad_request
+  - match: { error.root_cause.0.type: "action_request_validation_exception" }
+  - match: { error.root_cause.0.reason: "Validation Failed: 1: Expected map for runtime field [day_of_week] definition but got [String];" }