EQL: Adds "fields" request field to the eql request (elastic#68962) (elastic#69447)

(cherry picked from commit 22c880e)

Author: astefan

client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java

@@ -15,9 +15,11 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Objects;
 
 public class EqlSearchRequest implements Validatable, ToXContentObject {
@@ -29,6 +31,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
     private String timestampField = "@timestamp";
     private String eventCategoryField = "event.category";
     private String resultPosition = "tail";
+    private List<FieldAndFormat> fetchFields;
 
     private int size = 10;
     private int fetchSize = 1000;
@@ -51,6 +54,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
     static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout";
     static final String KEY_KEEP_ALIVE = "keep_alive";
     static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion";
+    static final String KEY_FETCH_FIELDS = "fields";
 
     public EqlSearchRequest(String indices, String query) {
         indices(indices);
@@ -80,6 +84,9 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
             builder.field(KEY_KEEP_ALIVE, keepAlive);
         }
         builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion);
+        if (fetchFields != null) {
+            builder.field(KEY_FETCH_FIELDS, fetchFields);
+        }
         builder.endObject();
         return builder;
     }
@@ -145,6 +152,15 @@ public EqlSearchRequest resultPosition(String position) {
         return this;
     }
 
+    public List<FieldAndFormat> fetchFields() {
+        return fetchFields;
+    }
+
+    public EqlSearchRequest fetchFields(List<FieldAndFormat> fetchFields) {
+        this.fetchFields = fetchFields;
+        return this;
+    }
+
     public int size() {
         return this.size;
     }
@@ -226,7 +242,8 @@ public boolean equals(Object o) {
             Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) &&
             Objects.equals(keepAlive, that.keepAlive) &&
             Objects.equals(keepOnCompletion, that.keepOnCompletion) &&
-            Objects.equals(resultPosition, that.resultPosition);
+            Objects.equals(resultPosition, that.resultPosition) &&
+            Objects.equals(fetchFields, that.fetchFields);
     }
 
     @Override
@@ -244,7 +261,8 @@ public int hashCode() {
             waitForCompletionTimeout,
             keepAlive,
             keepOnCompletion,
-            resultPosition);
+            resultPosition,
+            fetchFields);
     }
 
     public String[] indices() {

client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java

@@ -11,17 +11,23 @@
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.client.AbstractResponseTestCase;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.function.Supplier;
 
@@ -83,7 +89,16 @@ static List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event> randomEv
             hits = new ArrayList<>();
             for (int i = 0; i < size; i++) {
                 BytesReference bytes = new RandomSource(() -> randomAlphaOfLength(10)).toBytes(xType);
-                hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes));
+                Map<String, DocumentField> fetchFields = new HashMap<>();
+                int fieldsCount = randomIntBetween(0, 5);
+                for (int j = 0; j < fieldsCount; j++) {
+                    fetchFields.put(randomAlphaOfLength(10), randomDocumentField(xType).v1());
+                }
+                if (fetchFields.isEmpty() && randomBoolean()) {
+                    fetchFields = null;
+                }
+                hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes,
+                    fetchFields));
             }
         }
         if (randomBoolean()) {
@@ -92,6 +107,28 @@ static List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event> randomEv
         return hits;
     }
 
+    private static Tuple<DocumentField, DocumentField> randomDocumentField(XContentType xType) {
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                String fieldName = randomAlphaOfLengthBetween(3, 10);
+                Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xType);
+                DocumentField input = new DocumentField(fieldName, tuple.v1());
+                DocumentField expected = new DocumentField(fieldName, tuple.v2());
+                return Tuple.tuple(input, expected);
+            case 1:
+                List<Object> listValues = randomList(1, 5, () -> randomList(1, 5, ESTestCase::randomInt));
+                DocumentField listField = new DocumentField(randomAlphaOfLength(5), listValues);
+                return Tuple.tuple(listField, listField);
+            case 2:
+                List<Object> objectValues = randomList(1, 5,
+                    randomFrom(Arrays.asList(() -> randomAlphaOfLength(10), ESTestCase::randomInt, ESTestCase::randomBoolean)));
+                DocumentField objectField = new DocumentField(randomAlphaOfLength(5), objectValues);
+                return Tuple.tuple(objectField, objectField);
+            default:
+                throw new IllegalStateException();
+        }
+    }
+
     public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomEventsResponse(TotalHits totalHits, XContentType xType) {
         org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits hits = null;
         if (randomBoolean()) {

x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml

@@ -1,5 +1,15 @@
 ---
 setup:
+  - do:
+      indices.create:
+          index:  eql_test
+          body:
+            mappings:
+              runtime:
+                day_of_week:
+                  type: keyword
+                  script:
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
   - do:
       bulk:
         refresh: true
@@ -49,6 +59,55 @@ setup:
   - match: {hits.events.1._id: "2"}
   - match: {hits.events.2._id: "3"}
 
+---
+"Execute EQL events query with fields filtering":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 3}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.valid: [false]}
+  - match: {hits.events.0.fields.day_of_week: ["Monday"]}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.valid: [true]}
+  - match: {hits.events.1.fields.day_of_week: ["Tuesday"]}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2.fields.@timestamp: ["1580906096000"]}
+  - match: {hits.events.2.fields.id: [123]}
+  - match: {hits.events.2.fields.valid: [true]}
+  - match: {hits.events.2.fields.day_of_week: ["Wednesday"]}
+
+---
+"Execute EQL events query with filter_path":
+  - do:
+      eql.search:
+        index: eql_test
+        filter_path: "hits.events._source.event.category,hits.events.fields.user,hits.events.fields.id"
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","user"]
+
+  - match: {hits.events.0._source.event.0.category: "process"}
+  - match: {hits.events.0.fields.id: [123]}
+  - match: {hits.events.0.fields.user: ["SYSTEM"]}
+  - match: {hits.events.1._source.event.0.category: "process"}
+  - match: {hits.events.1.fields.id: [123]}
+  - match: {hits.events.1.fields.user: ["SYSTEM"]}
+  - match: {hits.events.2._source.event.0.category: "process"}
+  - match: {hits.events.2.fields.id: [123]}
+  - match: {hits.events.2.fields.user: ["SYSTEM"]}
+
 ---
 "Execute EQL sequence with string key.":
   - do:
@@ -124,6 +183,57 @@ setup:
   - match: {hits.sequences.0.join_keys.0: true}
   - match: {hits.sequences.0.events.0._id: "2"}
   - match: {hits.sequences.0.events.1._id: "3"}
+
+---
+"Execute EQL sequence with fields filtering.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]}
+  - match: {hits.sequences.0.events.0.fields.id: [123]}
+  - match: {hits.sequences.0.events.0.fields.valid: [false]}
+  - match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.sequences.0.events.1.fields.id: [123]}
+  - match: {hits.sequences.0.events.1.fields.valid: [true]}
+  - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0.fields.@timestamp: ["1580819696000"]}
+  - match: {hits.sequences.1.events.0.fields.id: [123]}
+  - match: {hits.sequences.1.events.0.fields.valid: [true]}
+  - match: {hits.sequences.1.events.0.fields.day_of_week: ["Tuesday"]}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1.fields.@timestamp: ["1580906096000"]}
+  - match: {hits.sequences.1.events.1.fields.id: [123]}
+  - match: {hits.sequences.1.events.1.fields.valid: [true]}
+  - match: {hits.sequences.1.events.1.fields.day_of_week: ["Wednesday"]}
+
+---
+"Execute EQL sequence with filter_path":
+  - do:
+      eql.search:
+        index: eql_test
+        filter_path: "hits.sequences.join_keys,hits.sequences.events.fields.valid"
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: ["id","valid"]
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0.fields.valid: [false]}
+  - match: {hits.sequences.0.events.1.fields.valid: [true]}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0.fields.valid: [true]}
+  - match: {hits.sequences.1.events.1.fields.valid: [true]}
+
 ---
 "Execute some EQL in async mode.":
   - do: