EQL: Remove "fields" section from sequence in-progress searches (#74824) (#74935)

Author: elasticsearchmachine

x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml

@@ -88,6 +88,25 @@ setup:
   - match: {hits.events.2.fields.valid: [true]}
   - match: {hits.events.2.fields.day_of_week: ["Wednesday"]}
 
+---
+"Execute EQL events query with wildcard (*) fields filtering.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'process where user == "SYSTEM"'
+          fields: [{"field":"*"}]
+
+  - match: {timed_out: false}
+  - match: {hits.total.value: 3}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.0.fields:{"valid":[false],"@timestamp":["2020-02-03T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Monday"]}}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.1.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}}
+  - match: {hits.events.2._id: "3"}
+  - match: {hits.events.2.fields:{"valid":[true],"@timestamp":["2020-02-05T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Wednesday"]}}
+
 ---
 "Execute EQL events query with filter_path":
   - do:
@@ -218,6 +237,50 @@ setup:
   - match: {hits.sequences.1.events.1.fields.valid: [true]}
   - match: {hits.sequences.1.events.1.fields.day_of_week: ["Wednesday"]}
 
+---
+"Execute EQL sequence with wildcard (*) fields filtering.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: [{"field":"*"}]
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields:{"valid":[false],"@timestamp":["2020-02-03T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Monday"]}}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0.fields:{"valid":[true],"@timestamp":["2020-02-04T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1.fields:{"valid":[true],"@timestamp":["2020-02-05T12:34:56.000Z"],"event.category":["process"],"event.category.keyword":["process"],"id":[123],"user.keyword":["SYSTEM"],"user":["SYSTEM"],"day_of_week":["Wednesday"]}}
+
+---
+"Execute EQL sequence with custom format for timestamp field.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
+          fields: [{"field":"@timestamp","format":"yyyy"},{"field":"day_of_week"}]
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.0.fields:{"@timestamp":["2020"],"day_of_week":["Monday"]}}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.0.events.1.fields:{"@timestamp":["2020"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.0.fields:{"@timestamp":["2020"],"day_of_week":["Tuesday"]}}
+  - match: {hits.sequences.1.events.1._id: "3"}
+  - match: {hits.sequences.1.events.1.fields:{"@timestamp":["2020"],"day_of_week":["Wednesday"]}}
+
 ---
 "Execute EQL sequence with filter_path":
   - do:

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java

@@ -92,7 +92,7 @@ public Executable assemble(List<List<Attribute>> listOfKeys,
             PhysicalPlan query = plans.get(i);
             // search query
             if (query instanceof EsQueryExec) {
-                SearchSourceBuilder source = ((EsQueryExec) query).source(session);
+                SearchSourceBuilder source = ((EsQueryExec) query).source(session, false);
                 QueryRequest original = () -> source;
                 BoxedQueryRequest boxedRequest = new BoxedQueryRequest(original, timestampName, keyFields);
                 Criterion<BoxedQueryRequest> criterion =

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java

@@ -52,16 +52,22 @@ public List<Attribute> output() {
         return output;
     }
 
-    public SearchSourceBuilder source(EqlSession session) {
+    
+    /*
+     * {@param includeFetchFields} should be true for event queries and false for in progress sequence queries
+     * Fetching fields during in progress sequence queries is unnecessary.
+     */
+    public SearchSourceBuilder source(EqlSession session, boolean includeFetchFields) {
         EqlConfiguration cfg = session.configuration();
         // by default use the configuration size
-        return SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), cfg.fetchFields(), cfg.runtimeMappings());
+        return SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), includeFetchFields ? cfg.fetchFields() : null,
+            cfg.runtimeMappings());
     }
 
     @Override
     public void execute(EqlSession session, ActionListener<Payload> listener) {
         // endpoint - fetch all source
-        QueryRequest request = () -> source(session).fetchSource(FetchSourceContext.FETCH_SOURCE);
+        QueryRequest request = () -> source(session, true).fetchSource(FetchSourceContext.FETCH_SOURCE);
         listener = shouldReverse(request) ? new ReverseListener(listener) : listener;
         new BasicQueryClient(session).query(request, new AsEventListener(listener));
     }