EQL: Add implicit ordering on timestamp

x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml

@@ -8,6 +8,7 @@ setup:
               _index: eql_test
               _id:    1
           - event_type: process
+            timestamp: 2020-02-03T12:34:56Z
             user: SYSTEM
 
 ---

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java

@@ -10,7 +10,7 @@ public final class RequestDefaults {
 
     private RequestDefaults() {}
 
-    public static final String FIELD_TIMESTAMP = "@timestamp";
+    public static final String FIELD_TIMESTAMP = "timestamp";
     public static final String FIELD_EVENT_TYPE = "event_type";
     public static final String IMPLICIT_JOIN_KEY = "agent.id";
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java

@@ -7,21 +7,26 @@
 
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.NestedSortBuilder;
+import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
+import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
+import org.elasticsearch.xpack.ql.expression.Attribute;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.querydsl.container.AttributeSort;
+import org.elasticsearch.xpack.ql.querydsl.container.ScriptSort;
+import org.elasticsearch.xpack.ql.querydsl.container.Sort;
 
-import java.util.List;
-
-import static java.util.Collections.singletonList;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
+import static org.elasticsearch.search.sort.SortBuilders.scriptSort;
 
 public abstract class SourceGenerator {
 
     private SourceGenerator() {}
 
-    private static final List<String> NO_STORED_FIELD = singletonList(StoredFieldsContext._NONE_);
-
     public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryBuilder filter, Integer size) {
         QueryBuilder finalQuery = null;
         // add the source
@@ -38,8 +43,9 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
         }
 
         final SearchSourceBuilder source = new SearchSourceBuilder();
-        source.query(finalQuery);
 
+        source.query(finalQuery);
+        sorting(container, source);
         source.fetchSource(FetchSourceContext.FETCH_SOURCE);
 
         // set fetch size
@@ -54,6 +60,60 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
         return source;
     }
 
+    private static void sorting(QueryContainer container, SearchSourceBuilder source) {
+        for (Sort sortable : container.sort().values()) {
+            SortBuilder<?> sortBuilder = null;
+
+            if (sortable instanceof AttributeSort) {
+                AttributeSort as = (AttributeSort) sortable;
+                Attribute attr = as.attribute();
+
+                // sorting only works on not-analyzed fields - look for a multi-field replacement
+                if (attr instanceof FieldAttribute) {
+                    FieldAttribute fa = ((FieldAttribute) attr).exactAttribute();
+
+                    sortBuilder = fieldSort(fa.name())
+                            .missing(as.missing().position())
+                            .unmappedType(fa.dataType().esType());
+
+                    if (fa.isNested()) {
+                        FieldSortBuilder fieldSort = fieldSort(fa.name())
+                                .missing(as.missing().position())
+                                .unmappedType(fa.dataType().esType());
+
+                        NestedSortBuilder newSort = new NestedSortBuilder(fa.nestedParent().name());
+                        NestedSortBuilder nestedSort = fieldSort.getNestedSort();
+
+                        if (nestedSort == null) {
+                            fieldSort.setNestedSort(newSort);
+                        } else {
+                            while (nestedSort.getNestedSort() != null) {
+                                nestedSort = nestedSort.getNestedSort();
+                            }
+                            nestedSort.setNestedSort(newSort);
+                        }
+
+                        nestedSort = newSort;
+
+                        if (container.query() != null) {
+                            container.query().enrichNestedSort(nestedSort);
+                        }
+                        sortBuilder = fieldSort;
+                    }
+                }
+            } else if (sortable instanceof ScriptSort) {
+                ScriptSort ss = (ScriptSort) sortable;
+                sortBuilder = scriptSort(ss.script().toPainless(),
+                        ss.script().outputType().isNumeric() ? ScriptSortType.NUMBER : ScriptSortType.STRING);
+            }
+
+            if (sortBuilder != null) {
+                sortBuilder.order(sortable.direction().asOrder());
+                source.sort(sortBuilder);
+            }
+        }
+    }
+
     private static void optimize(QueryContainer query, SearchSourceBuilder builder) {
         if (query.shouldTrackHits()) {
             builder.trackTotalHits(true);

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java

@@ -7,15 +7,19 @@
 
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.Order;
 import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
 import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
 import org.elasticsearch.xpack.ql.tree.Source;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
+import static java.util.Collections.singletonList;
+
 public abstract class LogicalPlanBuilder extends ExpressionBuilder {
 
     private final ParserParams params;
@@ -40,6 +44,11 @@ public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) {
             condition = new And(source, eventMatch, condition);
         }
 
-        return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);
+        Filter filter = new Filter(source, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);
+        // add implicit sorting - when pipes are added, this would better seat there (as a default pipe)
+        Order order = new Order(source, new UnresolvedAttribute(source, params.fieldTimestamp()), Order.OrderDirection.ASC,
+                Order.NullsPosition.FIRST);
+        OrderBy orderBy = new OrderBy(source, filter, singletonList(order));
+        return orderBy;
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java

@@ -6,12 +6,21 @@
 
 package org.elasticsearch.xpack.eql.planner;
 
+import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
 import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.eql.plan.physical.FilterExec;
+import org.elasticsearch.xpack.eql.plan.physical.OrderExec;
 import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
 import org.elasticsearch.xpack.ql.expression.Attribute;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.Order;
 import org.elasticsearch.xpack.ql.planner.ExpressionTranslators;
+import org.elasticsearch.xpack.ql.querydsl.container.AttributeSort;
+import org.elasticsearch.xpack.ql.querydsl.container.Sort.Direction;
+import org.elasticsearch.xpack.ql.querydsl.container.Sort.Missing;
 import org.elasticsearch.xpack.ql.querydsl.query.Query;
 import org.elasticsearch.xpack.ql.rule.Rule;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
@@ -27,7 +36,7 @@ PhysicalPlan fold(PhysicalPlan plan) {
     @Override
     protected Iterable<RuleExecutor<PhysicalPlan>.Batch> batches() {
         Batch fold = new Batch("Fold queries",
-                new FoldFilter()
+                new FoldFilter(), new FoldOrderBy()
         );
         Batch finish = new Batch("Finish query", Limiter.ONCE,
                 new PlanOutputToQueryRef()
@@ -57,6 +66,38 @@ protected PhysicalPlan rule(FilterExec plan) {
         }
     }
 
+    private static class FoldOrderBy extends FoldingRule<OrderExec> {
+        @Override
+        protected PhysicalPlan rule(OrderExec plan) {
+            if (plan.child() instanceof EsQueryExec) {
+                EsQueryExec exec = (EsQueryExec) plan.child();
+                QueryContainer qContainer = exec.queryContainer();
+
+                for (Order order : plan.order()) {
+                    Direction direction = Direction.from(order.direction());
+                    Missing missing = Missing.from(order.nullsPosition());
+
+                    // check whether sorting is on an group (and thus nested agg) or field
+                    Expression orderExpression = order.child();
+
+                    String lookup = Expressions.id(orderExpression);
+
+                    // field
+                    if (orderExpression instanceof FieldAttribute) {
+                        qContainer = qContainer.addSort(lookup, new AttributeSort((FieldAttribute) orderExpression, direction, missing));
+                    }
+                    // unknown
+                    else {
+                        throw new EqlIllegalArgumentException("unsupported sorting expression {}", orderExpression);
+                    }
+                }
+
+                return exec.with(qContainer);
+            }
+            return plan;
+        }
+    }
+
     private static class PlanOutputToQueryRef extends FoldingRule<EsQueryExec> {
         @Override
         protected PhysicalPlan rule(EsQueryExec exec) {

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java

@@ -19,14 +19,18 @@
 import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.ConstantInput;
+import org.elasticsearch.xpack.ql.querydsl.container.Sort;
 import org.elasticsearch.xpack.ql.querydsl.query.Query;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.io.IOException;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine;
 
 public class QueryContainer {
@@ -37,17 +41,19 @@ public class QueryContainer {
     // list of fields available in the output
     private final List<Tuple<FieldExtraction, String>> fields;
 
+    private final Map<String, Sort> sort;
     private final boolean trackHits;
     private final boolean includeFrozen;
 
     public QueryContainer() {
-        this(null, emptyList(), AttributeMap.emptyAttributeMap(), false, false);
+        this(null, emptyList(), AttributeMap.emptyAttributeMap(), emptyMap(), false, false);
     }
 
-    private QueryContainer(Query query, List<Tuple<FieldExtraction, String>> fields, AttributeMap<Expression> attributes, boolean trackHits,
-                           boolean includeFrozen) {
+    private QueryContainer(Query query, List<Tuple<FieldExtraction, String>> fields, AttributeMap<Expression> attributes,
+                           Map<String, Sort> sort, boolean trackHits, boolean includeFrozen) {
         this.query = query;
         this.fields = fields;
+        this.sort = sort;
         this.attributes = attributes;
         this.trackHits = trackHits;
         this.includeFrozen = includeFrozen;
@@ -65,12 +71,16 @@ public List<Tuple<FieldExtraction, String>> fields() {
         return fields;
     }
 
+    public Map<String, Sort> sort() {
+        return sort;
+    }
+
     public boolean shouldTrackHits() {
         return trackHits;
     }
 
     public QueryContainer with(Query q) {
-        return new QueryContainer(q, fields, attributes, trackHits, includeFrozen);
+        return new QueryContainer(q, fields, attributes, sort, trackHits, includeFrozen);
     }
 
     public QueryContainer addColumn(Attribute attr) {
@@ -98,6 +108,12 @@ private Tuple<QueryContainer, FieldExtraction> asFieldExtraction(Attribute attr)
         throw new EqlIllegalArgumentException("Unknown output attribute {}", attr);
     }
 
+    public QueryContainer addSort(String expressionId, Sort sortable) {
+        Map<String, Sort> newSort = new LinkedHashMap<>(this.sort);
+        newSort.put(expressionId, sortable);
+        return new QueryContainer(query, fields, attributes, newSort, trackHits, includeFrozen);
+    }
+
     //
     // reference methods
     //
@@ -139,7 +155,7 @@ private FieldExtraction topHitFieldRef(FieldAttribute fieldAttr) {
     }
 
     public QueryContainer addColumn(FieldExtraction ref, String id) {
-        return new QueryContainer(query, combine(fields, new Tuple<>(ref, id)), attributes, trackHits, includeFrozen);
+        return new QueryContainer(query, combine(fields, new Tuple<>(ref, id)), attributes, sort, trackHits, includeFrozen);
     }
 
     @Override

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java

@@ -8,11 +8,18 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Order;
+import org.elasticsearch.xpack.ql.expression.Order.NullsPosition;
+import org.elasticsearch.xpack.ql.expression.Order.OrderDirection;
+import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
 import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
 import org.elasticsearch.xpack.ql.tree.Source;
 
+import static java.util.Collections.singletonList;
+
 public class LogicalPlanTests extends ESTestCase {
 
     private final EqlParser parser = new EqlParser();
@@ -25,15 +32,20 @@ public void testEventQuery() {
         LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'");
         Expression fullExpression = expr("event_type == 'process' and process_name == 'net.exe'");
 
-        assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression));
+        LogicalPlan filter = new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression);
+        Order order = new Order(Source.EMPTY, new UnresolvedAttribute(Source.EMPTY, "timestamp"), OrderDirection.ASC, NullsPosition.FIRST);
+        LogicalPlan expected = new OrderBy(Source.EMPTY, filter, singletonList(order));
+        assertEquals(expected, fullQuery);
     }
 
     public void testParameterizedEventQuery() {
         ParserParams params = new ParserParams().fieldEventType("myCustomEvent");
         LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'", params);
         Expression fullExpression = expr("myCustomEvent == 'process' and process_name == 'net.exe'");
 
-        assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression));
+        LogicalPlan filter = new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression);
+        Order order = new Order(Source.EMPTY, new UnresolvedAttribute(Source.EMPTY, "timestamp"), OrderDirection.ASC, NullsPosition.FIRST);
+        LogicalPlan expected = new OrderBy(Source.EMPTY, filter, singletonList(order));
+        assertEquals(expected, fullQuery);
     }
-
 }

x-pack/plugin/eql/src/test/resources/mapping-binary.json

@@ -3,6 +3,9 @@
         "event_type" : {
             "type" : "keyword"
         },
+        "timestamp" : {
+            "type" : "keyword"
+        },
         "blob" : {
             "type" : "binary"
         }

x-pack/plugin/eql/src/test/resources/mapping-boolean.json

@@ -3,6 +3,9 @@
         "event_type" : {
             "type" : "keyword"
         },
+        "timestamp" : {
+            "type" : "date"
+        },
         "boolean_field" : {
             "type" : "boolean"
         }

x-pack/plugin/eql/src/test/resources/mapping-date.json

@@ -3,6 +3,9 @@
         "event_type" : {
             "type" : "keyword"
         },
+        "timestamp" : {
+            "type" : "date"
+        },
         "date" : {
             "type" : "date"
         },

x-pack/plugin/eql/src/test/resources/mapping-ip.json

@@ -3,6 +3,9 @@
         "event_type" : {
             "type" : "keyword"
         },
+        "timestamp" : {
+            "type" : "date"
+        },
         "ip_addr" : {
             "type" : "ip"
         }

x-pack/plugin/eql/src/test/resources/mapping-join.json

@@ -3,6 +3,9 @@
         "event_type" : {
             "type" : "keyword"
         },
+        "timestamp" : {
+            "type" : "date"
+        },
         "serial_event_id" : {
             "type" : "long"
         },