Skip to content

Commit 798015b

Browse files
costincostin
authored
EQL: Add implicit ordering on timestamp (#53004)
QL: Move Sort base class from SQL to QL
1 parent 4323817 commit 798015b

File tree

36 files changed

+221
-40
lines changed

36 files changed

+221
-40
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
3737
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, false);
3838

3939
private QueryBuilder filter = null;
40-
private String timestampField = "@timestamp";
40+
private String timestampField = "timestamp";
4141
private String eventTypeField = "event_type";
4242
private String implicitJoinKeyField = "agent.id";
4343
private int fetchSize = 50;

‎client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java

+1
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ public void testLargeMapping() throws Exception {
7979
sb.append(",");
8080
}
8181
sb.append("\"event_type\": \"process\",");
82+
sb.append("\"timestamp\": \"2020-02-03T12:34:56Z\",");
8283
sb.append("\"serial_event_id\": 1");
8384
sb.append("}");
8485
doc1.setJsonEntity(sb.toString());

‎docs/reference/eql/search.asciidoc

+6-2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ specified in the `query` parameter. The EQL query matches events with an
3636
GET sec_logs/_eql/search
3737
{
3838
"event_type_field": "event.category",
39+
"timestamp_field": "@timestamp",
3940
"query": """
4041
process where process.name == "cmd.exe"
4142
"""
@@ -61,7 +62,7 @@ The API returns the following response containing the matching event:
6162
{
6263
"_index": "sec_logs",
6364
"_id": "1",
64-
"_score": 0.9400072,
65+
"_score": null,
6566
"_source": {
6667
"@timestamp": "2020-12-07T11:06:07.000Z",
6768
"agent": {
@@ -74,7 +75,8 @@ The API returns the following response containing the matching event:
7475
"name": "cmd.exe",
7576
"path": "C:\\Windows\\System32\\cmd.exe"
7677
}
77-
}
78+
},
79+
"sort" : [1607339167000]
7880
}
7981
]
8082
}
@@ -98,6 +100,7 @@ field.
98100
GET sec_logs/_eql/search
99101
{
100102
"event_type_field": "file.type",
103+
"timestamp_field": "@timestamp",
101104
"query": """
102105
file where agent.id == "8a4f500d"
103106
"""
@@ -145,6 +148,7 @@ filtered documents.
145148
GET sec_logs/_eql/search
146149
{
147150
"event_type_field": "event.category",
151+
"timestamp_field": "@timestamp",
148152
"filter": {
149153
"range" : {
150154
"file.size" : {

‎x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ setup:
88
_index: eql_test
99
_id: 1
1010
- event_type: process
11+
timestamp: 2020-02-03T12:34:56Z
1112
user: SYSTEM
1213

1314
---

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public final class RequestDefaults {
1010

1111
private RequestDefaults() {}
1212

13-
public static final String FIELD_TIMESTAMP = "@timestamp";
13+
public static final String FIELD_TIMESTAMP = "timestamp";
1414
public static final String FIELD_EVENT_TYPE = "event_type";
1515
public static final String IMPLICIT_JOIN_KEY = "agent.id";
1616

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java

+67-7
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,26 @@
77

88
import org.elasticsearch.index.query.QueryBuilder;
99
import org.elasticsearch.search.builder.SearchSourceBuilder;
10-
import org.elasticsearch.search.fetch.StoredFieldsContext;
1110
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
11+
import org.elasticsearch.search.sort.FieldSortBuilder;
12+
import org.elasticsearch.search.sort.NestedSortBuilder;
13+
import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
14+
import org.elasticsearch.search.sort.SortBuilder;
1215
import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
16+
import org.elasticsearch.xpack.ql.expression.Attribute;
17+
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
18+
import org.elasticsearch.xpack.ql.querydsl.container.AttributeSort;
19+
import org.elasticsearch.xpack.ql.querydsl.container.ScriptSort;
20+
import org.elasticsearch.xpack.ql.querydsl.container.Sort;
1321

14-
import java.util.List;
15-
16-
import static java.util.Collections.singletonList;
1722
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
23+
import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
24+
import static org.elasticsearch.search.sort.SortBuilders.scriptSort;
1825

1926
public abstract class SourceGenerator {
2027

2128
private SourceGenerator() {}
2229

23-
private static final List<String> NO_STORED_FIELD = singletonList(StoredFieldsContext._NONE_);
24-
2530
public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryBuilder filter, Integer size) {
2631
QueryBuilder finalQuery = null;
2732
// add the source
@@ -38,8 +43,9 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
3843
}
3944

4045
final SearchSourceBuilder source = new SearchSourceBuilder();
41-
source.query(finalQuery);
4246

47+
source.query(finalQuery);
48+
sorting(container, source);
4349
source.fetchSource(FetchSourceContext.FETCH_SOURCE);
4450

4551
// set fetch size
@@ -54,6 +60,60 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
5460
return source;
5561
}
5662

63+
private static void sorting(QueryContainer container, SearchSourceBuilder source) {
64+
for (Sort sortable : container.sort().values()) {
65+
SortBuilder<?> sortBuilder = null;
66+
67+
if (sortable instanceof AttributeSort) {
68+
AttributeSort as = (AttributeSort) sortable;
69+
Attribute attr = as.attribute();
70+
71+
// sorting only works on not-analyzed fields - look for a multi-field replacement
72+
if (attr instanceof FieldAttribute) {
73+
FieldAttribute fa = ((FieldAttribute) attr).exactAttribute();
74+
75+
sortBuilder = fieldSort(fa.name())
76+
.missing(as.missing().position())
77+
.unmappedType(fa.dataType().esType());
78+
79+
if (fa.isNested()) {
80+
FieldSortBuilder fieldSort = fieldSort(fa.name())
81+
.missing(as.missing().position())
82+
.unmappedType(fa.dataType().esType());
83+
84+
NestedSortBuilder newSort = new NestedSortBuilder(fa.nestedParent().name());
85+
NestedSortBuilder nestedSort = fieldSort.getNestedSort();
86+
87+
if (nestedSort == null) {
88+
fieldSort.setNestedSort(newSort);
89+
} else {
90+
while (nestedSort.getNestedSort() != null) {
91+
nestedSort = nestedSort.getNestedSort();
92+
}
93+
nestedSort.setNestedSort(newSort);
94+
}
95+
96+
nestedSort = newSort;
97+
98+
if (container.query() != null) {
99+
container.query().enrichNestedSort(nestedSort);
100+
}
101+
sortBuilder = fieldSort;
102+
}
103+
}
104+
} else if (sortable instanceof ScriptSort) {
105+
ScriptSort ss = (ScriptSort) sortable;
106+
sortBuilder = scriptSort(ss.script().toPainless(),
107+
ss.script().outputType().isNumeric() ? ScriptSortType.NUMBER : ScriptSortType.STRING);
108+
}
109+
110+
if (sortBuilder != null) {
111+
sortBuilder.order(sortable.direction().asOrder());
112+
source.sort(sortBuilder);
113+
}
114+
}
115+
}
116+
57117
private static void optimize(QueryContainer query, SearchSourceBuilder builder) {
58118
if (query.shouldTrackHits()) {
59119
builder.trackTotalHits(true);

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java

+10-1
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,19 @@
77

88
import org.elasticsearch.xpack.ql.expression.Expression;
99
import org.elasticsearch.xpack.ql.expression.Literal;
10+
import org.elasticsearch.xpack.ql.expression.Order;
1011
import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
1112
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
1213
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
1314
import org.elasticsearch.xpack.ql.plan.logical.Filter;
1415
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
16+
import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
1517
import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
1618
import org.elasticsearch.xpack.ql.tree.Source;
1719
import org.elasticsearch.xpack.ql.type.DataTypes;
1820

21+
import static java.util.Collections.singletonList;
22+
1923
public abstract class LogicalPlanBuilder extends ExpressionBuilder {
2024

2125
private final ParserParams params;
@@ -40,6 +44,11 @@ public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) {
4044
condition = new And(source, eventMatch, condition);
4145
}
4246

43-
return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);
47+
Filter filter = new Filter(source, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);
48+
// add implicit sorting - when pipes are added, this would better seat there (as a default pipe)
49+
Order order = new Order(source, new UnresolvedAttribute(source, params.fieldTimestamp()), Order.OrderDirection.ASC,
50+
Order.NullsPosition.FIRST);
51+
OrderBy orderBy = new OrderBy(source, filter, singletonList(order));
52+
return orderBy;
4453
}
4554
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java

+42-1
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,21 @@
66

77
package org.elasticsearch.xpack.eql.planner;
88

9+
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
910
import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec;
1011
import org.elasticsearch.xpack.eql.plan.physical.FilterExec;
12+
import org.elasticsearch.xpack.eql.plan.physical.OrderExec;
1113
import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
1214
import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
1315
import org.elasticsearch.xpack.ql.expression.Attribute;
16+
import org.elasticsearch.xpack.ql.expression.Expression;
17+
import org.elasticsearch.xpack.ql.expression.Expressions;
18+
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
19+
import org.elasticsearch.xpack.ql.expression.Order;
1420
import org.elasticsearch.xpack.ql.planner.ExpressionTranslators;
21+
import org.elasticsearch.xpack.ql.querydsl.container.AttributeSort;
22+
import org.elasticsearch.xpack.ql.querydsl.container.Sort.Direction;
23+
import org.elasticsearch.xpack.ql.querydsl.container.Sort.Missing;
1524
import org.elasticsearch.xpack.ql.querydsl.query.Query;
1625
import org.elasticsearch.xpack.ql.rule.Rule;
1726
import org.elasticsearch.xpack.ql.rule.RuleExecutor;
@@ -27,7 +36,7 @@ PhysicalPlan fold(PhysicalPlan plan) {
2736
@Override
2837
protected Iterable<RuleExecutor<PhysicalPlan>.Batch> batches() {
2938
Batch fold = new Batch("Fold queries",
30-
new FoldFilter()
39+
new FoldFilter(), new FoldOrderBy()
3140
);
3241
Batch finish = new Batch("Finish query", Limiter.ONCE,
3342
new PlanOutputToQueryRef()
@@ -57,6 +66,38 @@ protected PhysicalPlan rule(FilterExec plan) {
5766
}
5867
}
5968

69+
private static class FoldOrderBy extends FoldingRule<OrderExec> {
70+
@Override
71+
protected PhysicalPlan rule(OrderExec plan) {
72+
if (plan.child() instanceof EsQueryExec) {
73+
EsQueryExec exec = (EsQueryExec) plan.child();
74+
QueryContainer qContainer = exec.queryContainer();
75+
76+
for (Order order : plan.order()) {
77+
Direction direction = Direction.from(order.direction());
78+
Missing missing = Missing.from(order.nullsPosition());
79+
80+
// check whether sorting is on an group (and thus nested agg) or field
81+
Expression orderExpression = order.child();
82+
83+
String lookup = Expressions.id(orderExpression);
84+
85+
// field
86+
if (orderExpression instanceof FieldAttribute) {
87+
qContainer = qContainer.addSort(lookup, new AttributeSort((FieldAttribute) orderExpression, direction, missing));
88+
}
89+
// unknown
90+
else {
91+
throw new EqlIllegalArgumentException("unsupported sorting expression {}", orderExpression);
92+
}
93+
}
94+
95+
return exec.with(qContainer);
96+
}
97+
return plan;
98+
}
99+
}
100+
60101
private static class PlanOutputToQueryRef extends FoldingRule<EsQueryExec> {
61102
@Override
62103
protected PhysicalPlan rule(EsQueryExec exec) {

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java

+21-5
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,18 @@
1919
import org.elasticsearch.xpack.ql.expression.Expressions;
2020
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
2121
import org.elasticsearch.xpack.ql.expression.gen.pipeline.ConstantInput;
22+
import org.elasticsearch.xpack.ql.querydsl.container.Sort;
2223
import org.elasticsearch.xpack.ql.querydsl.query.Query;
2324
import org.elasticsearch.xpack.ql.type.DataTypes;
2425

2526
import java.io.IOException;
27+
import java.util.LinkedHashMap;
2628
import java.util.List;
29+
import java.util.Map;
2730
import java.util.Objects;
2831

2932
import static java.util.Collections.emptyList;
33+
import static java.util.Collections.emptyMap;
3034
import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine;
3135

3236
public class QueryContainer {
@@ -37,17 +41,19 @@ public class QueryContainer {
3741
// list of fields available in the output
3842
private final List<Tuple<FieldExtraction, String>> fields;
3943

44+
private final Map<String, Sort> sort;
4045
private final boolean trackHits;
4146
private final boolean includeFrozen;
4247

4348
public QueryContainer() {
44-
this(null, emptyList(), AttributeMap.emptyAttributeMap(), false, false);
49+
this(null, emptyList(), AttributeMap.emptyAttributeMap(), emptyMap(), false, false);
4550
}
4651

47-
private QueryContainer(Query query, List<Tuple<FieldExtraction, String>> fields, AttributeMap<Expression> attributes, boolean trackHits,
48-
boolean includeFrozen) {
52+
private QueryContainer(Query query, List<Tuple<FieldExtraction, String>> fields, AttributeMap<Expression> attributes,
53+
Map<String, Sort> sort, boolean trackHits, boolean includeFrozen) {
4954
this.query = query;
5055
this.fields = fields;
56+
this.sort = sort;
5157
this.attributes = attributes;
5258
this.trackHits = trackHits;
5359
this.includeFrozen = includeFrozen;
@@ -65,12 +71,16 @@ public List<Tuple<FieldExtraction, String>> fields() {
6571
return fields;
6672
}
6773

74+
public Map<String, Sort> sort() {
75+
return sort;
76+
}
77+
6878
public boolean shouldTrackHits() {
6979
return trackHits;
7080
}
7181

7282
public QueryContainer with(Query q) {
73-
return new QueryContainer(q, fields, attributes, trackHits, includeFrozen);
83+
return new QueryContainer(q, fields, attributes, sort, trackHits, includeFrozen);
7484
}
7585

7686
public QueryContainer addColumn(Attribute attr) {
@@ -98,6 +108,12 @@ private Tuple<QueryContainer, FieldExtraction> asFieldExtraction(Attribute attr)
98108
throw new EqlIllegalArgumentException("Unknown output attribute {}", attr);
99109
}
100110

111+
public QueryContainer addSort(String expressionId, Sort sortable) {
112+
Map<String, Sort> newSort = new LinkedHashMap<>(this.sort);
113+
newSort.put(expressionId, sortable);
114+
return new QueryContainer(query, fields, attributes, newSort, trackHits, includeFrozen);
115+
}
116+
101117
//
102118
// reference methods
103119
//
@@ -139,7 +155,7 @@ private FieldExtraction topHitFieldRef(FieldAttribute fieldAttr) {
139155
}
140156

141157
public QueryContainer addColumn(FieldExtraction ref, String id) {
142-
return new QueryContainer(query, combine(fields, new Tuple<>(ref, id)), attributes, trackHits, includeFrozen);
158+
return new QueryContainer(query, combine(fields, new Tuple<>(ref, id)), attributes, sort, trackHits, includeFrozen);
143159
}
144160

145161
@Override

0 commit comments

Comments
 (0)