Skip to content

Commit 7087358

Browse files
costincostin
authored
EQL: Add field resolution and verification (#51872)
Add basic field resolution inside the Analyzer and a basic Verifier to check for any unresolved fields.
1 parent b97f1ca commit 7087358

File tree

15 files changed

+381
-26
lines changed

15 files changed

+381
-26
lines changed

‎x-pack/plugin/eql/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ dependencies {
2727
testCompile project(':test:framework')
2828
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
2929
testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
30+
testCompile project(path: xpackModule('ql'), configuration: 'testArtifacts')
3031
testCompile project(path: ':modules:reindex', configuration: 'runtime')
3132
testCompile project(path: ':modules:parent-join', configuration: 'runtime')
3233
testCompile project(path: ':modules:analysis-common', configuration: 'runtime')

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.analysis;
8+
9+
import org.elasticsearch.xpack.ql.expression.Attribute;
10+
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
11+
import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
12+
import org.elasticsearch.xpack.ql.type.DataTypes;
13+
import org.elasticsearch.xpack.ql.type.InvalidMappedField;
14+
import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
15+
16+
import java.util.ArrayList;
17+
import java.util.Collection;
18+
import java.util.List;
19+
import java.util.Objects;
20+
21+
import static java.util.stream.Collectors.toList;
22+
23+
public final class AnalysisUtils {
24+
25+
private AnalysisUtils() {}
26+
27+
//
28+
// Shared methods around the analyzer rules
29+
//
30+
static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<Attribute> attrList) {
31+
return resolveAgainstList(u, attrList, false);
32+
}
33+
34+
static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<Attribute> attrList, boolean allowCompound) {
35+
List<Attribute> matches = new ArrayList<>();
36+
37+
// first take into account the qualified version
38+
boolean qualified = u.qualifier() != null;
39+
40+
for (Attribute attribute : attrList) {
41+
if (!attribute.synthetic()) {
42+
boolean match = qualified ? Objects.equals(u.qualifiedName(), attribute.qualifiedName()) :
43+
// if the field is unqualified
44+
// first check the names directly
45+
(Objects.equals(u.name(), attribute.name())
46+
// but also if the qualifier might not be quoted and if there's any ambiguity with nested fields
47+
|| Objects.equals(u.name(), attribute.qualifiedName()));
48+
if (match) {
49+
matches.add(attribute.withLocation(u.source()));
50+
}
51+
}
52+
}
53+
54+
// none found
55+
if (matches.isEmpty()) {
56+
return null;
57+
}
58+
59+
if (matches.size() == 1) {
60+
return handleSpecialFields(u, matches.get(0), allowCompound);
61+
}
62+
63+
return u.withUnresolvedMessage(
64+
"Reference [" + u.qualifiedName() + "] is ambiguous (to disambiguate use quotes or qualifiers); matches any of "
65+
+ matches.stream().map(a -> "\"" + a.qualifier() + "\".\"" + a.name() + "\"").sorted().collect(toList()));
66+
}
67+
68+
private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) {
69+
// if it's a object/compound type, keep it unresolved with a nice error message
70+
if (named instanceof FieldAttribute) {
71+
FieldAttribute fa = (FieldAttribute) named;
72+
73+
// incompatible mappings
74+
if (fa.field() instanceof InvalidMappedField) {
75+
named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] due to ambiguities being "
76+
+ ((InvalidMappedField) fa.field()).errorMessage());
77+
}
78+
// unsupported types
79+
else if (DataTypes.isUnsupported(fa.dataType())) {
80+
UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
81+
if (unsupportedField.hasInherited()) {
82+
named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] with unsupported type ["
83+
+ unsupportedField.getOriginalType() + "] " + "in hierarchy (field [" + unsupportedField.getInherited() + "])");
84+
} else {
85+
named = u.withUnresolvedMessage(
86+
"Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
87+
}
88+
}
89+
// compound fields
90+
else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) {
91+
named = u.withUnresolvedMessage(
92+
"Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields");
93+
}
94+
}
95+
return named;
96+
}
97+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,20 @@
66

77
package org.elasticsearch.xpack.eql.analysis;
88

9+
import org.elasticsearch.xpack.ql.expression.Attribute;
10+
import org.elasticsearch.xpack.ql.expression.NamedExpression;
11+
import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
912
import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
1013
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
14+
import org.elasticsearch.xpack.ql.rule.Rule;
1115
import org.elasticsearch.xpack.ql.rule.RuleExecutor;
1216

17+
import java.util.ArrayList;
1318
import java.util.Collection;
19+
import java.util.List;
1420

1521
import static java.util.Arrays.asList;
22+
import static org.elasticsearch.xpack.eql.analysis.AnalysisUtils.resolveAgainstList;
1623

1724
public class Analyzer extends RuleExecutor<LogicalPlan> {
1825

@@ -26,7 +33,8 @@ public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) {
2633

2734
@Override
2835
protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
29-
Batch resolution = new Batch("Resolution");
36+
Batch resolution = new Batch("Resolution",
37+
new ResolveRefs());
3038

3139
return asList(resolution);
3240
}
@@ -42,4 +50,56 @@ private LogicalPlan verify(LogicalPlan plan) {
4250
}
4351
return plan;
4452
}
45-
}
53+
54+
private static class ResolveRefs extends AnalyzeRule<LogicalPlan> {
55+
56+
@Override
57+
protected LogicalPlan rule(LogicalPlan plan) {
58+
// if the children are not resolved, there's no way the node can be resolved
59+
if (!plan.childrenResolved()) {
60+
return plan;
61+
}
62+
63+
// okay, there's a chance so let's get started
64+
if (log.isTraceEnabled()) {
65+
log.trace("Attempting to resolve {}", plan.nodeString());
66+
}
67+
68+
return plan.transformExpressionsUp(e -> {
69+
if (e instanceof UnresolvedAttribute) {
70+
UnresolvedAttribute u = (UnresolvedAttribute) e;
71+
List<Attribute> childrenOutput = new ArrayList<>();
72+
for (LogicalPlan child : plan.children()) {
73+
childrenOutput.addAll(child.output());
74+
}
75+
NamedExpression named = resolveAgainstList(u, childrenOutput);
76+
// if resolved, return it; otherwise keep it in place to be resolved later
77+
if (named != null) {
78+
if (log.isTraceEnabled()) {
79+
log.trace("Resolved {} to {}", u, named);
80+
}
81+
return named;
82+
}
83+
}
84+
return e;
85+
});
86+
}
87+
}
88+
89+
abstract static class AnalyzeRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
90+
91+
// transformUp (post-order) - that is first children and then the node
92+
// but with a twist; only if the tree is not resolved or analyzed
93+
@Override
94+
public final LogicalPlan apply(LogicalPlan plan) {
95+
return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken());
96+
}
97+
98+
@Override
99+
protected abstract LogicalPlan rule(SubPlan plan);
100+
101+
protected boolean skipResolved() {
102+
return true;
103+
}
104+
}
105+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.analysis;
8+
9+
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
10+
import org.elasticsearch.xpack.ql.rule.Rule;
11+
12+
public abstract class AnalyzerRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
13+
14+
// transformUp (post-order) - that is first children and then the node
15+
// but with a twist; only if the tree is not resolved or analyzed
16+
@Override
17+
public final LogicalPlan apply(LogicalPlan plan) {
18+
return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken());
19+
}
20+
21+
@Override
22+
protected abstract LogicalPlan rule(SubPlan plan);
23+
24+
protected boolean skipResolved() {
25+
return true;
26+
}
27+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ Collection<Failure> verify(LogicalPlan plan) {
9999
});
100100
});
101101
}
102+
103+
failures.addAll(localFailures);
102104
});
103105

104106
return failures;

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql.expression.function;
8+
9+
import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
10+
11+
public class EqlFunctionRegistry extends FunctionRegistry {
12+
13+
public EqlFunctionRegistry() {
14+
}
15+
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,16 @@
1111
import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
1212
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
1313
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
14-
import org.elasticsearch.xpack.ql.index.EsIndex;
15-
import org.elasticsearch.xpack.ql.plan.logical.EsRelation;
1614
import org.elasticsearch.xpack.ql.plan.logical.Filter;
1715
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
16+
import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
1817
import org.elasticsearch.xpack.ql.tree.Source;
1918
import org.elasticsearch.xpack.ql.type.DataTypes;
2019

21-
import static java.util.Collections.emptyMap;
22-
2320
public abstract class LogicalPlanBuilder extends ExpressionBuilder {
2421

2522
// TODO: these need to be made configurable
26-
private static final String EVENT_TYPE = "event.category";
27-
private static final EsIndex esIndex = new EsIndex("<not-specified>", emptyMap());
23+
private static final String EVENT_TYPE = "event_type";
2824

2925
@Override
3026
public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) {
@@ -43,6 +39,6 @@ public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) {
4339

4440
}
4541

46-
return new Filter(source(ctx), new EsRelation(Source.EMPTY, esIndex, false), condition);
42+
return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);
4743
}
4844
}

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Configuration.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@ public class Configuration extends org.elasticsearch.xpack.ql.session.Configurat
2323
private QueryBuilder filter;
2424

2525
public Configuration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter,
26-
TimeValue requestTimeout,
27-
boolean includeFrozen, String clientId) {
26+
TimeValue requestTimeout, boolean includeFrozen, String clientId) {
2827

2928
super(zi, username, clusterName);
3029

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
1414
import org.elasticsearch.xpack.eql.execution.PlanExecutor;
1515
import org.elasticsearch.xpack.eql.optimizer.Optimizer;
16+
import org.elasticsearch.xpack.eql.parser.EqlParser;
1617
import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
1718
import org.elasticsearch.xpack.eql.planner.Planner;
1819
import org.elasticsearch.xpack.ql.index.IndexResolver;
@@ -97,7 +98,6 @@ private <T> void preAnalyze(LogicalPlan parsed, ActionListener<LogicalPlan> list
9798

9899
private LogicalPlan doParse(String eql, List<Object> params) {
99100
Check.isTrue(params.isEmpty(), "Parameters were given despite being ignored - server bug");
100-
//LogicalPlan plan = new EqlParser().createStatement(eql);
101-
throw new UnsupportedOperationException();
101+
return new EqlParser().createStatement(eql);
102102
}
103103
}

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.eql;
8+
9+
import org.elasticsearch.common.unit.TimeValue;
10+
import org.elasticsearch.xpack.eql.session.Configuration;
11+
12+
import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength;
13+
import static org.elasticsearch.test.ESTestCase.randomBoolean;
14+
import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong;
15+
import static org.elasticsearch.test.ESTestCase.randomZone;
16+
17+
public final class EqlTestUtils {
18+
19+
private EqlTestUtils() {}
20+
21+
public static final Configuration TEST_CFG = new Configuration(new String[] { "none" }, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
22+
"nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
23+
24+
public static Configuration randomConfiguration() {
25+
return new Configuration(new String[] {randomAlphaOfLength(16)},
26+
randomZone(),
27+
randomAlphaOfLength(16),
28+
randomAlphaOfLength(16),
29+
null,
30+
new TimeValue(randomNonNegativeLong()),
31+
randomBoolean(),
32+
randomAlphaOfLength(16));
33+
}
34+
}

0 commit comments

Comments
 (0)