Skip to content

EQL: Introduce ~ grammar for case-insensitive functions #67869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jan 28, 2021
Merged

EQL: Introduce ~ grammar for case-insensitive functions #67869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
df95fff
EQL: Introduce ~ grammar for case-insensitive functions
costin Jan 20, 2021
26fd397
Fix constructor visibility
costin Jan 24, 2021
419902b
Add back generic scalar function handling
costin Jan 24, 2021
8733f70
Fix default in SQL StartsWith
costin Jan 24, 2021
3917f37
Fix default in SQL StartsWith
costin Jan 24, 2021
0e65475
Change ~ from prefix to suffix
costin Jan 24, 2021
d8011c3
Address feedback
costin Jan 25, 2021
f4c7abf
Merge branch 'master' into eql/tilde
elasticmachine Jan 25, 2021
e75f2a4
Address feedback
costin Jan 26, 2021
d1c2f11
Address feedback
costin Jan 28, 2021
3495163
Merge branch 'master' into eql/tilde
elasticmachine Jan 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1445,7 +1445,7 @@ description = "check built-in string functions"
[[queries]]
name = "startsWithCaseInsensitive1"
query = '''
file where opcode==0 and startsWith(file_name, "explorer.")
file where opcode==0 and startsWith~(file_name, "explorer.")
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"
Expand All @@ -1454,15 +1454,15 @@ description = "check built-in string functions"
[[queries]]
name = "startsWithCaseInsensitive2"
query = '''
file where opcode==0 and startsWith(file_name, "exploRER.")
file where opcode==0 and startsWith~(file_name, "exploRER.")
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"

[[queries]]
name = "startsWithCaseInsensitive3"
query = '''
file where opcode==0 and startsWith(file_name, "expLORER.exe")
file where opcode==0 and startsWith~(file_name, "expLORER.ExE")
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"
Expand All @@ -1479,7 +1479,7 @@ description = "check built-in string functions"
[[queries]]
name = "endsWithCaseInsensitive"
query = '''
file where opcode==0 and endsWith(file_name, "loREr.exe")
file where opcode==0 and endsWith~(file_name, "loREr.exe")
'''
expected_event_ids = [88]
description = "check built-in string functions"
Expand All @@ -1495,7 +1495,7 @@ description = "check built-in string functions"
[[queries]]
name = "endsWithAndCondition"
query = '''
file where opcode==0 and serial_event_id == 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe")
file where opcode==0 and serial_event_id == 88 and startsWith~("explorer.exeaAAAA", "EXPLORER.exe")
'''
expected_event_ids = [88]
description = "check built-in string functions"
Expand All @@ -1511,7 +1511,7 @@ description = "check built-in string functions"
[[queries]]
name = "indexOfCaseInsensitive"
query = '''
file where opcode==0 and indexOf(file_name, "plore") == 2 and indexOf(file_name, ".pf") == null
file where opcode==0 and indexOf~(file_name, "plore") == 2 and indexOf~(file_name, ".pf") == null
'''
expected_event_ids = [88]
description = "check built-in string functions"
Expand All @@ -1535,7 +1535,7 @@ description = "check built-in string functions"
[[queries]]
name = "indexOf3"
query = '''
file where opcode==0 and indexOf(file_name, "plorer.", 0) == 2
file where opcode==0 and indexOf~(file_name, "plorer.", 0) == 2
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"
Expand All @@ -1551,7 +1551,7 @@ description = "check built-in string functions"
[[queries]]
name = "indexOf5"
query = '''
file where opcode==0 and indexOf(file_name, "plorer.", 2) != null
file where opcode==0 and indexOf~(file_name, "plorer.", 2) != null
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"
Expand All @@ -1575,7 +1575,7 @@ description = "check built-in string functions"
[[queries]]
name = "indexOf8"
query = '''
file where opcode==0 and indexOf(file_name, "plorer.", 2) == 2
file where opcode==0 and indexOf~(file_name, "pLOrer.", 2) == 2
'''
expected_event_ids = [88, 92]
description = "check substring ranges"
Expand All @@ -1599,7 +1599,7 @@ description = "check substring ranges"
[[queries]]
name = "indexOf11"
query = '''
file where opcode==0 and indexOf(file_name, "explorer.", 0) == 0
file where opcode==0 and indexOf~(file_name, "explorer.", 0) == 0
'''
expected_event_ids = [88, 92]
description = "check substring ranges"
Expand Down Expand Up @@ -1901,14 +1901,14 @@ query = "file where serial_event_id % 40 == 2"
name = "betweenCaseInsensitive1"
expected_event_ids = [1, 2]
query = '''
process where between(process_name, "s", "e") : "yst"
process where between~(process_name, "s", "E") : "yst"
'''

[[queries]]
name = "betweenCaseInsensitive2"
expected_event_ids = [1, 2]
query = '''
process where between(process_name, "s", "e", false) : "yst"
process where between~(process_name, "s", "E", false) : "yst"
'''

[[queries]]
Expand All @@ -1929,7 +1929,7 @@ process where between(process_name, "S", "e", true) : "ystem Idle Proc"
name = "betweenCaseInsensitive3"
expected_event_ids = [1]
query = '''
process where between(process_name, "s", "e", true) : "ystem Idle Proc"
process where between~(process_name, "s", "e", true) : "ystem Idle Proc"
'''

[[queries]]
Expand Down
93 changes: 0 additions & 93 deletions x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,99 +8,6 @@
# in order to allow at least one round-trip test with the test harness.
# This will be removed once the EQL implementation is wired and actually supports this query.

[[queries]]
name = "startsWithCaseInsensitive1"
query = '''
file where opcode==0 and startsWith(file_name, "explorer.")
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"

[[queries]]
name = "startsWithCaseInsensitive2"
query = '''
file where opcode==0 and startsWith(file_name, "exploRER.")
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"

[[queries]]
name = "startsWithCaseInsensitive3"
query = '''
file where opcode==0 and startsWith(file_name, "expLORER.exe")
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"

[[queries]]
name = "endsWithCaseInsensitive"
query = '''
file where opcode==0 and endsWith(file_name, "loREr.exe")
'''
expected_event_ids = [88]
description = "check built-in string functions"

[[queries]]
name = "endsWithAndCondition"
query = '''
file where opcode==0 and serial_event_id == 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe")
'''
expected_event_ids = [88]
description = "check built-in string functions"

[[queries]]
name = "indexOf3"
query = '''
file where opcode==0 and indexOf(file_name, "plorer.", 0) == 2
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"

[[queries]]
name = "indexOf5"
query = '''
file where opcode==0 and indexOf(file_name, "plorer.", 2) != null
'''
expected_event_ids = [88, 92]
description = "check built-in string functions"

[[queries]]
name = "indexOf8"
query = '''
file where opcode==0 and indexOf(file_name, "plorer.", 2) == 2
'''
expected_event_ids = [88, 92]
description = "check substring ranges"

[[queries]]
name = "indexOf11"
query = '''
file where opcode==0 and indexOf(file_name, "explorer.", 0) == 0
'''
expected_event_ids = [88, 92]
description = "check substring ranges"

[[queries]]
name = "betweenCaseInsensitive1"
expected_event_ids = [1, 2]
query = '''
process where between(process_name, "s", "e") : "yst"
'''

[[queries]]
name = "betweenCaseInsensitive2"
expected_event_ids = [1, 2]
query = '''
process where between(process_name, "s", "e", false) : "yst"
'''

[[queries]]
name = "betweenCaseInsensitive3"
expected_event_ids = [1]
query = '''
process where between(process_name, "s", "e", true) : "ystem Idle Proc"
'''

[[queries]]
name = "twoSequencesWithKeys"
query = '''
Expand Down
5 changes: 5 additions & 0 deletions x-pack/plugin/eql/src/main/antlr/EqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ functionExpression

functionName
: IDENTIFIER
| TILDE_IDENTIFIER
;

constant
Expand Down Expand Up @@ -226,6 +227,10 @@ QUOTED_IDENTIFIER
: '`' ( ~'`' | '``' )* '`'
;

TILDE_IDENTIFIER
: LETTER (LETTER | DIGIT | '_')* '~'
;

Comment on lines +230 to +233
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: I expected that the functionName will be IDENTIFIER '~'?. Why are function names allowed to start with _ and @ only if the function is not suffixed with ~?

I guess you don't want to allow ~ for @timestamp, but what about functions starting with _?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No. Defining a new token just for function names would overlap with the IDENTIIFER and the grammar ends up applying whoever is declared first due to ambiguity leading to rules not matching and thus broken queries.

eventValue
: STRING
| IDENTIFIER
Expand Down
7 changes: 4 additions & 3 deletions x-pack/plugin/eql/src/main/antlr/EqlBase.tokens
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ INTEGER_VALUE=38
DECIMAL_VALUE=39
IDENTIFIER=40
QUOTED_IDENTIFIER=41
LINE_COMMENT=42
BRACKETED_COMMENT=43
WS=44
TILDE_IDENTIFIER=42
LINE_COMMENT=43
BRACKETED_COMMENT=44
WS=45
'and'=1
'any'=2
'by'=3
Expand Down
7 changes: 4 additions & 3 deletions x-pack/plugin/eql/src/main/antlr/EqlBaseLexer.tokens
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ INTEGER_VALUE=38
DECIMAL_VALUE=39
IDENTIFIER=40
QUOTED_IDENTIFIER=41
LINE_COMMENT=42
BRACKETED_COMMENT=43
WS=44
TILDE_IDENTIFIER=42
LINE_COMMENT=43
BRACKETED_COMMENT=44
WS=45
'and'=1
'any'=2
'by'=3
Expand Down
39 changes: 39 additions & 0 deletions .../src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionDefinition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.eql.expression.function;

import org.elasticsearch.xpack.ql.expression.function.Function;
import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;

import java.util.List;

/**
* Dedicated wrapper for EQL specific function definitions.
* Used mainly for validating case sensitivity of wrapping functions.
*/
class EqlFunctionDefinition extends FunctionDefinition {

private final boolean caseAware;

protected EqlFunctionDefinition(String name,
List<String> aliases,
Class<? extends Function> clazz,
boolean caseAware,
Builder builder) {
super(name, aliases, clazz, builder);
this.caseAware = caseAware;
}

public boolean isCaseAware() {
return caseAware;
}

@Override
protected Builder builder() {
return super.builder();
}
Comment on lines +35 to +38
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this needed?

Copy link
Member Author

@costin costin Jan 26, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the parent method is protected and is not visible outside classes in the same package - that is nobody outside the package in ql has access to it. By overriding it, it becomes accessible in the eql package as well.

}
Loading