Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EQL: Remove "wildcard" function #76099

Merged
merged 5 commits into from
Aug 16, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -273,20 +273,6 @@ file where file_name : ("winini?.exe", "lsass.e?e") and opcode == 2
'''
expected_event_ids = [65, 86]

[[queries]]
name = "wildcardFunctionWildcardPattern"
query = '''
file where wildcard(file_name, "winini*.exe", "lsass.*") and opcode == 2
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this be replaced with like?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's already a like test for this (likeMultipleArgWithPattern).

'''
expected_event_ids = [65, 86]

[[queries]]
name = "wildcardFunctionQuestionMarkPattern"
query = '''
file where wildcard(file_name, "winini?.exe", "lsass.e?e") and opcode == 2
'''
expected_event_ids = [65, 86]

[[queries]]
name = "insensitiveInSingleArg"
query = 'process where string(serial_event_id) in~ ("1")'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -349,13 +349,13 @@ expected_event_ids = [3299718, 3364047]
filter_counts = [24, 3, 37]
filters = [
'process where process_name == "powershell.exe" and opcode == 1',
'powershell where wildcard(message, "*Get-NetShare*") == true',
'powershell where message like "*Get-NetShare*"',
'process where process_name == "powershell.exe" and opcode == 2'
]
query = '''
sequence by hostname, unique_pid
[process where process_name == "powershell.exe" and opcode == 1]
[powershell where wildcard(message, "*Get-NetShare*") == true]
[powershell where message like "*Get-NetShare*"]
until
[process where process_name == "powershell.exe" and opcode == 2]
'''
Expand Down Expand Up @@ -444,17 +444,17 @@ count = 1
expected_event_ids = [2732749, 2732788]
filter_counts = [89, 1]
filters = [
'''file where file_extension in ("exe", "EXE", "Exe", "scr") and wildcard(file_path, "C:\\Users*", "C:\\ProgramData*") and file_name != "DismHost.exe"''',
'''process where opcode == 1 and signature_status == "noSignature" and wildcard(process_path, "C:\\Users*", "C:\\ProgramData*")'''
'''file where file_extension in ("exe", "EXE", "Exe", "scr") and file_path like ("C:\\Users*", "C:\\ProgramData*") and file_name != "DismHost.exe"''',
'''process where opcode == 1 and signature_status == "noSignature" and process_path like ("C:\\Users*", "C:\\ProgramData*")'''
]
query = '''
sequence by hostname with maxspan=5m
[file where file_extension in ("exe", "EXE", "Exe", "scr")
and wildcard(file_path, "C:\\Users*", "C:\\ProgramData*")
and file_path like ("C:\\Users*", "C:\\ProgramData*")
and file_name != "DismHost.exe"
] by process_path
[process where opcode == 1 and signature_status == "noSignature"
and wildcard(process_path, "C:\\Users*", "C:\\ProgramData*")
and process_path like ("C:\\Users*", "C:\\ProgramData*")
] by parent_process_path
'''
time = 7.179757833480835
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StringContains;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
import org.elasticsearch.xpack.ql.ParsingException;
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.Expression;
Expand Down Expand Up @@ -66,7 +65,6 @@ private FunctionDefinition[][] functions() {
def(ToString.class, ToString::new, "string"),
def(StringContains.class, StringContains::new, "stringcontains"),
def(Substring.class, Substring::new, "substring"),
def(Wildcard.class, Wildcard::new, "wildcard"),
},
// Arithmetic
new FunctionDefinition[]{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@

/**
* EQL wildcard function. Matches the form:
* wildcard(field, "*wildcard*pattern*", ...)
* field like ("*wildcard*pattern*", ...)
*/
public class Wildcard extends BaseSurrogateFunction {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to keep this class around.
It's used inside the parser ExpressionBuilder#visitOperatorExpressionDefault however its only role is to call makeSubstitute which is similar to ExpressionBuilder#combineExpressions. See the handling of EqlBaseParser.IN_INSENSITIVE: in visitOperatorExpressionDefault.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Class removed.


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,35 +205,4 @@ public void testStringContainsWrongParams() {
assertEquals("1:15: second argument of [stringContains(process_name, 1)] must be [string], found value [1] type [integer]",
error("process where stringContains(process_name, 1)"));
}

public void testWildcardNotEnoughArguments() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please convert these to like if there aren't tests that already do that.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These tests become now syntactic failures. One case could however be indeed converted (ex testWildcardWithNumericField).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As long as we have equivalent semantical tests, I'm 👍 . Note that wildcard tests that rely on the syntax can be changed to like/like~ hence my point of replacing removed tests (or checking) there are semantical equivalents to them.

ParsingException e = expectThrows(ParsingException.class,
() -> plan("process where wildcard(process_name)"));
String msg = e.getMessage();
assertEquals("line 1:16: error building [wildcard]: expects at least two arguments", msg);
}

public void testWildcardAgainstVariable() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where wildcard(process_name, parent_process_name)"));
String msg = e.getMessage();
assertEquals("Found 1 problem\nline 1:15: second argument of [wildcard(process_name, parent_process_name)] " +
"must be a constant, received [parent_process_name]", msg);
}

public void testWildcardWithNumericPattern() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where wildcard(process_name, 1)"));
String msg = e.getMessage();
assertEquals("Found 1 problem\n" +
"line 1:15: second argument of [wildcard(process_name, 1)] must be [string], found value [1] type [integer]", msg);
}

public void testWildcardWithNumericField() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where wildcard(pid, \"*.exe\")"));
String msg = e.getMessage();
assertEquals("Found 1 problem\n" +
"line 1:15: first argument of [wildcard(pid, \"*.exe\")] must be [string], found value [pid] type [long]", msg);
}
}
2 changes: 1 addition & 1 deletion x-pack/plugin/eql/src/test/resources/queries-supported.eql
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
process where process_name : "svchost.exe" and command_line != "* -k *";
process where process_name in ("ipconfig.exe", "netstat.exe", "systeminfo.exe", "route.exe");
process where subtype.create and wildcard(command_line, "*.ost *", "*.pst *")
process where subtype.create and command_line like ("*.ost *", "*.pst *")
;

process where subtype.create and
Expand Down
43 changes: 0 additions & 43 deletions x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -463,49 +463,6 @@ process where serial_event_id == number("0x32", 16);
;


wildcardFunctionSingleArgument
process where wildcard(process_path, "*\\red_ttp\\wininit.*")
;
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","boost":1.0}}
;

wildcardFunctionTwoArguments
process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*")
;
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","boost":1.0}}
"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*","boost":1.0}}
;

wildcardFunctionThreeArguments
process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def*")
;
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","boost":1.0}}
"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*","boost":1.0}}
"wildcard":{"process_path":{"wildcard":"*def*","boost":1.0}}
;

wildcardFunctionSingleArgument-insensitive
process where wildcard~(process_path, "*\\red_ttp\\wininit.*")
;
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}
;

wildcardFunctionTwoArguments-insensitive
process where wildcard~(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*")
;
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}
"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*","case_insensitive":true,"boost":1.0}}
;

wildcardFunctionThreeArguments-insensitive
process where wildcard~(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def*")
;
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}
"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*","case_insensitive":true,"boost":1.0}}
"wildcard":{"process_path":{"wildcard":"*def*","case_insensitive":true,"boost":1.0}}
;


addOperator
process where serial_event_id + 2 == -2147483647
;
Expand Down
70 changes: 0 additions & 70 deletions x-pack/plugin/eql/src/test/resources/test_string_functions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -411,73 +411,3 @@ description = "Test the `substring` function when the case already matches"
[[substring.fold.tests]]
expression = '''substring("hello world", -5, -1)'''
expected = "worl"

[wildcard]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this one be replaced with like? is this file even used?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, file still in use.

description = "Test that `wildcard` folds with correct case matches."

[[wildcard.fold.tests]]
expression = 'wildcard(null, "f*o*o*")'
# expected = null

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "F*o*o*")'
expected = true

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "*Foo")'
expected = true

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "*Foo*")'
expected = true

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "*")'
expected = true

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "Bar*")'
expected = false

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "*Bar*")'
expected = false

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "*Bar*", "Baz*")'
expected = false

[[wildcard.fold.tests]]
expression = 'wildcard("Foo", "Foo*", "*Bar*", "Baz*")'
expected = true

[[wildcard.fold.tests]]
expression = 'wildcard("Bar", "Foo*", "*Bar*", "Baz*")'
expected = true

[[wildcard.fold.tests]]
expression = 'wildcard("Baz", "Foo*", "*Bar*", "Baz*")'
expected = true

[wildcard_case_insensitive]
description = "Test that `wildcard` function folds case insensitive as expected."

[[wildcard_case_insensitive.fold.tests]]
expression = 'wildcard("FOO", "f*o*o*")'
expected = false

[[wildcard_case_insensitive.fold.tests]]
expression = 'wildcard("bar", "f*o*o*")'
expected = false


[wildcard_case_sensitive]
description = "Test that `wildcard` folds case-sensitive matches."

[[wildcard_case_sensitive.fold.tests]]
expression = 'wildcard("Foo", "F*o*o*")'
expected = true

[[wildcard_case_sensitive.fold.tests]]
expression = 'wildcard("foo", "F*o*o*")'
expected = false
10 changes: 5 additions & 5 deletions x-pack/plugin/eql/src/test/resources/test_unsupported.toml
Original file line number Diff line number Diff line change
Expand Up @@ -92,15 +92,15 @@ case_insensitive = true
expected = false


[wildcard_case_insensitive]
[like_case_insensitive]
case_insensitive = true

[[wildcard_case_insensitive.fold.tests]]
expression = 'wildcard("FOO", "f*o*o*")'
[[like_case_insensitive.fold.tests]]
expression = 'field like~ ("FOO", "f*o*o*")'
expected = true

[[wildcard_case_insensitive.fold.tests]]
expression = 'wildcard("bar", "f*o*o*")'
[[like_case_insensitive.fold.tests]]
expression = 'field like~ ("bar", "f*o*o*")'
expected = false


Expand Down