[BUG] test_all_rule_queries_optimized does not run on rules (#2823)

* Fixed kql -> kuery in test_all_rule_queries_opt...

* all queries optimized

* manually reconciled all rules that failed due to toml escaped chars

* merge rules from main

* Rules needing optimization

* Fix optimized note

* fix another note

* another note fix

* fixing whitespace

* Updated for readability

---------

Co-authored-by: terrancedejesus <[email protected]>
Co-authored-by: Mika Ayenson <[email protected]>

Author: 3 people

rules/cross-platform/execution_pentest_eggshell_remote_admin_tool.toml

@@ -23,7 +23,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and event.type:(start or process_started) and process.name:espl and process.args:eyJkZWJ1ZyI6*
+event.category:process and event.type:(process_started or start) and process.name:espl and process.args:eyJkZWJ1ZyI6*
 '''
 
 

rules/cross-platform/guided_onboarding_sample_rule.toml

@@ -49,7 +49,7 @@ timestamp_override = "event.ingested"
 type = "threshold"
 
 query = '''
-event.kind:"event"
+event.kind:event
 '''
 
 

rules/integrations/kubernetes/initial_access_anonymous_request_authorized.toml

@@ -38,10 +38,10 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.dataset : "kubernetes.audit_logs"
-  and kubernetes.audit.annotations.authorization_k8s_io/decision:"allow"
-  and (kubernetes.audit.user.username:("system:anonymous" or "system:unauthenticated") or not kubernetes.audit.user.username:*)
-  and not kubernetes.audit.objectRef.resource:("healthz" or "livez" or "readyz")
+event.dataset:kubernetes.audit_logs
+  and kubernetes.audit.annotations.authorization_k8s_io/decision:allow
+  and kubernetes.audit.user.username:("system:anonymous" or "system:unauthenticated" or not *)
+  and not kubernetes.audit.objectRef.resource:(healthz or livez or readyz)
 '''
 
 

rules/linux/command_and_control_suspicious_network_activity_from_unknown_executable.toml

@@ -4,15 +4,15 @@ integration = ["endpoint"]
 maturity = "production"
 min_stack_comments = "Multiple field support in the New Terms rule type was added in Elastic 8.6"
 min_stack_version = "8.6.0"
-updated_date = "2023/06/14"
+updated_date = "2023/06/22"
 
 [rule]
 author = ["Elastic"]
 description = """
 This rule monitors for network connectivity to the internet from a previously unknown executable located in a suspicious
-directory to a previously unknown destination ip. An alert from this rule can indicate the presence of potentially 
-malicious activity, such as the execution of unauthorized or suspicious processes attempting to establish connections to 
-unknown or suspicious destinations such as a command and control server. Detecting and investigating such behavior can 
+directory to a previously unknown destination ip. An alert from this rule can indicate the presence of potentially
+malicious activity, such as the execution of unauthorized or suspicious processes attempting to establish connections to
+unknown or suspicious destinations such as a command and control server. Detecting and investigating such behavior can
 help identify and mitigate potential security threats, protecting the system and its data from potential compromise.
 """
 from = "now-59m"
@@ -26,64 +26,77 @@ severity = "low"
 tags = ["Domain: Endpoint", "OS: Linux", "Use Case: Threat Detection", "Tactic: Command and Control", "Data Source: Elastic Endgame"]
 timestamp_override = "event.ingested"
 type = "new_terms"
+
 query = '''
-host.os.type : "linux" and event.category : "network" and 
-event.action : ("connection_attempted" or "ipv4_connection_attempt_event") and 
+host.os.type:linux and event.category:network and 
+event.action:(connection_attempted or ipv4_connection_attempt_event) and 
 process.executable : ( 
-  /tmp/* or /var/tmp/* or /dev/shm/* or /etc/init.d/* or /etc/rc*.d/* or /etc/crontab or /etc/cron.*/* or 
-  /etc/update-motd.d/* or /usr/lib/update-notifier/* or /home/*/.* or /boot/* or /srv/* or /run/* or /etc/rc.local) and
-source.ip : (
-  127.0.0.0/8 or
-  10.0.0.0/8 or
-  172.16.0.0/12 or
-  192.168.0.0/16 
-  ) and not destination.ip : (
-  10.0.0.0/8 or
-  127.0.0.0/8 or
-  169.254.0.0/16 or
-  172.16.0.0/12 or
-  192.0.0.0/24 or
-  192.0.0.0/29 or
-  192.0.0.8/32 or
-  192.0.0.9/32 or
-  192.0.0.10/32 or
-  192.0.0.170/32 or
-  192.0.0.171/32 or
-  192.0.2.0/24 or
-  192.31.196.0/24 or
-  192.52.193.0/24 or
-  192.168.0.0/16 or
-  192.88.99.0/24 or
-  224.0.0.0/4 or
-  100.64.0.0/10 or
-  192.175.48.0/24 or
-  198.18.0.0/15 or
-  198.51.100.0/24 or
-  203.0.113.0/24 or
-  240.0.0.0/4 or
-  "::1" or
-  "FE80::/10" or
-  "FF00::/8"
-  ) and not process.executable : (
-  "/usr/bin/wget" or
-  "/usr/bin/curl" or 
-  "/usr/bin/apt" or 
-  "/usr/bin/dpkg" or 
-  "/usr/bin/yum" or
-  "/usr/bin/rpm" or
-  "/usr/bin/dnf" or
-  "/usr/bin/dockerd"
-  )
+    (/etc/crontab or 
+     /etc/rc.local or 
+     /boot/* or 
+     /dev/shm/* or 
+     /etc/cron.*/* or 
+     /etc/init.d/* or 
+     /etc/rc*.d/* or 
+     /etc/update-motd.d/* or 
+     /home/*/.* or 
+     /run/* or 
+     /srv/* or 
+     /tmp/* or 
+     /usr/lib/update-notifier/* or 
+     /var/tmp/*) and 
+     not (/usr/bin/apt or 
+          /usr/bin/curl or 
+          /usr/bin/dnf or 
+          /usr/bin/dockerd or 
+          /usr/bin/dpkg or 
+          /usr/bin/rpm or 
+          /usr/bin/wget or 
+          /usr/bin/yum) 
+    ) 
+and source.ip : ( 
+    10.0.0.0/8 or 
+    127.0.0.0/8 or 
+    172.16.0.0/12 or 
+    192.168.0.0/16) and 
+    not destination.ip : ( 
+        10.0.0.0/8 or 
+        100.64.0.0/10 or 
+        127.0.0.0/8 or 
+        169.254.0.0/16 or 
+        172.16.0.0/12 or 
+        192.0.0.0/24 or 
+        192.0.0.0/29 or 
+        192.0.0.10/32 or 
+        192.0.0.170/32 or 
+        192.0.0.171/32 or 
+        192.0.0.8/32 or 
+        192.0.0.9/32 or 
+        192.0.2.0/24 or 
+        192.168.0.0/16 or 
+        192.175.48.0/24 or 
+        192.31.196.0/24 or 
+        192.52.193.0/24 or 
+        192.88.99.0/24 or 
+        198.18.0.0/15 or 
+        198.51.100.0/24 or 
+        203.0.113.0/24 or 
+        224.0.0.0/4 or 
+        240.0.0.0/4 or 
+        "::1" or 
+        "FE80::/10" or 
+        "FF00::/8")
 '''
 
+
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-
 [[rule.threat.technique]]
 id = "T1071"
 name = "Application Layer Protocol"
 reference = "https://attack.mitre.org/techniques/T1071/"
 
+
 [rule.threat.tactic]
 id = "TA0011"
 name = "Command and Control"
@@ -92,7 +105,8 @@ reference = "https://attack.mitre.org/tactics/TA0011/"
 [rule.new_terms]
 field = "new_terms_fields"
 value = ["destination.ip", "process.executable"]
-
 [[rule.new_terms.history_window_start]]
 field = "history_window_start"
 value = "now-2d"
+
+

rules/linux/persistence_credential_access_modify_ssh_binaries.toml

@@ -29,10 +29,13 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:file and host.os.type:linux and event.type:change and
- process.name:* and
- (file.path:(/usr/sbin/sshd or /usr/bin/ssh or /usr/bin/sftp or /usr/bin/scp) or file.name:libkeyutils.so) and
- not process.name:("dpkg" or "yum" or "dnf" or "dnf-automatic")
+event.category:file and host.os.type:linux and event.type:change and 
+  process.name:(* and not (dnf or dnf-automatic or dpkg or yum)) and 
+  (file.path:(/usr/bin/scp or 
+                /usr/bin/sftp or 
+                /usr/bin/ssh or 
+                /usr/sbin/sshd) or 
+  file.name:libkeyutils.so)
 '''
 
 
@@ -48,7 +51,6 @@ reference = "https://attack.mitre.org/techniques/T1543/"
 id = "TA0003"
 name = "Persistence"
 reference = "https://attack.mitre.org/tactics/TA0003/"
-
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
@@ -61,12 +63,18 @@ reference = "https://attack.mitre.org/techniques/T1556/"
 id = "TA0006"
 name = "Credential Access"
 reference = "https://attack.mitre.org/tactics/TA0006/"
-
-
-
-
 [[rule.threat]]
 framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1021"
+name = "Remote Services"
+reference = "https://attack.mitre.org/techniques/T1021/"
+[[rule.threat.technique.subtechnique]]
+id = "T1021.004"
+name = "SSH"
+reference = "https://attack.mitre.org/techniques/T1021/004/"
+
+
 [[rule.threat.technique]]
 id = "T1563"
 name = "Remote Service Session Hijacking"
@@ -76,16 +84,10 @@ id = "T1563.001"
 name = "SSH Hijacking"
 reference = "https://attack.mitre.org/techniques/T1563/001/"
 
-[[rule.threat.technique]]
-id = "T1021"
-name = "Remote Services"
-reference = "https://attack.mitre.org/techniques/T1021/"
-[[rule.threat.technique.subtechnique]]
-id = "T1021.004"
-name = "SSH"
-reference = "https://attack.mitre.org/techniques/T1021/004/"
+
 
 [rule.threat.tactic]
 id = "TA0008"
 name = "Lateral Movement"
 reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/linux/persistence_shared_object_creation.toml

@@ -4,26 +4,24 @@ integration = ["endpoint"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup, New Term"
 min_stack_version = "8.6.0"
-updated_date = "2023/06/09"
+updated_date = "2023/06/22"
 
 [rule]
 author = ["Elastic"]
 description = """
-This rule monitors the creation of shared object files by previously unknown processes. The creation of a shared object 
-file involves compiling code into a dynamically linked library that can be loaded by other programs at runtime. While 
-this process is typically used for legitimate purposes, malicious actors can leverage shared object files to execute 
-unauthorized code, inject malicious functionality into legitimate processes, or bypass security controls. This allows 
-malware to persist on the system, evade detection, and potentially compromise the integrity and confidentiality of the 
+This rule monitors the creation of shared object files by previously unknown processes. The creation of a shared object
+file involves compiling code into a dynamically linked library that can be loaded by other programs at runtime. While
+this process is typically used for legitimate purposes, malicious actors can leverage shared object files to execute
+unauthorized code, inject malicious functionality into legitimate processes, or bypass security controls. This allows
+malware to persist on the system, evade detection, and potentially compromise the integrity and confidentiality of the
 affected system and its data.
 """
 from = "now-9m"
 index = ["logs-endpoint.events.*", "endgame-*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "Shared Object Created or Changed by Previously Unknown Process"
-references = [
-    "https://threatpost.com/sneaky-malware-backdoors-linux/180158/"
-]
+references = ["https://threatpost.com/sneaky-malware-backdoors-linux/180158/"]
 risk_score = 47
 rule_id = "aebaa51f-2a91-4f6a-850b-b601db2293f4"
 severity = "medium"
@@ -32,23 +30,25 @@ timestamp_override = "event.ingested"
 type = "new_terms"
 
 query = '''
-host.os.type : "linux" and event.action:("creation" or "file_create_event" or "rename" or "file_rename_event") and 
-file.path : (/usr/lib/* or /dev/shm/*) and file.extension : "so" and process.name : * and not 
-process.name : ("dpkg" or "dockerd" or "rpm" or "snapd" or "5")
+host.os.type:linux and event.action:(creation or file_create_event or file_rename_event or rename) and 
+file.path:(/dev/shm/* or /usr/lib/*) and file.extension:so and 
+process.name:(* and not (5 or dockerd or dpkg or rpm or snapd))
 '''
 
+
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
 id = "T1574"
 name = "Hijack Execution Flow"
 reference = "https://attack.mitre.org/techniques/T1574/"
-
 [[rule.threat.technique.subtechnique]]
 id = "T1574.006"
 name = "Dynamic Linker Hijacking"
 reference = "https://attack.mitre.org/techniques/T1574/006/"
 
+
+
 [rule.threat.tactic]
 id = "TA0003"
 name = "Persistence"
@@ -57,7 +57,8 @@ reference = "https://attack.mitre.org/tactics/TA0003/"
 [rule.new_terms]
 field = "new_terms_fields"
 value = ["file.path", "process.name"]
-
 [[rule.new_terms.history_window_start]]
 field = "history_window_start"
 value = "now-7d"
+
+

rules/macos/defense_evasion_modify_environment_launchctl.toml

@@ -29,21 +29,20 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and host.os.type:macos and event.type:start and
-  process.name:launchctl and
-  process.args:(setenv and not (JAVA*_HOME or
-                                RUNTIME_JAVA_HOME or
-                                DBUS_LAUNCHD_SESSION_BUS_SOCKET or
-                                ANT_HOME or
-                                LG_WEBOS_TV_SDK_HOME or
-                                WEBOS_CLI_TV or
-                                EDEN_ENV)
-                ) and
-  not process.parent.executable:("/Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin" or
-                                 "/usr/local/bin/kr" or
-                                 "/Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin" or
-                                 "/Applications/IntelliJ IDEA CE.app/Contents/jbr/Contents/Home/lib/jspawnhelper") and
-  not process.args : "*.vmoptions"
+event.category:process and host.os.type:macos and event.type:start and 
+  process.name:launchctl and 
+  process.args:(setenv and not (ANT_HOME or 
+                                DBUS_LAUNCHD_SESSION_BUS_SOCKET or 
+                                EDEN_ENV or 
+                                LG_WEBOS_TV_SDK_HOME or 
+                                RUNTIME_JAVA_HOME or 
+                                WEBOS_CLI_TV or 
+                                JAVA*_HOME) and 
+                not *.vmoptions) and 
+  not process.parent.executable:("/Applications/IntelliJ IDEA CE.app/Contents/jbr/Contents/Home/lib/jspawnhelper" or 
+                                  /Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin or 
+                                  /Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin or 
+                                  /usr/local/bin/kr)
 '''
 
 

rules/macos/defense_evasion_sandboxed_office_app_suspicious_zip_file.toml

@@ -31,7 +31,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:file and host.os.type:macos and not event.type:deletion and file.name:~$*.zip and host.os.type:macos
+event.category:file and host.os.type:(macos and macos) and not event.type:deletion and file.name:~$*.zip
 '''