[BUG] test_all_rule_queries_optimized does not run on rules (#2823)

* Fixed kql -> kuery in test_all_rule_queries_opt...

* all queries optimized

* manually reconciled all rules that failed due to toml escaped chars

* merge rules from main

* Rules needing optimization

* Fix optimized note

* fix another note

* another note fix

* fixing whitespace

* Updated for readability

---------

Co-authored-by: terrancedejesus <[email protected]>
Co-authored-by: Mika Ayenson <[email protected]>

Removed changes from:
- rules/cross-platform/guided_onboarding_sample_rule.toml
- rules/linux/command_and_control_suspicious_network_activity_from_unknown_executable.toml
- rules/linux/persistence_shared_object_creation.toml

(selectively cherry picked from commit aaa4ce2)

Author: eric-forte-elastic

rules/cross-platform/execution_pentest_eggshell_remote_admin_tool.toml

@@ -23,7 +23,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and event.type:(start or process_started) and process.name:espl and process.args:eyJkZWJ1ZyI6*
+event.category:process and event.type:(process_started or start) and process.name:espl and process.args:eyJkZWJ1ZyI6*
 '''
 
 

rules/integrations/kubernetes/initial_access_anonymous_request_authorized.toml

@@ -38,10 +38,10 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.dataset : "kubernetes.audit_logs"
-  and kubernetes.audit.annotations.authorization_k8s_io/decision:"allow"
-  and (kubernetes.audit.user.username:("system:anonymous" or "system:unauthenticated") or not kubernetes.audit.user.username:*)
-  and not kubernetes.audit.objectRef.resource:("healthz" or "livez" or "readyz")
+event.dataset:kubernetes.audit_logs
+  and kubernetes.audit.annotations.authorization_k8s_io/decision:allow
+  and kubernetes.audit.user.username:("system:anonymous" or "system:unauthenticated" or not *)
+  and not kubernetes.audit.objectRef.resource:(healthz or livez or readyz)
 '''
 
 

rules/linux/persistence_credential_access_modify_ssh_binaries.toml

@@ -29,10 +29,13 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:file and host.os.type:linux and event.type:change and
- process.name:* and
- (file.path:(/usr/sbin/sshd or /usr/bin/ssh or /usr/bin/sftp or /usr/bin/scp) or file.name:libkeyutils.so) and
- not process.name:("dpkg" or "yum" or "dnf" or "dnf-automatic")
+event.category:file and host.os.type:linux and event.type:change and 
+  process.name:(* and not (dnf or dnf-automatic or dpkg or yum)) and 
+  (file.path:(/usr/bin/scp or 
+                /usr/bin/sftp or 
+                /usr/bin/ssh or 
+                /usr/sbin/sshd) or 
+  file.name:libkeyutils.so)
 '''
 
 
@@ -48,7 +51,6 @@ reference = "https://attack.mitre.org/techniques/T1543/"
 id = "TA0003"
 name = "Persistence"
 reference = "https://attack.mitre.org/tactics/TA0003/"
-
 [[rule.threat]]
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
@@ -61,12 +63,18 @@ reference = "https://attack.mitre.org/techniques/T1556/"
 id = "TA0006"
 name = "Credential Access"
 reference = "https://attack.mitre.org/tactics/TA0006/"
-
-
-
-
 [[rule.threat]]
 framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1021"
+name = "Remote Services"
+reference = "https://attack.mitre.org/techniques/T1021/"
+[[rule.threat.technique.subtechnique]]
+id = "T1021.004"
+name = "SSH"
+reference = "https://attack.mitre.org/techniques/T1021/004/"
+
+
 [[rule.threat.technique]]
 id = "T1563"
 name = "Remote Service Session Hijacking"
@@ -76,16 +84,10 @@ id = "T1563.001"
 name = "SSH Hijacking"
 reference = "https://attack.mitre.org/techniques/T1563/001/"
 
-[[rule.threat.technique]]
-id = "T1021"
-name = "Remote Services"
-reference = "https://attack.mitre.org/techniques/T1021/"
-[[rule.threat.technique.subtechnique]]
-id = "T1021.004"
-name = "SSH"
-reference = "https://attack.mitre.org/techniques/T1021/004/"
+
 
 [rule.threat.tactic]
 id = "TA0008"
 name = "Lateral Movement"
 reference = "https://attack.mitre.org/tactics/TA0008/"
+

rules/macos/defense_evasion_modify_environment_launchctl.toml

@@ -29,21 +29,20 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and host.os.type:macos and event.type:start and
-  process.name:launchctl and
-  process.args:(setenv and not (JAVA*_HOME or
-                                RUNTIME_JAVA_HOME or
-                                DBUS_LAUNCHD_SESSION_BUS_SOCKET or
-                                ANT_HOME or
-                                LG_WEBOS_TV_SDK_HOME or
-                                WEBOS_CLI_TV or
-                                EDEN_ENV)
-                ) and
-  not process.parent.executable:("/Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin" or
-                                 "/usr/local/bin/kr" or
-                                 "/Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin" or
-                                 "/Applications/IntelliJ IDEA CE.app/Contents/jbr/Contents/Home/lib/jspawnhelper") and
-  not process.args : "*.vmoptions"
+event.category:process and host.os.type:macos and event.type:start and 
+  process.name:launchctl and 
+  process.args:(setenv and not (ANT_HOME or 
+                                DBUS_LAUNCHD_SESSION_BUS_SOCKET or 
+                                EDEN_ENV or 
+                                LG_WEBOS_TV_SDK_HOME or 
+                                RUNTIME_JAVA_HOME or 
+                                WEBOS_CLI_TV or 
+                                JAVA*_HOME) and 
+                not *.vmoptions) and 
+  not process.parent.executable:("/Applications/IntelliJ IDEA CE.app/Contents/jbr/Contents/Home/lib/jspawnhelper" or 
+                                  /Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin or 
+                                  /Applications/NoMachine.app/Contents/Frameworks/bin/nxserver.bin or 
+                                  /usr/local/bin/kr)
 '''
 
 

rules/macos/defense_evasion_sandboxed_office_app_suspicious_zip_file.toml

@@ -31,7 +31,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:file and host.os.type:macos and not event.type:deletion and file.name:~$*.zip and host.os.type:macos
+event.category:file and host.os.type:(macos and macos) and not event.type:deletion and file.name:~$*.zip
 '''
 
 

rules/windows/collection_posh_clipboard_capture.toml

@@ -74,8 +74,8 @@ reg add "hklm\\SOFTWARE\\Policies\\Microsoft\\Windows\\PowerShell\\ScriptBlockLo
 ```
 """
 references = [
-  "https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.management/get-clipboard",
-  "https://github.com/EmpireProject/Empire/blob/master/data/module_source/collection/Get-ClipboardContents.ps1",
+    "https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.management/get-clipboard",
+    "https://github.com/EmpireProject/Empire/blob/master/data/module_source/collection/Get-ClipboardContents.ps1",
 ]
 risk_score = 47
 rule_id = "92984446-aefb-4d5e-ad12-598042ca80ba"
@@ -85,17 +85,14 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and host.os.type:windows and
-  (powershell.file.script_block_text : (
-    "Windows.Clipboard" or
-    "Windows.Forms.Clipboard" or
-    "Windows.Forms.TextBox"
-   ) and
-   powershell.file.script_block_text : (
-    "]::GetText" or
-    ".Paste()"
-  )) or powershell.file.script_block_text : "Get-Clipboard"
-  and not user.id : "S-1-5-18"
+event.category:process and host.os.type:windows and 
+  powershell.file.script_block_text:((Windows.Clipboard or 
+                                      Windows.Forms.Clipboard or 
+                                      Windows.Forms.TextBox) and 
+                                    (".Paste()" or 
+                                    "]::GetText")) or 
+  powershell.file.script_block_text:Get-Clipboard and 
+  not user.id:S-1-5-18
 '''
 
 

rules/windows/collection_posh_keylogger.toml

@@ -85,14 +85,23 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and host.os.type:windows and
-  (
-   powershell.file.script_block_text : (GetAsyncKeyState or NtUserGetAsyncKeyState or GetKeyboardState or "Get-Keystrokes") or
-   powershell.file.script_block_text : (
-        (SetWindowsHookA or SetWindowsHookW or SetWindowsHookEx or SetWindowsHookExA or NtUserSetWindowsHookEx) and
-        (GetForegroundWindow or GetWindowTextA or GetWindowTextW or "WM_KEYBOARD_LL")
-   )
-   ) and not user.id : "S-1-5-18"
+event.category:process and host.os.type:windows and 
+powershell.file.script_block_text : ( 
+    Get-Keystrokes or GetAsyncKeyState or GetKeyboardState or NtUserGetAsyncKeyState or 
+        (
+            NtUserSetWindowsHookEx or 
+            SetWindowsHookA or 
+            SetWindowsHookEx or 
+            SetWindowsHookExA or 
+            SetWindowsHookW
+        ) and 
+        (
+            GetForegroundWindow or 
+            GetWindowTextA or 
+            GetWindowTextW or 
+            WM_KEYBOARD_LL)
+        ) 
+and not user.id:S-1-5-18
 '''
 
 

rules/windows/collection_posh_mailbox.toml

@@ -86,18 +86,13 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and host.os.type:windows and
-  (
-   powershell.file.script_block_text : (
-      "Microsoft.Office.Interop.Outlook" or
-      "Interop.Outlook.olDefaultFolders" or
-      "::olFolderInBox"
-   ) or
-   powershell.file.script_block_text : (
-      "Microsoft.Exchange.WebServices.Data.Folder" or
-      "Microsoft.Exchange.WebServices.Data.FileAttachment"
-   )
-   )
+event.category:process and host.os.type:windows and 
+powershell.file.script_block_text : ( 
+    "::olFolderInBox" or 
+    Interop.Outlook.olDefaultFolders or 
+    Microsoft.Exchange.WebServices.Data.FileAttachment or 
+    Microsoft.Exchange.WebServices.Data.Folder or 
+    Microsoft.Office.Interop.Outlook)
 '''
 
 
@@ -107,7 +102,6 @@ framework = "MITRE ATT&CK"
 id = "T1114"
 name = "Email Collection"
 reference = "https://attack.mitre.org/techniques/T1114/"
-
 [[rule.threat.technique.subtechnique]]
 id = "T1114.001"
 name = "Local Email Collection"
@@ -118,26 +112,27 @@ id = "T1114.002"
 name = "Remote Email Collection"
 reference = "https://attack.mitre.org/techniques/T1114/002/"
 
+
+
 [rule.threat.tactic]
 id = "TA0009"
 name = "Collection"
 reference = "https://attack.mitre.org/tactics/TA0009/"
-
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-
 [[rule.threat.technique]]
 id = "T1059"
 name = "Command and Scripting Interpreter"
 reference = "https://attack.mitre.org/techniques/T1059/"
-
 [[rule.threat.technique.subtechnique]]
 id = "T1059.001"
 name = "PowerShell"
 reference = "https://attack.mitre.org/techniques/T1059/001/"
 
 
+
 [rule.threat.tactic]
 id = "TA0002"
 name = "Execution"
 reference = "https://attack.mitre.org/tactics/TA0002/"
+

rules/windows/defense_evasion_amsi_bypass_powershell.toml

@@ -104,26 +104,25 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:"process" and host.os.type:windows and
- (powershell.file.script_block_text :
-        ("System.Management.Automation.AmsiUtils" or
-				 amsiInitFailed or 
-				 "Invoke-AmsiBypass" or 
-				 "Bypass.AMSI" or 
-				 "amsi.dll" or 
-				 AntimalwareProvider  or 
-				 amsiSession or 
-				 amsiContext or 
-				 "System.Management.Automation.ScriptBlock" or
-				 AmsiInitialize or 
-				 unloadobfuscated or 
-				 unloadsilent or 
-				 AmsiX64 or 
-				 AmsiX32 or 
-				 FindAmsiFun) or
-  powershell.file.script_block_text:("[System.Runtime.InteropServices.Marshal]::Copy" and "VirtualProtect") or
-  powershell.file.script_block_text:("[Ref].Assembly.GetType(('System.Management.Automation" and ".SetValue(")
- )
+event.category:process and host.os.type:windows and 
+powershell.file.script_block_text : (
+  AmsiInitialize or 
+  AmsiX32 or 
+  AmsiX64 or 
+  AntimalwareProvider or 
+  Bypass.AMSI or 
+  FindAmsiFun or 
+  Invoke-AmsiBypass or 
+  System.Management.Automation.AmsiUtils or 
+  System.Management.Automation.ScriptBlock or 
+  amsi.dll or 
+  amsiContext or 
+  amsiInitFailed or 
+  amsiSession or 
+  unloadobfuscated or 
+  unloadsilent or 
+  VirtualProtect and "[System.Runtime.InteropServices.Marshal]::Copy" or 
+  ".SetValue(" and "[Ref].Assembly.GetType(('System.Management.Automation")
 '''
 
 

rules/windows/defense_evasion_posh_encryption.toml

@@ -26,44 +26,39 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:process and host.os.type:windows and
-  powershell.file.script_block_text : (
-    (
-      "Cryptography.AESManaged" or
-      "Cryptography.RijndaelManaged" or
-      "Cryptography.SHA1Managed" or
-      "Cryptography.SHA256Managed" or
-      "Cryptography.SHA384Managed" or
-      "Cryptography.SHA512Managed" or
-      "Cryptography.SymmetricAlgorithm" or
-      "PasswordDeriveBytes" or
-      "Rfc2898DeriveBytes"
-    ) and
-    (
-      CipherMode and PaddingMode
-    ) and
-    (
-      ".CreateEncryptor" or
-      ".CreateDecryptor"
-    )
-  ) and not user.id : "S-1-5-18"
+event.category:process and host.os.type:windows and 
+powershell.file.script_block_text : (
+    CipherMode and 
+    PaddingMode and 
+    ( 
+        Cryptography.AESManaged or 
+        Cryptography.RijndaelManaged or 
+        Cryptography.SHA1Managed or 
+        Cryptography.SHA256Managed or 
+        Cryptography.SHA384Managed or 
+        Cryptography.SHA512Managed or 
+        Cryptography.SymmetricAlgorithm or 
+        PasswordDeriveBytes or 
+        Rfc2898DeriveBytes
+    ) and (.CreateDecryptor or .CreateEncryptor)) and not user.id:S-1-5-18
 '''
 
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"
-[[rule.threat.technique]]
-id = "T1140"
-name = "Deobfuscate/Decode Files or Information"
-reference = "https://attack.mitre.org/techniques/T1140/"
-
 [[rule.threat.technique]]
 id = "T1027"
 name = "Obfuscated Files or Information"
 reference = "https://attack.mitre.org/techniques/T1027/"
 
+[[rule.threat.technique]]
+id = "T1140"
+name = "Deobfuscate/Decode Files or Information"
+reference = "https://attack.mitre.org/techniques/T1140/"
+
 
 [rule.threat.tactic]
 id = "TA0005"
 name = "Defense Evasion"
 reference = "https://attack.mitre.org/tactics/TA0005/"
+

tests/test_all_rules.py

@@ -59,7 +59,7 @@ def test_file_names(self):
     def test_all_rule_queries_optimized(self):
         """Ensure that every rule query is in optimized form."""
         for rule in self.production_rules:
-            if rule.contents.data.get("language") == "kql":
+            if rule.contents.data.get("language") == "kuery":
                 source = rule.contents.data.query
                 tree = kql.parse(source, optimize=False)
                 optimized = tree.optimize(recursive=True)