[FR] Add support for BBR rules to the rule loader (#2968)

---------

Co-authored-by: eric-forte-elastic <[email protected]>

Author: Mikaayenson

detection_rules/packaging.py

@@ -21,7 +21,7 @@
 from .misc import JS_LICENSE, cached, load_current_package_version
 from .navigator import NavigatorBuilder, Navigator
 from .rule import TOMLRule, QueryRuleData, ThreatMapping
-from .rule_loader import DeprecatedCollection, RuleCollection, DEFAULT_RULES_DIR
+from .rule_loader import DeprecatedCollection, RuleCollection, DEFAULT_RULES_DIR, DEFAULT_BBR_DIR
 from .schemas import definitions
 from .utils import Ndjson, get_path, get_etc_path, load_etc_dump
 from .version_lock import default_version_lock
@@ -466,13 +466,19 @@ def create_bulk_index_body(self) -> Tuple[Ndjson, Ndjson]:
                 status = 'unmodified'
 
             bulk_upload_docs.append(create)
+
+            try:
+                relative_path = str(rule.path.resolve().relative_to(DEFAULT_RULES_DIR))
+            except ValueError:
+                relative_path = str(rule.path.resolve().relative_to(DEFAULT_BBR_DIR))
+
             rule_doc = dict(hash=rule.contents.sha256(),
                             source='repo',
                             datetime_uploaded=now,
                             status=status,
                             package_version=self.name,
                             flat_mitre=ThreatMapping.flatten(rule.contents.data.threat).to_dict(),
-                            relative_path=str(rule.path.resolve().relative_to(DEFAULT_RULES_DIR)))
+                            relative_path=relative_path)
             rule_doc.update(**rule.contents.to_api_format())
             bulk_upload_docs.append(rule_doc)
             importable_rules_docs.append(rule_doc)

detection_rules/rule_loader.py

@@ -349,6 +349,7 @@ def default(cls) -> 'RuleCollection':
         if cls.__default is None:
             collection = RuleCollection()
             collection.load_directory(DEFAULT_RULES_DIR)
+            collection.load_directory(DEFAULT_BBR_DIR)
             collection.freeze()
             cls.__default = collection
 

rules_building_block/command_and_control_non_standard_http_port.toml

@@ -4,7 +4,7 @@ integration = ["endpoint"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
-updated_date = "2023/07/10"
+updated_date = "2023/07/26"
 
 [rule]
 author = ["Elastic"]
@@ -37,18 +37,12 @@ network where process.name : ("http", "https")
 
 [[rule.threat]]
 framework = "MITRE ATT&CK"
+
 [[rule.threat.technique]]
 id = "T1571"
 name = "Non-Standard Port"
 reference = "https://attack.mitre.org/techniques/T1571/"
 
-
-[rule.threat.tactic]
-id = "TA0011"
-name = "Command and Control"
-reference = "https://attack.mitre.org/tactics/TA0011/"
-[[rule.threat]]
-framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
 id = "T1071"
 name = "Application Layer Protocol"
@@ -58,14 +52,6 @@ id = "T1071.001"
 name = "Web Protocols"
 reference = "https://attack.mitre.org/techniques/T1071/001/"
 
-
-
-[rule.threat.tactic]
-id = "TA0011"
-name = "Command and Control"
-reference = "https://attack.mitre.org/tactics/TA0011/"
-[[rule.threat]]
-framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
 id = "T1573"
 name = "Encrypted Channel"
@@ -75,7 +61,7 @@ id = "T1573.001"
 name = "Symmetric Cryptography"
 reference = "https://attack.mitre.org/techniques/T1573/001/"
 [[rule.threat.technique.subtechnique]]
-id = "T1573.001"
+id = "T1573.002"
 name = "Asymmetric Cryptography"
 reference = "https://attack.mitre.org/techniques/T1573/002/"
 

rules_building_block/discovery_posh_generic.toml

@@ -4,7 +4,7 @@ integration = ["windows"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
-updated_date = "2023/07/06"
+updated_date = "2023/07/26"
 
 
 [rule]
@@ -40,7 +40,7 @@ reg add "hklm\\SOFTWARE\\Policies\\Microsoft\\Windows\\PowerShell\\ScriptBlockLo
 risk_score = 21
 rule_id = "1e0a3f7c-21e7-4bb1-98c7-2036612fb1be"
 severity = "low"
-tags = ["Domain: Endpoint", "OS: Windows", "Use Case: Threat Detection", "Tactic: Collection", "Data Source: PowerShell Logs", "Rule Type: BBR"]
+tags = ["Domain: Endpoint", "OS: Windows", "Use Case: Threat Detection", "Tactic: Collection", "Tactic: Discovery", "Data Source: PowerShell Logs", "Rule Type: BBR"]
 timestamp_override = "event.ingested"
 type = "query"
 building_block_type = "default"

rules_building_block/discovery_windows_system_information_discovery.toml

@@ -4,7 +4,7 @@ integration = ["windows", "endpoint"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
-updated_date = "2023/07/06"
+updated_date = "2023/07/26"
 
 [rule]
 author = ["Elastic"]
@@ -50,7 +50,7 @@ process.parent.executable : (
 framework = "MITRE ATT&CK"
 [[rule.threat.technique]]
 id = "T1082"
-name = "System Service Discovery"
+name = "System Information Discovery"
 reference = "https://attack.mitre.org/techniques/T1082/"
 
 [rule.threat.tactic]

tests/test_all_rules.py

@@ -499,9 +499,10 @@ def test_non_bbr_in_correct_dir(self):
         for rule in self.all_rules:
             if rule.path.parent.name == 'rules_building_block':
                 self.assertIn(rule, self.bbr, f'{self.rule_str(rule)} should be in the {proper_directory}')
-            # Is the rule of type BBR
-            self.assertEqual(rule.contents.data.building_block_type, None,
-                             f'{self.rule_str(rule)} should not have building_block_type or be in {proper_directory}')
+            else:
+                # Is the rule of type BBR and not in the correct directory
+                self.assertEqual(rule.contents.data.building_block_type, None,
+                                 f'{self.rule_str(rule)} should be in {proper_directory}')
 
 
 class TestRuleMetadata(BaseRuleTest):
@@ -530,9 +531,15 @@ def test_deprecated_rules(self):
         rules_path = get_path('rules')
         deprecated_path = get_path("rules", "_deprecated")
 
-        misplaced_rules = [r for r in self.all_rules
-                           if r.path.relative_to(rules_path).parts[-2] == '_deprecated' and  # noqa: W504
-                           r.contents.metadata.maturity != 'deprecated']
+        misplaced_rules = []
+        for r in self.all_rules:
+            if "rules_building_block" in str(r.path):
+                if r.contents.metadata.maturity == "deprecated":
+                    misplaced_rules.append(r)
+            elif r.path.relative_to(rules_path).parts[-2] == "_deprecated" \
+                    and r.contents.metadata.maturity != "deprecated":
+                misplaced_rules.append(r)
+
         misplaced = '\n'.join(f'{self.rule_str(r)} {r.contents.metadata.maturity}' for r in misplaced_rules)
         err_str = f'The following rules are stored in {deprecated_path} but are not marked as deprecated:\n{misplaced}'
         self.assertListEqual(misplaced_rules, [], err_str)