[Rule Tuning] Change Network Rules to Use Network Packet Capture Integration (#2665)

terrancedejesus · web-flow · commit 48cf95c8ebde · 2023-06-26T17:35:49.000-04:00
* updated indexes and updated dates

* added network_traffic integration tag to rules

* reverting changes to resolve conflicts

* metadata changes; indexes changed; schemas and manifest updated

* updated default telnet port connection rule

* updating integration manifests

* adjusted rules; updated integrations; deduplicate packages
diff --git a/detection_rules/etc/integration-manifests.json.gz b/detection_rules/etc/integration-manifests.json.gz
diff --git a/detection_rules/etc/integration-schemas.json.gz b/detection_rules/etc/integration-schemas.json.gz
diff --git a/detection_rules/rule.py b/detection_rules/rule.py
@@ -1018,6 +1018,9 @@ def _convert_add_related_integrations(self, obj: dict) -> None:
                             if package["integration"] not in policy_templates:
                                 del package["integration"]
 
+                # remove duplicate entries
+                package_integrations = list({json.dumps(d, sort_keys=True):
+                                            d for d in package_integrations}.values())
                 obj.setdefault("related_integrations", package_integrations)
 
     def _convert_add_required_fields(self, obj: dict) -> None:
diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
@@ -27,7 +27,7 @@
 MINOR_SEMVER = r'^\d+\.\d+$'
 BRANCH_PATTERN = f'{VERSION_PATTERN}|^master$'
 
-NON_DATASET_PACKAGES = ['apm', 'endpoint', 'system', 'windows', 'cloud_defend']
+NON_DATASET_PACKAGES = ['apm', 'endpoint', 'system', 'windows', 'cloud_defend', 'network_traffic']
 INTERVAL_PATTERN = r'^\d+[mshd]$'
 TACTIC_URL = r'^https://attack.mitre.org/tactics/TA[0-9]+/$'
 TECHNIQUE_URL = r'^https://attack.mitre.org/techniques/T[0-9]+/$'
diff --git a/rules/network/command_and_control_accepted_default_telnet_port_connection.toml b/rules/network/command_and_control_accepted_default_telnet_port_connection.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -24,7 +24,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "Accepted Default Telnet Port Connection"
@@ -45,8 +45,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and destination.port:23
-  and network.direction:(inbound or ingress or outbound or egress)
+event.dataset: network_traffic.flow and event.type: connection
   and not event.action:(
       flow_dropped or denied or deny or
       flow_terminated or timeout or Reject or network_flow)
diff --git a/rules/network/command_and_control_cobalt_strike_beacon.toml b/rules/network/command_and_control_cobalt_strike_beacon.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/07/06"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -20,7 +20,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "lucene"
 license = "Elastic License v2"
 name = "Cobalt Strike Command and Control Beacon"
@@ -40,7 +40,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-z]{3}.stage.[0-9]{8}\..*/
+event.dataset: (network_traffic.tls or network_traffic.http) AND destination.domain:/[a-z]{3}.stage.[0-9]{8}\..*/
 '''
 
 
diff --git a/rules/network/command_and_control_cobalt_strike_default_teamserver_cert.toml b/rules/network/command_and_control_cobalt_strike_default_teamserver_cert.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/10/05"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -16,7 +16,7 @@ SHA256 hashing algorithms (the default is SHA1). See the References section for
 configuration.
 """
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "Default Cobalt Strike Team Server Certificate"
@@ -39,7 +39,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and (tls.server.hash.md5:950098276A495286EB2A2556FBAB6D83 or
+event.dataset: network_traffic.tls and (tls.server.hash.md5:950098276A495286EB2A2556FBAB6D83 or
   tls.server.hash.sha1:6ECE5ECE4192683D2D84E25B0BA7E04F9CB7EB7C or
   tls.server.hash.sha256:87F2085C32B6A2CC709B365F55873E207A9CAA10BFFECF2FD16D3CF9D94D390C)
 '''
diff --git a/rules/network/command_and_control_download_rar_powershell_from_internet.toml b/rules/network/command_and_control_download_rar_powershell_from_internet.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/07/02"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -21,7 +21,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "Roshal Archive (RAR) or PowerShell File Downloaded from the Internet"
@@ -41,7 +41,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.protocol:http and
+event.dataset: (network_traffic.http or network_traffic.tls) and
   (url.extension:(ps1 or rar) or url.path:(*.ps1 or *.rar)) and
     not destination.ip:(
       10.0.0.0/8 or
diff --git a/rules/network/command_and_control_fin7_c2_behavior.toml b/rules/network/command_and_control_fin7_c2_behavior.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/07/06"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -19,7 +19,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "lucene"
 license = "Elastic License v2"
 name = "Possible FIN7 DGA Command and Control Behavior"
@@ -37,8 +37,8 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp
-AND destination.domain:/[a-zA-Z]{4,5}\.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us
+event.dataset: (network_traffic.tls or network_traffic.http) AND
+destination.domain:/[a-zA-Z]{4,5}\.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us
 '''
 
 
diff --git a/rules/network/command_and_control_halfbaked_beacon.toml b/rules/network/command_and_control_halfbaked_beacon.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/07/06"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -19,7 +19,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "lucene"
 license = "Elastic License v2"
 name = "Halfbaked Command and Control Beacon"
@@ -38,7 +38,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network OR network_traffic) AND network.protocol:http AND
+event.dataset: (network_traffic.tls or network_traffic.http) AND
   network.transport:tcp AND url.full:/http:\/\/[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}\/cd/ AND
   destination.port:(53 OR 80 OR 8080 OR 443)
 '''
diff --git a/rules/network/command_and_control_nat_traversal_port_activity.toml b/rules/network/command_and_control_nat_traversal_port_activity.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -23,7 +23,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "IPSEC NAT Traversal Port Activity"
@@ -35,7 +35,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:udp and destination.port:4500
+event.dataset: network_traffic.flow and network.transport:udp and destination.port:4500
 '''
 
 
diff --git a/rules/network/command_and_control_port_26_activity.toml b/rules/network/command_and_control_port_26_activity.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -20,7 +20,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "SMTP on Port 26/TCP"
@@ -36,7 +36,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:tcp and (destination.port:26 or (event.dataset:zeek.smtp and destination.port:26))
+event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:26 or (event.dataset:zeek.smtp and destination.port:26))
 '''
 
 
diff --git a/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml b/rules/network/command_and_control_rdp_remote_desktop_protocol_from_the_internet.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -25,7 +25,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "RDP (Remote Desktop Protocol) from the Internet"
@@ -40,7 +40,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:tcp and (destination.port:3389 or event.dataset:zeek.rdp) and
+event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:3389 or event.dataset:zeek.rdp) and
   not source.ip:(
     10.0.0.0/8 or
     127.0.0.0/8 or
diff --git a/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml b/rules/network/command_and_control_vnc_virtual_network_computing_from_the_internet.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -23,7 +23,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "VNC (Virtual Network Computing) from the Internet"
@@ -36,7 +36,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
+event.dataset: network_traffic.flow and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
   not source.ip:(
     10.0.0.0/8 or
     127.0.0.0/8 or
diff --git a/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml b/rules/network/command_and_control_vnc_virtual_network_computing_to_the_internet.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -23,7 +23,7 @@ false_positives = [
     """,
 ]
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "VNC (Virtual Network Computing) to the Internet"
@@ -36,7 +36,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
+event.dataset: network_traffic.flow and network.transport:tcp and destination.port >= 5800 and destination.port <= 5810 and
   source.ip:(
     10.0.0.0/8 or
     172.16.0.0/12 or
diff --git a/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml b/rules/network/initial_access_rpc_remote_procedure_call_from_the_internet.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -15,7 +15,7 @@ directly exposed to the Internet, as it is frequently targeted and exploited by
 backdoor vector.
 """
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "RPC (Remote Procedure Call) from the Internet"
@@ -28,7 +28,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and
+event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and
   not source.ip:(
     10.0.0.0/8 or
     127.0.0.0/8 or
diff --git a/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml b/rules/network/initial_access_rpc_remote_procedure_call_to_the_internet.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -15,7 +15,7 @@ directly exposed to the Internet, as it is frequently targeted and exploited by
 backdoor vector.
 """
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "RPC (Remote Procedure Call) to the Internet"
@@ -28,7 +28,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and
+event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:135 or event.dataset:zeek.dce_rpc) and
   source.ip:(
     10.0.0.0/8 or
     172.16.0.0/12 or
diff --git a/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml b/rules/network/initial_access_smb_windows_file_sharing_activity_to_the_internet.toml
@@ -1,6 +1,6 @@
 [metadata]
 creation_date = "2020/02/18"
-integration = ["endpoint"]
+integration = ["network_traffic"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
@@ -15,7 +15,7 @@ systems. It should almost never be directly exposed to the Internet, as it is fr
 threat actors as an initial access or backdoor vector or for data exfiltration.
 """
 from = "now-9m"
-index = ["auditbeat-*", "filebeat-*", "packetbeat-*", "logs-endpoint.events.*"]
+index = ["packetbeat-*", "logs-network_traffic.*"]
 language = "kuery"
 license = "Elastic License v2"
 name = "SMB (Windows File Sharing) Activity to the Internet"
@@ -28,7 +28,7 @@ timestamp_override = "event.ingested"
 type = "query"
 
 query = '''
-event.category:(network or network_traffic) and network.transport:tcp and (destination.port:(139 or 445) or event.dataset:zeek.smb) and
+event.dataset: network_traffic.flow and network.transport:tcp and (destination.port:(139 or 445) or event.dataset:zeek.smb) and
   source.ip:(
     10.0.0.0/8 or
     172.16.0.0/12 or
diff --git a/rules/network/initial_access_unsecure_elasticsearch_node.toml b/rules/network/initial_access_unsecure_elasticsearch_node.toml
diff --git a/rules/network/lateral_movement_dns_server_overflow.toml b/rules/network/lateral_movement_dns_server_overflow.toml
