[FR] Recommend Latest Compatible Integration Versions

detection_rules/integrations.py

@@ -8,8 +8,8 @@
 import json
 import os
 import re
-from collections import OrderedDict
 from pathlib import Path
+from typing import Union
 
 import requests
 from marshmallow import EXCLUDE, Schema, fields, post_load
@@ -62,29 +62,43 @@ def build_integrations_manifest(overwrite: bool, rule_integrations: list) -> Non
     print(f"final integrations manifests dumped: {MANIFEST_FILE_PATH}")
 
 
-def find_least_compatible_version(package: str, integration: str,
-                                  current_stack_version: str, packages_manifest: dict) -> str:
+def find_latest_compatible_version(package: str, integration: str,
+                                   rule_stack_version: str, packages_manifest: dict) -> Union[None, str]:
     """Finds least compatible version for specified integration based on stack version supplied."""
-    integration_manifests = {k: v for k, v in sorted(packages_manifest[package].items(),
-                             key=lambda x: Version(str(x[0])))}
-
-    # filter integration_manifests to only the latest major entries
-    major_versions = sorted(list(set([Version(manifest_version)[0] for manifest_version in integration_manifests])),
-                            reverse=True)
-    for max_major in major_versions:
-        major_integration_manifests = \
-            {k: v for k, v in integration_manifests.items() if Version(k)[0] == max_major}
-
-        # iterates through ascending integration manifests
-        # returns latest major version that is least compatible
-        for version, manifest in OrderedDict(sorted(major_integration_manifests.items(),
-                                                    key=lambda x: Version(str(x[0])))).items():
-            compatible_versions = re.sub(r"\>|\<|\=|\^", "", manifest["conditions"]["kibana"]["version"]).split(" || ")
-            for kibana_ver in compatible_versions:
-                # check versions have the same major
-                if int(kibana_ver[0]) == int(current_stack_version[0]):
-                    if Version(kibana_ver) <= Version(current_stack_version + ".0"):
-                        return f"^{version}"
+
+    if not package:
+        raise ValueError("Package must be specified")
+
+    package_manifest = packages_manifest.get(package)
+    if package_manifest is None:
+        raise ValueError(f"Package {package} not found in manifest.")
+
+    # Converts the dict keys (version numbers) to Version objects for proper sorting (descending)
+    integration_manifests = sorted(package_manifest.items(), key=lambda x: Version(str(x[0])), reverse=True)
+
+    for version, manifest in integration_manifests:
+        kibana_conditions = manifest.get("conditions", {}).get("kibana", {})
+        version_requirement = kibana_conditions.get("version")
+        if not version_requirement:
+            raise ValueError(f"Manifest for {package}:{integration} version {version} is missing conditions.")
+
+        compatible_versions = re.sub(r"\>|\<|\=|\^", "", version_requirement).split(" || ")
+
+        if not compatible_versions:
+            raise ValueError(f"Manifest for {package}:{integration} version {version} is missing compatible versions")
+
+        highest_compatible_version = max(compatible_versions, key=lambda x: Version(x))
+
+        if Version(highest_compatible_version) > Version(rule_stack_version):
+            # TODO: Determine if we should raise an error here or not
+            integration = f" {integration}" if integration else ""
+            print(f"Integration {package}{integration} version {version} has a higher stack version requirement.",
+                  f"Consider updating min_stack version from {rule_stack_version} to "
+                  f"{highest_compatible_version} to support this version.")
+
+        elif int(highest_compatible_version[0]) == int(rule_stack_version[0]):
+            # TODO: Should we add a ^ for kibana
+            return f"^{version}"
 
     raise ValueError(f"no compatible version for integration {package}:{integration}")
 

detection_rules/rule.py

@@ -23,7 +23,7 @@
 from marshmallow import ValidationError, validates_schema
 
 from . import beats, ecs, endgame, utils
-from .integrations import (find_least_compatible_version,
+from .integrations import (find_latest_compatible_version,
                            load_integrations_manifests)
 from .misc import load_current_package_version
 from .mixins import MarshmallowDataclassMixin, StackCompatMixin
@@ -841,9 +841,8 @@ def _add_related_integrations(self, obj: dict) -> None:
         field_name = "related_integrations"
         package_integrations = obj.get(field_name, [])
 
-        if not package_integrations and self.metadata.integration:
+        if not package_integrations and self.metadata.integration and self.metadata.maturity == "production":
             packages_manifest = load_integrations_manifests()
-            current_stack_version = load_current_package_version()
 
             if self.check_restricted_field_version(field_name):
                 if isinstance(self.data, QueryRuleData) and self.data.language != 'lucene':
@@ -853,10 +852,10 @@ def _add_related_integrations(self, obj: dict) -> None:
                         return
 
                     for package in package_integrations:
-                        package["version"] = find_least_compatible_version(
+                        package["version"] = find_latest_compatible_version(
                             package=package["package"],
                             integration=package["integration"],
-                            current_stack_version=current_stack_version,
+                            rule_stack_version=self.metadata.min_stack_version,
                             packages_manifest=packages_manifest)
 
                         # if integration is not a policy template remove