[FR] Adapt PyPi semver Library and Remove Custom

detection_rules/attack.py

@@ -13,7 +13,7 @@
 import requests
 from collections import OrderedDict
 
-from .semver import Version
+from semver import Version
 from .utils import cached, clear_caches, get_etc_path, get_etc_glob_path, read_gzip, gzip_compress
 
 PLATFORMS = ['Windows', 'macOS', 'Linux']
@@ -105,16 +105,17 @@ def get_version_from_tag(name, pattern='att&ck-v'):
         _, version = name.lower().split(pattern, 1)
         return version
 
-    current_version = get_version_from_tag(filename, 'attack-v')
+    current_version = Version.parse(get_version_from_tag(filename, 'attack-v'), optional_minor_and_patch=True)
 
     r = requests.get('https://api.github.com/repos/mitre/cti/tags')
     r.raise_for_status()
     releases = [t for t in r.json() if t['name'].startswith('ATT&CK-v')]
-    latest_release = max(releases, key=lambda release: Version(get_version_from_tag(release['name'])))
+    latest_release = max(releases, key=lambda release: Version.parse(get_version_from_tag(release['name']),
+                         optional_minor_and_patch=True))
     release_name = latest_release['name']
-    latest_version = get_version_from_tag(release_name)
+    latest_version = Version.parse(get_version_from_tag(release_name), optional_minor_and_patch=True)
 
-    if Version(current_version) >= Version(latest_version):
+    if current_version >= latest_version:
         print(f'No versions newer than the current detected: {current_version}')
         return None, None
 

detection_rules/beats.py

@@ -4,18 +4,20 @@
 # 2.0.
 
 """ECS Schemas management."""
+import json
 import os
 import re
 from typing import List, Optional
 
-import kql
 import eql
-import json
 import requests
+from semver import Version
 import yaml
 
-from .semver import Version
-from .utils import DateTimeEncoder, unzip, get_etc_path, gzip_compress, read_gzip, cached
+import kql
+
+from .utils import (DateTimeEncoder, cached, get_etc_path, gzip_compress,
+                    read_gzip, unzip)
 
 
 def _decompress_and_save_schema(url, release_name):
@@ -91,7 +93,7 @@ def download_latest_beats_schema():
     url = 'https://api.github.com/repos/elastic/beats/releases'
     releases = requests.get(url)
 
-    latest_release = max(releases.json(), key=lambda release: Version(release["tag_name"].lstrip("v")))
+    latest_release = max(releases.json(), key=lambda release: Version.parse(release["tag_name"].lstrip("v")))
     download_beats_schema(latest_release["tag_name"])
 
 
@@ -198,7 +200,7 @@ def get_versions() -> List[Version]:
     for filename in os.listdir(get_etc_path("beats_schemas")):
         version_match = re.match(r'v(.+)\.json\.gz', filename)
         if version_match:
-            versions.append(Version(version_match.groups()[0]))
+            versions.append(Version.parse(version_match.groups()[0]))
 
     return versions
 
@@ -213,7 +215,7 @@ def read_beats_schema(version: str = None):
     if version and version.lower() == 'main':
         return json.loads(read_gzip(get_etc_path('beats_schemas', 'main.json.gz')))
 
-    version = Version(version) if version else None
+    version = Version.parse(version) if version else None
     beats_schemas = get_versions()
 
     if version and version not in beats_schemas:

detection_rules/devtools.py

@@ -21,6 +21,7 @@
 
 import click
 import requests.exceptions
+from semver import Version
 import yaml
 from elasticsearch import Elasticsearch
 from eql.table import Table
@@ -33,7 +34,10 @@
 from .endgame import EndgameSchemaManager
 from .eswrap import CollectEvents, add_range_to_dsl
 from .ghwrap import GithubClient, update_gist
-from .integrations import (build_integrations_manifest, build_integrations_schemas, find_latest_compatible_version,
+from .integrations import (build_integrations_manifest,
+                           build_integrations_schemas,
+                           find_latest_compatible_version,
+                           find_latest_integration_version,
                            load_integrations_manifests)
 from .main import root
 from .misc import PYTHON_LICENSE, add_client, client_error
@@ -43,9 +47,8 @@
                    ThreatMapping, TOMLRule)
 from .rule_loader import RuleCollection, production_filter
 from .schemas import definitions, get_stack_versions
-from .semver import Version
-from .utils import (dict_hash, get_etc_path, get_path, load_dump, save_etc_dump,
-                    load_etc_dump)
+from .utils import (dict_hash, get_etc_path, get_path, load_dump,
+                    load_etc_dump, save_etc_dump)
 from .version_lock import VersionLockFile, default_version_lock
 
 RULES_DIR = get_path('rules')
@@ -152,43 +155,45 @@ def build_integration_docs(ctx: click.Context, registry_version: str, pre: str,
     return docs
 
 
-@dev_group.command("bump-versions")
-@click.option("--major", is_flag=True, help="bump the major version")
-@click.option("--minor", is_flag=True, help="bump the minor version")
-@click.option("--patch", is_flag=True, help="bump the patch version")
-@click.option("--package", is_flag=True, help="Update the package version in the packages.yml file")
-@click.option("--kibana", is_flag=True, help="Update the kibana version in the packages.yml file")
-@click.option("--registry", is_flag=True, help="Update the registry version in the packages.yml file")
-def bump_versions(major, minor, patch, package, kibana, registry):
+@dev_group.command("bump-pkg-versions")
+@click.option("--major-release", is_flag=True, help="bump the major version")
+@click.option("--minor-release", is_flag=True, help="bump the minor version")
+@click.option("--patch-release", is_flag=True, help="bump the patch version")
+@click.option("--maturity", type=click.Choice(['beta', 'ga'], case_sensitive=False),
+              required=True, help="beta or production versions")
+def bump_versions(major_release: bool, minor_release: bool, patch_release: bool, maturity: str):
     """Bump the versions"""
 
-    package_data = load_etc_dump('packages.yml')['package']
-    ver = package_data["name"]
-    new_version = Version(ver).bump(major, minor, patch)
-
-    kibana_version = f"^{new_version}.0" if not patch else f"^{new_version}"
-    registry_version = f"{new_version}.0-dev.0" if not patch else f"{new_version}-dev.0"
-
-    # print the new versions
-    click.echo(f"New package version: {new_version}")
-    click.echo(f"New registry data version: {registry_version}")
-    click.echo(f"New Kibana version: {kibana_version}")
-
-    if package:
-        # update package version
-        package_data["name"] = str(new_version)
-
-    if kibana:
-        # update kibana version
-        package_data["registry_data"]["conditions"]["kibana.version"] = kibana_version
+    pkg_data = load_etc_dump('packages.yml')['package']
+    kibana_ver = Version.parse(pkg_data["name"], optional_minor_and_patch=True)
+    pkg_ver = Version.parse(pkg_data["registry_data"]["version"])
+    pkg_kibana_ver = Version.parse(pkg_data["registry_data"]["conditions"]["kibana.version"].lstrip("^"))
+    if major_release:
+        pkg_data["name"] = str(kibana_ver.bump_major()).rstrip(".0")
+        pkg_data["registry_data"]["conditions"]["kibana.version"] = f"^{pkg_kibana_ver.bump_major()}"
+        pkg_data["registry_data"]["version"] = str(pkg_ver.bump_major().bump_prerelease("beta"))
+    if minor_release:
+        pkg_data["name"] = str(kibana_ver.bump_minor()).rstrip(".0")
+        pkg_data["registry_data"]["conditions"]["kibana.version"] = f"^{pkg_kibana_ver.bump_minor()}"
+        pkg_data["registry_data"]["version"] = str(pkg_ver.bump_minor().bump_prerelease("beta"))
+        pkg_data["registry_data"]["release"] = maturity
+    if patch_release:
+        latest_patch_release_ver = find_latest_integration_version("security_detection_engine",
+                                                                   maturity, pkg_data["name"])
+        if maturity == "ga":
+            pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_patch())
+            pkg_data["registry_data"]["release"] = maturity
+        else:
+            pkg_data["registry_data"]["version"] = str(latest_patch_release_ver.bump_prerelease("beta"))
+            pkg_data["registry_data"]["release"] = maturity
 
-    if registry:
-        # update registry version
-        package_data["registry_data"]["version"] = registry_version
-        # update packages.yml
+    click.echo(f"Kibana version: {pkg_data['name']}")
+    click.echo(f"Package Kibana version: {pkg_data['registry_data']['conditions']['kibana.version']}")
+    click.echo(f"Package version: {pkg_data['registry_data']['version']}")
 
-    if package or kibana or registry:
-        save_etc_dump({"package": package_data}, "packages.yml")
+    # we only save major and minor version bumps
+    # patch version bumps are OOB packages and thus we keep the base versioning
+    save_etc_dump({"package": pkg_data}, "packages.yml")
 
 
 @dataclasses.dataclass
@@ -249,7 +254,7 @@ def prune_staging_area(target_stack_version: str, dry_run: bool, exception_list:
     }
     exceptions.update(exception_list.split(","))
 
-    target_stack_version = Version(target_stack_version)[:2]
+    target_stack_version = Version.parse(target_stack_version, optional_minor_and_patch=True)
 
     # load a structured summary of the diff from git
     git_output = subprocess.check_output(["git", "diff", "--name-status", "HEAD"])
@@ -270,7 +275,8 @@ def prune_staging_area(target_stack_version: str, dry_run: bool, exception_list:
             dict_contents = RuleCollection.deserialize_toml_string(change.read())
             min_stack_version: Optional[str] = dict_contents.get("metadata", {}).get("min_stack_version")
 
-            if min_stack_version is not None and target_stack_version < Version(min_stack_version)[:2]:
+            if min_stack_version is not None and \
+                    (target_stack_version < Version.parse(min_stack_version, optional_minor_and_patch=True)):
                 # rule is incompatible, add to the list of reversions to make later
                 reversions.append(change)
 
@@ -896,13 +902,13 @@ def trim_version_lock(min_version: str, dry_run: bool):
     stack_versions = get_stack_versions()
     assert min_version in stack_versions, f'Unknown min_version ({min_version}), expected: {", ".join(stack_versions)}'
 
-    min_version = Version(min_version)
+    min_version = Version.parse(min_version)
     version_lock_dict = default_version_lock.version_lock.to_dict()
     removed = {}
 
     for rule_id, lock in version_lock_dict.items():
         if 'previous' in lock:
-            prev_vers = [Version(v) for v in list(lock['previous'])]
+            prev_vers = [Version.parse(v, optional_minor_and_patch=True) for v in list(lock['previous'])]
             outdated_vers = [v for v in prev_vers if v <= min_version]
 
             if not outdated_vers:
@@ -1212,7 +1218,9 @@ def show_latest_compatible_version(package: str, stack_version: str) -> None:
         return
 
     try:
-        version = find_latest_compatible_version(package, "", stack_version, packages_manifest)
+        version = find_latest_compatible_version(package, "",
+                                                 Version.parse(stack_version, optional_minor_and_patch=True),
+                                                 packages_manifest)
         click.echo(f"Compatible integration {version=}")
     except Exception as e:
         click.echo(f"Error finding compatible version: {str(e)}")

detection_rules/docs.py

@@ -5,6 +5,7 @@
 
 """Create summary documents for a rule package."""
 import itertools
+import json
 import re
 import shutil
 import textwrap
@@ -13,14 +14,13 @@
 from pathlib import Path
 from typing import Dict, Iterable, Optional, Union
 
-import json
+from semver import Version
 import xlsxwriter
 
 from .attack import attack_tm, matrix, tactics, technique_lookup
 from .packaging import Package
-from .rule_loader import DeprecatedCollection, RuleCollection
 from .rule import ThreatMapping, TOMLRule
-from .semver import Version
+from .rule_loader import DeprecatedCollection, RuleCollection
 
 
 class PackageDocument(xlsxwriter.Workbook):
@@ -304,7 +304,7 @@ def __init__(self, registry_version: str, directory: Path, overwrite=False,
 
     @staticmethod
     def parse_registry(registry_version: str) -> (str, str, str):
-        registry_version = Version(registry_version)
+        registry_version = Version.parse(registry_version)
         short_registry_version = [str(n) for n in registry_version[:3]]
         registry_version_str = '.'.join(short_registry_version)
         base_name = "-".join(short_registry_version)

detection_rules/ecs.py

@@ -6,18 +6,19 @@
 """ECS Schemas management."""
 import copy
 import glob
+import json
 import os
 import shutil
-import json
 from pathlib import Path
 
-import requests
 import eql
 import eql.types
+import requests
+from semver import Version
 import yaml
 
-from .semver import Version
-from .utils import DateTimeEncoder, cached, load_etc_dump, get_etc_path, gzip_compress, read_gzip, unzip
+from .utils import (DateTimeEncoder, cached, get_etc_path, gzip_compress,
+                    load_etc_dump, read_gzip, unzip)
 
 ETC_NAME = "ecs_schemas"
 ECS_SCHEMAS_DIR = get_etc_path(ETC_NAME)
@@ -87,7 +88,7 @@ def get_max_version(include_master=False):
     if include_master and any([v.startswith('master') for v in versions]):
         return list(Path(ECS_SCHEMAS_DIR).glob('master*'))[0].name
 
-    return str(max([Version(v) for v in versions if not v.startswith('master')]))
+    return str(max([Version.parse(v) for v in versions if not v.startswith('master')]))
 
 
 @cached
@@ -205,12 +206,12 @@ def get_kql_schema(version=None, indexes=None, beat_schema=None) -> dict:
 
 def download_schemas(refresh_master=True, refresh_all=False, verbose=True):
     """Download additional schemas from ecs releases."""
-    existing = [Version(v) for v in get_schema_map()] if not refresh_all else []
+    existing = [Version.parse(v) for v in get_schema_map()] if not refresh_all else []
     url = 'https://api.github.com/repos/elastic/ecs/releases'
     releases = requests.get(url)
 
     for release in releases.json():
-        version = Version(release.get('tag_name', '').lstrip('v'))
+        version = Version.parse(release.get('tag_name', '').lstrip('v'))
 
         # we don't ever want beta
         if not version or version < (1, 0, 1) or version in existing:
@@ -247,7 +248,7 @@ def download_schemas(refresh_master=True, refresh_all=False, verbose=True):
     # handle working master separately
     if refresh_master:
         master_ver = requests.get('https://raw.githubusercontent.com/elastic/ecs/master/version')
-        master_ver = Version(master_ver.text.strip())
+        master_ver = Version.parse(master_ver.text.strip())
         master_schema = requests.get('https://raw.githubusercontent.com/elastic/ecs/master/generated/ecs/ecs_flat.yml')
         master_schema = yaml.safe_load(master_schema.text)