From b1ca536fbdbac169cc76a4d57e122d6ae7d69f70 Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Wed, 25 Nov 2020 01:16:20 -0900 Subject: [PATCH 1/7] Add export-rule command to CLI --- detection_rules/main.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/detection_rules/main.py b/detection_rules/main.py index 8e3abe1b4b3..2aea7ab684a 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -130,7 +130,7 @@ def mass_update(ctx, query, metadata, language, field): @click.option('--rule-file', '-f', type=click.Path(dir_okay=False), help='Optionally view a rule from a specified file') @click.option('--api-format/--rule-format', default=True, help='Print the rule in final api or rule format') @click.pass_context -def view_rule(ctx, rule_id, rule_file, api_format): +def view_rule(ctx, rule_id, rule_file, api_format, verbose=True): """View an internal rule or specified rule file.""" rule = None @@ -149,12 +149,36 @@ def view_rule(ctx, rule_id, rule_file, api_format): if not rule: client_error('Unknown format!') - click.echo(toml_write(rule.rule_format()) if not api_format else - json.dumps(rule.contents, indent=2, sort_keys=True)) + if verbose: + click.echo(toml_write(rule.rule_format()) if not api_format else + json.dumps(rule.contents, indent=2, sort_keys=True)) return rule +@root.command('export-rule') +@click.argument('rule-id', required=False) +@click.option('--rule-file', '-f', type=click.Path(dir_okay=False), help='Optionally view a rule from a specified file') +@click.option('--ndjson/--json', default=True, help='Output format') +@click.pass_context +def export_rule(ctx, rule_id, rule_file, ndjson): + """Export a rule as json/ndjson.""" + from .packaging import manage_versions + + rule = ctx.invoke(view_rule, rule_id=rule_id, rule_file=rule_file, verbose=False) + + ext = '.ndjson' if ndjson else '.json' + base, _ = os.path.splitext(rule.path) + outfile = base + ext + + # add version + manage_versions([rule], verbose=False) + + with open(outfile, 'w') as f: + json.dump(rule.contents, f, sort_keys=True, indent=None if ndjson else 2) + click.echo(f'Rule: {rule.name} saved to: {outfile}') + + @root.command('validate-rule') @click.argument('rule-id', required=False) @click.option('--rule-name', '-n') From 0da2e5574dc93c459fb13aa9b217c96361bf74eb Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Wed, 25 Nov 2020 23:59:08 -0900 Subject: [PATCH 2/7] move export rule to packaging method --- CLI.md | 16 ++++++++++++ detection_rules/main.py | 48 +++++++++++++++++++++++------------- detection_rules/packaging.py | 25 ++++++++++++++----- 3 files changed, 66 insertions(+), 23 deletions(-) diff --git a/CLI.md b/CLI.md index 367846573b9..0e333c6e50e 100644 --- a/CLI.md +++ b/CLI.md @@ -167,6 +167,22 @@ Options: -h, --help Show this message and exit. ``` +Alternatively, rules can be exported into a consolidated ndjson file which can be imported in the Kibana security app +directly. + +```console +Usage: detection_rules export-rules [OPTIONS] [RULE_ID]... + + Export rule(s) into an importable ndjson file. + +Options: + -f, --rule-file FILE Export specified rule files + -d, --directory DIRECTORY Recursively export rules from a directory + -o, --outfile FILE Name of file for exported rules + -r, --randomize-id Randomize rule IDs before export + -h, --help Show this message and exit. +``` + _*To load a custom rule, the proper index must be setup first. The simplest way to do this is to click the `Load prebuilt detection rules and timeline templates` button on the `detections` page in the Kibana security app._ diff --git a/detection_rules/main.py b/detection_rules/main.py index 2aea7ab684a..fd202cd5271 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -7,6 +7,8 @@ import json import os import re +import time +from pathlib import Path import click import jsonschema @@ -156,27 +158,39 @@ def view_rule(ctx, rule_id, rule_file, api_format, verbose=True): return rule -@root.command('export-rule') -@click.argument('rule-id', required=False) -@click.option('--rule-file', '-f', type=click.Path(dir_okay=False), help='Optionally view a rule from a specified file') -@click.option('--ndjson/--json', default=True, help='Output format') -@click.pass_context -def export_rule(ctx, rule_id, rule_file, ndjson): - """Export a rule as json/ndjson.""" - from .packaging import manage_versions +@root.command('export-rules') +@click.argument('rule-id', nargs=-1, required=False) +@click.option('--rule-file', '-f', multiple=True, type=click.Path(dir_okay=False), help='Export specified rule files') +@click.option('--directory', '-d', multiple=True, type=click.Path(file_okay=False), + help='Recursively export rules from a directory') +@click.option('--outfile', '-o', default=get_path('exports', f'{time.strftime("%Y%m%dT%H%M%SL")}.ndjson'), + type=click.Path(dir_okay=False), help='Name of file for exported rules') +@click.option('--randomize-id', '-r', is_flag=True, help='Randomize rule IDs before export') +def export_rules(rule_id, rule_file, directory, outfile, randomize_id): + """Export rule(s) into an importable ndjson file.""" + from .packaging import Package + + if not (rule_id or rule_file or directory): + client_error('Must specify a rule_id, rule_file, or directory') + + rules = [r for r in rule_loader.load_rules(verbose=False).values() if r.id in rule_id] if rule_id else [] - rule = ctx.invoke(view_rule, rule_id=rule_id, rule_file=rule_file, verbose=False) + rule_files = list(rule_file) + for _dir in directory: + rule_files.extend(list(Path(_dir).rglob('*.toml'))) - ext = '.ndjson' if ndjson else '.json' - base, _ = os.path.splitext(rule.path) - outfile = base + ext + file_lookup = rule_loader.load_rule_files(verbose=False, paths=rule_files) + rules.extend(rule_loader.load_rules(file_lookup=file_lookup).values()) - # add version - manage_versions([rule], verbose=False) + if randomize_id: + from uuid import uuid4 + for rule in rules: + rule.contents['rule_id'] = str(uuid4()) - with open(outfile, 'w') as f: - json.dump(rule.contents, f, sort_keys=True, indent=None if ndjson else 2) - click.echo(f'Rule: {rule.name} saved to: {outfile}') + Path(outfile).parent.mkdir(exist_ok=True) + package = Package(rules, '_', verbose=False) + package.export(outfile) + return package.rules @root.command('validate-rule') diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index 064c962a49e..71399ee6d9b 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -10,6 +10,7 @@ import os import shutil from collections import defaultdict, OrderedDict +from typing import List import click @@ -137,24 +138,25 @@ class Package(object): """Packaging object for siem rules and releases.""" def __init__(self, rules, name, deprecated_rules=None, release=False, current_versions=None, min_version=None, - max_version=None, update_version_lock=False): + max_version=None, update_version_lock=False, verbose=True): """Initialize a package.""" - self.rules = [r.copy() for r in rules] # type: list[Rule] + self.rules: List[Rule] = [r.copy() for r in rules] self.name = name - self.deprecated_rules = [r.copy() for r in deprecated_rules or []] # type: list[Rule] + self.deprecated_rules: List[Rule] = [r.copy() for r in deprecated_rules or []] self.release = release self.changed_rule_ids, self.new_rules_ids, self.removed_rule_ids = self._add_versions(current_versions, - update_version_lock) + update_version_lock, + verbose=verbose) if min_version or max_version: self.rules = [r for r in self.rules if (min_version or 0) <= r.contents['version'] <= (max_version or r.contents['version'])] - def _add_versions(self, current_versions, update_versions_lock=False): + def _add_versions(self, current_versions, update_versions_lock=False, verbose=True): """Add versions to rules at load time.""" return manage_versions(self.rules, deprecated_rules=self.deprecated_rules, current_versions=current_versions, - save_changes=update_versions_lock) + save_changes=update_versions_lock, verbose=verbose) @staticmethod def _package_notice_file(save_dir): @@ -245,6 +247,17 @@ def save(self, verbose=True): if verbose: click.echo('Package saved to: {}'.format(save_dir)) + def export(self, outfile, verbose=True): + """Export rules into a consolidated ndjson file.""" + base, _ = os.path.splitext(outfile) + outfile = base + '.ndjson' + + with open(outfile, 'w') as f: + f.write('\n'.join(json.dumps(r.contents, sort_keys=True) for r in self.rules)) + + if verbose: + click.echo(f'Exported {len(self.rules)} rules into {outfile}') + def get_package_hash(self, as_api=True, verbose=True): """Get hash of package contents.""" contents = base64.b64encode(self.get_consolidated(as_api=as_api).encode('utf-8')) From 7de7f437d58088d06d55b954d0a4df87f2d3f5c7 Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Tue, 1 Dec 2020 14:13:35 -0900 Subject: [PATCH 3/7] add checks for missing rule ID and duplicate rules --- CLI.md | 2 +- detection_rules/main.py | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CLI.md b/CLI.md index 0e333c6e50e..fe82f5b0fe6 100644 --- a/CLI.md +++ b/CLI.md @@ -179,7 +179,7 @@ Options: -f, --rule-file FILE Export specified rule files -d, --directory DIRECTORY Recursively export rules from a directory -o, --outfile FILE Name of file for exported rules - -r, --randomize-id Randomize rule IDs before export + -r, --replace-id Replace rule IDs with new IDs before export -h, --help Show this message and exit. ``` diff --git a/detection_rules/main.py b/detection_rules/main.py index fd202cd5271..451380f343e 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -165,24 +165,44 @@ def view_rule(ctx, rule_id, rule_file, api_format, verbose=True): help='Recursively export rules from a directory') @click.option('--outfile', '-o', default=get_path('exports', f'{time.strftime("%Y%m%dT%H%M%SL")}.ndjson'), type=click.Path(dir_okay=False), help='Name of file for exported rules') -@click.option('--randomize-id', '-r', is_flag=True, help='Randomize rule IDs before export') -def export_rules(rule_id, rule_file, directory, outfile, randomize_id): +@click.option('--replace-id', '-r', is_flag=True, help='Replace rule IDs with new IDs before export') +def export_rules(rule_id, rule_file, directory, outfile, replace_id): """Export rule(s) into an importable ndjson file.""" from .packaging import Package if not (rule_id or rule_file or directory): - client_error('Must specify a rule_id, rule_file, or directory') + client_error('Required: at least one of --rule-id, --rule-file, or --directory') - rules = [r for r in rule_loader.load_rules(verbose=False).values() if r.id in rule_id] if rule_id else [] + if rule_id: + all_rules = {r.id: r for r in rule_loader.load_rules(verbose=False).values()} + missing = [rid for rid in rule_id if rid not in all_rules] + + if missing: + client_error(f'Unknown rules for rule IDs: {", ".join(missing)}') + + rules = [r for r in all_rules.values() if r.id in rule_id] + rule_ids = [r.id for r in rules] + else: + rules = [] + rule_ids = [] rule_files = list(rule_file) - for _dir in directory: - rule_files.extend(list(Path(_dir).rglob('*.toml'))) + for dirpath in directory: + rule_files.extend(list(Path(dirpath).rglob('*.toml'))) file_lookup = rule_loader.load_rule_files(verbose=False, paths=rule_files) - rules.extend(rule_loader.load_rules(file_lookup=file_lookup).values()) + rules_from_files = rule_loader.load_rules(file_lookup=file_lookup).values() + + # rule_loader.load_rules handles checks for duplicate rule IDs - this means rules loaded by ID are de-duped and + # rules loaded from files and directories are de-duped from each other, so this check is to ensure that there is + # no overlap between the two sets of rules + duplicates = [r.id for r in rules_from_files if r.id in rule_ids] + if duplicates: + client_error(f'Duplicate rules for rule IDs: {", ".join(duplicates)}') + + rules.extend(rules_from_files) - if randomize_id: + if replace_id: from uuid import uuid4 for rule in rules: rule.contents['rule_id'] = str(uuid4()) From 65cf1950b4dbcedd3b0a517d063e92bbcaaee1bb Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Fri, 18 Dec 2020 23:05:34 -0900 Subject: [PATCH 4/7] add --downgrade-version option --- .gitignore | 2 ++ CLI.md | 16 ++++++++++----- detection_rules/kbwrap.py | 14 ++++++-------- detection_rules/main.py | 11 ++++++++--- detection_rules/packaging.py | 30 +++++++++++++++++++++++++---- detection_rules/rule.py | 13 ++++++++++++- detection_rules/schemas/__init__.py | 14 ++++++++------ 7 files changed, 73 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index e588a35d396..9f9e91c6ce2 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,5 @@ ENV/ # Siem rules releases/ collections/ +exports/ +surveys/ diff --git a/CLI.md b/CLI.md index fe82f5b0fe6..9194d6583fc 100644 --- a/CLI.md +++ b/CLI.md @@ -176,11 +176,17 @@ Usage: detection_rules export-rules [OPTIONS] [RULE_ID]... Export rule(s) into an importable ndjson file. Options: - -f, --rule-file FILE Export specified rule files - -d, --directory DIRECTORY Recursively export rules from a directory - -o, --outfile FILE Name of file for exported rules - -r, --replace-id Replace rule IDs with new IDs before export - -h, --help Show this message and exit. + -f, --rule-file FILE Export specified rule files + -d, --directory DIRECTORY Recursively export rules from a directory + -o, --outfile FILE Name of file for exported rules + -r, --replace-id Replace rule IDs with new IDs before export + --downgrade-version [7.8|7.9|7.10|7.11] + Downgrade a rule version to be compatible + with older instances of Kibana + -s, --skip-unsupported If `--downgrade-version` is passed, skip + rule types which are unsupported (an error + will be raised otherwise) + -h, --help Show this message and exit. ``` _*To load a custom rule, the proper index must be setup first. The simplest way to do this is to click diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index dec19acf4aa..78adc049ac8 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -52,9 +52,8 @@ def kibana_group(ctx: click.Context, **kibana_kwargs): @click.pass_context def upload_rule(ctx, toml_files): """Upload a list of rule .toml files to Kibana.""" - from uuid import uuid4 from .packaging import manage_versions - from .schemas import downgrade + from .rule import downgrade_contents_from_rule kibana = ctx.obj['kibana'] file_lookup = load_rule_files(paths=toml_files) @@ -68,12 +67,11 @@ def upload_rule(ctx, toml_files): api_payloads = [] for rule in rules: - payload = rule.contents.copy() - meta = payload.setdefault("meta", {}) - meta["original"] = dict(id=rule.id, **rule.metadata) - payload["rule_id"] = str(uuid4()) - payload = downgrade(payload, kibana.version) - rule = RuleResource(payload) + try: + rule = RuleResource(downgrade_contents_from_rule(rule, kibana.version)) + except ValueError as e: + client_error(f'{e} in version:{kibana.version}, for rule: {rule.name}', e, ctx=ctx) + api_payloads.append(rule) with kibana: diff --git a/detection_rules/main.py b/detection_rules/main.py index 451380f343e..b4c36800df9 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -18,7 +18,7 @@ from .misc import client_error, nested_set, parse_config from .rule import Rule from .rule_formatter import toml_write -from .schemas import CurrentSchema +from .schemas import CurrentSchema, schema_map from .utils import get_path, clear_caches, load_rule_contents @@ -166,7 +166,12 @@ def view_rule(ctx, rule_id, rule_file, api_format, verbose=True): @click.option('--outfile', '-o', default=get_path('exports', f'{time.strftime("%Y%m%dT%H%M%SL")}.ndjson'), type=click.Path(dir_okay=False), help='Name of file for exported rules') @click.option('--replace-id', '-r', is_flag=True, help='Replace rule IDs with new IDs before export') -def export_rules(rule_id, rule_file, directory, outfile, replace_id): +@click.option('--downgrade-version', type=click.Choice(list(schema_map)), + help='Downgrade a rule version to be compatible with older instances of Kibana') +@click.option('--skip-unsupported', '-s', is_flag=True, + help='If `--downgrade-version` is passed, skip rule types which are unsupported ' + '(an error will be raised otherwise)') +def export_rules(rule_id, rule_file, directory, outfile, replace_id, downgrade_version, skip_unsupported): """Export rule(s) into an importable ndjson file.""" from .packaging import Package @@ -209,7 +214,7 @@ def export_rules(rule_id, rule_file, directory, outfile, replace_id): Path(outfile).parent.mkdir(exist_ok=True) package = Package(rules, '_', verbose=False) - package.export(outfile) + package.export(outfile, downgrade_version=downgrade_version, skip_unsupported=skip_unsupported) return package.rules diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index 71399ee6d9b..3770274776a 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -16,7 +16,7 @@ from . import rule_loader from .misc import JS_LICENSE -from .rule import Rule # noqa: F401 +from .rule import Rule, downgrade_contents_from_rule # noqa: F401 from .utils import get_path, get_etc_path, load_etc_dump, save_etc_dump RELEASE_DIR = get_path("releases") @@ -247,16 +247,38 @@ def save(self, verbose=True): if verbose: click.echo('Package saved to: {}'.format(save_dir)) - def export(self, outfile, verbose=True): + def export(self, outfile, downgrade_version=None, verbose=True, skip_unsupported=False): """Export rules into a consolidated ndjson file.""" base, _ = os.path.splitext(outfile) outfile = base + '.ndjson' + unsupported = [] with open(outfile, 'w') as f: - f.write('\n'.join(json.dumps(r.contents, sort_keys=True) for r in self.rules)) + if downgrade_version: + if skip_unsupported: + export_str = '' + + for rule in self.rules: + try: + export_str += json.dumps(downgrade_contents_from_rule(rule, downgrade_version), + sort_keys=True) + '\n' + except ValueError as e: + unsupported.append(f'{e}: {rule.id} - {rule.name}') + continue + + f.write(export_str) + else: + f.write('\n'.join(json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True) + for r in self.rules)) + else: + f.write('\n'.join(json.dumps(r.contents, sort_keys=True) for r in self.rules)) if verbose: - click.echo(f'Exported {len(self.rules)} rules into {outfile}') + click.echo(f'Exported {len(self.rules) - len(unsupported)} rules into {outfile}') + + if skip_unsupported and unsupported: + unsupported_str = '\n- '.join(unsupported) + click.echo(f'Skipped {len(unsupported)} unsupported rules: \n- {unsupported_str}') def get_package_hash(self, as_api=True, verbose=True): """Get hash of package contents.""" diff --git a/detection_rules/rule.py b/detection_rules/rule.py index b4bfc604887..3ab2fc91621 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -7,6 +7,7 @@ import hashlib import json import os +from uuid import uuid4 import click import kql @@ -15,7 +16,7 @@ from . import ecs, beats from .attack import tactics, build_threat_map_entry, technique_lookup from .rule_formatter import nested_normalize, toml_write -from .schemas import CurrentSchema, TomlMetadata # RULE_TYPES, metadata_schema, schema_validate, get_schema +from .schemas import CurrentSchema, TomlMetadata, downgrade from .utils import get_path, clear_caches, cached @@ -439,3 +440,13 @@ def build(cls, path=None, rule_type=None, required_only=True, save=True, verbose click.echo(' - to have a rule validate against a specific beats schema, add it to metadata->beats_version') return rule + + +def downgrade_contents_from_rule(rule: Rule, target_version: str) -> dict: + """Generate the downgraded contents from a rule.""" + payload = rule.contents.copy() + meta = payload.setdefault("meta", {}) + meta["original"] = dict(id=rule.id, **rule.metadata) + payload["rule_id"] = str(uuid4()) + payload = downgrade(payload, target_version) + return payload diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py index 15c859c7aa2..96adcfef077 100644 --- a/detection_rules/schemas/__init__.py +++ b/detection_rules/schemas/__init__.py @@ -17,15 +17,17 @@ "downgrade", "CurrentSchema", "validate_rta_mapping", + "schema_map", "TomlMetadata", ) -all_schemas = [ - ApiSchema78, - ApiSchema79, - ApiSchema710, - ApiSchema711, -] +schema_map = { + '7.8': ApiSchema78, + '7.9': ApiSchema79, + '7.10': ApiSchema710, + '7.11': ApiSchema711 +} +all_schemas = list(schema_map.values()) CurrentSchema = all_schemas[-1] From c90e4b56cf58fd05ef6bbcbce1e6237b0ea19796 Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Fri, 18 Dec 2020 23:33:40 -0900 Subject: [PATCH 5/7] open file more lazily to avoid erroring out with open handle --- detection_rules/packaging.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index 3770274776a..e6a615cb78f 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -10,6 +10,7 @@ import os import shutil from collections import defaultdict, OrderedDict +from pathlib import Path from typing import List import click @@ -249,28 +250,29 @@ def save(self, verbose=True): def export(self, outfile, downgrade_version=None, verbose=True, skip_unsupported=False): """Export rules into a consolidated ndjson file.""" - base, _ = os.path.splitext(outfile) - outfile = base + '.ndjson' + outfile = Path(outfile).with_suffix('.ndjson') unsupported = [] - with open(outfile, 'w') as f: - if downgrade_version: - if skip_unsupported: - export_str = '' + if downgrade_version: + if skip_unsupported: + export_str = '' - for rule in self.rules: - try: - export_str += json.dumps(downgrade_contents_from_rule(rule, downgrade_version), - sort_keys=True) + '\n' - except ValueError as e: - unsupported.append(f'{e}: {rule.id} - {rule.name}') - continue + for rule in self.rules: + try: + export_str += json.dumps(downgrade_contents_from_rule(rule, downgrade_version), + sort_keys=True) + '\n' + except ValueError as e: + unsupported.append(f'{e}: {rule.id} - {rule.name}') + continue + with open(outfile, 'w') as f: f.write(export_str) - else: + else: + with open(outfile, 'w') as f: f.write('\n'.join(json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True) for r in self.rules)) - else: + else: + with open(outfile, 'w') as f: f.write('\n'.join(json.dumps(r.contents, sort_keys=True) for r in self.rules)) if verbose: From 3681b7cc3ddc817cb4009d687081ceb1f957b4f5 Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Wed, 3 Feb 2021 14:05:04 -0900 Subject: [PATCH 6/7] tweaks varname changes from feedback --- CLI.md | 5 +++-- detection_rules/kbwrap.py | 5 +++-- detection_rules/main.py | 10 +++++----- detection_rules/misc.py | 5 +++-- detection_rules/packaging.py | 12 +++++------- detection_rules/schemas/__init__.py | 17 ++++++++--------- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/CLI.md b/CLI.md index 9194d6583fc..30910e57b68 100644 --- a/CLI.md +++ b/CLI.md @@ -164,6 +164,7 @@ Usage: detection_rules kibana upload-rule [OPTIONS] TOML_FILES... Upload a list of rule .toml files to Kibana. Options: + -r, --replace-id Replace rule IDs with new IDs before export -h, --help Show this message and exit. ``` @@ -180,10 +181,10 @@ Options: -d, --directory DIRECTORY Recursively export rules from a directory -o, --outfile FILE Name of file for exported rules -r, --replace-id Replace rule IDs with new IDs before export - --downgrade-version [7.8|7.9|7.10|7.11] + --stack-version [7.8|7.9|7.10|7.11] Downgrade a rule version to be compatible with older instances of Kibana - -s, --skip-unsupported If `--downgrade-version` is passed, skip + -s, --skip-unsupported If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise) -h, --help Show this message and exit. diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index eb2fa0abc8d..3b1e83ad769 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -49,8 +49,9 @@ def kibana_group(ctx: click.Context, **kibana_kwargs): @kibana_group.command("upload-rule") @click.argument("toml-files", nargs=-1, required=True) +@click.option('--replace-id', '-r', is_flag=True, help='Replace rule IDs with new IDs before export') @click.pass_context -def upload_rule(ctx, toml_files): +def upload_rule(ctx, toml_files, replace_id): """Upload a list of rule .toml files to Kibana.""" from .packaging import manage_versions @@ -67,7 +68,7 @@ def upload_rule(ctx, toml_files): for rule in rules: try: - payload = rule.get_payload(include_version=True, replace_id=True, embed_metadata=True, + payload = rule.get_payload(include_version=True, replace_id=replace_id, embed_metadata=True, target_version=kibana.version) except ValueError as e: client_error(f'{e} in version:{kibana.version}, for rule: {rule.name}', e, ctx=ctx) diff --git a/detection_rules/main.py b/detection_rules/main.py index 826f7c1a629..c450c053bc2 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -18,7 +18,7 @@ from .misc import client_error, nested_set, parse_config from .rule import Rule from .rule_formatter import toml_write -from .schemas import CurrentSchema, schema_map +from .schemas import CurrentSchema, available_versions from .utils import get_path, clear_caches, load_rule_contents @@ -166,12 +166,12 @@ def view_rule(ctx, rule_id, rule_file, api_format, verbose=True): @click.option('--outfile', '-o', default=get_path('exports', f'{time.strftime("%Y%m%dT%H%M%SL")}.ndjson'), type=click.Path(dir_okay=False), help='Name of file for exported rules') @click.option('--replace-id', '-r', is_flag=True, help='Replace rule IDs with new IDs before export') -@click.option('--downgrade-version', type=click.Choice(list(schema_map)), +@click.option('--stack-version', type=click.Choice(available_versions), help='Downgrade a rule version to be compatible with older instances of Kibana') @click.option('--skip-unsupported', '-s', is_flag=True, - help='If `--downgrade-version` is passed, skip rule types which are unsupported ' + help='If `--stack-version` is passed, skip rule types which are unsupported ' '(an error will be raised otherwise)') -def export_rules(rule_id, rule_file, directory, outfile, replace_id, downgrade_version, skip_unsupported): +def export_rules(rule_id, rule_file, directory, outfile, replace_id, stack_version, skip_unsupported): """Export rule(s) into an importable ndjson file.""" from .packaging import Package @@ -214,7 +214,7 @@ def export_rules(rule_id, rule_file, directory, outfile, replace_id, downgrade_v Path(outfile).parent.mkdir(exist_ok=True) package = Package(rules, '_', verbose=False) - package.export(outfile, downgrade_version=downgrade_version, skip_unsupported=skip_unsupported) + package.export(outfile, downgrade_version=stack_version, skip_unsupported=skip_unsupported) return package.rules diff --git a/detection_rules/misc.py b/detection_rules/misc.py index 6dd4de0ee5c..35d5da54fc4 100644 --- a/detection_rules/misc.py +++ b/detection_rules/misc.py @@ -17,7 +17,7 @@ from datetime import datetime from functools import wraps from pathlib import Path -from typing import Dict, Tuple +from typing import Dict, NoReturn, Tuple from zipfile import ZipFile import click @@ -359,7 +359,8 @@ def show(self, file=None, err=True): click.echo(msg, err=err, file=file) -def client_error(message, exc: Exception = None, debug=None, ctx: click.Context = None, file=None, err=None): +def client_error(message, exc: Exception = None, debug=None, ctx: click.Context = None, file=None, + err=None) -> NoReturn: config_debug = True if ctx and ctx.ensure_object(dict) and ctx.obj.get('debug') is True else False debug = debug if debug is not None else config_debug diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index d4d9dd44892..a780e4efcbf 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -270,15 +270,13 @@ def export(self, outfile, downgrade_version=None, verbose=True, skip_unsupported unsupported.append(f'{e}: {rule.id} - {rule.name}') continue - with open(outfile, 'w') as f: - f.write(export_str) + outfile.write_text(export_str) else: - with open(outfile, 'w') as f: - f.write('\n'.join(json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True) - for r in self.rules)) + outfile.write_text( + '\n'.join(json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True) + for r in self.rules)) else: - with open(outfile, 'w') as f: - f.write('\n'.join(json.dumps(r.contents, sort_keys=True) for r in self.rules)) + outfile.write_text('\n'.join(json.dumps(r.contents, sort_keys=True) for r in self.rules)) if verbose: click.echo(f'Exported {len(self.rules) - len(unsupported)} rules into {outfile}') diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py index 96adcfef077..e80fd8c7d9a 100644 --- a/detection_rules/schemas/__init__.py +++ b/detection_rules/schemas/__init__.py @@ -14,22 +14,21 @@ __all__ = ( "all_schemas", + "available_versions", "downgrade", "CurrentSchema", "validate_rta_mapping", - "schema_map", "TomlMetadata", ) -schema_map = { - '7.8': ApiSchema78, - '7.9': ApiSchema79, - '7.10': ApiSchema710, - '7.11': ApiSchema711 -} -all_schemas = list(schema_map.values()) - +all_schemas = [ + ApiSchema78, + ApiSchema79, + ApiSchema710, + ApiSchema711, +] CurrentSchema = all_schemas[-1] +available_versions = [cls.STACK_VERSION for cls in all_schemas] def downgrade(api_contents: dict, target_version: str): From 7b2f1eaa03459be9053d2fa94ed2fe6b1b1fb805 Mon Sep 17 00:00:00 2001 From: brokensound77 Date: Mon, 8 Feb 2021 20:26:37 -0900 Subject: [PATCH 7/7] buffer exported rules and save --- detection_rules/main.py | 2 +- detection_rules/packaging.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/detection_rules/main.py b/detection_rules/main.py index c450c053bc2..b31f6aca234 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -196,7 +196,7 @@ def export_rules(rule_id, rule_file, directory, outfile, replace_id, stack_versi rule_files.extend(list(Path(dirpath).rglob('*.toml'))) file_lookup = rule_loader.load_rule_files(verbose=False, paths=rule_files) - rules_from_files = rule_loader.load_rules(file_lookup=file_lookup).values() + rules_from_files = rule_loader.load_rules(file_lookup=file_lookup).values() if file_lookup else [] # rule_loader.load_rules handles checks for duplicate rule IDs - this means rules loaded by ID are de-duped and # rules loaded from files and directories are de-duped from each other, so this check is to ensure that there is diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index a780e4efcbf..78542a3d494 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -260,23 +260,23 @@ def export(self, outfile, downgrade_version=None, verbose=True, skip_unsupported if downgrade_version: if skip_unsupported: - export_str = '' + output_lines = [] for rule in self.rules: try: - export_str += json.dumps(downgrade_contents_from_rule(rule, downgrade_version), - sort_keys=True) + '\n' + output_lines.append(json.dumps(downgrade_contents_from_rule(rule, downgrade_version), + sort_keys=True)) except ValueError as e: unsupported.append(f'{e}: {rule.id} - {rule.name}') continue - outfile.write_text(export_str) else: - outfile.write_text( - '\n'.join(json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True) - for r in self.rules)) + output_lines = [json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True) + for r in self.rules] else: - outfile.write_text('\n'.join(json.dumps(r.contents, sort_keys=True) for r in self.rules)) + output_lines = [json.dumps(r.contents, sort_keys=True) for r in self.rules] + + outfile.write_text('\n'.join(output_lines) + '\n') if verbose: click.echo(f'Exported {len(self.rules) - len(unsupported)} rules into {outfile}')