Merge pull request #20 from gilles-peskine-arm/search_outcomes_config-create

Script to search the outcome file for configurations with given requirements

Author: gilles-peskine-arm

scripts/search_outcomes_config.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python
+"""Search an outcome file for configurations with given settings.
+
+Read an outcome file and report the configurations in which test_suite_config
+runs with the required settings (compilation option enabled or disabled).
+"""
+
+import argparse
+import os
+import re
+import subprocess
+from typing import Dict, FrozenSet, Iterator, List, Set
+import tempfile
+import unittest
+
+from mbedtls_framework import build_tree
+
+
+def make_regexp_for_settings(settings: List[str]) -> str:
+    """Construct a regexp matching the interesting outcome lines.
+
+    Interesting outcome lines are from test_suite_config where the given
+    setting is passing.
+
+    We assume that the elements of settings don't contain regexp special
+    characters.
+    """
+    return (r';test_suite_config[^;]*;Config: (' +
+            '|'.join(settings) +
+            r');PASS;')
+
+def run_grep(regexp: str, outcome_file: str) -> List[str]:
+    """Run grep on the outcome file and return the matching lines."""
+    env = os.environ.copy()
+    env['LC_ALL'] = 'C' # Speeds up some versions of GNU grep
+    try:
+        return subprocess.check_output(['grep', '-E', regexp, outcome_file],
+                                       encoding='ascii',
+                                       env=env).splitlines()
+    except subprocess.CalledProcessError as exn:
+        if exn.returncode == 1:
+            return [] # No results. We don't consider this an error.
+        raise
+
+OUTCOME_LINE_RE = re.compile(r'[^;]*;'
+                             r'([^;]*);'
+                             r'test_suite_config\.(?:[^;]*);'
+                             r'Config: ([^;]*);'
+                             r'PASS;')
+
+def extract_configuration_data(outcome_lines: List[str]) -> Dict[str, FrozenSet[str]]:
+    """Extract the configuration data from outcome lines.
+
+    The result maps a configuration name to the list of passing settings
+    in that configuration.
+    """
+    config_data = {} #type: Dict[str, Set[str]]
+    for line in outcome_lines:
+        m = OUTCOME_LINE_RE.match(line)
+        # Assuming a well-formed outcome file, make_regexp_for_settings()
+        # arranges to only return lines that should match OUTCOME_LINE_RE.
+        # So this assertion can't fail unless there is an unexpected
+        # divergence between OUTCOME_LINE_RE, make_regexp_for_settings()
+        # and the format of the given outcome file
+        assert m is not None
+        config_name, setting = m.groups()
+        if config_name not in config_data:
+            config_data[config_name] = set()
+        config_data[config_name].add(setting)
+    return dict((name, frozenset(settings))
+                for name, settings in config_data.items())
+
+
+def matching_configurations(config_data: Dict[str, FrozenSet[str]],
+                            required: List[str]) -> Iterator[str]:
+    """Search configurations with the given passing settings.
+
+    config_data maps a configuration name to the list of passing settings
+    in that configuration.
+
+    Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or
+    PSA_xxx), optionally prefixed with "!".
+    """
+    required_set = frozenset(required)
+    for config, observed in config_data.items():
+        if required_set.issubset(observed):
+            yield config
+
+def search_config_outcomes(outcome_file: str, settings: List[str]) -> List[str]:
+    """Search the given outcome file for reports of the given settings.
+
+    Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or
+    PSA_xxx), optionally prefixed with "!".
+    """
+    # The outcome file is large enough (hundreds of MB) that parsing it
+    # in Python is slow. Use grep to speed this up considerably.
+    regexp = make_regexp_for_settings(settings)
+    outcome_lines = run_grep(regexp, outcome_file)
+    config_data = extract_configuration_data(outcome_lines)
+    return sorted(matching_configurations(config_data, settings))
+
+
+class TestSearch(unittest.TestCase):
+    """Tests of search functionality."""
+
+    OUTCOME_FILE_CONTENT = """\
+whatever;foobar;test_suite_config.part;Config: MBEDTLS_FOO;PASS;
+whatever;foobar;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP;
+whatever;foobar;test_suite_config.part;Config: MBEDTLS_BAR;PASS;
+whatever;foobar;test_suite_config.part;Config: !MBEDTLS_BAR;SKIP;
+whatever;foobar;test_suite_config.part;Config: MBEDTLS_QUX;SKIP;
+whatever;foobar;test_suite_config.part;Config: !MBEDTLS_QUX;PASS;
+whatever;fooqux;test_suite_config.part;Config: MBEDTLS_FOO;PASS;
+whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP;
+whatever;fooqux;test_suite_config.part;Config: MBEDTLS_BAR;SKIP;
+whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_BAR;PASS;
+whatever;fooqux;test_suite_config.part;Config: MBEDTLS_QUX;PASS;
+whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_QUX;SKIP;
+whatever;fooqux;test_suite_something.else;Config: MBEDTLS_BAR;PASS;
+whatever;boring;test_suite_config.part;Config: BORING;PASS;
+whatever;parasite;not_test_suite_config.not;Config: MBEDTLS_FOO;PASS;
+whatever;parasite;test_suite_config.but;Config: MBEDTLS_QUX with bells on;PASS;
+whatever;parasite;test_suite_config.but;Not Config: MBEDTLS_QUX;PASS;
+"""
+
+    def search(self, settings: List[str], expected: List[str]) -> None:
+        """Test the search functionality.
+
+        * settings: settings to search.
+        * expected: expected search results.
+        """
+        with tempfile.NamedTemporaryFile() as tmp:
+            tmp.write(self.OUTCOME_FILE_CONTENT.encode())
+            tmp.flush()
+            actual = search_config_outcomes(tmp.name, settings)
+            self.assertEqual(actual, expected)
+
+    def test_foo(self) -> None:
+        self.search(['MBEDTLS_FOO'], ['foobar', 'fooqux'])
+
+    def test_bar(self) -> None:
+        self.search(['MBEDTLS_BAR'], ['foobar'])
+
+    def test_foo_bar(self) -> None:
+        self.search(['MBEDTLS_FOO', 'MBEDTLS_BAR'], ['foobar'])
+
+    def test_foo_notbar(self) -> None:
+        self.search(['MBEDTLS_FOO', '!MBEDTLS_BAR'], ['fooqux'])
+
+
+class TestOutcome(unittest.TestCase):
+    """Tests of outcome file format expectations.
+
+    This class builds and runs the config tests in the current configuration.
+    The configuration must have at least one feature enabled and at least
+    one feature disabled in each category: MBEDTLS_xxx and PSA_WANT_xxx.
+    It needs a C compiler.
+    """
+
+    outcome_content = '' # Let mypy know this field can be used in test case methods
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        """Generate, build and run the config tests."""
+        root_dir = build_tree.guess_project_root()
+        tests_dir = os.path.join(root_dir, 'tests')
+        suites = ['test_suite_config.mbedtls_boolean',
+                  'test_suite_config.psa_boolean']
+        _output = subprocess.check_output(['make'] + suites,
+                                          cwd=tests_dir,
+                                          stderr=subprocess.STDOUT)
+        with tempfile.NamedTemporaryFile(dir=tests_dir) as outcome_file:
+            env = os.environ.copy()
+            env['MBEDTLS_TEST_PLATFORM'] = 'some_platform'
+            env['MBEDTLS_TEST_CONFIGURATION'] = 'some_configuration'
+            env['MBEDTLS_TEST_OUTCOME_FILE'] = outcome_file.name
+            for suite in suites:
+                _output = subprocess.check_output([os.path.join(os.path.curdir, suite)],
+                                                  cwd=tests_dir,
+                                                  env=env,
+                                                  stderr=subprocess.STDOUT)
+            cls.outcome_content = outcome_file.read().decode('ascii')
+
+    def test_outcome_format(self) -> None:
+        """Check that there are outcome lines matching the expected general format."""
+        def regex(prefix: str, result: str) -> str:
+            return (r'(?:\A|\n)some_platform;some_configuration;'
+                    r'test_suite_config\.\w+;Config: {}_\w+;{};'
+                    .format(prefix, result))
+        self.assertRegex(self.outcome_content, regex('MBEDTLS', 'PASS'))
+        self.assertRegex(self.outcome_content, regex('MBEDTLS', 'SKIP'))
+        self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'PASS'))
+        self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'SKIP'))
+        self.assertRegex(self.outcome_content, regex('PSA_WANT', 'PASS'))
+        self.assertRegex(self.outcome_content, regex('PSA_WANT', 'SKIP'))
+        self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'PASS'))
+        self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'SKIP'))
+
+    def test_outcome_lines(self) -> None:
+        """Look for some sample outcome lines."""
+        def regex(setting: str) -> str:
+            return (r'(?:\A|\n)some_platform;some_configuration;'
+                    r'test_suite_config\.\w+;Config: {};(PASS|SKIP);'
+                    .format(setting))
+        self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_C'))
+        self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_ROM_TABLES'))
+        self.assertRegex(self.outcome_content, regex('MBEDTLS_SSL_CLI_C'))
+        self.assertRegex(self.outcome_content, regex('MBEDTLS_X509_CRT_PARSE_C'))
+        self.assertRegex(self.outcome_content, regex('PSA_WANT_ALG_HMAC'))
+        self.assertRegex(self.outcome_content, regex('PSA_WANT_KEY_TYPE_AES'))
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--outcome-file', '-f', metavar='FILE',
+                        default='outcomes.csv',
+                        help='Outcome file to read (default: outcomes.csv)')
+    parser.add_argument('settings', metavar='SETTING', nargs='+',
+                        help='Required setting (e.g. "MBEDTLS_RSA_C" or "!PSA_WANT_ALG_SHA256")')
+    options = parser.parse_args()
+    found = search_config_outcomes(options.outcome_file, options.settings)
+    for name in found:
+        print(name)
+
+if __name__ == '__main__':
+    main()