SoftLayerDetector (Yelp#169)

justineyster · justineyster · commit ca961f4a36dd · 2020-09-09T16:05:13.000-04:00
Detects Softlayer API Keys

Supports git-defenders/detect-secrets-discuss#120
diff --git a/detect_secrets/core/baseline.py b/detect_secrets/core/baseline.py
@@ -3,11 +3,9 @@
 import re
 import subprocess
 
-from detect_secrets import util
 from detect_secrets.core.log import get_logger
 from detect_secrets.core.secrets_collection import SecretsCollection
 
-
 log = get_logger(format_string='%(message)s')
 
 
@@ -294,8 +292,8 @@ def _get_git_tracked_files(rootdir='.'):
             git_files = subprocess.check_output(
                 [
                     'git',
-                    '-C', rootdir,
                     'ls-files',
+                    rootdir,
                 ],
                 stderr=fnull,
             )
diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py
@@ -498,6 +498,12 @@ class PluginOptions:
             disable_help_text='Disable scanning for GH credentials',
             is_default=False,
         ),
+        PluginDescriptor(
+            classname='SoftLayerDetector',
+            disable_flag_text='--no-sl-scan',
+            disable_help_text='Disable scanning for SoftLayer keys',
+            is_default=True,
+        ),
     ]
 
     default_plugins_list = [
diff --git a/detect_secrets/plugins/common/initialize.py b/detect_secrets/plugins/common/initialize.py
@@ -10,6 +10,7 @@
 from ..keyword import KeywordDetector                       # noqa: F401
 from ..private_key import PrivateKeyDetector                # noqa: F401
 from ..slack import SlackDetector                           # noqa: F401
+from ..softlayer import SoftLayerDetector                   # noqa: F401
 from ..stripe import StripeDetector                         # noqa: F401
 from detect_secrets.core.log import log
 from detect_secrets.core.usage import PluginOptions
diff --git a/detect_secrets/plugins/softlayer.py b/detect_secrets/plugins/softlayer.py
@@ -1,35 +1,47 @@
 import re
-
 import requests
 
+from .base import RegexBasedDetector
 from detect_secrets.core.constants import VerifiedResult
-from detect_secrets.plugins.base import RegexBasedDetector
 
 
-class SoftlayerDetector(RegexBasedDetector):
-    """Scans for Softlayer credentials."""
+class SoftLayerDetector(RegexBasedDetector):
 
     secret_type = 'SoftLayer Credentials'
 
     # opt means optional
-    sl = r'(?:softlayer|sl)(?:_|-|)(?:api|)'
-    key_or_pass = r'(?:key|pwd|password|pass|token)'
+    opt_quote = r'(?:"|\'|)'
+    opt_dashes = r'(?:--|)'
+    sl = r'(?:softlayer|sl)'
+    opt_dash_undrscr = r'(?:_|-|)'
+    opt_api = r'(?:api|)'
+    key_or_pass = r'(?:key|pwd|password|pass)'
+    opt_space = r'(?: |)'
+    opt_equals = r'(?:=|:|:=|=>|)'
     secret = r'([a-z0-9]{64})'
     denylist = [
-        RegexBasedDetector.assign_regex_generator(
-            prefix_regex=sl,
-            secret_keyword_regex=key_or_pass,
-            secret_regex=secret,
+        re.compile(
+            r'{opt_quote}{opt_dashes}{sl}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}{key_or_pass}'
+            '{opt_quote}{opt_space}{opt_equals}{opt_space}{opt_quote}{secret}{opt_quote}'.format(
+                opt_quote=opt_quote,
+                opt_dashes=opt_dashes,
+                sl=sl,
+                opt_dash_undrscr=opt_dash_undrscr,
+                opt_api=opt_api,
+                key_or_pass=key_or_pass,
+                opt_space=opt_space,
+                opt_equals=opt_equals,
+                secret=secret,
+            ), flags=re.IGNORECASE,
         ),
-
         re.compile(
             r'(?:http|https)://api.softlayer.com/soap/(?:v3|v3.1)/([a-z0-9]{64})',
             flags=re.IGNORECASE,
         ),
     ]
 
     def verify(self, token, content):
-        usernames = find_username(content)
+        usernames = get_username(content)
         if not usernames:
             return VerifiedResult.UNVERIFIED
 
@@ -39,21 +51,32 @@ def verify(self, token, content):
         return VerifiedResult.VERIFIED_FALSE
 
 
-def find_username(content):
+def get_username(content):
     # opt means optional
-    username_keyword = (
-        r'(?:'
-        r'username|id|user|userid|user-id|user-name|'
-        r'name|user_id|user_name|uname'
-        r')'
-    )
+    opt_quote = r'(?:"|\'|)'
+    opt_dashes = r'(?:--|)'
+    opt_sl = r'(?:softlayer|sl|)'
+    opt_dash_undrscr = r'(?:_|-|)'
+    opt_api = r'(?:api|)'
+    username_keyword = r'(?:username|id|user|userid|user-id|user-name|name|user_id|user_name|uname)'
+    opt_space = r'(?: |)'
+    opt_equals = r'(?:=|:|:=|=>|)'
+    seperator = r'(?: |=|:|:=|=>)+'
     username = r'(\w(?:\w|_|@|\.|-)+)'
     regex = re.compile(
-        RegexBasedDetector.assign_regex_generator(
-            prefix_regex=SoftlayerDetector.sl,
-            secret_keyword_regex=username_keyword,
-            secret_regex=username,
-        ),
+        r'{opt_quote}{opt_dashes}{opt_sl}{opt_dash_undrscr}{opt_api}{opt_dash_undrscr}'
+        '{username_keyword}{opt_quote}{seperator}{opt_quote}{username}{opt_quote}'.format(
+            opt_quote=opt_quote,
+            opt_dashes=opt_dashes,
+            opt_sl=opt_sl,
+            opt_dash_undrscr=opt_dash_undrscr,
+            opt_api=opt_api,
+            username_keyword=username_keyword,
+            opt_space=opt_space,
+            opt_equals=opt_equals,
+            username=username,
+            seperator=seperator,
+        ), flags=re.IGNORECASE,
     )
 
     return [
@@ -64,16 +87,16 @@ def find_username(content):
 
 
 def verify_softlayer_key(username, token):
-    headers = {'Content-type': 'application/json'}
     try:
+        headers = {'Content-type': 'application/json'}
         response = requests.get(
             'https://api.softlayer.com/rest/v3/SoftLayer_Account.json',
             auth=(username, token), headers=headers,
         )
-    except requests.exceptions.RequestException:
-        return VerifiedResult.UNVERIFIED
 
-    if response.status_code == 200:
-        return VerifiedResult.VERIFIED_TRUE
-    else:
-        return VerifiedResult.VERIFIED_FALSE
+        if response.status_code == 200:
+            return VerifiedResult.VERIFIED_TRUE
+        else:
+            return VerifiedResult.VERIFIED_FALSE
+    except Exception:
+        return VerifiedResult.UNVERIFIED
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -12,3 +12,4 @@ responses
 tox-pip-extensions
 tox>=3.8
 unidiff
+responses
diff --git a/testing/factories.py b/testing/factories.py
@@ -3,15 +3,13 @@
 
 
 def potential_secret_factory(
-    type_='type',
-    filename='filename',
-    secret='secret',
-    lineno=1,
+    type_='type', filename='filename', secret='secret',
+    lineno=1, output_raw=False,
 ):
     """This is only marginally better than creating PotentialSecret objects directly,
     because of the default values.
     """
-    return PotentialSecret(type_, filename, secret, lineno)
+    return PotentialSecret(type_, filename, secret, lineno, output_raw=output_raw)
 
 
 def secrets_collection_factory(
diff --git a/tests/core/potential_secret_test.py b/tests/core/potential_secret_test.py
@@ -39,3 +39,12 @@ def test_json(self):
         secret = potential_secret_factory(secret='blah')
         for value in secret.json().values():
             assert value != 'blah'
+
+    def test_json_output_raw(self):
+        secret = potential_secret_factory(secret='blah', output_raw=True)
+        assert 'blah' in secret.json().values()
+
+    def test_other_factors(self):
+        secret = potential_secret_factory(secret='blah')
+        secret.other_factors['second factor'] = 'another one'
+        assert {'second factor': 'another one'} == secret.json()['other_factors']
diff --git a/tests/core/usage_test.py b/tests/core/usage_test.py
@@ -35,6 +35,7 @@ def test_consolidates_output_basic(self):
             'StripeDetector': {},
             'ArtifactoryDetector': {},
             'GHDetector': {},
+            'SoftLayerDetector': {},
         }
 
     def test_consolidates_removes_disabled_plugins(self):
diff --git a/tests/main_test.py b/tests/main_test.py
@@ -327,6 +327,9 @@ def test_old_baseline_ignored_with_update_flag(
                     {
                         'name': 'SlackDetector',
                     },
+                    {
+                        'name': 'SoftLayerDetector',
+                    },
                     {
                         'name': 'StripeDetector',
                     },
@@ -373,6 +376,9 @@ def test_old_baseline_ignored_with_update_flag(
                     {
                         'name': 'SlackDetector',
                     },
+                    {
+                        'name': 'SoftLayerDetector',
+                    },
                     {
                         'name': 'StripeDetector',
                     },
@@ -481,6 +487,9 @@ def test_old_baseline_ignored_with_update_flag(
                     {
                         'name': 'SlackDetector',
                     },
+                    {
+                        'name': 'SoftLayerDetector',
+                    },
                     {
                         'name': 'StripeDetector',
                     },
@@ -535,6 +544,9 @@ def test_old_baseline_ignored_with_update_flag(
                     {
                         'name': 'SlackDetector',
                     },
+                    {
+                        'name': 'SoftLayerDetector',
+                    },
                     {
                         'name': 'StripeDetector',
                     },
@@ -653,33 +665,32 @@ def test_audit_short_file(self, filename, expected_output):
                 expected_output,
             )
 
-    @pytest.mark.parametrize(
-        'filename, expected_output',
-        [
-            (
-                'test_data/short_files/first_line.php',
-                {
-                    'KeywordDetector': {
-                        'config': {
-                            'name': 'KeywordDetector',
-                            'keyword_exclude': None,
-                        },
-                        'results': {
-                            'false-positives': {},
-                            'true-positives': {},
-                            'unknowns': {
-                                'test_data/short_files/first_line.php': [{
-                                    'line': "secret = 'notHighEnoughEntropy'",
-                                    'plaintext': 'nothighenoughentropy',
-                                }],
-                            },
-                        },
-                    },
-                },
-            ),
-        ],
-    )
-    def test_audit_display_results(self, filename, expected_output):
+    def test_scan_with_default_plugin(self):
+        filename = 'test_data/short_files/last_line.ini'
+        plugins_used = [
+            {
+                'name': 'AWSKeyDetector',
+            },
+            {
+                'name': 'ArtifactoryDetector',
+            },
+            {
+                'name': 'BasicAuthDetector',
+            },
+            {
+                'name': 'PrivateKeyDetector',
+            },
+            {
+                'name': 'SlackDetector',
+            },
+            {
+                'name': 'SoftLayerDetector',
+            },
+            {
+                'name': 'StripeDetector',
+            },
+        ]
+
         with mock_stdin(), mock_printer(
             main_module,
         ) as printer_shim:
diff --git a/tests/plugins/softlayer_test.py b/tests/plugins/softlayer_test.py
diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py

Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,7 @@ def test_consolidates_output_basic(self):`
`35`	`35`	`'StripeDetector': {},`
`36`	`36`	`'ArtifactoryDetector': {},`
`37`	`37`	`'GHDetector': {},`
	`38`	`+ 'SoftLayerDetector': {},`
`38`	`39`	`}`
`39`	`40`
`40`	`41`	`def test_consolidates_removes_disabled_plugins(self):`