Skip to content

Commit 02620e1

Browse files
committed
DB2 Verification (Yelp#196)
* DB2 Verification Supports git-defenders/detect-secrets-discuss#190 * Tests * Fixing tests... * Fixing tests * Fix db2 on mac * Fixing tests * Consistent formatting * Fix tests * No more pypy * comments in makefile * Added ibm_db to setup.py * Requires gcc * Don't purge until after easy_install * Don't delete gcc
1 parent 866cfcf commit 02620e1

File tree

8 files changed

+280
-34
lines changed

8 files changed

+280
-34
lines changed

.travis.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ matrix:
3333
- env: TOXENV=py37
3434
python: 3.7
3535
dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069)
36-
- env: TOXENV=pypy
37-
python: pypy
36+
# - env: TOXENV=pypy
37+
# python: pypy
3838
install:
3939
- pip install tox
4040
script: make test && docker build -t $DOCKER_IMAGE:$DOCKER_IMAGE_TAG --no-cache .

Dockerfile

+2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ RUN apk add --no-cache jq git curl bash openssl
44
RUN mkdir -p /code
55
COPY . /usr/src/app
66
WORKDIR /usr/src/app
7+
RUN apk add --no-cache --virtual .build-deps gcc musl-dev
8+
RUN pip install cython
79
RUN easy_install /usr/src/app
810
WORKDIR /code
911
ENTRYPOINT [ "/usr/src/app/run-scan.sh" ]

Makefile

+12
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
PROJECT_DIR := $(shell pwd)
2+
13
.PHONY: minimal
24
minimal: setup
35

@@ -22,3 +24,13 @@ clean:
2224
super-clean: clean
2325
rm -rf .tox
2426
rm -rf venv
27+
28+
.PHONY: fix-db2-mac
29+
fix-db2-mac:
30+
# comment out lines for any interpreters that aren't installed on your machine
31+
install_name_tool -change libdb2.dylib $(PROJECT_DIR)/.tox/py27/lib/python2.7/site-packages/clidriver/lib/libdb2.dylib $(PROJECT_DIR)/.tox/py27/lib/python2.7/site-packages/ibm_db.so
32+
install_name_tool -change libbd2.dylib $(PROJECT_DIR)/.tox/py35/lib/python3.5/site-packages/clidriver/lib/libdb2.dylib $(PROJECT_DIR)/.tox/py35/lib/python3.5/site-packages/ibm_db.cpython-35m-darwin.so
33+
install_name_tool -change libbd2.dylib $(PROJECT_DIR)/.tox/py36/lib/python3.6/site-packages/clidriver/lib/libdb2.dylib $(PROJECT_DIR)/.tox/py36/lib/python3.6/site-packages/ibm_db.cpython-36m-darwin.so
34+
install_name_tool -change libdb2.dylib $(PROJECT_DIR)/.tox/py37/lib/python3.7/site-packages/clidriver/lib/libdb2.dylib $(PROJECT_DIR)/.tox/py37/lib/python3.7/site-packages/ibm_db.cpython-37m-darwin.so
35+
install_name_tool -change libdb2.dylib $(PROJECT_DIR)/.tox/pypy/site-packages/clidriver/lib/libdb2.dylib $(PROJECT_DIR)/.tox/pypy/site-packages/ibm_db.pypy-41.so
36+
install_name_tool -change libdb2.dylib $(PROJECT_DIR)/.tox/pypy3/site-packages/clidriver/lib/libdb2.dylib $(PROJECT_DIR)/.tox/pypy3/site-packages/ibm_db.pypy3-71-darwin.so

detect_secrets/plugins/db2.py

+101
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
1+
from __future__ import absolute_import
2+
13
import re
24

5+
import ibm_db
6+
37
from .base import RegexBasedDetector
8+
from detect_secrets.core.constants import VerifiedResult
49

510

611
class DB2Detector(RegexBasedDetector):
@@ -30,3 +35,99 @@ class DB2Detector(RegexBasedDetector):
3035
), flags=re.IGNORECASE,
3136
),
3237
)
38+
39+
username_keyword_regex = r'(?:user|user(?:_|-|)name|uid|user(?:_|-|)id)'
40+
username_regex = r'([a-zA-Z0-9_]+)'
41+
42+
database_keyword_regex = r'(?:database|db|database(?:_|-|)name|db(?:_|-|)name)'
43+
database_regex = r'([a-zA-Z0-9_-]+)'
44+
45+
port_keyword_regex = r'(?:port|port(?:_|-|)number)'
46+
port_regex = r'([0-9]{1,5})'
47+
48+
hostname_keyword_regex = (
49+
r'(?:host|host(?:_|-|)name|host(?:_|-|)address|'
50+
r'host(?:_|-|)ip|host(?:_|-|)ip(?:_|-|)address)'
51+
)
52+
hostname_regex = (
53+
r'((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)'
54+
r'*(?:.\[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]))'
55+
)
56+
57+
def verify(self, token, content, potential_secret=None):
58+
59+
username_matches = get_other_factor(
60+
content, self.username_keyword_regex,
61+
self.username_regex,
62+
)
63+
database_matches = get_other_factor(
64+
content, self.database_keyword_regex,
65+
self.database_regex,
66+
)
67+
port_matches = get_other_factor(
68+
content, self.port_keyword_regex,
69+
self.port_regex,
70+
)
71+
hostname_matches = get_other_factor(
72+
content, self.hostname_keyword_regex,
73+
self.hostname_regex,
74+
)
75+
76+
if not username_matches or not database_matches or not port_matches or not hostname_matches:
77+
return VerifiedResult.UNVERIFIED
78+
79+
for username in username_matches: # pragma: no cover
80+
for database in database_matches: # pragma: no cover
81+
for port in port_matches: # pragma: no cover
82+
for hostname in hostname_matches: # pragma: no cover
83+
try:
84+
conn_str = 'database={database};hostname={hostname};port={port};' + \
85+
'protocol=tcpip;uid={username};pwd={token}'
86+
conn_str = conn_str.format(
87+
database=database,
88+
hostname=hostname,
89+
port=port,
90+
username=username,
91+
token=token,
92+
)
93+
ibm_db_conn = ibm_db.connect(conn_str, '', '')
94+
if ibm_db_conn:
95+
potential_secret.other_factors['database'] = database
96+
potential_secret.other_factors['hostname'] = hostname
97+
potential_secret.other_factors['port'] = port
98+
potential_secret.other_factors['username'] = username
99+
return VerifiedResult.VERIFIED_TRUE
100+
else:
101+
return VerifiedResult.VERIFIED_FALSE
102+
except Exception:
103+
return VerifiedResult.UNVERIFIED
104+
105+
return VerifiedResult.VERIFIED_FALSE
106+
107+
108+
def get_other_factor(content, factor_keyword_regex, factor_regex):
109+
begin = r'(?:(?<=\W)|(?<=^))'
110+
opt_quote = r'(?:"|\'|)'
111+
opt_db = r'(?:db2|dashdb|db|)'
112+
opt_dash_undrscr = r'(?:_|-|)'
113+
opt_space = r'(?: *)'
114+
assignment = r'(?:=|:|:=|=>|::)'
115+
regex = re.compile(
116+
r'{begin}{opt_quote}{opt_db}{opt_dash_undrscr}{factor_keyword}{opt_quote}{opt_space}'
117+
'{assignment}{opt_space}{opt_quote}{factor}{opt_quote}'.format(
118+
begin=begin,
119+
opt_quote=opt_quote,
120+
opt_db=opt_db,
121+
opt_dash_undrscr=opt_dash_undrscr,
122+
factor_keyword=factor_keyword_regex,
123+
opt_space=opt_space,
124+
assignment=assignment,
125+
factor=factor_regex,
126+
), flags=re.IGNORECASE,
127+
)
128+
129+
return [
130+
match
131+
for line in content.splitlines()
132+
for match in regex.findall(line)
133+
]

requirements-dev.txt

+1
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ tox-pip-extensions
1010
tox>=3.8
1111
unidiff
1212
responses
13+
ibm_db

setup.py

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
install_requires=[
1919
'pyyaml',
2020
'requests',
21+
'ibm_db',
2122
],
2223
extras_require={
2324
':python_version=="2.7"': [

tests/plugins/common/db2_test.py

-32
This file was deleted.

tests/plugins/db2_test.py

+161
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
from __future__ import absolute_import
2+
3+
import textwrap
4+
5+
import pytest
6+
from mock import MagicMock
7+
from mock import patch
8+
9+
from detect_secrets.core.constants import VerifiedResult
10+
from detect_secrets.core.potential_secret import PotentialSecret
11+
from detect_secrets.plugins.db2 import DB2Detector
12+
from detect_secrets.plugins.db2 import get_other_factor
13+
14+
15+
DB2_USER = 'fake_user'
16+
DB2_PASSWORD = 'fake_password'
17+
DB2_PORT = '1234'
18+
DB2_HOSTNAME = 'fake.host.name'
19+
DB2_DATABASE = 'fake_database'
20+
DB2_CONN_STRING = 'database={DB2_DATABASE};hostname={DB2_HOSTNAME};port={DB2_PORT};' + \
21+
'protocol=tcpip;uid={DB2_USER};pwd={DB2_PASSWORD}'
22+
DB2_CONN_STRING = DB2_CONN_STRING.format(
23+
DB2_DATABASE=DB2_DATABASE,
24+
DB2_HOSTNAME=DB2_HOSTNAME,
25+
DB2_PORT=DB2_PORT,
26+
DB2_USER=DB2_USER,
27+
DB2_PASSWORD=DB2_PASSWORD,
28+
)
29+
30+
31+
class TestGHDetector(object):
32+
33+
@pytest.mark.parametrize(
34+
'payload, should_flag',
35+
[
36+
(
37+
'database=test;hostname=host.test.com;'
38+
'port=1;protocol=tcpip;uid=testid;pwd=secret', True,
39+
),
40+
('dbpwd=$omespeci@!ch@r$', True),
41+
('db2_password = "astring"', True),
42+
('"password": "Iusedb2!"', True),
43+
('password = "ilikespaces"', True),
44+
('pwd::anothersyntax!', True),
45+
('DB2_PASSWORD = "@#!%#"', True),
46+
('dashdb-password = "pass"', True),
47+
('dashdb_host = notapassword', False),
48+
('someotherpassword = "doesnt start right"', False),
49+
],
50+
)
51+
def test_analyze_string(self, payload, should_flag):
52+
logic = DB2Detector()
53+
54+
output = logic.analyze_string(payload, 1, 'mock_filename')
55+
assert len(output) == int(should_flag)
56+
57+
@patch('detect_secrets.plugins.db2.ibm_db.connect')
58+
def test_verify_invalid_secret(self, mock_db2_connect):
59+
mock_db2_connect.return_value = None
60+
61+
potential_secret = PotentialSecret('test db2', 'test filename', DB2_PASSWORD)
62+
assert DB2Detector().verify(
63+
DB2_PASSWORD,
64+
'''user={},
65+
password={},
66+
database={},
67+
host={},
68+
port={}'''.format(DB2_USER, DB2_PASSWORD, DB2_DATABASE, DB2_HOSTNAME, DB2_PORT),
69+
potential_secret,
70+
) == VerifiedResult.VERIFIED_FALSE
71+
72+
mock_db2_connect.assert_called_with(DB2_CONN_STRING, '', '')
73+
74+
@patch('detect_secrets.plugins.db2.ibm_db.connect')
75+
def test_verify_valid_secret(self, mock_db2_connect):
76+
mock_db2_connect.return_value = MagicMock()
77+
78+
potential_secret = PotentialSecret('test db2', 'test filename', DB2_PASSWORD)
79+
assert DB2Detector().verify(
80+
DB2_PASSWORD,
81+
'''user={},
82+
password={},
83+
database={},
84+
host={},
85+
port={}'''.format(DB2_USER, DB2_PASSWORD, DB2_DATABASE, DB2_HOSTNAME, DB2_PORT),
86+
potential_secret,
87+
) == VerifiedResult.VERIFIED_TRUE
88+
89+
mock_db2_connect.assert_called_with(DB2_CONN_STRING, '', '')
90+
91+
@patch('detect_secrets.plugins.db2.ibm_db.connect')
92+
def test_verify_unverified_secret(self, mock_db2_connect):
93+
mock_db2_connect.side_effect = Exception('oops')
94+
95+
potential_secret = PotentialSecret('test db2', 'test filename', DB2_PASSWORD)
96+
assert DB2Detector().verify(
97+
DB2_PASSWORD,
98+
'''user={},
99+
password={},
100+
database={},
101+
host={},
102+
port={}'''.format(DB2_USER, DB2_PASSWORD, DB2_DATABASE, DB2_HOSTNAME, DB2_PORT),
103+
potential_secret,
104+
) == VerifiedResult.UNVERIFIED
105+
106+
mock_db2_connect.assert_called_with(DB2_CONN_STRING, '', '')
107+
108+
def test_verify_no_other_factors(self):
109+
assert DB2Detector().verify(
110+
DB2_PASSWORD,
111+
'password={}'.format(DB2_PASSWORD),
112+
) == VerifiedResult.UNVERIFIED
113+
114+
115+
@pytest.mark.parametrize(
116+
'content, factor_keyword_regex, factor_regex, expected_output',
117+
(
118+
(
119+
textwrap.dedent("""
120+
user = {}
121+
""")[1:-1].format(
122+
DB2_USER,
123+
),
124+
DB2Detector().username_keyword_regex,
125+
DB2Detector().username_regex,
126+
[DB2_USER],
127+
),
128+
(
129+
textwrap.dedent("""
130+
port = {}
131+
""")[1:-1].format(
132+
DB2_PORT,
133+
),
134+
DB2Detector().port_keyword_regex,
135+
DB2Detector().port_regex,
136+
[DB2_PORT],
137+
),
138+
(
139+
textwrap.dedent("""
140+
database = {}
141+
""")[1:-1].format(
142+
DB2_DATABASE,
143+
),
144+
DB2Detector().database_keyword_regex,
145+
DB2Detector().database_regex,
146+
[DB2_DATABASE],
147+
),
148+
(
149+
textwrap.dedent("""
150+
host = {}
151+
""")[1:-1].format(
152+
DB2_HOSTNAME,
153+
),
154+
DB2Detector().hostname_keyword_regex,
155+
DB2Detector().hostname_regex,
156+
[DB2_HOSTNAME],
157+
),
158+
),
159+
)
160+
def test_get_other_factor(content, factor_keyword_regex, factor_regex, expected_output):
161+
assert get_other_factor(content, factor_keyword_regex, factor_regex) == expected_output

0 commit comments

Comments
 (0)