Skip to content

Commit 58e3ff3

Browse files
NavonilDashaikoschol
NavonilDas
authored and
haikoschol
committed
Collect NPM
closes #101 Signed-off-by: Navonil Das <[email protected]>
1 parent f56b604 commit 58e3ff3

File tree

7 files changed

+328
-1
lines changed

7 files changed

+328
-1
lines changed

‎requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ django==2.2.4
66
djangorestframework==3.9.2
77
django-filter==2.1.0
88
packageurl-python==0.8.7
9+
semantic-version==2.8.2
910

1011
# Tests
1112
pytest==3.2.3

‎vulnerabilities/data_dump.py

+33
Original file line numberDiff line numberDiff line change
@@ -159,3 +159,36 @@ def archlinux_dump(extract_data):
159159
package=package_fixed,
160160
repository=f'https://security.archlinux.org/package/{package_name}',
161161
)
162+
163+
164+
def npm_dump(extract_data):
165+
for data in extract_data:
166+
vulnerability = Vulnerability.objects.create(
167+
summary=data.get('summary'),
168+
)
169+
VulnerabilityReference.objects.create(
170+
vulnerability=vulnerability,
171+
reference_id=data.get('vulnerability_id'),
172+
)
173+
174+
affected_versions = data.get('affected_version', [])
175+
for version in affected_versions:
176+
package_affected = Package.objects.create(
177+
name=data.get('package_name'),
178+
version=version,
179+
)
180+
ImpactedPackage.objects.create(
181+
vulnerability=vulnerability,
182+
package=package_affected
183+
)
184+
185+
fixed_versions = data.get('fixed_version', [])
186+
for version in fixed_versions:
187+
package_fixed = Package.objects.create(
188+
name=data.get('package_name'),
189+
version=version
190+
)
191+
ResolvedPackage.objects.create(
192+
vulnerability=vulnerability,
193+
package=package_fixed
194+
)

‎vulnerabilities/management/commands/import.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@
2424
from django.core.management.base import BaseCommand, CommandError
2525

2626
from vulnerabilities import data_dump as dd
27-
from vulnerabilities.scraper import debian, ubuntu, archlinux
27+
from vulnerabilities.scraper import debian, ubuntu, archlinux, npm
2828

2929
IMPORTERS = {
30+
'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()),
3031
'debian': lambda: dd.debian_dump(debian.scrape_vulnerabilities()),
3132
'ubuntu': lambda: dd.ubuntu_dump(ubuntu.scrape_cves()),
3233
'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities()),

‎vulnerabilities/scraper/npm.py

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# Author: Navonil Das (@NavonilDas)
2+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
4+
# The VulnerableCode software is licensed under the Apache License version 2.0.
5+
# Data generated with VulnerableCode require an acknowledgment.
6+
#
7+
# You may not use this software except in compliance with the License.
8+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
9+
# Unless required by applicable law or agreed to in writing, software distributed
10+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
# specific language governing permissions and limitations under the License.
13+
#
14+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
15+
# derivative work, you must accompany this data with the following acknowledgment:
16+
#
17+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
18+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
19+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
20+
# for any legal advice.
21+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
22+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23+
24+
import json
25+
import re
26+
import semantic_version
27+
from urllib.request import urlopen
28+
29+
NPM_URL = 'https://registry.npmjs.org{}'
30+
PAGE = '/-/npm/v1/security/advisories?page=0'
31+
32+
33+
def remove_spaces(x):
34+
"""
35+
Remove Multiple Space, spaces after relational operator
36+
and remove v charecter in front of version string (ex v1.2.3)
37+
"""
38+
x = re.sub(r' +', ' ', x)
39+
x = re.sub(r'< +', '<', x)
40+
x = re.sub(r'> +', '>', x)
41+
x = re.sub(r'<= +', '<=', x)
42+
x = re.sub(r'>= +', '>=', x)
43+
x = re.sub(r'>=[vV]', '>=', x)
44+
x = re.sub(r'<=[vV]', '<=', x)
45+
x = re.sub(r'>[vV]', '>', x)
46+
x = re.sub(r'<[vV]', '<', x)
47+
return x
48+
49+
50+
def get_all_version(package_name):
51+
"""
52+
Returns all available for a module
53+
"""
54+
package_url = NPM_URL.format('/'+package_name)
55+
response = urlopen(package_url).read()
56+
data = json.loads(response)
57+
versions = data.get('versions', {})
58+
all_version = [obj for obj in versions]
59+
return all_version
60+
61+
62+
def extract_version(package_name, aff_version_range, fixed_version_range):
63+
"""
64+
Seperate list of Affected version and fixed version from all version
65+
using the range specified
66+
"""
67+
68+
if aff_version_range == '' or fixed_version_range == '':
69+
return ([], [])
70+
71+
aff_spec = semantic_version.NpmSpec(remove_spaces(aff_version_range))
72+
fix_spec = semantic_version.NpmSpec(remove_spaces(fixed_version_range))
73+
all_ver = get_all_version(package_name)
74+
aff_ver = []
75+
fix_ver = []
76+
for ver in all_ver:
77+
cur_version = semantic_version.Version(ver)
78+
if cur_version in aff_spec:
79+
aff_ver.append(ver)
80+
else:
81+
if cur_version in fix_spec:
82+
fix_ver.append(ver)
83+
84+
return (aff_ver, fix_ver)
85+
86+
87+
def extract_data(JSON):
88+
"""
89+
Extract module name, summary, vulnerability id,severity
90+
"""
91+
package_vulnerabilities = []
92+
for obj in JSON.get('objects', []):
93+
if 'module_name' not in obj:
94+
continue
95+
package_name = obj['module_name']
96+
summary = obj.get('overview', '')
97+
severity = obj.get('severity', '')
98+
99+
vulnerability_id = obj.get('cves', [])
100+
if len(vulnerability_id) > 0:
101+
vulnerability_id = vulnerability_id[0]
102+
else:
103+
vulnerability_id = ''
104+
105+
affected_version, fixed_version = extract_version(
106+
package_name,
107+
obj.get('vulnerable_versions', ''),
108+
obj.get('patched_versions', '')
109+
)
110+
111+
package_vulnerabilities.append({
112+
'package_name': package_name,
113+
'summary': summary,
114+
'vulnerability_id': vulnerability_id,
115+
'fixed_version': fixed_version,
116+
'affected_version': affected_version,
117+
'severity': severity
118+
})
119+
return package_vulnerabilities
120+
121+
122+
def scrape_vulnerabilities():
123+
"""
124+
Extract JSON From NPM registry
125+
"""
126+
cururl = NPM_URL.format(PAGE)
127+
response = urlopen(cururl).read()
128+
package_vulnerabilities = []
129+
while True:
130+
data = json.loads(response)
131+
package_vulnerabilities = package_vulnerabilities + extract_data(data)
132+
next_page = data.get('urls', {}).get('next', False)
133+
if next_page:
134+
cururl = NPM_URL.format(next_page)
135+
response = urlopen(cururl).read()
136+
else:
137+
break
138+
return package_vulnerabilities

‎vulnerabilities/tests/test_data/npm_test.json

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"objects": [
3+
{
4+
"id": 12,
5+
"created": "2015-10-17T19:41:46.382Z",
6+
"updated": "2019-06-24T14:13:54.355Z",
7+
"deleted": null,
8+
"title": "Rosetta-Flash JSONP Vulnerability",
9+
"found_by": {
10+
"name": "Michele Spagnuolo"
11+
},
12+
"reported_by": {
13+
"name": "Michele Spagnuolo"
14+
},
15+
"module_name": "hapi",
16+
"cves": [
17+
"CVE-2014-4671"
18+
],
19+
"vulnerable_versions": "< 6.1.0",
20+
"patched_versions": ">= 6.1.0",
21+
"overview": "This description taken from the pull request provided by Patrick Kettner.\n\n\n\nVersions 6.1.0 and earlier of hapi are vulnerable to a rosetta-flash attack, which can be used by attackers to send data across domains and break the browser same-origin-policy.\n\n\n",
22+
"recommendation": "- Update hapi to version 6.1.1 or later.\n\nAlternatively, a solution previously implemented by Google, Facebook, and Github is to prepend callbacks with an empty inline comment. This will cause the flash parser to break on invalid inputs and prevent the issue, and how the issue has been resolved internally in hapi.",
23+
"references": "- [PR #1766 - prepend jsonp callbacks with a comment to prevent the rosetta-flash vulnerability](https://github.com/spumko/hapi/pull/1766)\n\n- [Background from Michele Spagnuolo](http://miki.it/blog/2014/7/8/abusing-jsonp-with-rosetta-flash/)\n\nThanks to [Patrick Kettner](https://github.com/patrickkettner) for submitting a pull request to address this in hapi.",
24+
"access": "public",
25+
"severity": "moderate",
26+
"cwe": "CWE-538",
27+
"metadata": {
28+
"module_type": "Network.Library",
29+
"exploitability": 3,
30+
"affected_components": ""
31+
},
32+
"url": "https://npmjs.com/advisories/12"
33+
}
34+
],
35+
"total": 1179,
36+
"urls": {
37+
"next": "/-/npm/v1/security/advisories?page=2",
38+
"prev": "/-/npm/v1/security/advisories?page=0"
39+
}
40+
}

‎vulnerabilities/tests/test_import_cli.py

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ def test_list_sources(self):
3535
call_command('import', '--list', stdout=buf)
3636

3737
out = buf.getvalue()
38+
self.assertIn('npm', out)
3839
self.assertIn('debian', out)
3940
self.assertIn('ubuntu', out)
4041
self.assertIn('archlinux', out)

‎vulnerabilities/tests/test_npm.py

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Author: Navonil Das (@NavonilDas)
2+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
4+
# The VulnerableCode software is licensed under the Apache License version 2.0.
5+
# Data generated with VulnerableCode require an acknowledgment.
6+
#
7+
# You may not use this software except in compliance with the License.
8+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
9+
# Unless required by applicable law or agreed to in writing, software distributed
10+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
# specific language governing permissions and limitations under the License.
13+
#
14+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
15+
# derivative work, you must accompany this data with the following acknowledgment:
16+
#
17+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
18+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
19+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
20+
# for any legal advice.
21+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
22+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23+
24+
25+
from django.test import TestCase
26+
from vulnerabilities.scraper.npm import remove_spaces, get_all_version, extract_data
27+
import os
28+
import json
29+
30+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
31+
TEST_DATA = os.path.join(BASE_DIR, 'test_data/')
32+
33+
34+
class NPMScrapperTest(TestCase):
35+
def test_remove_space(self):
36+
res = remove_spaces(">= 1.2.1 || <= 2.1.1")
37+
self.assertEqual(res, '>=1.2.1 || <=2.1.1')
38+
39+
res = remove_spaces(">= v1.2.1 || <= V2.1.1")
40+
self.assertEqual(res, '>=1.2.1 || <=2.1.1')
41+
42+
def test_get_all_version(self):
43+
x = get_all_version('electron')
44+
expected = ['0.1.2', '2.0.0', '3.0.0',
45+
'4.0.0', '5.0.0', '6.0.0', '7.0.0']
46+
self.assertTrue(set(expected) <= set(x))
47+
48+
def test_extract_data(self):
49+
with open(os.path.join(TEST_DATA, 'npm_test.json')) as f:
50+
test_data = json.loads(f.read())
51+
52+
expected = {
53+
'package_name': 'hapi',
54+
'vulnerability_id': 'CVE-2014-4671',
55+
'fixed_version': [
56+
'6.1.0', '6.2.0', '6.2.1', '6.2.2', '6.3.0', '6.4.0',
57+
'6.5.0', '6.5.1', '6.6.0', '6.7.0', '6.7.1', '6.8.0',
58+
'6.8.1', '6.9.0', '6.10.0', '6.11.0', '6.11.1', '7.0.0',
59+
'7.0.1', '7.1.0', '7.1.1', '7.2.0', '7.3.0', '7.4.0',
60+
'7.5.0', '7.5.1', '7.5.2', '8.0.0', '7.5.3', '8.1.0',
61+
'8.2.0', '8.3.0', '8.3.1', '8.4.0', '8.5.0', '8.5.1',
62+
'8.5.2', '8.5.3', '8.6.0', '8.6.1', '8.8.0', '8.8.1',
63+
'9.0.0', '9.0.1', '9.0.2', '9.0.3', '9.0.4', '9.1.0',
64+
'9.2.0', '9.3.0', '9.3.1', '10.0.0', '10.0.1', '10.1.0',
65+
'10.2.1', '10.4.0', '10.4.1', '10.5.0', '11.0.0', '11.0.1',
66+
'11.0.2', '11.0.3', '11.0.4', '11.0.5', '11.1.0', '11.1.1',
67+
'11.1.2', '11.1.3', '11.1.4', '12.0.0', '12.0.1', '12.1.0',
68+
'9.5.1', '13.0.0', '13.1.0', '13.2.0', '13.2.1', '13.2.2',
69+
'13.3.0', '13.4.0', '13.4.1', '13.4.2', '13.5.0', '14.0.0',
70+
'13.5.3', '14.1.0', '14.2.0', '15.0.1', '15.0.2', '15.0.3',
71+
'15.1.0', '15.1.1', '15.2.0', '16.0.0', '16.0.1', '16.0.2',
72+
'16.0.3', '16.1.0', '16.1.1', '16.2.0', '16.3.0', '16.3.1',
73+
'16.4.0', '16.4.1', '16.4.2', '16.4.3', '16.5.0', '16.5.1',
74+
'16.5.2', '16.6.0', '16.6.1', '16.6.2', '17.0.0', '17.0.1',
75+
'17.0.2', '17.1.0', '17.1.1', '17.2.0', '17.2.1', '16.6.3',
76+
'17.2.2', '17.2.3', '17.3.0', '17.3.1', '17.4.0', '17.5.0',
77+
'17.5.1', '17.5.2', '17.5.3', '17.5.4', '17.5.5', '17.6.0',
78+
'17.6.1', '17.6.2', '17.6.3', '16.6.4', '17.6.4', '16.6.5',
79+
'17.7.0', '16.7.0', '17.8.0', '17.8.1', '18.0.0', '17.8.2',
80+
'17.8.3', '18.0.1', '17.8.4', '18.1.0', '17.8.5'],
81+
'affected_version': [
82+
'0.0.1', '0.0.2', '0.0.3', '0.0.4', '0.0.5', '0.0.6', '0.1.0',
83+
'0.1.1', '0.1.2', '0.1.3', '0.2.0', '0.2.1', '0.3.0', '0.4.0',
84+
'0.4.1', '0.4.2', '0.4.3', '0.4.4', '0.5.0', '0.5.1', '0.6.0',
85+
'0.6.1', '0.5.2', '0.7.0', '0.7.1', '0.8.0', '0.8.1', '0.8.2',
86+
'0.8.3', '0.8.4', '0.9.0', '0.9.1', '0.9.2', '0.10.0', '0.10.1',
87+
'0.11.0', '0.11.1', '0.11.2', '0.11.3', '0.12.0', '0.13.0',
88+
'0.13.1', '0.13.2', '0.11.4', '0.13.3', '0.14.0', '0.14.1',
89+
'0.14.2', '0.15.0', '0.15.1', '0.15.2', '0.15.3', '0.15.4',
90+
'0.15.5', '0.15.6', '0.15.7', '0.15.8', '0.15.9', '0.16.0',
91+
'1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.2.0', '1.3.0',
92+
'1.4.0', '1.5.0', '1.6.0', '1.6.1', '1.6.2', '1.7.0', '1.7.1',
93+
'1.7.2', '1.7.3', '1.8.0', '1.8.1', '1.8.2', '1.8.3', '1.9.0',
94+
'1.9.1', '1.9.2', '1.9.3', '1.9.4', '1.9.5', '1.9.6', '1.9.7',
95+
'1.10.0', '1.11.0', '1.11.1', '1.12.0', '1.13.0', '1.14.0',
96+
'1.15.0', '1.16.0', '1.16.1', '1.17.0', '1.18.0', '1.19.0',
97+
'1.19.1', '1.19.2', '1.19.3', '1.19.4', '1.19.5', '1.20.0',
98+
'2.0.0', '2.1.0', '2.1.1', '2.1.2', '2.2.0', '2.3.0', '2.4.0',
99+
'2.5.0', '2.6.0', '3.0.0', '3.0.1', '3.0.2', '3.1.0', '4.0.0',
100+
'4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3',
101+
'4.1.4', '5.0.0', '5.1.0', '6.0.0', '6.0.1', '6.0.2'],
102+
'severity': 'moderate'
103+
}
104+
got = extract_data(test_data)[0]
105+
# Check if expected affected version and fixed version is subset of what we get from online
106+
self.assertTrue(set(expected['fixed_version'])
107+
<= set(got['fixed_version']))
108+
self.assertTrue(set(expected['affected_version']) <= set(
109+
got['affected_version']))
110+
111+
self.assertEqual(expected['package_name'], got['package_name'])
112+
self.assertEqual(expected['severity'], got['severity'])
113+
self.assertEqual(expected['vulnerability_id'], got['vulnerability_id'])

0 commit comments

Comments
 (0)