Collect NPM #101

Signed-off-by: Navonil Das <[email protected]>

Author: NavonilDas

requirements.txt

@@ -5,6 +5,7 @@ lxml==4.3.3
 django==2.2.4
 djangorestframework==3.9.2
 django-filter==2.1.0
+semantic-version==2.8.2
 
 # Tests
 pytest==3.2.3

vulnerabilities/data_dump.py

@@ -147,3 +147,36 @@ def archlinux_dump(extract_data):
                 package=package_fixed,
                 repository='https://security.archlinux.org/package/{}'.format(package_name)
             )
+
+
+def npm_dump(extract_data):
+    for data in extract_data:
+        vulnerability = Vulnerability.objects.create(
+            summary=data.get('summary'),
+        )
+        VulnerabilityReference.objects.create(
+            vulnerability=vulnerability,
+            reference_id=data.get('vulnerability_id'),
+        )
+
+        affected_versions = data.get('affected_version',[])
+        for version in affected_versions:
+            package_affected = Package.objects.create(
+                name = data.get('package_name'),
+                version = version,
+            )
+            ImpactedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package_affected
+            )
+
+        fixed_versions = data.get('fixed_version',[])
+        for version in fixed_versions:
+            package_fixed = Package.objects.create(
+                name = data.get('package_name'),
+                version = version
+            )
+            ResolvedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package_fixed
+            )

vulnerabilities/management/commands/import.py

@@ -24,9 +24,10 @@
 from django.core.management.base import BaseCommand, CommandError
 
 from vulnerabilities import data_dump as dd
-from vulnerabilities.scraper import debian, ubuntu, archlinux
+from vulnerabilities.scraper import debian, ubuntu, archlinux , npm
 
 IMPORTERS = {
+    'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()),
     'debian': lambda: dd.debian_dump(debian.scrape_vulnerabilities()),
     'ubuntu': lambda: dd.ubuntu_dump(ubuntu.scrape_cves()),
     'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities()),

vulnerabilities/scraper/npm.py

@@ -0,0 +1,105 @@
+# Author: Navonil Das (@NavonilDas)
+import semantic_version
+import re
+import json
+from urllib.request import urlopen
+
+NPM_URL = 'https://registry.npmjs.org{}'
+PAGE = '/-/npm/v1/security/advisories?page=1'
+    
+# Removes spaces and v Charecter in front of version 
+def remove_spaces(x):
+    x = re.sub(r" +"," ",x)     # Replace Multiple spaces to one
+    # Remove space after the Relational operator
+    x = re.sub(r'< +','<',x)
+    x = re.sub(r'> +','>',x)
+    x = re.sub(r'<= +','<=',x)
+    x = re.sub(r'>= +','>=',x)
+    # Remove v at starting of version
+    x = re.sub(r'>=[vV]','>=',x)
+    x = re.sub(r'<=[vV]','<=',x)
+    x = re.sub(r'>[vV]','>',x)
+    x = re.sub(r'<[vV]','<',x)
+    return x
+
+
+# Returns all available for a module
+def get_all_version(package_name):
+    package_url = NPM_URL.format('/'+package_name)
+    response = urlopen(package_url).read()
+    data = json.loads(response)
+    versions = data.get('versions',{})
+    all_version = [obj for obj in versions]
+    return all_version
+
+# Seperate list of Affected version and fixed version from all version
+# using the range specified
+def extract_version(package_name,aff_version_range,fixed_version_range):
+    if aff_version_range == '' or fixed_version_range == '':
+        return ([],[])
+    
+    aff_spec = semantic_version.NpmSpec(remove_spaces(aff_version_range))
+    fix_spec = semantic_version.NpmSpec(remove_spaces(fixed_version_range))
+    all_ver = get_all_version(package_name)
+    aff_ver = []
+    fix_ver = []
+    for ver in all_ver:
+        cur_version = semantic_version.Version(ver)
+        if cur_version in aff_spec:
+            aff_ver.append(ver)
+        else:
+            if cur_version in fix_spec:
+                fix_ver.append(ver)
+    
+    return (aff_ver,fix_ver)
+
+
+# Extract module name, summary, vulnerability id,severity
+def extract_data(JSON):
+    package_vulnerabilities = []
+    for obj in JSON.get('objects',[]):
+        if 'module_name' not in obj:
+            continue
+        package_name = obj['module_name']
+        summary = obj.get('overview','')
+        severity = obj.get('severity','')
+        
+        vulnerability_id = obj.get('cves',[])
+        if len(vulnerability_id) > 0:
+            vulnerability_id = vulnerability_id[0]
+        else:
+            vulnerability_id = ''
+
+        affected_version,fixed_version = extract_version(
+            package_name,
+            obj.get('vulnerable_versions',''),
+            obj.get('patched_versions','')
+        )
+
+        package_vulnerabilities.append({
+            'package_name': package_name,
+            'summary': summary,
+            'vulnerability_id':vulnerability_id,
+            'fixed_version':fixed_version,
+            'affected_version':affected_version,
+            'severity':severity
+        })
+    return package_vulnerabilities
+
+
+# Extract JSON From NPM registry
+def scrape_vulnerabilities():
+    cururl = NPM_URL.format(PAGE)
+    response = urlopen(cururl).read()
+    package_vulnerabilities = []
+    while True:
+        data = json.loads(response)
+        package_vulnerabilities = package_vulnerabilities + extract_data(data)
+        next_page = data.get('urls',{}).get('next',False)
+        if next_page:
+            cururl = NPM_URL.format(next_page)
+            response = urlopen(cururl).read()
+        else:
+            break
+    return package_vulnerabilities
+