Update scan_for_application_packages #436 #444 #447

JonoYang · JonoYang · commit 8963d7fd53bf · 2022-07-20T17:03:03.000-07:00
* Update scan_for_application_packages to save detected Package data to the CodebaseResource it is from, then iterate through the CodebaseResources with Package data and use the proper Package handler to process the Package data
    * Create DiscoveredDependency model
    * Add package_data JSON field to CodebaseResource

Signed-off-by: Jono Yang &lt;jyang@nexb.com&gt;
diff --git a/scanpipe/migrations/0019_codebaseresource_package_data_discovereddependency.py b/scanpipe/migrations/0019_codebaseresource_package_data_discovereddependency.py
@@ -0,0 +1,41 @@
+# Generated by Django 4.0.6 on 2022-07-20 23:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+import scanpipe.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0018_codebaseresource_tag'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='codebaseresource',
+            name='package_data',
+            field=models.JSONField(blank=True, default=dict, help_text='List of Package data detected from this CodebaseResource'),
+        ),
+        migrations.CreateModel(
+            name='DiscoveredDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('purl', models.CharField(help_text='The Package URL of this dependency.', max_length=1024)),
+                ('extracted_requirement', models.CharField(help_text='The version requirements of this dependency.', max_length=32)),
+                ('scope', models.CharField(help_text='The scope of this dependency, how it is used in a project.', max_length=32)),
+                ('is_runtime', models.BooleanField(default=False)),
+                ('is_optional', models.BooleanField(default=False)),
+                ('is_resolved', models.BooleanField(default=False)),
+                ('dependency_uid', models.CharField(help_text='The unique identifier of this dependency.', max_length=1024)),
+                ('for_package_uid', models.CharField(help_text='The unique identifier of the package this dependency is for.', max_length=1024)),
+                ('datafile_path', models.CharField(blank=True, help_text='The relative path to the datafile where this dependency was detected from.', max_length=1024)),
+                ('datasource_id', models.CharField(help_text='The identifier for the datafile handler used to obtain this dependency.', max_length=64)),
+                ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)ss', to='scanpipe.project')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, scanpipe.models.SaveProjectErrorMixin),
+        ),
+    ]
diff --git a/scanpipe/models.py b/scanpipe/models.py
@@ -54,6 +54,7 @@
 import django_rq
 import redis
 import requests
+from commoncode.fileutils import parent_directory
 from commoncode.hash import multi_checksums
 from packageurl import PackageURL
 from packageurl import normalize_qualifiers
@@ -1466,6 +1467,12 @@ class Compliance(models.TextChoices):
         ),
     )
 
+    package_data = models.JSONField(
+        default=dict,
+        blank=True,
+        help_text=_("List of Package data detected from this CodebaseResource"),
+    )
+
     objects = CodebaseResourceQuerySet.as_manager()
 
     class Meta:
@@ -1582,6 +1589,56 @@ def unique_license_expressions(self):
         """
         return sorted(set(self.license_expressions))
 
+    def parent_path(self):
+        """
+        Return the parent path for this CodebaseResource or None.
+        """
+        return parent_directory(self.path, with_trail=False)
+
+    def has_parent(self):
+        """
+        Return True if this CodebaseResource has a parent CodebaseResource or
+        False otherwise.
+        """
+        parent_path = self.parent_path()
+        if not parent_path:
+            return False
+        if self.project.codebaseresources.filter(path=parent_path).exists():
+            return True
+        return False
+
+    def parent(self, codebase=None):
+        """
+        Return the parent CodebaseResource object for this CodebaseResource or
+        None.
+
+        `codebase` is not used in this context but required for compatibility
+        with the commoncode.resource.Codebase class API.
+        """
+        parent_path = self.parent_path()
+        return parent_path and self.project.codebaseresources.get(path=parent_path)
+
+    def has_siblings(self, codebase=None):
+        """
+        Return True is this CodebaseResource has siblings.
+
+        `codebase` is not used in this context but required for compatibility
+        with the commoncode.resource.Codebase class API.
+        """
+        return self.has_parent() and self.parent(codebase).has_children()
+
+    def siblings(self, codebase=None):
+        """
+        Return a sequence of sibling Resource objects for this Resource
+        or an empty sequence.
+
+        `codebase` is not used in this context but required for compatibility
+        with the commoncode.resource.Codebase class API.
+        """
+        if self.has_parent():
+            return self.parent(codebase).children(codebase)
+        return []
+
     def descendants(self):
         """
         Returns a QuerySet of descendant CodebaseResource objects using a
@@ -1847,6 +1904,103 @@ def update_from_data(self, package_data, override=False):
         return updated_fields
 
 
+class DiscoveredDependency(
+    ProjectRelatedModel,
+    SaveProjectErrorMixin,
+):
+    """
+    A project's Discovered Dependencies are records of the dependencies used by
+    system and application packages discovered in the code under analysis.
+    """
+    purl = models.CharField(
+        max_length=1024,
+        help_text=_(
+            "The Package URL of this dependency."
+        ),
+    )
+    extracted_requirement = models.CharField(
+        max_length=32,
+        help_text=_(
+            "The version requirements of this dependency."
+        ),
+    )
+    scope = models.CharField(
+        max_length=32,
+        help_text=_(
+            "The scope of this dependency, how it is used in a project."
+        ),
+    )
+
+    is_runtime = models.BooleanField(default=False)
+    is_optional = models.BooleanField(default=False)
+    is_resolved = models.BooleanField(default=False)
+
+    dependency_uid = models.CharField(
+        max_length=1024,
+        help_text=_(
+            "The unique identifier of this dependency."
+        ),
+    )
+    for_package_uid = models.CharField(
+        max_length=1024,
+        help_text=_(
+            "The unique identifier of the package this dependency is for."
+        ),
+    )
+    datafile_path = models.CharField(
+        max_length=1024,
+        blank=True,
+        help_text=_(
+            "The relative path to the datafile where this dependency was detected from."
+        ),
+    )
+    datasource_id = models.CharField(
+        max_length=64,
+        help_text=_(
+            "The identifier for the datafile handler used to obtain this dependency."
+        )
+    )
+
+    @classmethod
+    def create_from_data(cls, project, dependency_data):
+        """
+        Creates and returns a DiscoveredPackage for a `project` from the `dependency_data`.
+        """
+        if "resolved_package" in dependency_data:
+            dependency_data.pop("resolved_package")
+        discovered_dependency = cls(project=project, **dependency_data)
+        discovered_dependency.save()
+        return discovered_dependency
+
+    def update_from_data(self, dependency_data):
+        """
+        Update this discovered dependency instance with the provided `dependency_data`.
+        The `save()` is called only if at least one field was modified.
+        """
+        model_fields = DiscoveredPackage.model_fields()
+        updated_fields = []
+
+        for field_name, value in dependency_data.items():
+            skip_reasons = [
+                not value,
+                field_name not in model_fields,
+            ]
+            if any(skip_reasons):
+                continue
+
+            current_value = getattr(self, field_name, None)
+            if not current_value:
+                setattr(self, field_name, value)
+                updated_fields.append(field_name)
+            elif current_value != value:
+                pass  # TODO: handle this case
+
+        if updated_fields:
+            self.save()
+
+        return updated_fields
+
+
 class WebhookSubscription(UUIDPKModel, ProjectRelatedModel):
     target_url = models.URLField(_("Target URL"), max_length=1024)
     sent = models.BooleanField(default=False)
diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py
@@ -30,6 +30,7 @@
 from django.db.models import Count
 
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.pipes import scancode
 
@@ -104,6 +105,29 @@ def update_or_create_package(project, package_data, codebase_resource=None):
     return package
 
 
+def update_or_create_dependencies(project, dependency_data):
+    """
+    Gets, updates or creates a DiscoveredDependency then returns it.
+    Uses the `project` and `dependency_data` mapping to lookup and creates the
+    DiscoveredDependency using its dependency_uid and for_package_uid as a unique key.
+    """
+    try:
+        dependency = DiscoveredDependency.objects.get(
+            project=project,
+            dependency_uid=dependency_data.get("dependency_uid"),
+            for_package_uid=dependency_data.get("for_package_uid"),
+        )
+    except DiscoveredDependency.DoesNotExist:
+        dependency = None
+
+    if dependency:
+        dependency.update_from_data(dependency_data)
+    else:
+        dependency = DiscoveredDependency.create_from_data(project, dependency_data)
+
+    return dependency
+
+
 def analyze_scanned_files(project):
     """
     Sets the status for CodebaseResource to unknown or no license.
diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py
@@ -35,13 +35,16 @@
 
 from commoncode import fileutils
 from commoncode.resource import VirtualCodebase
+from packagedcode import get_package_handler
+from packagedcode import models as packagedcode_models
 from extractcode import api as extractcode_api
 from scancode import ScancodeError
 from scancode import Scanner
 from scancode import api as scancode_api
 from scancode import cli as scancode_cli
 
 from scanpipe import pipes
+from scanpipe.pipes.codebase import ProjectCodebase
 from scanpipe.models import CodebaseResource
 
 logger = logging.getLogger("scanpipe.pipes")
@@ -230,10 +233,9 @@ def save_scan_package_results(codebase_resource, scan_results, scan_errors):
     Saves the resource scan package results in the database.
     Creates project errors if any occurred during the scan.
     """
-    packages = scan_results.get("package_data", [])
-    if packages:
-        for package_data in packages:
-            codebase_resource.create_and_add_package(package_data)
+    package_data = scan_results.get("package_data", [])
+    if package_data:
+        codebase_resource.package_data = package_data
         codebase_resource.status = "application-package"
         codebase_resource.save()
 
@@ -310,18 +312,89 @@ def scan_for_files(project):
 
 def scan_for_application_packages(project):
     """
-    Runs a package scan on files without a status for a `project`.
+    Runs a package scan on files without a status for a `project`,
+    then create DiscoveredPackage and DiscoveredDependency instances
+    from the detected package data
 
     Multiprocessing is enabled by default on this pipe, the number of processes can be
     controlled through the SCANCODEIO_PROCESSES setting.
     """
     resource_qs = project.codebaseresources.no_status()
+
+    # Collect detected Package data and save it to the CodebaseResource it was
+    # detected from
     _scan_and_save(
         resource_qs=resource_qs,
         scan_func=scan_for_package_data,
         save_func=save_scan_package_results,
     )
 
+    # Iterate through CodebaseResources with Package data and handle them using
+    # the proper Package handler from packagedcode
+    assemble_packages(project=project)
+
+
+def assemble_packages(project):
+    """
+    Create instances of DiscoveredPackage and DiscoveredDependency for `project`
+    from the parsed package data present in the CodebaseResources of `project`.
+    """
+    seen_resource_paths = set()
+    package_data_resources_qs = project.codebaseresources.filter(package_data__isnull=False)
+    for resource in package_data_resources_qs:
+        if resource.path in seen_resource_paths:
+            continue
+
+        logger.info(
+            f"project: {project}:\n"
+            "function: assemble_packages\n"
+            f"Processing: CodebaseResource {resource.path}\n"
+        )
+
+        for package_data in resource.package_data:
+            package_data = packagedcode_models.PackageData.from_dict(mapping=package_data)
+
+            logger.info(
+                f"project: {project}:\n"
+                "function: assemble_packages\n"
+                f"Processing: PackageData {package_data.purl}\n"
+            )
+
+            handler = get_package_handler(package_data)
+
+            logger.info(
+                f"project: {project}:\n"
+                "function: assemble_packages\n"
+                f"Selected: Package handler {handler}\n"
+            )
+
+            items = handler.assemble(
+                package_data=package_data,
+                resource=resource,
+                codebase=None,
+            )
+
+            for item in items:
+                logger.info(
+                    f"project: {project}:\n"
+                    "function: assemble_packages\n"
+                    f"Processing: item {item}\n"
+                )
+                if isinstance(item, packagedcode_models.Package):
+                    package_data = item.to_dict()
+                    pipes.update_or_create_package(project, package_data)
+                elif isinstance(item, packagedcode_models.Dependency):
+                    dependency_data = item.to_dict()
+                    pipes.update_or_create_dependencies(project, dependency_data)
+                elif isinstance(item, CodebaseResource):
+                    seen_resource_paths.add(item.path)
+                else:
+                    logger.info(
+                        f"project: {project}:\n"
+                        "function: assemble_packages\n"
+                        f"Unknown Package assembly item type: {item!r}\n"
+                    )
+
 
 def run_scancode(location, output_file, options, raise_on_error=False):
     """
