Re-add update_or_create_dependencies #447

    * Create DiscoveredDependencies in load_codebase pipeline
    * Update tests to check for DiscoveredDependencies
    * Update test expectations

Signed-off-by: Jono Yang <[email protected]>

Author: JonoYang

CHANGELOG.rst

@@ -56,13 +56,32 @@ v31.0.0 (next)
 
   https://github.com/nexB/scancode.io/issues/413
 
+- Update application Package scanning step to reflect the updates in
+  scancode-toolkit package scanning.
+
+  - Package data detected from a file are now stored on the
+    CodebaseResource.package_data field.
+  - A second processing step is now done after scanning for Package data, where
+    Package Resources are determined and DiscoveredPackages and
+    DiscoveredDependencies are created.
+
+  https://github.com/nexB/scancode.io/issues/444
+
 - CodebaseResource.name now contains both the bare file name with extension, as
   opposed to just the bare file name without extension.
-  https://github.com/nexB/scancode.io/issues/467
 
   - Using a name stripped from its extension was something that was not used in
     other AboutCode project or tools.
 
+  https://github.com/nexB/scancode.io/issues/467
+
+- Add the model DiscoveredDependency. This represents Package dependencies
+  discovered in a Project. The ``scan_codebase`` and ``scan_packages`` pipelines
+  have been updated to create DiscoveredDepdendency objects.
+  https://github.com/nexB/scancode.io/issues/447
+
+- The ``dependencies`` field has been removed from the DiscoveredPackage model.
+
 v30.2.0 (2021-12-17)
 --------------------
 

scanpipe/models.py

@@ -1983,12 +1983,67 @@ def create_from_data(cls, project, dependency_data):
         Creates and returns a DiscoveredDependency for a `project` from the
         `dependency_data`.
         """
+        required_fields = ["purl", "dependency_uid"]
+        missing_values = [
+            field_name
+            for field_name in required_fields
+            if not dependency_data.get(field_name)
+        ]
+
+        if missing_values:
+            message = (
+                f"No values for the following required fields: "
+                f"{', '.join(missing_values)}"
+            )
+
+            project.add_error(error=message, model=cls, details=dependency_data)
+            return
+
         if "resolved_package" in dependency_data:
             dependency_data.pop("resolved_package")
-        discovered_dependency = cls(project=project, **dependency_data)
+
+        cleaned_dependency_data = {
+            field_name: value
+            for field_name, value in dependency_data.items()
+            if field_name in DiscoveredDependency.model_fields() and value
+        }
+        discovered_dependency = cls(
+            project=project,
+            **cleaned_dependency_data
+        )
         discovered_dependency.save()
+
         return discovered_dependency
 
+    def update_from_data(self, dependency_data):
+        """
+        Update this discovered dependency instance with the provided `dependency_data`.
+        The `save()` is called only if at least one field was modified.
+        """
+        model_fields = DiscoveredDependency.model_fields()
+        updated_fields = []
+
+        for field_name, value in dependency_data.items():
+            skip_reasons = [
+                not value,
+                field_name not in model_fields,
+            ]
+            if any(skip_reasons):
+                continue
+
+            current_value = getattr(self, field_name, None)
+            if (
+                not current_value
+                or current_value != value
+            ):
+                setattr(self, field_name, value)
+                updated_fields.append(field_name)
+
+        if updated_fields:
+            self.save()
+
+        return updated_fields
+
 
 class WebhookSubscription(UUIDPKModel, ProjectRelatedModel):
     target_url = models.URLField(_("Target URL"), max_length=1024)

scanpipe/pipes/__init__.py

@@ -105,6 +105,29 @@ def update_or_create_package(project, package_data, codebase_resource=None):
     return package
 
 
+def update_or_create_dependencies(project, dependency_data):
+    """
+    Gets, updates or creates a DiscoveredDependency then returns it.
+    Uses the `project` and `dependency_data` mapping to lookup and creates the
+    DiscoveredDependency using its dependency_uid and for_package_uid as a unique key.
+    """
+    try:
+        dependency = DiscoveredDependency.objects.get(
+            project=project,
+            dependency_uid=dependency_data.get("dependency_uid"),
+            for_package_uid=dependency_data.get("for_package_uid"),
+        )
+    except DiscoveredDependency.DoesNotExist:
+        dependency = None
+
+    if dependency:
+        dependency.update_from_data(dependency_data)
+    else:
+        dependency = DiscoveredDependency.create_from_data(project, dependency_data)
+
+    return dependency
+
+
 def analyze_scanned_files(project):
     """
     Sets the status for CodebaseResource to unknown or no license.

scanpipe/pipes/scancode.py

@@ -488,6 +488,16 @@ def create_discovered_packages(project, scanned_codebase):
             pipes.update_or_create_package(project, package_data)
 
 
+def create_discovered_dependencies(project, scanned_codebase):
+    """
+    Saves the dependencies of a ScanCode `scanned_codebase` scancode.resource.Codebase
+    object to the database as a DiscoveredDependency of `project`.
+    """
+    if hasattr(scanned_codebase.attributes, "dependencies"):
+        for dependency_data in scanned_codebase.attributes.dependencies:
+            pipes.update_or_create_dependencies(project, dependency_data)
+
+
 def set_codebase_resource_for_package(codebase_resource, discovered_package):
     """
     Assigns the `discovered_package` to the `codebase_resource` and set its
@@ -578,4 +588,5 @@ def create_inventory_from_scan(project, input_location):
     """
     scanned_codebase = get_virtual_codebase(project, input_location)
     create_discovered_packages(project, scanned_codebase)
+    create_discovered_dependencies(project, scanned_codebase)
     create_codebase_resources(project, scanned_codebase)

scanpipe/tests/__init__.py

@@ -89,6 +89,19 @@
     "package_uid": "pkg:deb/debian/[email protected]?uuid=610bed29-ce39-40e7-92d6-fd8b",
 }
 
+dependency_data1 = {
+  "purl": "pkg:pypi/dask",
+  "extracted_requirement": "dask<2023.0.0,>=2022.6.0",
+  "scope": "install",
+  "is_runtime": True,
+  "is_optional": False,
+  "is_resolved": False,
+  "dependency_uid": "pkg:pypi/dask?uuid=e656b571-7d3f-46d1-b95b-8f037aef9692",
+  "for_package_uid": "pkg:pypi/[email protected]?uuid=4d1f048b-a155-4f95-8cf9-185ab872ab4c",
+  "datafile_path": "daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO",
+  "datasource_id": "pypi_sdist_pkginfo"
+}
+
 license_policies = [
     {
         "license_key": "apache-2.0",

scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json

@@ -119,7 +119,56 @@
       "source_packages": []
     }
   ],
-  "dependencies": [],
+  "dependencies": [
+    {
+      "purl": "pkg:pypi/pytest",
+      "extracted_requirement": "",
+      "scope": "tests",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false,
+      "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl",
+      "datasource_id": "pypi_wheel"
+    },
+    {
+      "purl": "pkg:pypi/pytest",
+      "extracted_requirement": "",
+      "scope": "tests",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false,
+      "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA",
+      "datasource_id": "pypi_wheel_metadata"
+    },
+    {
+      "purl": "pkg:pypi/pytest-asyncio",
+      "extracted_requirement": "",
+      "scope": "tests",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false,
+      "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl",
+      "datasource_id": "pypi_wheel"
+    },
+    {
+      "purl": "pkg:pypi/pytest-asyncio",
+      "extracted_requirement": "",
+      "scope": "tests",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false,
+      "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA",
+      "datasource_id": "pypi_wheel_metadata"
+    }
+  ],
   "files": [
     {
       "for_packages": [

scanpipe/tests/test_models.py

@@ -46,6 +46,7 @@
 
 from scancodeio import __version__ as scancodeio_version
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectError
@@ -57,6 +58,7 @@
 from scanpipe.pipes.input import copy_inputs
 from scanpipe.tests import license_policies_index
 from scanpipe.tests import mocked_now
+from scanpipe.tests import dependency_data1
 from scanpipe.tests import package_data1
 from scanpipe.tests.pipelines.do_nothing import DoNothing
 
@@ -1416,6 +1418,37 @@ def test_scanpipe_discovered_package_model_create_from_data(self):
         self.assertEqual(package_count, DiscoveredPackage.objects.count())
         self.assertEqual(project_error_count, ProjectError.objects.count())
 
+    @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.")
+    def test_scanpipe_discovered_dependency_model_create_from_data(self):
+        project1 = Project.objects.create(name="Analysis")
+
+        dependency = DiscoveredDependency.create_from_data(project1, dependency_data1)
+        self.assertEqual(project1, dependency.project)
+        self.assertEqual("pkg:pypi/dask", dependency.purl)
+        self.assertEqual("dask<2023.0.0,>=2022.6.0", dependency.extracted_requirement)
+        self.assertEqual("install", dependency.scope)
+        self.assertTrue(dependency.is_runtime)
+        self.assertFalse(dependency.is_optional)
+        self.assertFalse(dependency.is_resolved)
+        self.assertEqual("pkg:pypi/dask?uuid=e656b571-7d3f-46d1-b95b-8f037aef9692", dependency.dependency_uid)
+        self.assertEqual("pkg:pypi/[email protected]?uuid=4d1f048b-a155-4f95-8cf9-185ab872ab4c", dependency.for_package_uid)
+        self.assertEqual("daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO", dependency.datafile_path)
+        self.assertEqual("pypi_sdist_pkginfo", dependency.datasource_id)
+
+        dependency_count = DiscoveredDependency.objects.count()
+        incomplete_data = dict(dependency_data1)
+        incomplete_data["dependency_uid"] = ""
+        self.assertIsNone(DiscoveredDependency.create_from_data(project1, incomplete_data))
+        self.assertEqual(dependency_count, DiscoveredDependency.objects.count())
+        error = project1.projecterrors.latest("created_date")
+        self.assertEqual("DiscoveredDependency", error.model)
+        expected_message = "No values for the following required fields: dependency_uid"
+        self.assertEqual(expected_message, error.message)
+        self.assertEqual(dependency_data1["purl"], error.details["purl"])
+        self.assertEqual("", error.details["dependency_uid"])
+        self.assertEqual("", error.traceback)
+
+
     def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self):
         project1 = Project.objects.create(name="Analysis")
 

scanpipe/tests/test_pipelines.py

@@ -302,7 +302,7 @@ def test_scanpipe_scan_package_pipeline_integration_test(self):
 
         self.assertEqual(4, project1.codebaseresources.count())
         self.assertEqual(1, project1.discoveredpackages.count())
-        self.assertEqual(0, project1.discovereddependencys.count())
+        self.assertEqual(1, project1.discovereddependencys.count())
 
         scancode_file = project1.get_latest_output(filename="scancode")
         expected_file = self.data_location / "is-npm-1.0.0_scan_package.json"
@@ -336,7 +336,7 @@ def test_scanpipe_scan_package_pipeline_integration_test_multiple_packages(self)
 
         self.assertEqual(9, project1.codebaseresources.count())
         self.assertEqual(2, project1.discoveredpackages.count())
-        self.assertEqual(0, project1.discovereddependencys.count())
+        self.assertEqual(2, project1.discovereddependencys.count())
 
         scancode_file = project1.get_latest_output(filename="scancode")
         expected_file = self.data_location / "multiple-is-npm-1.0.0_scan_package.json"
@@ -500,7 +500,7 @@ def test_scanpipe_load_inventory_pipeline_integration_test(self):
 
         self.assertEqual(18, project1.codebaseresources.count())
         self.assertEqual(2, project1.discoveredpackages.count())
-        self.assertEqual(0, project1.discovereddependencys.count())
+        self.assertEqual(4, project1.discovereddependencys.count())
 
         result_file = output.to_json(project1)
         expected_file = (

scanpipe/tests/test_pipes.py

@@ -39,6 +39,7 @@
 from scancode.interrupt import TimeoutError as InterruptTimeoutError
 
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectError
@@ -528,9 +529,11 @@ def test_scanpipe_pipes_scancode_virtual_codebase(self):
 
         scancode.create_codebase_resources(project, virtual_codebase)
         scancode.create_discovered_packages(project, virtual_codebase)
+        scancode.create_discovered_dependencies(project, virtual_codebase)
 
         self.assertEqual(18, CodebaseResource.objects.count())
         self.assertEqual(1, DiscoveredPackage.objects.count())
+        self.assertEqual(1, DiscoveredDependency.objects.count())
         # Make sure the root is not created as a CodebaseResource, walk(skip_root=True)
         self.assertFalse(CodebaseResource.objects.filter(path="codebase").exists())
 
@@ -547,8 +550,10 @@ def test_scanpipe_pipes_scancode_virtual_codebase(self):
         # The functions can be called again and existing objects are skipped
         scancode.create_codebase_resources(project, virtual_codebase)
         scancode.create_discovered_packages(project, virtual_codebase)
+        scancode.create_discovered_dependencies(project, virtual_codebase)
         self.assertEqual(18, CodebaseResource.objects.count())
         self.assertEqual(1, DiscoveredPackage.objects.count())
+        self.assertEqual(1, DiscoveredDependency.objects.count())
 
     def test_scanpipe_pipes_scancode_create_codebase_resources_inject_policy(self):
         project = Project.objects.create(name="asgiref")
@@ -560,6 +565,7 @@ def test_scanpipe_pipes_scancode_create_codebase_resources_inject_policy(self):
 
         scanpipe_app.license_policies_index = license_policies_index
         scancode.create_discovered_packages(project, virtual_codebase)
+        scancode.create_discovered_dependencies(project, virtual_codebase)
         scancode.create_codebase_resources(project, virtual_codebase)
         resources = project.codebaseresources