Add test for custom package_adder #3034

JonoYang · JonoYang · commit ec6f47abeb07 · 2022-08-01T12:36:33.000-07:00
* Update CHANGELOG.rst

Signed-off-by: Jono Yang &lt;jyang@nexb.com&gt;
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -166,8 +166,12 @@ Package detection:
     of the new format where there is no root conceptually, just a list of files for each
     package.
 
-  - There is a new resource-level attribute `for_packages` which refers to packages
-    through package_uuids (pURL + uuid string).
+  - There is a new resource-level attribute `for_packages` which refers to
+    packages through package_uids (pURL + uuid string). A `package_adder`
+    function is now used to associate a Package to a Resource that is part of
+    it. This gives us the flexibility to use the packagedcode Package handlers
+    in other contexts where `for_packages` on Resource is not implemented in the
+    same way as scancode-toolkit.
 
   - The package_data attribute `dependencies` (which is a list of DependentPackages),
     now has a new attribute `resolved_package` with a package data mapping.
@@ -331,8 +335,8 @@ Miscellaneous
 
 - Added support for usage of shortcut flags
   - `-A` or `--about`
-  - `-q` or `--quiet` 
-  - `-v` or `--verbose` 
+  - `-q` or `--quiet`
+  - `-v` or `--verbose`
   - `-V` or `--version` can be used.
 
 
diff --git a/tests/packagedcode/data/models/get_package_resources.scan.json b/tests/packagedcode/data/models/get_package_resources.scan.json
@@ -0,0 +1,213 @@
+{
+  "headers": [
+    {
+      "tool_name": "scancode-toolkit",
+      "tool_version": "31.0.0rc3",
+      "options": {
+        "input": [
+          "."
+        ],
+        "--info": true,
+        "--json-pp": "../get_package_resources.scan.json",
+        "--package": true
+      },
+      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
+      "start_timestamp": "2022-08-01T185859.869487",
+      "end_timestamp": "2022-08-01T185902.232500",
+      "output_format_version": "2.0.0",
+      "duration": 2.363025665283203,
+      "message": null,
+      "errors": [],
+      "warnings": [],
+      "extra_data": {
+        "system_environment": {
+          "operating_system": "linux",
+          "cpu_architecture": "64",
+          "platform": "Linux-5.4.0-122-generic-x86_64-with-glibc2.27",
+          "platform_version": "#138~18.04.1-Ubuntu SMP Fri Jun 24 14:14:03 UTC 2022",
+          "python_version": "3.10.4 (main, May 22 2022, 00:46:26) [GCC 7.5.0]"
+        },
+        "spdx_license_list_version": "3.17",
+        "files_count": 3
+      }
+    }
+  ],
+  "dependencies": [],
+  "packages": [],
+  "files": [
+    {
+      "path": "get_package_resources",
+      "type": "directory",
+      "name": "get_package_resources",
+      "base_name": "get_package_resources",
+      "extension": "",
+      "size": 0,
+      "date": null,
+      "sha1": null,
+      "md5": null,
+      "sha256": null,
+      "mime_type": null,
+      "file_type": null,
+      "programming_language": null,
+      "is_binary": false,
+      "is_text": false,
+      "is_archive": false,
+      "is_media": false,
+      "is_source": false,
+      "is_script": false,
+      "package_data": [],
+      "for_packages": [],
+      "files_count": 3,
+      "dirs_count": 1,
+      "size_count": 63,
+      "scan_errors": []
+    },
+    {
+      "path": "get_package_resources/package.json",
+      "type": "file",
+      "name": "package.json",
+      "base_name": "package",
+      "extension": ".json",
+      "size": 63,
+      "date": "2022-02-15",
+      "sha1": "47ad6746a065a22a4c2a119cb425c250bddfbf88",
+      "md5": "5cbb43ca814ec801e8673e348e8eec19",
+      "sha256": "aac331f407d4d662a2897eafeb4f05a1c2343218220d99779c9553d3b92bd4ac",
+      "mime_type": "application/json",
+      "file_type": "JSON data",
+      "programming_language": null,
+      "is_binary": false,
+      "is_text": true,
+      "is_archive": false,
+      "is_media": false,
+      "is_source": false,
+      "is_script": false,
+      "package_data": [
+        {
+          "type": "npm",
+          "namespace": null,
+          "name": "test",
+          "version": "0.1.0",
+          "qualifiers": {},
+          "subpath": null,
+          "primary_language": "JavaScript",
+          "description": null,
+          "release_date": null,
+          "parties": [],
+          "keywords": [],
+          "homepage_url": null,
+          "download_url": "https://registry.npmjs.org/test/-/test-0.1.0.tgz",
+          "size": null,
+          "sha1": null,
+          "md5": null,
+          "sha256": null,
+          "sha512": null,
+          "bug_tracking_url": null,
+          "code_view_url": null,
+          "vcs_url": null,
+          "copyright": null,
+          "license_expression": "mit",
+          "declared_license": [
+            "MIT"
+          ],
+          "notice_text": null,
+          "source_packages": [],
+          "file_references": [],
+          "extra_data": {},
+          "dependencies": [],
+          "repository_homepage_url": "https://www.npmjs.com/package/test",
+          "repository_download_url": "https://registry.npmjs.org/test/-/test-0.1.0.tgz",
+          "api_data_url": "https://registry.npmjs.org/test/0.1.0",
+          "datasource_id": "npm_package_json",
+          "purl": "pkg:npm/test@0.1.0"
+        }
+      ],
+      "for_packages": [],
+      "files_count": 0,
+      "dirs_count": 0,
+      "size_count": 0,
+      "scan_errors": []
+    },
+    {
+      "path": "get_package_resources/this-should-be-returned",
+      "type": "file",
+      "name": "this-should-be-returned",
+      "base_name": "this-should-be-returned",
+      "extension": "",
+      "size": 0,
+      "date": "2022-02-15",
+      "sha1": null,
+      "md5": null,
+      "sha256": null,
+      "mime_type": "inode/x-empty",
+      "file_type": "empty",
+      "programming_language": null,
+      "is_binary": false,
+      "is_text": true,
+      "is_archive": false,
+      "is_media": false,
+      "is_source": false,
+      "is_script": false,
+      "package_data": [],
+      "for_packages": [],
+      "files_count": 0,
+      "dirs_count": 0,
+      "size_count": 0,
+      "scan_errors": []
+    },
+    {
+      "path": "get_package_resources/node_modules",
+      "type": "directory",
+      "name": "node_modules",
+      "base_name": "node_modules",
+      "extension": "",
+      "size": 0,
+      "date": null,
+      "sha1": null,
+      "md5": null,
+      "sha256": null,
+      "mime_type": null,
+      "file_type": null,
+      "programming_language": null,
+      "is_binary": false,
+      "is_text": false,
+      "is_archive": false,
+      "is_media": false,
+      "is_source": false,
+      "is_script": false,
+      "package_data": [],
+      "for_packages": [],
+      "files_count": 1,
+      "dirs_count": 0,
+      "size_count": 0,
+      "scan_errors": []
+    },
+    {
+      "path": "get_package_resources/node_modules/this-should-not-be-returned",
+      "type": "file",
+      "name": "this-should-not-be-returned",
+      "base_name": "this-should-not-be-returned",
+      "extension": "",
+      "size": 0,
+      "date": "2022-02-15",
+      "sha1": null,
+      "md5": null,
+      "sha256": null,
+      "mime_type": "inode/x-empty",
+      "file_type": "empty",
+      "programming_language": null,
+      "is_binary": false,
+      "is_text": true,
+      "is_archive": false,
+      "is_media": false,
+      "is_source": false,
+      "is_script": false,
+      "package_data": [],
+      "for_packages": [],
+      "files_count": 0,
+      "dirs_count": 0,
+      "size_count": 0,
+      "scan_errors": []
+    }
+  ]
+}
diff --git a/tests/packagedcode/test_package_models.py b/tests/packagedcode/test_package_models.py
@@ -7,18 +7,22 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+from cgi import test
 import os.path
 
 from packagedcode import misc
 from packagedcode import models
 from packagedcode import ALL_DATAFILE_HANDLERS
 from packagedcode.models import PackageData
 from packagedcode.models import Party
+from packagedcode.plugin_package import get_package_and_deps
 from packagedcode.plugin_package import PackageScanner
 from packages_test_utils import PackageTester
+from scancode.cli_test_utils import purl_with_fake_uuid
 from scancode_config import REGEN_TEST_FIXTURES
 
 from commoncode.resource import Codebase
+from commoncode.resource import VirtualCodebase
 
 
 class TestModels(PackageTester):
@@ -177,3 +181,33 @@ def test_add_to_package(self):
             test_codebase
         )
         assert test_package.package_uid in test_resource.for_packages
+
+    def test_assembly_custom_package_adder(self):
+        def test_package_adder(package_uid, resource, codebase):
+            """
+            Add `package_uid` to `resource.extra_data`
+            """
+            if 'for_packages' in resource.extra_data:
+                resource.extra_data['for_packages'].append(package_uid)
+            else:
+                resource.extra_data['for_packages'] = [package_uid]
+            resource.save(codebase)
+
+        # This scan does not contain top-level Packages or Dependencies since we
+        # want to run `get_packages_and_deps` to create them
+        test_loc = self.get_test_loc('models/get_package_resources.scan.json')
+        test_codebase = VirtualCodebase(location=test_loc)
+        packages, dependencies = get_package_and_deps(test_codebase, test_package_adder)
+
+        assert len(packages) == 1
+        assert not dependencies
+
+        package = packages[0]
+        package_uid = package.package_uid
+        test_package_uid = purl_with_fake_uuid(package_uid)
+
+        for resource in test_codebase.walk():
+            for_packages = resource.extra_data.get('for_packages', [])
+            for package_uid in for_packages:
+                normalized_package_uid = purl_with_fake_uuid(package_uid)
+                assert normalized_package_uid == test_package_uid
