Correctly compute_normalized_license for debian

* The license of Debian installed system packages was not detected
  correctly. In particular merging copyright detection details with
  other metadata was not working

Reference: aboutcode-org/scancode.io#478
Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/packagedcode/debian.py

@@ -28,9 +28,11 @@
 
 TRACE = SCANCODE_DEBUG_PACKAGE_API
 
+
 def logger_debug(*args):
     pass
 
+
 logger = logging.getLogger(__name__)
 
 if TRACE:
@@ -43,7 +45,6 @@ def logger_debug(*args):
             ' '.join(isinstance(a, str) and a or repr(a) for a in args)
         )
 
-
 # TODO: add dependencies
 
 
@@ -282,14 +283,27 @@ def assemble(cls, package_data, resource, codebase):
         resources = []
         # TODO: keep track of missing files
         for res in root_resource.walk(codebase):
+            if TRACE:
+                logger_debug(f'   debian: assemble: root_walk: res: {res}')
             if not res.path.endswith(assemblable_paths):
                 continue
 
             for pkgdt in res.package_data:
                 package_data = models.PackageData.from_dict(pkgdt)
+                if TRACE:
+                    # logger_debug(f'     debian: assemble: root_walk: package_data: {package_data}')
+                    logger_debug(f'     debian: assemble: root_walk: package_data: {package_data.license_expression}')
+
+                # Most debian secondary files are only specific to a name. We
+                # have a few cases where the arch is included in the lists and
+                # md5sums.
                 package.update(
                     package_data=package_data,
                     datafile_path=res.path,
+                    replace=False,
+                    include_version=False,
+                    include_qualifiers=False,
+                    include_subpath=False,
                 )
                 package_file_references.extend(package_data.file_references)
 

src/packagedcode/models.py

@@ -1207,6 +1207,13 @@ def __attrs_post_init__(self, *args, **kwargs):
     def to_dict(self):
         return super().to_dict(with_details=False)
 
+    def to_package_data(self):
+        mapping = super().to_dict(with_details=True)
+        mapping.pop('package_uid', None)
+        mapping.pop('datafile_paths', None)
+        mapping.pop('datasource_ids', None)
+        return PackageData.from_dict(mapping)
+
     @classmethod
     def from_package_data(cls, package_data, datafile_path):
         """
@@ -1255,7 +1262,15 @@ def is_compatible(self, package_data, include_qualifiers=True):
             and self.primary_language == package_data.primary_language
         )
 
-    def update(self, package_data, datafile_path, replace=False):
+    def update(
+        self,
+        package_data,
+        datafile_path,
+        replace=False,
+        include_version=True,
+        include_qualifiers=False,
+        include_subpath=False,
+    ):
         """
         Update this Package with data from the ``package_data`` PackageData.
 
@@ -1281,9 +1296,15 @@ def update(self, package_data, datafile_path, replace=False):
         if isinstance(package_data, dict):
             package_data = PackageData.from_dict(package_data)
 
-        if not self.is_compatible(package_data, include_qualifiers=False):
+        if not is_compatible(
+            purl1=self,
+            purl2=package_data,
+            include_version=include_version,
+            include_qualifiers=include_qualifiers,
+            include_subpath=include_subpath,
+        ):
             if TRACE_UPDATE:
-                logger_debug(f'update: {self.purl} not compatible with: {package_data.purl}')
+                logger_debug(f'update: skipping: {self.purl} is not compatible with: {package_data.purl}')
             return False
 
         # always append these new items
@@ -1345,6 +1366,56 @@ def get_packages_files(self, codebase):
                     yield resource
 
 
+def is_compatible(
+    purl1,
+    purl2,
+    include_version=True,
+    include_qualifiers=True,
+    include_subpath=True,
+):
+    """
+    Return True if the ``purl1`` PackageURL-like object is compatible with
+    the ``purl2`` PackageURL-like object, e.g. it is about the same package.
+    PackageData objectys are PackageURL-like.
+
+    For example::
+    >>> p1 = PackageURL.from_string('pkg:deb/[email protected]?arch=arm64')
+    >>> p2 = PackageURL.from_string('pkg:deb/[email protected]')
+    >>> p3 = PackageURL.from_string('pkg:deb/libssl')
+    >>> p4 = PackageURL.from_string('pkg:deb/libncurses5')
+    >>> p5 = PackageURL.from_string('pkg:deb/[email protected]?arch=arm64#/sbin')
+    >>> is_compatible(p1, p2)
+    False
+    >>> is_compatible(p1, p2, include_qualifiers=False)
+    True
+    >>> is_compatible(p1, p4)
+    False
+    >>> is_compatible(p1, p4, include_version=False, include_qualifiers=False)
+    True
+    >>> is_compatible(p3, p4)
+    False
+    >>> is_compatible(p1, p5)
+    False
+    >>> is_compatible(p1, p5, include_subpath=False)
+    True
+    """
+    is_compatible = (
+        purl1.type == purl2.type
+        and purl1.namespace == purl2.namespace
+        and purl1.name == purl2.name
+    )
+    if include_version:
+        is_compatible = is_compatible and (purl1.version == purl2.version)
+
+    if include_qualifiers:
+        is_compatible = is_compatible and (purl1.qualifiers == purl2.qualifiers)
+
+    if include_subpath:
+        is_compatible = is_compatible and (purl1.subpath == purl2.subpath)
+
+    return is_compatible
+
+
 @attr.attributes(slots=True)
 class PackageWithResources(Package):
     """
@@ -1384,7 +1455,15 @@ def merge_sequences(list1, list2, **kwargs):
     merged = []
     existing = set()
     for item in list1 + list2:
-        key = item.to_tuple(**kwargs)
+        try:
+            if hasattr(item, 'to_tuple'):
+                key = item.to_tuple(**kwargs)
+            else:
+                key = to_tuple(kwargs)
+
+        except Exception as e:
+            raise Exception(f'Failed to merge sequences: {item}', f'kwargs: {kwargs}') from e
+
         if not key in existing:
             merged.append(item)
             existing.add(key)

tests/packagedcode/data/chef/package.scan.expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 2
-      }
-    }
-  ],
   "dependencies": [
     {
       "purl": "pkg:chef/nodejs",
@@ -95,10 +69,12 @@
       "api_data_url": "https://supermarket.chef.io/api/v1/cookbooks/301/versions/0.1.0",
       "package_uid": "pkg:chef/[email protected]?uuid=fixed-uid-done-for-testing-5642512d1758",
       "datafile_paths": [
-        "package/metadata.rb"
+        "package/metadata.rb",
+        "package/metadata.json"
       ],
       "datasource_ids": [
-        "chef_cookbook_metadata_rb"
+        "chef_cookbook_metadata_rb",
+        "chef_cookbook_metadata_json"
       ],
       "purl": "pkg:chef/[email protected]"
     }

tests/packagedcode/data/debian/basic-rootfs-expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json-pp": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 5
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {
@@ -61,8 +35,8 @@
       "bug_tracking_url": null,
       "code_view_url": null,
       "vcs_url": null,
-      "copyright": null,
-      "license_expression": null,
+      "copyright": "Copyright (c) 1998-2016 Free Software Foundation, Inc.\nCopyright (c) 2001 by Pradeep Padala\nCopyright (c) 1994 X Consortium\nCopyright (c) 1980, 1991, 1992, 1993 The Regents of the University of California\nCopyright 1996-2007 by Thomas E. Dickey",
+      "license_expression": "x11-fsf AND x11-xconsortium AND bsd-new AND x11-fsf",
       "declared_license": null,
       "notice_text": null,
       "source_packages": [
@@ -114,10 +88,14 @@
       "api_data_url": null,
       "package_uid": "pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
       "datafile_paths": [
-        "basic-rootfs.tar.gz/var/lib/dpkg/status"
+        "basic-rootfs.tar.gz/var/lib/dpkg/status",
+        "basic-rootfs.tar.gz/usr/share/doc/libncurses5/copyright",
+        "basic-rootfs.tar.gz/var/lib/dpkg/info/libncurses5:amd64.md5sums"
       ],
       "datasource_ids": [
-        "debian_installed_status_db"
+        "debian_installed_status_db",
+        "debian_copyright_in_package",
+        "debian_installed_md5sums"
       ],
       "purl": "pkg:deb/[email protected]?architecture=amd64"
     },
@@ -155,9 +133,13 @@
       "bug_tracking_url": null,
       "code_view_url": null,
       "vcs_url": null,
-      "copyright": null,
-      "license_expression": null,
-      "declared_license": null,
+      "copyright": "Copyright 2013 Jiri Pirko <[email protected]>\nCopyright 2014 Andrew Ayer <[email protected]>",
+      "license_expression": "(lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1) AND (lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1)",
+      "declared_license": [
+        "LGPL-2.1+",
+        "LGPL-2.1+",
+        "LGPL-2.1+"
+      ],
       "notice_text": null,
       "source_packages": [
         "pkg:deb/libndp"
@@ -190,10 +172,14 @@
       "api_data_url": null,
       "package_uid": "pkg:deb/[email protected]?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
       "datafile_paths": [
-        "basic-rootfs.tar.gz/var/lib/dpkg/status"
+        "basic-rootfs.tar.gz/var/lib/dpkg/status",
+        "basic-rootfs.tar.gz/usr/share/doc/libndp0/copyright",
+        "basic-rootfs.tar.gz/var/lib/dpkg/info/libndp0:amd64.md5sums"
       ],
       "datasource_ids": [
-        "debian_installed_status_db"
+        "debian_installed_status_db",
+        "debian_copyright_in_package",
+        "debian_installed_md5sums"
       ],
       "purl": "pkg:deb/[email protected]?architecture=amd64"
     }