Add a "Inspect Manifest" pipeline #284

Signed-off-by: Thomas Druez <[email protected]>

Author: tdruez

docs/built-in-pipelines.rst

@@ -31,6 +31,16 @@ Docker Windows Image Analysis
     :members:
     :member-order: bysource
 
+.. _pipeline_inspect_manifest:
+
+Inspect Manifest
+----------------
+.. autoclass:: scanpipe.pipelines.inspect_manifest.InspectManifest()
+    :members:
+    :member-order: bysource
+
+.. _pipeline_load_inventory:
+
 Load Inventory From Scan
 ------------------------
 .. autoclass:: scanpipe.pipelines.load_inventory.LoadInventory()

scanpipe/pipelines/inspect_manifest.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from packagedcode import APPLICATION_PACKAGE_DATAFILE_HANDLERS
+from python_inspector.resolve_cli import resolver_api
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import update_or_create_package
+
+
+def resolve_pypi_packages(input_location):
+    """
+    https://github.com/nexB/scancode-toolkit/blob/develop/requirements.txt
+    https://raw.githubusercontent.com/nexB/python-inspector/main/requirements.txt
+    """
+    inspector_output = resolver_api(requirement_files=[input_location])
+    resolved_packages = inspector_output.packages
+    return resolved_packages
+
+
+# `default_package_type`: resolver callable
+resolver_registry = {
+    "pypi": resolve_pypi_packages,
+}
+
+
+def get_default_package_type(input_location):
+    for handler in APPLICATION_PACKAGE_DATAFILE_HANDLERS:
+        if handler.is_datafile(input_location):
+            return handler.default_package_type
+
+
+class InspectManifest(Pipeline):
+    """
+    A pipeline to inspect one or more manifest files and resolve its packages.
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.get_manifest_inputs,
+            cls.create_packages_from_manifest,
+        )
+
+    def get_manifest_inputs(self):
+        """
+        Locates all the manifest files from the project's input/ directory.
+        """
+        self.input_locations = [
+            str(input.absolute()) for input in self.project.inputs()
+        ]
+
+    def create_packages_from_manifest(self):
+        """
+        Resolves manifest files into packages.
+        """
+        for input_location in self.input_locations:
+            default_package_type = get_default_package_type(input_location)
+
+            resolver = resolver_registry.get(default_package_type)
+            if not resolver:
+                raise Exception(f"No resolver for {default_package_type}")
+
+            resolved_packages = resolver(input_location=input_location)
+
+            if not resolved_packages:
+                raise Exception("No packages could be resolved.")
+
+            for package_data in resolved_packages:
+                update_or_create_package(self.project, package_data)

setup.cfg

@@ -75,6 +75,8 @@ install_requires =
     commoncode==31.0.0
     # FetchCode
     fetchcode-container==1.2.3.210512; sys_platform == "linux"
+    # Python-inspector
+    python-inspector==0.8.2
     # Utilities
     XlsxWriter==3.0.3
     requests==2.28.1
@@ -106,6 +108,7 @@ console_scripts =
 scancodeio_pipelines =
     docker = scanpipe.pipelines.docker:Docker
     docker_windows = scanpipe.pipelines.docker_windows:DockerWindows
+    inspect_manifest = scanpipe.pipelines.inspect_manifest:InspectManifest
     load_inventory = scanpipe.pipelines.load_inventory:LoadInventory
     root_filesystems = scanpipe.pipelines.root_filesystems:RootFS
     scan_codebase = scanpipe.pipelines.scan_codebase:ScanCodebase