Package scanning update

CHANGELOG.rst

@@ -68,6 +68,50 @@ v31.0.0 (next)
   https://github.com/nexB/scancode.io/issues/164
   https://github.com/nexB/scancode.io/issues/464
 
+- Update application Package scanning step to reflect the updates in
+  scancode-toolkit package scanning.
+
+  - Package data detected from a file are now stored on the
+    CodebaseResource.package_data field.
+  - A second processing step is now done after scanning for Package data, where
+    Package Resources are determined and DiscoveredPackages and
+    DiscoveredDependencies are created.
+
+  https://github.com/nexB/scancode.io/issues/444
+
+- CodebaseResource.name now contains both the bare file name with extension, as
+  opposed to just the bare file name without extension.
+
+  - Using a name stripped from its extension was something that was not used in
+    other AboutCode project or tools.
+
+  https://github.com/nexB/scancode.io/issues/467
+
+- Add the model DiscoveredDependency. This represents Package dependencies
+  discovered in a Project. The ``scan_codebase`` and ``scan_packages`` pipelines
+  have been updated to create DiscoveredDepdendency objects. The Project API has
+  been updated with new fields:
+
+  - ``dependency_count``
+    - The number of DiscoveredDependencies associated with the project.
+
+  - ``discovered_dependency_summary``
+    - A mapping that contains following fields:
+
+      - ``total``
+        - The number of DiscoveredDependencies associated with the project.
+      - ``is_runtime``
+        - The number of runtime dependencies.
+      - ``is_optional``
+        - The number of optional dependencies.
+      - ``is_resolved``
+        - The number of resolved dependencies.
+
+  These values are also available on the Project view.
+  https://github.com/nexB/scancode.io/issues/447
+
+- The ``dependencies`` field has been removed from the DiscoveredPackage model.
+
 v30.2.0 (2021-12-17)
 --------------------
 

scanpipe/api/serializers.py

@@ -26,6 +26,7 @@
 
 from scanpipe.api import ExcludeFromListViewMixin
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectError
@@ -113,6 +114,7 @@ class ProjectSerializer(
     input_sources = serializers.JSONField(source="input_sources_list", read_only=True)
     codebase_resources_summary = serializers.SerializerMethodField()
     discovered_package_summary = serializers.SerializerMethodField()
+    discovered_dependency_summary = serializers.SerializerMethodField()
 
     class Meta:
         model = Project
@@ -136,8 +138,10 @@ class Meta:
             "error_count",
             "resource_count",
             "package_count",
+            "dependency_count",
             "codebase_resources_summary",
             "discovered_package_summary",
+            "discovered_dependency_summary",
         )
 
         exclude_from_list_view = [
@@ -147,8 +151,10 @@ class Meta:
             "error_count",
             "resource_count",
             "package_count",
+            "dependency_count",
             "codebase_resources_summary",
             "discovered_package_summary",
+            "discovered_dependency_summary",
         ]
 
     def get_codebase_resources_summary(self, project):
@@ -163,6 +169,15 @@ def get_discovered_package_summary(self, project):
             "with_modified_resources": base_qs.exclude(modified_resources=[]).count(),
         }
 
+    def get_discovered_dependency_summary(self, project):
+        base_qs = project.discovereddependencys
+        return {
+            "total": base_qs.count(),
+            "is_runtime": base_qs.filter(is_runtime=True).count(),
+            "is_optional": base_qs.filter(is_optional=True).count(),
+            "is_resolved": base_qs.filter(is_resolved=True).count(),
+        }
+
     def create(self, validated_data):
         """
         Creates a new `project` with `upload_file` and `pipeline` as optional.
@@ -216,6 +231,16 @@ class Meta:
             "filename",
             "last_modified_date",
             "codebase_resources",
+            "dependencies",
+        ]
+
+
+class DiscoveredDependencySerializer(serializers.ModelSerializer):
+    class Meta:
+        model = DiscoveredDependency
+        exclude = [
+            "id",
+            "project",
         ]
 
 
@@ -257,6 +282,7 @@ def get_model_serializer(model_class):
     serializer = {
         CodebaseResource: CodebaseResourceSerializer,
         DiscoveredPackage: DiscoveredPackageSerializer,
+        DiscoveredDependency: DiscoveredDependencySerializer,
         ProjectError: ProjectErrorSerializer,
     }.get(model_class, None)
 

scanpipe/api/views.py

@@ -36,6 +36,7 @@
 from rest_framework.response import Response
 
 from scanpipe.api.serializers import CodebaseResourceSerializer
+from scanpipe.api.serializers import DiscoveredDependencySerializer
 from scanpipe.api.serializers import DiscoveredPackageSerializer
 from scanpipe.api.serializers import PipelineSerializer
 from scanpipe.api.serializers import ProjectErrorSerializer
@@ -180,6 +181,16 @@ def packages(self, request, *args, **kwargs):
 
         return Response(serializer.data)
 
+    @action(detail=True)
+    def dependencies(self, request, *args, **kwargs):
+        project = self.get_object()
+        queryset = project.discovereddependencys.all()
+
+        paginated_qs = self.paginate_queryset(queryset)
+        serializer = DiscoveredDependencySerializer(paginated_qs, many=True)
+
+        return Response(serializer.data)
+
     @action(detail=True)
     def errors(self, request, *args, **kwargs):
         project = self.get_object()

scanpipe/filters.py

@@ -28,6 +28,7 @@
 from packageurl.contrib.django.filters import PackageURLFilter
 
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectError
@@ -275,6 +276,14 @@ class Meta:
         ]
 
 
+class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
+    class Meta:
+        model = DiscoveredDependency
+        fields = [
+            "purl",
+        ]
+
+
 class ErrorFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     search = django_filters.CharFilter(
         label="Search", field_name="message", lookup_expr="icontains"

scanpipe/migrations/0019_codebaseresource_package_data_and_more.py

@@ -0,0 +1,55 @@
+# Generated by Django 4.0.6 on 2022-08-03 18:36
+
+from django.db import migrations, models
+import django.db.models.deletion
+import scanpipe.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0018_codebaseresource_tag'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='codebaseresource',
+            name='package_data',
+            field=models.JSONField(blank=True, default=list, help_text='List of Package data detected from this CodebaseResource'),
+        ),
+        migrations.AlterField(
+            model_name='codebaseresource',
+            name='name',
+            field=models.CharField(blank=True, help_text='File or directory name of this resource with its extension.', max_length=255),
+        ),
+        migrations.RemoveField(
+            model_name='discoveredpackage',
+            name='dependencies',
+        ),
+        migrations.CreateModel(
+            name='DiscoveredDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('purl', models.CharField(help_text='The Package URL of this dependency.', max_length=1024)),
+                ('extracted_requirement', models.CharField(blank=True, help_text='The version requirements of this dependency.', max_length=64)),
+                ('scope', models.CharField(blank=True, help_text='The scope of this dependency, how it is used in a project.', max_length=64)),
+                ('is_runtime', models.BooleanField(default=False)),
+                ('is_optional', models.BooleanField(default=False)),
+                ('is_resolved', models.BooleanField(default=False)),
+                ('dependency_uid', models.CharField(help_text='The unique identifier of this dependency.', max_length=1024)),
+                ('for_package_uid', models.CharField(blank=True, help_text='The unique identifier of the package this dependency is for.', max_length=1024)),
+                ('datafile_path', models.CharField(blank=True, help_text='The relative path to the datafile where this dependency was detected from.', max_length=1024)),
+                ('datasource_id', models.CharField(blank=True, help_text='The identifier for the datafile handler used to obtain this dependency.', max_length=64)),
+                ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)ss', to='scanpipe.project')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, scanpipe.models.SaveProjectErrorMixin),
+        ),
+        migrations.AddField(
+            model_name='discoveredpackage',
+            name='dependencies',
+            field=models.ManyToManyField(related_name='discovered_packages', to='scanpipe.discovereddependency'),
+        ),
+    ]