From 8963d7fd53bf77d699388ddc98b657091141ce58 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 20 Jul 2022 17:03:03 -0700 Subject: [PATCH 01/24] Update scan_for_application_packages #436 #444 #447 * Update scan_for_application_packages to save detected Package data to the CodebaseResource it is from, then iterate through the CodebaseResources with Package data and use the proper Package handler to process the Package data * Create DiscoveredDependency model * Add package_data JSON field to CodebaseResource Signed-off-by: Jono Yang --- ...ource_package_data_discovereddependency.py | 41 +++++ scanpipe/models.py | 154 ++++++++++++++++++ scanpipe/pipes/__init__.py | 24 +++ scanpipe/pipes/scancode.py | 83 +++++++++- 4 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 scanpipe/migrations/0019_codebaseresource_package_data_discovereddependency.py diff --git a/scanpipe/migrations/0019_codebaseresource_package_data_discovereddependency.py b/scanpipe/migrations/0019_codebaseresource_package_data_discovereddependency.py new file mode 100644 index 000000000..b0f95c8c2 --- /dev/null +++ b/scanpipe/migrations/0019_codebaseresource_package_data_discovereddependency.py @@ -0,0 +1,41 @@ +# Generated by Django 4.0.6 on 2022-07-20 23:58 + +from django.db import migrations, models +import django.db.models.deletion +import scanpipe.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0018_codebaseresource_tag'), + ] + + operations = [ + migrations.AddField( + model_name='codebaseresource', + name='package_data', + field=models.JSONField(blank=True, default=dict, help_text='List of Package data detected from this CodebaseResource'), + ), + migrations.CreateModel( + name='DiscoveredDependency', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('purl', models.CharField(help_text='The Package URL of this dependency.', max_length=1024)), + ('extracted_requirement', models.CharField(help_text='The version requirements of this dependency.', max_length=32)), + ('scope', models.CharField(help_text='The scope of this dependency, how it is used in a project.', max_length=32)), + ('is_runtime', models.BooleanField(default=False)), + ('is_optional', models.BooleanField(default=False)), + ('is_resolved', models.BooleanField(default=False)), + ('dependency_uid', models.CharField(help_text='The unique identifier of this dependency.', max_length=1024)), + ('for_package_uid', models.CharField(help_text='The unique identifier of the package this dependency is for.', max_length=1024)), + ('datafile_path', models.CharField(blank=True, help_text='The relative path to the datafile where this dependency was detected from.', max_length=1024)), + ('datasource_id', models.CharField(help_text='The identifier for the datafile handler used to obtain this dependency.', max_length=64)), + ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)ss', to='scanpipe.project')), + ], + options={ + 'abstract': False, + }, + bases=(models.Model, scanpipe.models.SaveProjectErrorMixin), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 06100f3a1..81d7d2979 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -54,6 +54,7 @@ import django_rq import redis import requests +from commoncode.fileutils import parent_directory from commoncode.hash import multi_checksums from packageurl import PackageURL from packageurl import normalize_qualifiers @@ -1466,6 +1467,12 @@ class Compliance(models.TextChoices): ), ) + package_data = models.JSONField( + default=dict, + blank=True, + help_text=_("List of Package data detected from this CodebaseResource"), + ) + objects = CodebaseResourceQuerySet.as_manager() class Meta: @@ -1582,6 +1589,56 @@ def unique_license_expressions(self): """ return sorted(set(self.license_expressions)) + def parent_path(self): + """ + Return the parent path for this CodebaseResource or None. + """ + return parent_directory(self.path, with_trail=False) + + def has_parent(self): + """ + Return True if this CodebaseResource has a parent CodebaseResource or + False otherwise. + """ + parent_path = self.parent_path() + if not parent_path: + return False + if self.project.codebaseresources.filter(path=parent_path).exists(): + return True + return False + + def parent(self, codebase=None): + """ + Return the parent CodebaseResource object for this CodebaseResource or + None. + + `codebase` is not used in this context but required for compatibility + with the commoncode.resource.Codebase class API. + """ + parent_path = self.parent_path() + return parent_path and self.project.codebaseresources.get(path=parent_path) + + def has_siblings(self, codebase=None): + """ + Return True is this CodebaseResource has siblings. + + `codebase` is not used in this context but required for compatibility + with the commoncode.resource.Codebase class API. + """ + return self.has_parent() and self.parent(codebase).has_children() + + def siblings(self, codebase=None): + """ + Return a sequence of sibling Resource objects for this Resource + or an empty sequence. + + `codebase` is not used in this context but required for compatibility + with the commoncode.resource.Codebase class API. + """ + if self.has_parent(): + return self.parent(codebase).children(codebase) + return [] + def descendants(self): """ Returns a QuerySet of descendant CodebaseResource objects using a @@ -1847,6 +1904,103 @@ def update_from_data(self, package_data, override=False): return updated_fields +class DiscoveredDependency( + ProjectRelatedModel, + SaveProjectErrorMixin, +): + """ + A project's Discovered Dependencies are records of the dependencies used by + system and application packages discovered in the code under analysis. + """ + purl = models.CharField( + max_length=1024, + help_text=_( + "The Package URL of this dependency." + ), + ) + extracted_requirement = models.CharField( + max_length=32, + help_text=_( + "The version requirements of this dependency." + ), + ) + scope = models.CharField( + max_length=32, + help_text=_( + "The scope of this dependency, how it is used in a project." + ), + ) + + is_runtime = models.BooleanField(default=False) + is_optional = models.BooleanField(default=False) + is_resolved = models.BooleanField(default=False) + + dependency_uid = models.CharField( + max_length=1024, + help_text=_( + "The unique identifier of this dependency." + ), + ) + for_package_uid = models.CharField( + max_length=1024, + help_text=_( + "The unique identifier of the package this dependency is for." + ), + ) + datafile_path = models.CharField( + max_length=1024, + blank=True, + help_text=_( + "The relative path to the datafile where this dependency was detected from." + ), + ) + datasource_id = models.CharField( + max_length=64, + help_text=_( + "The identifier for the datafile handler used to obtain this dependency." + ) + ) + + @classmethod + def create_from_data(cls, project, dependency_data): + """ + Creates and returns a DiscoveredPackage for a `project` from the `dependency_data`. + """ + if "resolved_package" in dependency_data: + dependency_data.pop("resolved_package") + discovered_dependency = cls(project=project, **dependency_data) + discovered_dependency.save() + return discovered_dependency + + def update_from_data(self, dependency_data): + """ + Update this discovered dependency instance with the provided `dependency_data`. + The `save()` is called only if at least one field was modified. + """ + model_fields = DiscoveredPackage.model_fields() + updated_fields = [] + + for field_name, value in dependency_data.items(): + skip_reasons = [ + not value, + field_name not in model_fields, + ] + if any(skip_reasons): + continue + + current_value = getattr(self, field_name, None) + if not current_value: + setattr(self, field_name, value) + updated_fields.append(field_name) + elif current_value != value: + pass # TODO: handle this case + + if updated_fields: + self.save() + + return updated_fields + + class WebhookSubscription(UUIDPKModel, ProjectRelatedModel): target_url = models.URLField(_("Target URL"), max_length=1024) sent = models.BooleanField(default=False) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 50f42d4a4..70d924ab0 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -30,6 +30,7 @@ from django.db.models import Count from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.pipes import scancode @@ -104,6 +105,29 @@ def update_or_create_package(project, package_data, codebase_resource=None): return package +def update_or_create_dependencies(project, dependency_data): + """ + Gets, updates or creates a DiscoveredDependency then returns it. + Uses the `project` and `dependency_data` mapping to lookup and creates the + DiscoveredDependency using its dependency_uid and for_package_uid as a unique key. + """ + try: + dependency = DiscoveredDependency.objects.get( + project=project, + dependency_uid=dependency_data.get("dependency_uid"), + for_package_uid=dependency_data.get("for_package_uid"), + ) + except DiscoveredDependency.DoesNotExist: + dependency = None + + if dependency: + dependency.update_from_data(dependency_data) + else: + dependency = DiscoveredDependency.create_from_data(project, dependency_data) + + return dependency + + def analyze_scanned_files(project): """ Sets the status for CodebaseResource to unknown or no license. diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 34bf8ebdc..7ce039fe1 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -35,6 +35,8 @@ from commoncode import fileutils from commoncode.resource import VirtualCodebase +from packagedcode import get_package_handler +from packagedcode import models as packagedcode_models from extractcode import api as extractcode_api from scancode import ScancodeError from scancode import Scanner @@ -42,6 +44,7 @@ from scancode import cli as scancode_cli from scanpipe import pipes +from scanpipe.pipes.codebase import ProjectCodebase from scanpipe.models import CodebaseResource logger = logging.getLogger("scanpipe.pipes") @@ -230,10 +233,9 @@ def save_scan_package_results(codebase_resource, scan_results, scan_errors): Saves the resource scan package results in the database. Creates project errors if any occurred during the scan. """ - packages = scan_results.get("package_data", []) - if packages: - for package_data in packages: - codebase_resource.create_and_add_package(package_data) + package_data = scan_results.get("package_data", []) + if package_data: + codebase_resource.package_data = package_data codebase_resource.status = "application-package" codebase_resource.save() @@ -310,18 +312,89 @@ def scan_for_files(project): def scan_for_application_packages(project): """ - Runs a package scan on files without a status for a `project`. + Runs a package scan on files without a status for a `project`, + then create DiscoveredPackage and DiscoveredDependency instances + from the detected package data Multiprocessing is enabled by default on this pipe, the number of processes can be controlled through the SCANCODEIO_PROCESSES setting. """ resource_qs = project.codebaseresources.no_status() + + # Collect detected Package data and save it to the CodebaseResource it was + # detected from _scan_and_save( resource_qs=resource_qs, scan_func=scan_for_package_data, save_func=save_scan_package_results, ) + # Iterate through CodebaseResources with Package data and handle them using + # the proper Package handler from packagedcode + assemble_packages(project=project) + + +def assemble_packages(project): + """ + Create instances of DiscoveredPackage and DiscoveredDependency for `project` + from the parsed package data present in the CodebaseResources of `project`. + """ + seen_resource_paths = set() + package_data_resources_qs = project.codebaseresources.filter(package_data__isnull=False) + for resource in package_data_resources_qs: + if resource.path in seen_resource_paths: + continue + + logger.info( + f"project: {project}:\n" + "function: assemble_packages\n" + f"Processing: CodebaseResource {resource.path}\n" + ) + + for package_data in resource.package_data: + package_data = packagedcode_models.PackageData.from_dict(mapping=package_data) + + logger.info( + f"project: {project}:\n" + "function: assemble_packages\n" + f"Processing: PackageData {package_data.purl}\n" + ) + + handler = get_package_handler(package_data) + + logger.info( + f"project: {project}:\n" + "function: assemble_packages\n" + f"Selected: Package handler {handler}\n" + ) + + items = handler.assemble( + package_data=package_data, + resource=resource, + codebase=None, + ) + + for item in items: + logger.info( + f"project: {project}:\n" + "function: assemble_packages\n" + f"Processing: item {item}\n" + ) + if isinstance(item, packagedcode_models.Package): + package_data = item.to_dict() + pipes.update_or_create_package(project, package_data) + elif isinstance(item, packagedcode_models.Dependency): + dependency_data = item.to_dict() + pipes.update_or_create_dependencies(project, dependency_data) + elif isinstance(item, CodebaseResource): + seen_resource_paths.add(item.path) + else: + logger.info( + f"project: {project}:\n" + "function: assemble_packages\n" + f"Unknown Package assembly item type: {item!r}\n" + ) + def run_scancode(location, output_file, options, raise_on_error=False): """ From f60f6c27b289d7ecd8904c00d5b1c5e50257a639 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 21 Jul 2022 12:55:51 -0700 Subject: [PATCH 02/24] Add has_single_resource to Project #436 #447 * Increase field sizes in DiscoveredDependency Signed-off-by: Jono Yang --- ...pendency_extracted_requirement_and_more.py | 23 +++++++++++++++++++ scanpipe/models.py | 12 ++++++++-- scanpipe/pipes/scancode.py | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 scanpipe/migrations/0020_alter_discovereddependency_extracted_requirement_and_more.py diff --git a/scanpipe/migrations/0020_alter_discovereddependency_extracted_requirement_and_more.py b/scanpipe/migrations/0020_alter_discovereddependency_extracted_requirement_and_more.py new file mode 100644 index 000000000..ff639340a --- /dev/null +++ b/scanpipe/migrations/0020_alter_discovereddependency_extracted_requirement_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.0.6 on 2022-07-21 19:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0019_codebaseresource_package_data_discovereddependency'), + ] + + operations = [ + migrations.AlterField( + model_name='discovereddependency', + name='extracted_requirement', + field=models.CharField(help_text='The version requirements of this dependency.', max_length=64), + ), + migrations.AlterField( + model_name='discovereddependency', + name='scope', + field=models.CharField(help_text='The scope of this dependency, how it is used in a project.', max_length=64), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 81d7d2979..82096c252 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -853,6 +853,14 @@ def error_count(self): """ return self.projecterrors.count() + @property + def has_single_resource(self): + """ + Return True if we only have a single CodebaseResource associated to this + project, False otherwise. + """ + return self.codebaseresources.count() == 1 + class ProjectRelatedQuerySet(models.QuerySet): def project(self, project): @@ -1919,13 +1927,13 @@ class DiscoveredDependency( ), ) extracted_requirement = models.CharField( - max_length=32, + max_length=64, help_text=_( "The version requirements of this dependency." ), ) scope = models.CharField( - max_length=32, + max_length=64, help_text=_( "The scope of this dependency, how it is used in a project." ), diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 7ce039fe1..e8eff25dc 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -371,7 +371,7 @@ def assemble_packages(project): items = handler.assemble( package_data=package_data, resource=resource, - codebase=None, + codebase=project, ) for item in items: From 51ed7ce6139951da82cbdc5f15b6371724b8f8cc Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 21 Jul 2022 13:36:30 -0700 Subject: [PATCH 03/24] Add codebase argument to CodebaseResource.save #447 Signed-off-by: Jono Yang --- scanpipe/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 82096c252..2829e7091 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1503,11 +1503,14 @@ def from_db(cls, db, field_names, values): return new - def save(self, *args, **kwargs): + def save(self, codebase=None, *args, **kwargs): """ Saves the current resource instance. Injects policies—if the feature is enabled—when the `licenses` field value is changed. + + `codebase` is not used in this context but required for compatibility + with the commoncode.resource.Codebase class API. """ if scanpipe_app.policies_enabled: loaded_licenses = getattr(self, "loaded_licenses", []) From 9632137251521b0ea7500e1153a8141ce3b7238d Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 21 Jul 2022 17:35:46 -0700 Subject: [PATCH 04/24] Display DiscoveredDependencies in JSON output #437 * Create DiscoveredDependencySerializer Signed-off-by: Jono Yang --- scanpipe/api/serializers.py | 10 ++++++++++ scanpipe/pipes/output.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index 07917a709..26c095fdc 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -26,6 +26,7 @@ from scanpipe.api import ExcludeFromListViewMixin from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import ProjectError @@ -219,6 +220,15 @@ class Meta: ] +class DiscoveredDependencySerializer(serializers.ModelSerializer): + class Meta: + model = DiscoveredDependency + exclude = [ + "id", + "project", + ] + + class ProjectErrorSerializer(serializers.ModelSerializer): traceback = serializers.SerializerMethodField() diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 1dd04579b..fee4f771f 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -125,6 +125,7 @@ def __iter__(self): yield "{\n" yield from self.serialize(label="headers", generator=self.get_headers) yield from self.serialize(label="packages", generator=self.get_packages) + yield from self.serialize(label="dependencies", generator=self.get_dependencies) yield from self.serialize(label="files", generator=self.get_files, latest=True) yield "}" @@ -178,6 +179,16 @@ def get_packages(self, project): for obj in packages.iterator(): yield self.encode(DiscoveredPackageSerializer(obj).data) + def get_dependencies(self, project): + from scanpipe.api.serializers import DiscoveredDependencySerializer + + dependencies = project.discovereddependencys.all().order_by( + "purl", + ) + + for obj in dependencies.iterator(): + yield self.encode(DiscoveredDependencySerializer(obj).data) + def get_files(self, project): from scanpipe.api.serializers import CodebaseResourceSerializer From d6473907bf6160056181c242784e38cb960b4601 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 22 Jul 2022 15:36:41 -0700 Subject: [PATCH 05/24] Create initial Project dependency view #447 Signed-off-by: Jono Yang --- scanpipe/api/serializers.py | 12 +++ scanpipe/api/views.py | 11 +++ scanpipe/filters.py | 9 +++ scanpipe/models.py | 7 ++ .../templates/scanpipe/dependency_list.html | 76 +++++++++++++++++++ scanpipe/urls.py | 5 ++ scanpipe/views.py | 13 ++++ 7 files changed, 133 insertions(+) create mode 100644 scanpipe/templates/scanpipe/dependency_list.html diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index 26c095fdc..496bdca09 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -114,6 +114,7 @@ class ProjectSerializer( input_sources = serializers.JSONField(source="input_sources_list", read_only=True) codebase_resources_summary = serializers.SerializerMethodField() discovered_package_summary = serializers.SerializerMethodField() + discovered_dependency_summary = serializers.SerializerMethodField() class Meta: model = Project @@ -137,8 +138,10 @@ class Meta: "error_count", "resource_count", "package_count", + "dependency_count", "codebase_resources_summary", "discovered_package_summary", + "discovered_dependency_summary", ) exclude_from_list_view = [ @@ -148,8 +151,10 @@ class Meta: "error_count", "resource_count", "package_count", + "dependency_count", "codebase_resources_summary", "discovered_package_summary", + "discovered_dependency_summary", ] def get_codebase_resources_summary(self, project): @@ -164,6 +169,12 @@ def get_discovered_package_summary(self, project): "with_modified_resources": base_qs.exclude(modified_resources=[]).count(), } + def get_discovered_dependency_summary(self, project): + base_qs = project.discovereddependencys + return { + "total": base_qs.count(), + } + def create(self, validated_data): """ Creates a new `project` with `upload_file` and `pipeline` as optional. @@ -267,6 +278,7 @@ def get_model_serializer(model_class): serializer = { CodebaseResource: CodebaseResourceSerializer, DiscoveredPackage: DiscoveredPackageSerializer, + DiscoveredDependency: DiscoveredDependencySerializer, ProjectError: ProjectErrorSerializer, }.get(model_class, None) diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index 66bbc68c8..2a7156963 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -36,6 +36,7 @@ from rest_framework.response import Response from scanpipe.api.serializers import CodebaseResourceSerializer +from scanpipe.api.serializers import DiscoveredDependencySerializer from scanpipe.api.serializers import DiscoveredPackageSerializer from scanpipe.api.serializers import PipelineSerializer from scanpipe.api.serializers import ProjectErrorSerializer @@ -181,6 +182,16 @@ def packages(self, request, *args, **kwargs): return Response(serializer.data) + @action(detail=True) + def dependencies(self, request, *args, **kwargs): + project = self.get_object() + queryset = project.discovereddependencys.all() + + paginated_qs = self.paginate_queryset(queryset) + serializer = DiscoveredDependencySerializer(paginated_qs, many=True) + + return Response(serializer.data) + @action(detail=True) def errors(self, request, *args, **kwargs): project = self.get_object() diff --git a/scanpipe/filters.py b/scanpipe/filters.py index ba9a2b2a5..e7d36d211 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -29,6 +29,7 @@ from packageurl.contrib.django.filters import PackageURLFilter from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import ProjectError @@ -270,6 +271,14 @@ class Meta: ] +class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): + class Meta: + model = DiscoveredDependency + fields = [ + "purl", + ] + + class ErrorFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): search = django_filters.CharFilter(field_name="message", lookup_expr="icontains") diff --git a/scanpipe/models.py b/scanpipe/models.py index 2829e7091..6a12db790 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -846,6 +846,13 @@ def package_count(self): """ return self.discoveredpackages.count() + @cached_property + def dependency_count(self): + """ + Returns the number of dependencies related to this project. + """ + return self.discovereddependencys.count() + @cached_property def error_count(self): """ diff --git a/scanpipe/templates/scanpipe/dependency_list.html b/scanpipe/templates/scanpipe/dependency_list.html new file mode 100644 index 000000000..6a957dee0 --- /dev/null +++ b/scanpipe/templates/scanpipe/dependency_list.html @@ -0,0 +1,76 @@ +{% extends "scanpipe/base.html" %} + +{% block content %} +
+ {% include 'scanpipe/includes/navbar_header.html' %} +
+
+ {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True %} + {% include 'scanpipe/includes/search_field.html' with extra_class="is-small" %} +
+ {% include 'scanpipe/includes/pagination_header.html' with verbose_name="Packages" %} + {% include 'scanpipe/includes/filters_breadcrumb.html' with filterset=filter only %} +
+
+ +
+
+ + + + + + + + + + + + + + + + + {% for dependency in object_list %} + + + + + + + + + + + + + {% endfor %} + +
purlextracted_requirementscopeis_runtimeis_optionalis_resolveddependency_uidfor_package_uiddatafile_pathdatasource_id
+ {{ dependency.purl }} + + {{ dependency.extracted_requirement }} + + {{ dependency.scope }} + + {{ dependency.is_runtime }} + + {{ dependency.is_optional }} + + {{ dependency.is_resolved }} + + {{ dependency.dependency_uid }} + + {{ dependency.for_package_uid }} + + {{ dependency.datafile_path }} + + {{ dependency.datasource_id }} +
+
+ + {% if is_paginated %} + {% include 'scanpipe/includes/pagination.html' with page_obj=page_obj %} + {% endif %} +
+{% endblock %} \ No newline at end of file diff --git a/scanpipe/urls.py b/scanpipe/urls.py index 0fa962bb2..6445df409 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -46,6 +46,11 @@ views.DiscoveredPackageListView.as_view(), name="project_packages", ), + path( + "project//dependencies/", + views.DiscoveredDependencyListView.as_view(), + name="project_dependencies", + ), path( "project//errors/", views.ProjectErrorListView.as_view(), diff --git a/scanpipe/views.py b/scanpipe/views.py index d7c766be0..8bffcce81 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -44,6 +44,7 @@ from scancodeio.auth import ConditionalLoginRequired from scancodeio.auth import conditional_login_required from scanpipe.filters import ErrorFilterSet +from scanpipe.filters import DependencyFilterSet from scanpipe.filters import PackageFilterSet from scanpipe.filters import ProjectFilterSet from scanpipe.filters import ResourceFilterSet @@ -52,6 +53,7 @@ from scanpipe.forms import ArchiveProjectForm from scanpipe.forms import ProjectForm from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import ProjectError @@ -595,6 +597,17 @@ class DiscoveredPackageListView( prefetch_related = ["codebase_resources"] +class DiscoveredDependencyListView( + ConditionalLoginRequired, + ProjectRelatedViewMixin, + PaginatedFilterView, +): + model = DiscoveredDependency + filterset_class = DependencyFilterSet + template_name = "scanpipe/dependency_list.html" + paginate_by = 100 + + class ProjectErrorListView( ConditionalLoginRequired, ProjectRelatedViewMixin, FilterView ): From 8e618dee350f9df7b2062bc88c7271e27da9c709 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 22 Jul 2022 17:06:00 -0700 Subject: [PATCH 06/24] Update test expectations #447 Signed-off-by: Jono Yang --- .../data/alpine_3_15_4_scan_codebase.json | 250 ++++++---- ...asgiref-3.3.0_load_inventory_expected.json | 451 +++++++++++++++++- .../data/basic-rootfs_root_filesystems.json | 167 ++++--- scanpipe/tests/data/centos_scan_codebase.json | 76 ++- scanpipe/tests/data/debian_scan_codebase.json | 19 +- .../is-npm-1.0.0_scan_package_summary.json | 67 ++- ...ple-is-npm-1.0.0_scan_package_summary.json | 67 ++- scanpipe/tests/test_api.py | 11 +- scanpipe/tests/test_pipelines.py | 7 +- scanpipe/tests/test_pipes.py | 2 +- 10 files changed, 898 insertions(+), 219 deletions(-) diff --git a/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json b/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json index 4c241e89f..dfeaa496b 100644 --- a/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json +++ b/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json @@ -866,6 +866,7 @@ ] } ], + "dependencies": [], "files": [ { "for_packages": [ @@ -893,7 +894,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -919,7 +921,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -945,7 +948,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -973,7 +977,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1001,7 +1006,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1029,7 +1035,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1057,7 +1064,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1085,7 +1093,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -1111,7 +1120,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -1137,7 +1147,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1165,7 +1176,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1193,7 +1205,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1221,7 +1234,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1249,7 +1263,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1277,7 +1292,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1305,7 +1321,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -1331,7 +1348,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1359,7 +1377,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1387,7 +1406,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1415,7 +1435,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1443,7 +1464,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1471,7 +1493,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1499,7 +1522,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1527,7 +1551,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1555,7 +1580,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -1581,7 +1607,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1609,7 +1636,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1637,7 +1665,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1665,7 +1694,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1693,7 +1723,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1721,7 +1752,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1749,7 +1781,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1777,7 +1810,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1805,7 +1839,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1833,7 +1868,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1861,7 +1897,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1889,7 +1926,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1917,7 +1955,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1945,7 +1984,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -1973,7 +2013,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2001,7 +2042,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2029,7 +2071,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -2055,7 +2098,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -2081,7 +2125,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -2107,7 +2152,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -2133,7 +2179,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2161,7 +2208,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2189,7 +2237,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2217,7 +2266,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2245,7 +2295,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2273,7 +2324,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2301,7 +2353,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2329,7 +2382,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2357,7 +2411,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2385,7 +2440,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2413,7 +2469,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2441,7 +2498,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2469,7 +2527,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2497,7 +2556,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2525,7 +2585,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2553,7 +2614,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2581,7 +2643,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2609,7 +2672,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2637,7 +2701,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2665,7 +2730,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2693,7 +2759,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2721,7 +2788,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2749,7 +2817,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2777,7 +2846,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2805,7 +2875,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2833,7 +2904,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2861,7 +2933,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2889,7 +2962,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2917,7 +2991,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2945,7 +3020,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -2973,7 +3049,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -3001,7 +3078,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -3029,7 +3107,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -3057,7 +3136,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -3085,7 +3165,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -3113,7 +3194,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -3141,7 +3223,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -3169,7 +3252,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} } ] } \ No newline at end of file diff --git a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json index 2b2336fce..cc6780ced 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json +++ b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json @@ -121,6 +121,7 @@ "source_packages": [] } ], + "dependencies": [], "files": [ { "for_packages": [ @@ -148,7 +149,206 @@ "is_text": false, "is_archive": true, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [ + { + "md5": null, + "name": "asgiref", + "purl": "pkg:pypi/asgiref@3.3.0", + "sha1": null, + "type": "pypi", + "sha256": null, + "sha512": null, + "parties": [ + { + "url": null, + "name": "Django Software Foundation", + "role": "author", + "type": "person", + "email": "foundation@djangoproject.com" + } + ], + "subpath": null, + "vcs_url": null, + "version": "3.3.0", + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Topic :: Internet :: WWW/HTTP" + ], + "copyright": null, + "namespace": null, + "extra_data": { + "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt", + "Documentation": "Documentation, https://asgi.readthedocs.io/", + "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" + }, + "qualifiers": {}, + "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", + "notice_text": null, + "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", + "dependencies": [ + { + "purl": "pkg:pypi/pytest", + "scope": "tests", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "extracted_requirement": null + }, + { + "purl": "pkg:pypi/pytest-asyncio", + "scope": "tests", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "extracted_requirement": null + } + ], + "download_url": null, + "homepage_url": "https://github.com/django/asgiref/", + "release_date": null, + "code_view_url": null, + "datasource_id": "pypi_wheel", + "file_references": [ + { + "md5": null, + "path": "asgiref/__init__.py", + "sha1": null, + "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/compatibility.py", + "sha1": null, + "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/current_thread_executor.py", + "sha1": null, + "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/local.py", + "sha1": null, + "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/server.py", + "sha1": null, + "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/sync.py", + "sha1": null, + "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/testing.py", + "sha1": null, + "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/timeout.py", + "sha1": null, + "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/wsgi.py", + "sha1": null, + "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/LICENSE", + "sha1": null, + "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/METADATA", + "sha1": null, + "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/WHEEL", + "sha1": null, + "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/top_level.txt", + "sha1": null, + "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/RECORD", + "sha1": null, + "sha256": null, + "sha512": null, + "extra_data": {} + } + ], + "source_packages": [], + "bug_tracking_url": null, + "declared_license": { + "license": "BSD", + "classifiers": [ + "License :: OSI Approved :: BSD License" + ] + }, + "primary_language": "Python", + "license_expression": "bsd-new AND bsd-new", + "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", + "repository_homepage_url": "https://pypi.org/project/asgiref" + } + ] }, { "for_packages": [], @@ -174,7 +374,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -200,7 +401,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -226,7 +428,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -305,7 +508,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [ @@ -419,7 +623,206 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [ + { + "md5": null, + "name": "asgiref", + "purl": "pkg:pypi/asgiref@3.3.0", + "sha1": null, + "type": "pypi", + "sha256": null, + "sha512": null, + "parties": [ + { + "url": null, + "name": "Django Software Foundation", + "role": "author", + "type": "person", + "email": "foundation@djangoproject.com" + } + ], + "subpath": null, + "vcs_url": null, + "version": "3.3.0", + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Topic :: Internet :: WWW/HTTP" + ], + "copyright": null, + "namespace": null, + "extra_data": { + "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt", + "Documentation": "Documentation, https://asgi.readthedocs.io/", + "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" + }, + "qualifiers": {}, + "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", + "notice_text": null, + "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", + "dependencies": [ + { + "purl": "pkg:pypi/pytest", + "scope": "tests", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "extracted_requirement": null + }, + { + "purl": "pkg:pypi/pytest-asyncio", + "scope": "tests", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "extracted_requirement": null + } + ], + "download_url": null, + "homepage_url": "https://github.com/django/asgiref/", + "release_date": null, + "code_view_url": null, + "datasource_id": "pypi_wheel_metadata", + "file_references": [ + { + "md5": null, + "path": "asgiref/__init__.py", + "sha1": null, + "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/compatibility.py", + "sha1": null, + "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/current_thread_executor.py", + "sha1": null, + "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/local.py", + "sha1": null, + "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/server.py", + "sha1": null, + "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/sync.py", + "sha1": null, + "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/testing.py", + "sha1": null, + "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/timeout.py", + "sha1": null, + "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref/wsgi.py", + "sha1": null, + "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/LICENSE", + "sha1": null, + "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/METADATA", + "sha1": null, + "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/WHEEL", + "sha1": null, + "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/top_level.txt", + "sha1": null, + "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", + "sha512": null, + "extra_data": {} + }, + { + "md5": null, + "path": "asgiref-3.3.0.dist-info/RECORD", + "sha1": null, + "sha256": null, + "sha512": null, + "extra_data": {} + } + ], + "source_packages": [], + "bug_tracking_url": null, + "declared_license": { + "license": "BSD", + "classifiers": [ + "License :: OSI Approved :: BSD License" + ] + }, + "primary_language": "Python", + "license_expression": "bsd-new AND bsd-new", + "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", + "repository_homepage_url": "https://pypi.org/project/asgiref" + } + ] }, { "for_packages": [], @@ -445,7 +848,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -471,7 +875,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -497,7 +902,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -523,7 +929,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -549,7 +956,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -575,7 +983,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -601,7 +1010,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -627,7 +1037,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -653,7 +1064,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -679,7 +1091,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -746,7 +1159,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] }, { "for_packages": [], @@ -772,7 +1186,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [] } ] } \ No newline at end of file diff --git a/scanpipe/tests/data/basic-rootfs_root_filesystems.json b/scanpipe/tests/data/basic-rootfs_root_filesystems.json index 6bc24d065..d22e6fc28 100644 --- a/scanpipe/tests/data/basic-rootfs_root_filesystems.json +++ b/scanpipe/tests/data/basic-rootfs_root_filesystems.json @@ -48,38 +48,6 @@ } ], "packages": [ - { - "purl": "pkg:deb/libncurses5", - "type": "deb", - "namespace": "", - "name": "libncurses5", - "version": "", - "qualifiers": "", - "subpath": "", - "primary_language": "", - "description": "", - "release_date": null, - "homepage_url": "", - "download_url": "", - "sha1": "", - "md5": "", - "bug_tracking_url": "", - "code_view_url": "", - "vcs_url": "", - "copyright": "Copyright (c) 1998-2016 Free Software Foundation, Inc.\nCopyright (c) 2001 by Pradeep Padala\nCopyright (c) 1994 X Consortium\nCopyright (c) 1980, 1991, 1992, 1993 The Regents of the University of California\nCopyright 1996-2007 by Thomas E. Dickey", - "license_expression": "x11-fsf AND x11-xconsortium AND bsd-new AND x11-fsf", - "declared_license": "", - "notice_text": "", - "manifest_path": "", - "contains_source_code": null, - "extra_data": {}, - "missing_resources": [], - "modified_resources": [], - "dependencies": [], - "package_uid": "", - "keywords": [], - "source_packages": [] - }, { "purl": "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64", "type": "deb", @@ -120,38 +88,6 @@ "pkg:deb/ncurses" ] }, - { - "purl": "pkg:deb/libndp0", - "type": "deb", - "namespace": "", - "name": "libndp0", - "version": "", - "qualifiers": "", - "subpath": "", - "primary_language": "", - "description": "", - "release_date": null, - "homepage_url": "", - "download_url": "", - "sha1": "", - "md5": "", - "bug_tracking_url": "", - "code_view_url": "", - "vcs_url": "", - "copyright": "Copyright 2013 Jiri Pirko \nCopyright 2014 Andrew Ayer ", - "license_expression": "(lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1) AND (lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1)", - "declared_license": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']", - "notice_text": "", - "manifest_path": "", - "contains_source_code": null, - "extra_data": {}, - "missing_resources": [], - "modified_resources": [], - "dependencies": [], - "package_uid": "", - "keywords": [], - "source_packages": [] - }, { "purl": "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64", "type": "deb", @@ -193,6 +129,7 @@ ] } ], + "dependencies": [], "files": [ { "for_packages": [], @@ -218,12 +155,11 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { - "for_packages": [ - "pkg:deb/libncurses5?uuid=fixed-uid-done-for-testing-5642512d1758" - ], + "for_packages": [], "path": "basic-rootfs.tar.gz-extract/usr/share/doc/libncurses5/copyright", "sha1": "3ffa530d4b2e0ef318cf12f68cb7106a255ea27b", "md5": "bd73d1dbbd2e6374358baa205d6d9e66", @@ -246,12 +182,47 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [ + { + "md5": null, + "name": "libncurses5", + "purl": "pkg:deb/libncurses5", + "sha1": null, + "type": "deb", + "sha256": null, + "sha512": null, + "parties": [], + "subpath": null, + "vcs_url": null, + "version": null, + "keywords": [], + "copyright": "Copyright (c) 1998-2016 Free Software Foundation, Inc.\nCopyright (c) 2001 by Pradeep Padala\nCopyright (c) 1994 X Consortium\nCopyright (c) 1980, 1991, 1992, 1993 The Regents of the University of California\nCopyright 1996-2007 by Thomas E. Dickey", + "namespace": null, + "extra_data": {}, + "qualifiers": {}, + "description": null, + "notice_text": null, + "api_data_url": null, + "dependencies": [], + "download_url": null, + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "debian_copyright_in_package", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "declared_license": null, + "primary_language": null, + "license_expression": "x11-fsf AND x11-xconsortium AND bsd-new AND x11-fsf", + "repository_download_url": null, + "repository_homepage_url": null + } + ] }, { - "for_packages": [ - "pkg:deb/libndp0?uuid=fixed-uid-done-for-testing-5642512d1758" - ], + "for_packages": [], "path": "basic-rootfs.tar.gz-extract/usr/share/doc/libndp0/copyright", "sha1": "0d667249e3a0f605589e7607ae6cb29024ae59b5", "md5": "3f4ecdd67d5b9427cdc66847bdd11cf4", @@ -274,7 +245,48 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": [ + { + "md5": null, + "name": "libndp0", + "purl": "pkg:deb/libndp0", + "sha1": null, + "type": "deb", + "sha256": null, + "sha512": null, + "parties": [], + "subpath": null, + "vcs_url": null, + "version": null, + "keywords": [], + "copyright": "Copyright 2013 Jiri Pirko \nCopyright 2014 Andrew Ayer ", + "namespace": null, + "extra_data": {}, + "qualifiers": {}, + "description": null, + "notice_text": null, + "api_data_url": null, + "dependencies": [], + "download_url": null, + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "debian_copyright_in_package", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "declared_license": [ + "LGPL-2.1+", + "LGPL-2.1+", + "LGPL-2.1+" + ], + "primary_language": null, + "license_expression": "(lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1) AND (lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1)", + "repository_download_url": null, + "repository_homepage_url": null + } + ] }, { "for_packages": [], @@ -300,7 +312,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -326,7 +339,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -379,7 +393,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} } ] } \ No newline at end of file diff --git a/scanpipe/tests/data/centos_scan_codebase.json b/scanpipe/tests/data/centos_scan_codebase.json index f63c30add..691b58dcc 100644 --- a/scanpipe/tests/data/centos_scan_codebase.json +++ b/scanpipe/tests/data/centos_scan_codebase.json @@ -186542,6 +186542,7 @@ "source_packages": [] } ], + "dependencies": [], "files": [ { "for_packages": [ @@ -186569,7 +186570,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186597,7 +186599,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186625,7 +186628,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186653,7 +186657,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186681,7 +186686,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186709,7 +186715,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -186735,7 +186742,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186763,7 +186771,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -186789,7 +186798,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -186815,7 +186825,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186843,7 +186854,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186871,7 +186883,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186899,7 +186912,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186927,7 +186941,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186955,7 +186970,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -186983,7 +186999,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -187009,7 +187026,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -187037,7 +187055,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -187063,7 +187082,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -187091,7 +187111,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -187119,7 +187140,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -187145,7 +187167,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -187171,7 +187194,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -187197,7 +187221,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -187225,7 +187250,8 @@ "is_text": false, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} } ] } \ No newline at end of file diff --git a/scanpipe/tests/data/debian_scan_codebase.json b/scanpipe/tests/data/debian_scan_codebase.json index 1698e7250..6a0114b0f 100644 --- a/scanpipe/tests/data/debian_scan_codebase.json +++ b/scanpipe/tests/data/debian_scan_codebase.json @@ -227,6 +227,7 @@ ] } ], + "dependencies": [], "files": [ { "for_packages": [], @@ -252,7 +253,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -280,7 +282,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -308,7 +311,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -336,7 +340,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [ @@ -364,7 +369,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} }, { "for_packages": [], @@ -417,7 +423,8 @@ "is_text": true, "is_archive": false, "is_key_file": false, - "is_media": false + "is_media": false, + "package_data": {} } ] } \ No newline at end of file diff --git a/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json b/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json index 50df68acd..27720d434 100644 --- a/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json +++ b/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json @@ -108,12 +108,75 @@ "extension": ".json", "programming_language": "", "mime_type": "application/json", - "file_type": "JSON data", "is_binary": false, "is_text": true, "is_archive": false, "is_key_file": true, "is_media": false, + "package_data": [ + { + "md5": null, + "name": "is-npm", + "purl": "pkg:npm/is-npm@1.0.0", + "sha1": null, + "type": "npm", + "sha256": null, + "sha512": null, + "parties": [ + { + "url": "http://sindresorhus.com", + "name": "Sindre Sorhus", + "role": "author", + "type": "person", + "email": "sindresorhus@gmail.com" + } + ], + "subpath": null, + "vcs_url": "https://github.com/sindresorhus/is-npm", + "version": "1.0.0", + "keywords": [ + "npm", + "is", + "check", + "detect", + "env", + "environment" + ], + "copyright": null, + "namespace": null, + "extra_data": {}, + "qualifiers": {}, + "description": "Check if your code is running as an npm script", + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/is-npm/1.0.0", + "dependencies": [ + { + "purl": "pkg:npm/ava", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_resolved": false, + "resolved_package": {}, + "extracted_requirement": "0.0.3" + } + ], + "download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "npm_package_json", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "declared_license": [ + "MIT" + ], + "primary_language": "JavaScript", + "license_expression": "mit", + "repository_download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/is-npm" + } + ], "content": "{\n \"name\": \"is-npm\",\n \"version\": \"1.0.0\",\n \"description\": \"Check if your code is running as an npm script\",\n \"license\": \"MIT\",\n \"repository\": \"sindresorhus/is-npm\",\n \"author\": {\n \"name\": \"Sindre Sorhus\",\n \"email\": \"sindresorhus@gmail.com\",\n \"url\": \"http://sindresorhus.com\"\n },\n \"engines\": {\n \"node\": \">=0.10.0\"\n },\n \"scripts\": {\n \"test\": \"node test.js\"\n },\n \"files\": [\n \"index.js\"\n ],\n \"keywords\": [\n \"npm\",\n \"is\",\n \"check\",\n \"detect\",\n \"env\",\n \"environment\"\n ],\n \"devDependencies\": {\n \"ava\": \"0.0.3\"\n }\n}\n" }, { @@ -213,12 +276,12 @@ "extension": ".md", "programming_language": "", "mime_type": "text/plain", - "file_type": "UTF-8 Unicode text", "is_binary": false, "is_text": true, "is_archive": false, "is_key_file": true, "is_media": false, + "package_data": [], "content": "# is-npm [![Build Status](https://travis-ci.org/sindresorhus/is-npm.svg?branch=master)](https://travis-ci.org/sindresorhus/is-npm)\n\n> Check if your code is running as an [npm script](https://www.npmjs.org/doc/misc/npm-scripts.html)\n\n\n## Install\n\n```sh\n$ npm install --save is-npm\n```\n\n\n## Usage\n\n```js\nvar isNpm = require('is-npm');\nconsole.log(isNpm);\n```\n\n```sh\n$ node foo.js\n#=> false\n$ npm run foo\n#=> true\n```\n\n\n## License\n\nMIT \u00a9 [Sindre Sorhus](http://sindresorhus.com)\n" } ], diff --git a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json index 781e079e8..dce0883cf 100644 --- a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json +++ b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json @@ -116,12 +116,75 @@ "extension": ".json", "programming_language": "", "mime_type": "application/json", - "file_type": "JSON data", "is_binary": false, "is_text": true, "is_archive": false, "is_key_file": true, "is_media": false, + "package_data": [ + { + "md5": null, + "name": "is-npm", + "purl": "pkg:npm/is-npm@1.0.0", + "sha1": null, + "type": "npm", + "sha256": null, + "sha512": null, + "parties": [ + { + "url": "http://sindresorhus.com", + "name": "Sindre Sorhus", + "role": "author", + "type": "person", + "email": "sindresorhus@gmail.com" + } + ], + "subpath": null, + "vcs_url": "https://github.com/sindresorhus/is-npm", + "version": "1.0.0", + "keywords": [ + "npm", + "is", + "check", + "detect", + "env", + "environment" + ], + "copyright": null, + "namespace": null, + "extra_data": {}, + "qualifiers": {}, + "description": "Check if your code is running as an npm script", + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/is-npm/1.0.0", + "dependencies": [ + { + "purl": "pkg:npm/ava", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_resolved": false, + "resolved_package": {}, + "extracted_requirement": "0.0.3" + } + ], + "download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "npm_package_json", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "declared_license": [ + "MIT" + ], + "primary_language": "JavaScript", + "license_expression": "mit", + "repository_download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/is-npm" + } + ], "content": "{\n \"name\": \"is-npm\",\n \"version\": \"1.0.0\",\n \"description\": \"Check if your code is running as an npm script\",\n \"license\": \"MIT\",\n \"repository\": \"sindresorhus/is-npm\",\n \"author\": {\n \"name\": \"Sindre Sorhus\",\n \"email\": \"sindresorhus@gmail.com\",\n \"url\": \"http://sindresorhus.com\"\n },\n \"engines\": {\n \"node\": \">=0.10.0\"\n },\n \"scripts\": {\n \"test\": \"node test.js\"\n },\n \"files\": [\n \"index.js\"\n ],\n \"keywords\": [\n \"npm\",\n \"is\",\n \"check\",\n \"detect\",\n \"env\",\n \"environment\"\n ],\n \"devDependencies\": {\n \"ava\": \"0.0.3\"\n }\n}\n" }, { @@ -221,12 +284,12 @@ "extension": ".md", "programming_language": "", "mime_type": "text/plain", - "file_type": "UTF-8 Unicode text", "is_binary": false, "is_text": true, "is_archive": false, "is_key_file": true, "is_media": false, + "package_data": [], "content": "# is-npm [![Build Status](https://travis-ci.org/sindresorhus/is-npm.svg?branch=master)](https://travis-ci.org/sindresorhus/is-npm)\n\n> Check if your code is running as an [npm script](https://www.npmjs.org/doc/misc/npm-scripts.html)\n\n\n## Install\n\n```sh\n$ npm install --save is-npm\n```\n\n\n## Usage\n\n```js\nvar isNpm = require('is-npm');\nconsole.log(isNpm);\n```\n\n```sh\n$ node foo.js\n#=> false\n$ npm run foo\n#=> true\n```\n\n\n## License\n\nMIT \u00a9 [Sindre Sorhus](http://sindresorhus.com)\n" } ], diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 79cbb9f09..f89bd234c 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -42,6 +42,7 @@ from scanpipe.api.serializers import get_model_serializer from scanpipe.api.serializers import get_serializer_fields from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import ProjectError @@ -269,9 +270,10 @@ def test_scanpipe_api_project_results_generator(self): results_generator = JSONResultsGenerator(self.project1) results = json.loads("".join(results_generator)) - expected = ["files", "headers", "packages"] + expected = ["dependencies", "files", "headers", "packages"] self.assertEqual(expected, sorted(results.keys())) + self.assertEqual(0, len(results["dependencies"])) self.assertEqual(1, len(results["headers"])) self.assertEqual(1, len(results["files"])) self.assertEqual(1, len(results["packages"])) @@ -282,7 +284,7 @@ def test_scanpipe_api_project_action_results(self): response_value = response.getvalue() results = json.loads(response_value) - expected = ["files", "headers", "packages"] + expected = ["dependencies", "files", "headers", "packages"] self.assertEqual(expected, sorted(results.keys())) self.assertEqual(1, len(results["headers"])) @@ -299,7 +301,7 @@ def test_scanpipe_api_project_action_results_download(self): response_value = response.getvalue() results = json.loads(response_value) - expected = ["files", "headers", "packages"] + expected = ["dependencies", "files", "headers", "packages"] self.assertEqual(expected, sorted(results.keys())) def test_scanpipe_api_project_action_pipelines(self): @@ -628,7 +630,8 @@ def test_scanpipe_api_serializer_get_model_serializer(self): def test_scanpipe_api_serializer_get_serializer_fields(self): self.assertEqual(31, len(get_serializer_fields(DiscoveredPackage))) - self.assertEqual(27, len(get_serializer_fields(CodebaseResource))) + self.assertEqual(10, len(get_serializer_fields(DiscoveredDependency))) + self.assertEqual(28, len(get_serializer_fields(CodebaseResource))) with self.assertRaises(LookupError): get_serializer_fields(None) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index ac23d64da..9acda8c17 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -363,6 +363,7 @@ def test_scanpipe_scan_codebase_pipeline_integration_test(self): self.assertEqual(6, project1.codebaseresources.count()) self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(6, project1.discovereddependencys.count()) result_file = output.to_json(project1) expected_file = self.data_location / "is-npm-1.0.0_scan_codebase.json" @@ -451,7 +452,8 @@ def test_scanpipe_docker_pipeline_distroless_debian_integration_test(self): self.assertEqual(0, exitcode, msg=out) self.assertEqual(2302, project1.codebaseresources.count()) - self.assertEqual(8, project1.discoveredpackages.count()) + self.assertEqual(6, project1.discoveredpackages.count()) + self.assertEqual(0, project1.discovereddependencys.count()) result_file = output.to_json(project1) expected_file = self.data_location / "gcr_io_distroless_base_scan_codebase.json" @@ -471,7 +473,8 @@ def test_scanpipe_rootfs_pipeline_integration_test(self): self.assertEqual(0, exitcode, msg=out) self.assertEqual(6, project1.codebaseresources.count()) - self.assertEqual(4, project1.discoveredpackages.count()) + self.assertEqual(2, project1.discoveredpackages.count()) + self.assertEqual(0, project1.discovereddependencys.count()) result_file = output.to_json(project1) expected_file = self.data_location / "basic-rootfs_root_filesystems.json" diff --git a/scanpipe/tests/test_pipes.py b/scanpipe/tests/test_pipes.py index 8441f63da..92c131487 100644 --- a/scanpipe/tests/test_pipes.py +++ b/scanpipe/tests/test_pipes.py @@ -202,7 +202,7 @@ def test_scanpipe_pipes_outputs_to_json(self): with output_file.open() as f: results = json.loads(f.read()) - expected = ["files", "headers", "packages"] + expected = ["dependencies", "files", "headers", "packages"] self.assertEqual(expected, sorted(results.keys())) self.assertEqual(1, len(results["headers"])) From 3f03d966287fa96cf029fca3546ae08f60b5273e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 26 Jul 2022 15:27:02 -0700 Subject: [PATCH 07/24] Show dependency stats in project detail #447 Signed-off-by: Jono Yang --- .../templates/scanpipe/project_detail.html | 31 +++++++++++++++++++ scanpipe/views.py | 12 +++++++ 2 files changed, 43 insertions(+) diff --git a/scanpipe/templates/scanpipe/project_detail.html b/scanpipe/templates/scanpipe/project_detail.html index ad1d47682..807f1a27f 100644 --- a/scanpipe/templates/scanpipe/project_detail.html +++ b/scanpipe/templates/scanpipe/project_detail.html @@ -120,6 +120,28 @@

{% endif %} + {% if project.dependency_count %} +
+

+ Discovered Dependencies + {{ project.dependency_count|intcomma }} +

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {% endif %} + {% if project.resource_count %}

@@ -174,6 +196,11 @@

{{ package_licenses|json_script:"package_licenses" }} {{ package_types|json_script:"package_types" }} {% endif %} + {% if project.dependency_count %} + {{ dependency_is_runtime|json_script:"dependency_is_runtime" }} + {{ dependency_is_optional|json_script:"dependency_is_optional" }} + {{ dependency_is_resolved|json_script:"dependency_is_resolved" }} + {% endif %}