Skip to content

Yield Packages before other yieldables #3028 #3031

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 28, 2022
Merged

Yield Packages before other yieldables #3028 #3031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/packagedcode/alpine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def assemble(cls, package_data, resource, codebase):
for ref in package.file_references
}

resources = []
for res in root_resource.walk(codebase):
ref = file_references_by_path.get(res.path)
if not ref:
Expand All @@ -102,15 +103,15 @@ def assemble(cls, package_data, resource, codebase):
del file_references_by_path[res.path]
res.for_packages.append(package_uid)
res.save(codebase)

yield res
resources.append(res)

# if we have left over file references, add these to extra data
if file_references_by_path:
missing = sorted(file_references_by_path.values(), key=lambda r:r.path)
package.extra_data['missing_file_references'] = missing

yield package
yield from resources


class AlpineApkbuildHandler(models.DatafileHandler):
Expand Down
2 changes: 1 addition & 1 deletion src/packagedcode/cocoapods.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,6 @@ def assemble(cls, package_data, resource, codebase):

for resource in sibling_podspecs:
datafile_path = resource.path
yield resource
for package_data in resource.package_data:
package_data = models.PackageData.from_dict(package_data)
package = models.Package.from_package_data(
Expand All @@ -173,6 +172,7 @@ def assemble(cls, package_data, resource, codebase):
)
cls.assign_package_to_resources(package, resource, codebase)
yield package
yield resource

else:
# has_no_podspec:
Expand Down
36 changes: 20 additions & 16 deletions src/packagedcode/debian.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ def assemble(cls, package_data, resource, codebase):

f'usr/share/doc/{package_name}/copyright',
]))
resources = []
# TODO: keep track of missing files
for res in root_resource.walk(codebase):
if not res.path.endswith(assemblable_paths):
Expand All @@ -305,8 +306,7 @@ def assemble(cls, package_data, resource, codebase):
package_uid=package_uid,
)

# we yield this as we do not want this further processed
yield res
resources.append(res)

root_path = Path(root_resource.path)

Expand All @@ -332,14 +332,15 @@ def assemble(cls, package_data, resource, codebase):
res.for_packages.append(package_uid)
res.save(codebase)

yield res
resources.append(res)

# if we have left over file references, add these to extra data
if file_references_by_path:
missing = sorted(file_references_by_path.values(), key=lambda r:r.path)
package.extra_data['missing_file_references'] = missing

yield package
yield from resources


class DebianDistrolessInstalledDatabaseHandler(models.DatafileHandler):
Expand Down Expand Up @@ -383,38 +384,41 @@ def assemble(cls, package_data, resource, codebase):
)
package_uid = package.package_uid

dependent_packages = package_data.dependencies
if dependent_packages:
yield from models.Dependency.from_dependent_packages(
dependent_packages=dependent_packages,
datafile_path=resource.path,
datasource_id=package_data.datasource_id,
package_uid=package_uid,
)

# collect copyright file for this package
# and merge in package data
assemblable_paths = (
f'usr/share/doc/{package_name}/copyright',
)
resources = []
if package_uid:
for res in root_resource.walk(codebase):
if not res.path.endswith(assemblable_paths):
continue

for pkgdt in res.package_data:
package.update(
package_data=pkgdt,
datafile_path=res.path,
)

res.for_packages.append(package_uid)
res.save(codebase)
yield res

resources.append(res)

yield package

dependent_packages = package_data.dependencies
if dependent_packages:
yield from models.Dependency.from_dependent_packages(
dependent_packages=dependent_packages,
datafile_path=resource.path,
datasource_id=package_data.datasource_id,
package_uid=package_uid,
)

yield from resources


class DebianInstalledFilelistHandler(models.DatafileHandler):
# seen in installed rootfs in:
Expand Down
25 changes: 17 additions & 8 deletions src/packagedcode/pypi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,16 @@ def assign_package_to_resources(cls, package, resource, codebase):
return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)


def create_package_from_package_data(package_data, datafile_path):
package = models.Package.from_package_data(
package_data=package_data,
datafile_path=datafile_path,
)
if not package.license_expression:
package.license_expression = compute_normalized_license(package.declared_license)
return package


class BaseExtractedPythonLayout(BasePypiHandler):
"""
Base class for development repos, sdist tarballs and other related extracted
Expand Down Expand Up @@ -152,10 +162,12 @@ def assemble(cls, package_data, resource, codebase):
pkg_data = package_resource.package_data[0]
pkg_data = models.PackageData.from_dict(pkg_data)
if pkg_data.purl:
package = models.Package.from_package_data(
package = create_package_from_package_data(
package_data=pkg_data,
datafile_path=package_resource.path,
datafile_path=package_resource.path
)
yield package

package_resource.for_packages.append(package.package_uid)
package_resource.save(codebase)
yield package_resource
Expand Down Expand Up @@ -183,10 +195,11 @@ def assemble(cls, package_data, resource, codebase):
for setup_resource, setup_pkg_data in setup_package_data:
if setup_pkg_data.purl:
if not package:
package = models.Package.from_package_data(
package = create_package_from_package_data(
package_data=setup_pkg_data,
datafile_path=setup_resource.path,
datafile_path=setup_resource.path
)
yield package
package_resource = setup_resource
else:
package.update(setup_pkg_data, setup_resource.path)
Expand All @@ -203,8 +216,6 @@ def assemble(cls, package_data, resource, codebase):
)

if package:
if not package.license_expression:
package.license_expression = compute_normalized_license(package.declared_license)
package_uid = package.package_uid

root = package_resource.parent(codebase)
Expand All @@ -221,8 +232,6 @@ def assemble(cls, package_data, resource, codebase):
package_resource.for_packages.append(package_uid)
package_resource.save(codebase)

yield package

else:
package_uid = None

Expand Down
30 changes: 17 additions & 13 deletions src/packagedcode/rpm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,20 +188,8 @@ def assemble(cls, package_data, resource, codebase):
_declared, detected = detect_declared_license(package.declared_license)
package.license_expression = detected

# yield deps
dependent_packages = package_data.dependencies
if dependent_packages:
for dep in models.Dependency.from_dependent_packages(
dependent_packages=dependent_packages,
datafile_path=resource.path,
datasource_id=package_data.datasource_id,
package_uid=package_uid,
):
if not dep.namespace:
dep.namespace = namespace
yield dep

# tag files from refs
resources = []
missing_file_references = []
# a file ref extends from the root of the filesystem
for ref in package.file_references:
Expand All @@ -215,6 +203,7 @@ def assemble(cls, package_data, resource, codebase):
# found all of them
res.for_packages.append(package_uid)
res.save(codebase)
resources.append(res)

# if we have left over file references, add these to extra data
if missing_file_references:
Expand All @@ -223,6 +212,21 @@ def assemble(cls, package_data, resource, codebase):

yield package

# yield deps
dependent_packages = package_data.dependencies
if dependent_packages:
for dep in models.Dependency.from_dependent_packages(
dependent_packages=dependent_packages,
datafile_path=resource.path,
datasource_id=package_data.datasource_id,
package_uid=package_uid,
):
if not dep.namespace:
dep.namespace = namespace
yield dep

yield from resources


# TODO: add dependencies!!!
class RpmInstalledNdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):
Expand Down