diff --git a/tests/conftest.py b/tests/conftest.py index b66e02672934..90ec528d9dbc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,8 @@ from warehouse.accounts import services as account_services from warehouse.accounts.interfaces import ITokenService, IUserService from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations import services as attestations_services +from warehouse.attestations.interfaces import IIntegrityService from warehouse.email import services as email_services from warehouse.email.interfaces import IEmailSender from warehouse.helpdesk import services as helpdesk_services @@ -174,6 +176,7 @@ def pyramid_services( project_service, github_oidc_service, activestate_oidc_service, + integrity_service, macaroon_service, helpdesk_service, ): @@ -195,6 +198,7 @@ def pyramid_services( services.register_service( activestate_oidc_service, IOIDCPublisherService, None, name="activestate" ) + services.register_service(integrity_service, IIntegrityService, None, name="") services.register_service(macaroon_service, IMacaroonService, None, name="") services.register_service(helpdesk_service, IHelpDeskService, None) @@ -326,6 +330,7 @@ def get_app_config(database, nondefaults=None): "docs.backend": "warehouse.packaging.services.LocalDocsStorage", "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage", "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService", + "integrity.backend": "warehouse.attestations.services.NullIntegrityService", "billing.api_base": "http://stripe:12111", "billing.api_version": "2020-08-27", "mail.backend": "warehouse.email.services.SMTPEmailSender", @@ -557,6 +562,11 @@ def dummy_attestation(): ) +@pytest.fixture +def integrity_service(db_session): + return attestations_services.NullIntegrityService(db_session) + + @pytest.fixture def macaroon_service(db_session): return macaroon_services.DatabaseMacaroonService(db_session) diff --git a/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation new file mode 100644 index 000000000000..76ecef058bf5 --- /dev/null +++ b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation @@ -0,0 +1,49 @@ +{ + "version": 1, + "verification_material": { + "certificate": "MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5MTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4OtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXaAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYDVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQBg78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcKHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/S7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpuGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJWxaC/uOFEpyiYQ==", + "transparency_entries": [ + { + "logIndex": "125970014", + "logId": { + "keyId": "wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0=" + }, + "kindVersion": { + "kind": "dsse", + "version": "0.0.1" + }, + "integratedTime": "1724951379", + "inclusionPromise": { + "signedEntryTimestamp": "MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U=" + }, + "inclusionProof": { + "logIndex": "4065752", + "rootHash": "7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=", + "treeSize": "4065754", + "hashes": [ + "NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=", + "kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=", + "FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=", + "mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=", + "5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=", + "mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=", + "MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=", + "g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=", + "RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=", + "QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=", + "Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=", + "ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8=" + ], + "checkpoint": { + "envelope": "rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n" + } + }, + "canonicalizedBody": "eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0=" + } + ] + }, + "envelope": { + "statement": "eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJuYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3ZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4ZjcwOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRpb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9", + "signature": "MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaDGZE0UWBn0Gp5ZF35/Sc=" + } +} diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 7d97f8dc284a..81fb1d015ef5 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -11,8 +11,24 @@ # limitations under the License. from http import HTTPStatus +from pathlib import Path -from ...common.db.packaging import FileFactory, ProjectFactory, ReleaseFactory +import pymacaroons + +from warehouse.macaroons import caveats + +from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.macaroons import MacaroonFactory +from ...common.db.oidc import GitHubPublisherFactory +from ...common.db.packaging import ( + FileFactory, + ProjectFactory, + ReleaseFactory, + RoleFactory, +) + +_HERE = Path(__file__).parent +_ASSETS = _HERE.parent / "_fixtures" def test_simple_api_html(webtest): @@ -31,6 +47,91 @@ def test_simple_api_detail(webtest): assert resp.content_type == "text/html" assert "X-PyPI-Last-Serial" in resp.headers - assert resp.html.h1.string == f"Links for {project.normalized_name}" - # There should be a link for every file - assert len(resp.html.find_all("a")) == 2 + assert f"Links for {project.normalized_name}" in resp.text + + +def test_simple_attestations_from_upload(webtest): + user = UserFactory.create( + password=( # 'password' + "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" + "HOJaqfBroT0JCieHug281c" + ) + ) + EmailFactory.create(user=user, verified=True) + project = ProjectFactory.create(name="sampleproject") + RoleFactory.create(user=user, project=project, role_name="Owner") + publisher = GitHubPublisherFactory.create(projects=[project]) + + # Construct the macaroon. This needs to be based on a Trusted Publisher, which is + # required to upload attestations + dm = MacaroonFactory.create( + oidc_publisher_id=publisher.id, + caveats=[ + caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)), + caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]), + ], + additional={"oidc": {"ref": "someref", "sha": "somesha"}}, + ) + + m = pymacaroons.Macaroon( + location="localhost", + identifier=str(dm.id), + key=dm.key, + version=pymacaroons.MACAROON_V2, + ) + for caveat in dm.caveats: + m.add_first_party_caveat(caveats.serialize(caveat)) + serialized_macaroon = f"pypi-{m.serialize()}" + + with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f: + content = f.read() + + with open( + _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation", + ) as f: + attestation = f.read() + + webtest.set_authorization(("Basic", ("__token__", serialized_macaroon))) + webtest.post( + "/legacy/?:action=file_upload", + params={ + "name": "sampleproject", + "sha256_digest": ( + "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d" + ), + "filetype": "sdist", + "metadata_version": "2.1", + "version": "3.0.0", + "attestations": f"[{attestation}]", + }, + upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)], + status=HTTPStatus.OK, + ) + + assert len(project.releases) == 1 + assert project.releases[0].files.count() == 1 + assert project.releases[0].files[0].provenance is not None + # While we needed to be authenticated to upload a project, this is no longer + # required to view it. + webtest.authorization = None + expected_provenance = project.releases[0].files[0].provenance.provenance_digest + expected_filename = "sampleproject-3.0.0.tar.gz" + + response = webtest.get("/simple/sampleproject/", status=HTTPStatus.OK) + link = response.html.find("a", text=expected_filename) + + assert "data-provenance" in link.attrs + assert link.get("data-provenance") == expected_provenance + + response = webtest.get( + "/simple/sampleproject/", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + status=HTTPStatus.OK, + ) + + assert response.content_type == "application/vnd.pypi.simple.v1+json" + + json_content = response.json + assert len(json_content["files"]) == 1 + assert json_content["files"][0]["filename"] == expected_filename + assert json_content["files"][0]["provenance"] == expected_provenance diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 518a3dea2b95..359f0dcfc604 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -17,6 +17,7 @@ from pyramid.httpexceptions import HTTPMovedPermanently from pyramid.testing import DummyRequest +from tests.common.db.oidc import GitHubPublisherFactory from warehouse.api import simple from warehouse.packaging.utils import API_VERSION, _valid_simple_detail_context @@ -298,6 +299,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override) "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -349,6 +351,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -445,6 +448,7 @@ def test_with_files_with_version_multi_digit( if f.metadata_file_sha256_digest is not None else False ), + "provenance": None, } for f in files ], @@ -460,6 +464,92 @@ def test_with_files_with_version_multi_digit( if renderer_override is not None: assert db_request.override_renderer == renderer_override + @pytest.mark.parametrize( + ("content_type", "renderer_override"), + CONTENT_TYPE_PARAMS, + ) + def test_with_files_varying_provenance( + self, + db_request, + integrity_service, + dummy_attestation, + content_type, + renderer_override, + ): + db_request.accept = content_type + db_request.oidc_publisher = GitHubPublisherFactory.create() + + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0.0") + + # wheel with provenance, sdist with no provenance + wheel = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.whl", + packagetype="bdist_wheel", + metadata_file_sha256_digest="deadbeefdeadbeefdeadbeefdeadbeef", + ) + + provenance = integrity_service.build_provenance( + db_request, wheel, [dummy_attestation] + ) + assert wheel.provenance == provenance + assert wheel.provenance.provenance_digest is not None + + sdist = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.tar.gz", + packagetype="sdist", + ) + + files = [sdist, wheel] + + urls_iter = (f"/file/{f.filename}" for f in files) + db_request.matchdict["name"] = project.normalized_name + db_request.route_url = lambda *a, **kw: next(urls_iter) + user = UserFactory.create() + je = JournalEntryFactory.create(name=project.name, submitted_by=user) + + context = { + "meta": {"_last-serial": je.id, "api-version": API_VERSION}, + "name": project.normalized_name, + "versions": ["1.0.0"], + "files": [ + { + "filename": f.filename, + "url": f"/file/{f.filename}", + "hashes": {"sha256": f.sha256_digest}, + "requires-python": f.requires_python, + "yanked": False, + "size": f.size, + "upload-time": f.upload_time.isoformat() + "Z", + "data-dist-info-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "core-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "provenance": ( + f.provenance.provenance_digest + if f.provenance is not None + else None + ), + } + for f in files + ], + "alternate-locations": [], + } + context = _update_context(context, content_type, renderer_override) + + assert simple.simple_detail(project, db_request) == context + + if renderer_override is not None: + assert db_request.override_renderer == renderer_override + @pytest.mark.parametrize( ("content_type", "renderer_override"), CONTENT_TYPE_PARAMS, diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 4264b503767d..6b9475f980ff 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -13,6 +13,7 @@ import base64 import hashlib import io +import json import re import tarfile import tempfile @@ -24,15 +25,8 @@ import pretend import pytest -from pypi_attestations import ( - Attestation, - Distribution, - Envelope, - VerificationError, - VerificationMaterial, -) +from pypi_attestations import Attestation, Envelope, VerificationMaterial from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPTooManyRequests -from sigstore.verify import Verifier from sqlalchemy import and_, exists from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload @@ -43,6 +37,7 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations.interfaces import IIntegrityService from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -2458,6 +2453,7 @@ def test_upload_attestation_fails_without_oidc_publisher( metrics, project_service, macaroon_service, + integrity_service, ): project = ProjectFactory.create() owner = UserFactory.create() @@ -2516,6 +2512,7 @@ def test_upload_attestation_fails_without_oidc_publisher( IMacaroonService: macaroon_service, IMetricsService: metrics, IProjectService: project_service, + IIntegrityService: integrity_service, }.get(svc) db_request.user_agent = "warehouse-tests/6.6.6" @@ -2526,8 +2523,7 @@ def test_upload_attestation_fails_without_oidc_publisher( assert resp.status_code == 400 assert resp.status == ( - "400 Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions." + "400 Attestations are only supported when using Trusted Publishing" ) @pytest.mark.parametrize( @@ -3439,12 +3435,13 @@ def test_upload_succeeds_creates_release( ), ] - def test_upload_with_valid_attestation_succeeds( + def test_upload_succeeds_with_valid_attestation( self, monkeypatch, pyramid_config, db_request, metrics, + integrity_service, ): from warehouse.events.models import HasEvents @@ -3494,9 +3491,9 @@ def test_upload_with_valid_attestation_succeeds( ), } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: integrity_service, IFileStorage: storage_service, IMetricsService: metrics, }.get(svc) @@ -3505,286 +3502,49 @@ def test_upload_with_valid_attestation_succeeds( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - resp = legacy.file_upload(db_request) assert resp.status_code == 200 - assert len(verify.calls) == 1 - verified_distribution = verify.calls[0].args[3] - assert verified_distribution == Distribution( - name=filename, digest=_TAR_GZ_PKG_SHA256 - ) - - def test_upload_with_invalid_attestation_predicate_type_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - invalid_predicate_type = "Unsupported predicate type" - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: (invalid_predicate_type, None) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - f"400 Attestation with unsupported predicate type: {invalid_predicate_type}" - ) - - def test_upload_with_multiple_attestations_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}," - f" {attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Only a single attestation per-file is supported at the moment." - ) - - def test_upload_with_malformed_attestation_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": "[{'a_malformed_attestation': 3}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None + assert ( + pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Error while decoding the included attestation:" - ) + # The file was created and has an associated provenance object. + file = db_request.db.query(File).filter(File.filename == filename).one() + assert file.provenance is not None @pytest.mark.parametrize( - ("verify_exception", "expected_msg"), + "invalid_attestations", [ - ( - VerificationError, - "400 Could not verify the uploaded artifact using the included " - "attestation", - ), - ( - ValueError, - "400 Unknown error while trying to verify included attestations", - ), + # Bad top-level types. + "", + {}, + 1, + # Empty attestation sets not permitted. + [], + # Wrong version number. + [ + { + "version": 2, + "verification_material": { + "certificate": "somebase64string", + "transparency_entries": [{}], + }, + "envelope": { + "statement": "somebase64string", + "signature": "somebase64string", + }, + }, + ], ], ) - def test_upload_with_failing_attestation_verification( + def test_upload_fails_attestation_error( self, monkeypatch, pyramid_config, db_request, - metrics, - verify_exception, - expected_msg, + invalid_attestations, ): from warehouse.events.models import HasEvents @@ -3804,16 +3564,6 @@ def test_upload_with_failing_attestation_verification( db_request.db.add(Classifier(classifier="Programming Language :: Python")) filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) pyramid_config.testing_securitypolicy(identity=identity) db_request.user = None @@ -3822,7 +3572,7 @@ def test_upload_with_failing_attestation_verification( { "metadata_version": "1.2", "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", + "attestations": json.dumps(invalid_attestations), "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3835,30 +3585,18 @@ def test_upload_with_failing_attestation_verification( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - def failing_verify(_self, _verifier, _policy, _dist): - raise verify_exception("error") - - monkeypatch.setattr(Attestation, "verify", failing_verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - with pytest.raises(HTTPBadRequest) as excinfo: legacy.file_upload(db_request) resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(expected_msg) + assert resp.status.startswith("400 Malformed attestations") @pytest.mark.parametrize( ("url", "expected"), diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index e256ca0300e1..0e25ce7c5a37 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,6 +15,8 @@ import pretend +from tests.common.db.oidc import GitHubPublisherFactory +from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import ( _simple_detail, @@ -36,6 +38,27 @@ def test_simple_detail_empty_string(db_request): assert expected_content["files"][0]["requires-python"] is None +def test_simple_detail_with_provenance( + db_request, integrity_service, dummy_attestation +): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + file = FileFactory.create(release=release) + + db_request.route_url = lambda *a, **kw: "the-url" + + # Provenance objects are generated at upload time, so we + # are mocking it here. + provenance = integrity_service.build_provenance( + pretend.stub(oidc_publisher=GitHubPublisherFactory.create()), + file, + [dummy_attestation], + ) + + expected_content = _simple_detail(project, db_request) + assert expected_content["files"][0]["provenance"] == provenance.provenance_digest + + def test_render_simple_detail(db_request, monkeypatch, jinja): project = ProjectFactory.create() release1 = ReleaseFactory.create(project=project, version="1.0") @@ -53,6 +76,7 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) db_request.route_url = lambda *a, **kw: "the-url" + template = jinja.get_template("templates/api/simple/detail.html") context = _simple_detail(project, db_request) context = _valid_simple_detail_context(context) @@ -71,7 +95,9 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): ) -def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): +def test_render_simple_detail_with_store( + db_request, monkeypatch, jinja, integrity_service +): project = ProjectFactory.create() storage_service = pretend.stub( @@ -82,6 +108,7 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, + IIntegrityService: integrity_service, }.get(svc) ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index ba312034fefb..0c174501b8a4 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -434,6 +434,7 @@ def __init__(self): pretend.call(".accounts"), pretend.call(".macaroons"), pretend.call(".oidc"), + pretend.call(".attestations"), pretend.call(".manage"), pretend.call(".organizations"), pretend.call(".subscriptions"), diff --git a/warehouse/config.py b/warehouse/config.py index 32065f72933c..6f4d43fa7991 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -524,7 +524,7 @@ def configure(settings=None): "warehouse.account.accounts_search_ratelimit_string", "ACCOUNTS_SEARCH_RATELIMIT_STRING", default="100 per hour", - ), + ) maybe_set( settings, "warehouse.account.password_reset_ratelimit_string", @@ -805,6 +805,9 @@ def configure(settings=None): # Register support for OIDC based authentication config.include(".oidc") + # Register support for attestations + config.include(".attestations") + # Register logged-in views config.include(".manage") diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 44521a6d20b0..4244684f5ca9 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -29,13 +29,7 @@ import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) +from pypi_attestations import Attestation, Distribution from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -47,11 +41,12 @@ ) from pyramid.request import Request from pyramid.view import view_config -from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue +from warehouse.attestations import IIntegrityService +from warehouse.attestations.errors import AttestationUploadError from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -361,88 +356,6 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, distribution: Distribution): - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported, and - attestations are discarded after verification. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.new_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - - def _sort_releases(request: Request, project: Project): releases = ( request.db.query(Release) @@ -1252,14 +1165,6 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } - if "attestations" in request.POST and not request.flags.enabled( - AdminFlagValue.DISABLE_PEP740 - ): - _process_attestations( - request=request, - distribution=Distribution(name=filename, digest=file_hashes["sha256"]), - ) - # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. @@ -1328,6 +1233,29 @@ def file_upload(request): ) ) + # If the user provided attestations, verify and store them + if "attestations" in request.POST and not request.flags.enabled( + AdminFlagValue.DISABLE_PEP740 + ): + integrity_service: IIntegrityService = request.find_service( + IIntegrityService, context=None + ) + + try: + attestations: list[Attestation] = integrity_service.parse_attestations( + request, + Distribution(name=filename, digest=file_hashes["sha256"]), + ) + integrity_service.build_provenance(request, file_, attestations) + except AttestationUploadError as e: + raise _exc_with_message( + HTTPBadRequest, + str(e), + ) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") + # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is diff --git a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py index 31a1b5636bf4..eb36da1dfdd9 100644 --- a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py +++ b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py @@ -17,7 +17,6 @@ Create Date: 2024-08-21 19:52:40.084048 """ - from alembic import op revision = "7ca0f1f5e7b3" diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 949635f6e7c9..19f1d06f409c 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -62,7 +62,6 @@ from warehouse import db from warehouse.accounts.models import User -from warehouse.attestations.models import Provenance from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.events.models import HasEvents @@ -82,6 +81,7 @@ from warehouse.utils.db.types import bool_false, datetime_now if typing.TYPE_CHECKING: + from warehouse.attestations.models import Provenance from warehouse.oidc.models import OIDCPublisher _MONOTONIC_SEQUENCE = 42 diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 282c45747f75..7ad9a8b295db 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -101,6 +101,9 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), + "provenance": ( + file.provenance.provenance_digest if file.provenance else None + ), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 28a8f49c815d..77a73dc63fb6 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -23,7 +23,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%}