Provenance retrieval route

requirements/main.in

@@ -61,7 +61,6 @@ requests
 requests-aws4auth
 redis>=2.8.0,<6.0.0
 rfc3986
-rfc8785
 sentry-sdk
 setuptools
 sigstore~=3.3.0

requirements/main.txt

@@ -1993,9 +1993,7 @@ rfc3986==2.0.0 \
 rfc8785==0.1.3 \
     --hash=sha256:167efe3b5cdd09dded9d0cfc8fec1f48f5cd9f8f13b580ada4efcac138925048 \
     --hash=sha256:6116062831c62e7ac5d027973a1fe07b601ccd854bca4a2b401938a00a20b0c0
-    # via
-    #   -r requirements/main.in
-    #   sigstore
+    # via sigstore
 rich==13.8.1 \
     --hash=sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06 \
     --hash=sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a

tests/conftest.py

@@ -45,6 +45,8 @@
 from warehouse.accounts import services as account_services
 from warehouse.accounts.interfaces import ITokenService, IUserService
 from warehouse.admin.flags import AdminFlag, AdminFlagValue
+from warehouse.attestations import services as attestations_services
+from warehouse.attestations.interfaces import IIntegrityService
 from warehouse.email import services as email_services
 from warehouse.email.interfaces import IEmailSender
 from warehouse.helpdesk import services as helpdesk_services
@@ -174,6 +176,7 @@ def pyramid_services(
     project_service,
     github_oidc_service,
     activestate_oidc_service,
+    integrity_service,
     macaroon_service,
     helpdesk_service,
 ):
@@ -195,6 +198,7 @@ def pyramid_services(
     services.register_service(
         activestate_oidc_service, IOIDCPublisherService, None, name="activestate"
     )
+    services.register_service(integrity_service, IIntegrityService, None)
     services.register_service(macaroon_service, IMacaroonService, None, name="")
     services.register_service(helpdesk_service, IHelpDeskService, None)
 
@@ -326,6 +330,7 @@ def get_app_config(database, nondefaults=None):
         "docs.backend": "warehouse.packaging.services.LocalDocsStorage",
         "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
         "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService",
+        "integrity.backend": "warehouse.attestations.services.NullIntegrityService",
         "billing.api_base": "http://stripe:12111",
         "billing.api_version": "2020-08-27",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
@@ -557,6 +562,11 @@ def dummy_attestation():
     )
 
 
+@pytest.fixture
+def integrity_service(db_session):
+    return attestations_services.NullIntegrityService(db_session)
+
+
 @pytest.fixture
 def macaroon_service(db_session):
     return macaroon_services.DatabaseMacaroonService(db_session)

tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation

@@ -0,0 +1,49 @@
+{
+  "version": 1,
+  "verification_material": {
+    "certificate": "MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5MTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4OtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXaAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYDVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQBg78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcKHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/S7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpuGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJWxaC/uOFEpyiYQ==",
+    "transparency_entries": [
+      {
+        "logIndex": "125970014",
+        "logId": {
+          "keyId": "wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="
+        },
+        "kindVersion": {
+          "kind": "dsse",
+          "version": "0.0.1"
+        },
+        "integratedTime": "1724951379",
+        "inclusionPromise": {
+          "signedEntryTimestamp": "MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U="
+        },
+        "inclusionProof": {
+          "logIndex": "4065752",
+          "rootHash": "7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=",
+          "treeSize": "4065754",
+          "hashes": [
+            "NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=",
+            "kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=",
+            "FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=",
+            "mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=",
+            "5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=",
+            "mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=",
+            "MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=",
+            "g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=",
+            "RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=",
+            "QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=",
+            "Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=",
+            "ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8="
+          ],
+          "checkpoint": {
+            "envelope": "rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n"
+          }
+        },
+        "canonicalizedBody": "eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0="
+      }
+    ]
+  },
+  "envelope": {
+    "statement": "eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJuYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3ZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4ZjcwOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRpb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9",
+    "signature": "MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaDGZE0UWBn0Gp5ZF35/Sc="
+  }
+}

tests/functional/forklift/test_legacy.py

@@ -13,17 +13,24 @@
 import base64
 
 from http import HTTPStatus
+from pathlib import Path
 
 import pymacaroons
 import pytest
 
 from webob.multidict import MultiDict
 
+from tests.common.db.oidc import GitHubPublisherFactory
+from tests.common.db.packaging import ProjectFactory, RoleFactory
 from warehouse.macaroons import caveats
 
-from ...common.db.accounts import UserFactory
+from ...common.db.accounts import EmailFactory, UserFactory
 from ...common.db.macaroons import MacaroonFactory
 
+_HERE = Path(__file__).parent
+_ASSETS = _HERE.parent / "_fixtures"
+assert _ASSETS.is_dir()
+
 
 def test_incorrect_post_redirect(webtest):
     """
@@ -270,3 +277,77 @@ def test_invalid_classifier_upload_error(webtest):
         status=HTTPStatus.BAD_REQUEST,
     )
     assert "'This :: Is :: Invalid' is not a valid classifier" in resp.body.decode()
+
+
+def test_provenance_available_after_upload(webtest):
+    user = UserFactory.create(
+        password=(  # 'password'
+            "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
+            "HOJaqfBroT0JCieHug281c"
+        )
+    )
+    EmailFactory.create(user=user, verified=True)
+    project = ProjectFactory.create(name="sampleproject")
+    RoleFactory.create(user=user, project=project, role_name="Owner")
+    publisher = GitHubPublisherFactory.create(projects=[project])
+
+    # Construct the macaroon. This needs to be based on a Trusted Publisher, which is
+    # required to upload attestations
+    dm = MacaroonFactory.create(
+        oidc_publisher_id=publisher.id,
+        caveats=[
+            caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)),
+            caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]),
+        ],
+        additional={"oidc": {"ref": "someref", "sha": "somesha"}},
+    )
+
+    m = pymacaroons.Macaroon(
+        location="localhost",
+        identifier=str(dm.id),
+        key=dm.key,
+        version=pymacaroons.MACAROON_V2,
+    )
+    for caveat in dm.caveats:
+        m.add_first_party_caveat(caveats.serialize(caveat))
+    serialized_macaroon = f"pypi-{m.serialize()}"
+
+    with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f:
+        content = f.read()
+
+    with open(
+        _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation",
+    ) as f:
+        attestation = f.read()
+
+    webtest.set_authorization(("Basic", ("__token__", serialized_macaroon)))
+    webtest.post(
+        "/legacy/?:action=file_upload",
+        params={
+            "name": "sampleproject",
+            "sha256_digest": (
+                "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d"
+            ),
+            "filetype": "sdist",
+            "metadata_version": "2.1",
+            "version": "3.0.0",
+            "attestations": f"[{attestation}]",
+        },
+        upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)],
+        status=HTTPStatus.OK,
+    )
+
+    assert len(project.releases) == 1
+    assert project.releases[0].files.count() == 1
+    assert project.releases[0].files[0].provenance is not None
+
+    # While we needed to be authenticated to upload a project, this is no longer
+    # required to view it.
+    webtest.authorization = None
+    expected_filename = "sampleproject-3.0.0.tar.gz"
+
+    response = webtest.get(
+        f"/integrity/{project.name}/3.0.0/{expected_filename}/provenance",
+        status=HTTPStatus.OK,
+    )
+    assert response.json == project.releases[0].files[0].provenance.provenance

tests/unit/api/test_integrity.py

@@ -0,0 +1,54 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+import pytest
+
+from warehouse.api import integrity
+
+
+def test_select_content_type(db_request):
+    db_request.accept = "application/json"
+
+    assert (
+        integrity._select_content_type(db_request)
+        == integrity.MIME_PYPI_INTEGRITY_V1_JSON
+    )
+
+
+# Backstop; can be removed/changed once this view supports HTML.
+@pytest.mark.parametrize(
+    "content_type",
+    [integrity.MIME_TEXT_HTML, integrity.MIME_PYPI_INTEGRITY_V1_HTML],
+)
+def test_provenance_for_file_bad_accept(db_request, content_type):
+    db_request.accept = content_type
+    response = integrity.provenance_for_file(pretend.stub(), db_request)
+    assert response.status_code == 406
+    assert response.json == {"message": "Request not acceptable"}
+
+
+def test_provenance_for_file_not_enabled(db_request, monkeypatch):
+    monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: True))
+
+    response = integrity.provenance_for_file(pretend.stub(), db_request)
+    assert response.status_code == 403
+    assert response.json == {"message": "Attestations temporarily disabled"}
+
+
+def test_provenance_for_file_not_present(db_request, monkeypatch):
+    monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: False))
+    file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz")
+
+    response = integrity.provenance_for_file(file, db_request)
+    assert response.status_code == 404
+    assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"}

tests/unit/attestations/test_services.py

@@ -9,12 +9,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import hashlib
 import json
 
 import pretend
 import pytest
-import rfc8785
 
 from pydantic import TypeAdapter
 from pypi_attestations import (
@@ -52,10 +50,6 @@ def test_build_provenance(self, db_request, dummy_attestation):
 
         provenance = service.build_provenance(db_request, file, [dummy_attestation])
         assert isinstance(provenance, DatabaseProvenance)
-        assert (
-            provenance.provenance_digest
-            == hashlib.sha256(rfc8785.dumps(provenance.provenance)).hexdigest()
-        )
         assert provenance.file == file
         assert file.provenance == provenance