sigstore: stream input into signing

Closes #158.

Signed-off-by: William Woodruff <[email protected]>

Author: woodruffw

sigstore/_cli.py

@@ -447,10 +447,11 @@ def _sign(args: argparse.Namespace) -> None:
 
     for file, outputs in output_map.items():
         logger.debug(f"signing for {file.name}")
-        result = signer.sign(
-            input_=file.read_bytes(),
-            identity_token=args.identity_token,
-        )
+        with file.open(mode="rb", buffering=0) as io:
+            result = signer.sign(
+                input_=io,
+                identity_token=args.identity_token,
+            )
 
         print("Using ephemeral certificate:")
         print(result.cert_pem)

sigstore/_sign.py

@@ -15,19 +15,21 @@
 from __future__ import annotations
 
 import base64
-import hashlib
 import logging
+from typing import IO
 
 import cryptography.x509 as x509
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
 from cryptography.x509.oid import NameOID
 from pydantic import BaseModel
 
 from sigstore._internal.fulcio import FulcioClient
 from sigstore._internal.oidc import Identity
 from sigstore._internal.rekor import RekorClient, RekorEntry
 from sigstore._internal.sct import verify_sct
+from sigstore._utils import sha256_streaming
 
 logger = logging.getLogger(__name__)
 
@@ -56,11 +58,11 @@ def staging(cls) -> Signer:
 
     def sign(
         self,
-        input_: bytes,
+        input_: IO[bytes],
         identity_token: str,
     ) -> SigningResult:
         """Public API for signing blobs"""
-        sha256_artifact_hash = hashlib.sha256(input_).hexdigest()
+        input_digest = sha256_streaming(input_)
 
         logger.debug("Generating ephemeral keys...")
         private_key = ec.generate_private_key(ec.SECP384R1())
@@ -102,7 +104,9 @@ def sign(
         logger.debug("Successfully verified SCT...")
 
         # Sign artifact
-        artifact_signature = private_key.sign(input_, ec.ECDSA(hashes.SHA256()))
+        artifact_signature = private_key.sign(
+            input_digest, ec.ECDSA(Prehashed(hashes.SHA256()))
+        )
         b64_artifact_signature = base64.b64encode(artifact_signature).decode()
 
         # Prepare inputs
@@ -113,7 +117,7 @@ def sign(
         # Create the transparency log entry
         entry = self._rekor.log.entries.post(
             b64_artifact_signature=b64_artifact_signature,
-            sha256_artifact_hash=sha256_artifact_hash,
+            sha256_artifact_hash=input_digest.hex(),
             b64_cert=b64_cert.decode(),
         )
 

sigstore/_utils.py

@@ -20,7 +20,7 @@
 
 import base64
 import hashlib
-from typing import Union
+from typing import IO, Union
 
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import ec, rsa
@@ -104,3 +104,24 @@ def split_certificate_chain(chain_pem: str) -> list[bytes]:
     certificate_chain = [(pem_header + c).encode() for c in certificate_chain]
 
     return certificate_chain
+
+
+def sha256_streaming(io: IO[bytes]) -> bytes:
+    """
+    Compute the SHA256 of a stream.
+
+    This function does its own internal buffering, so an unbuffered stream
+    should be supplied for optimal performance.
+    """
+
+    sha256 = hashlib.sha256()
+    # Per coreutils' ioblksize.h: 128KB performs optimally across a range
+    # of systems in terms of minimizing syscall overhead.
+    view = memoryview(bytearray(128 * 1024))
+
+    nbytes = io.readinto(view)
+    while nbytes:
+        sha256.update(view[:nbytes])
+        nbytes = io.readinto(view)
+
+    return sha256.digest()