Extend private.ReusedBlob to allow zstd:chunked reuses

- Add a CompressionAnnotations field
- Allow turning a known-zstd blob into a zstd:chunked one if we
  know the right annotations

This just adds the fields, nothing sets them yet, should not change behavior.

Signed-off-by: Miloslav Trmač <[email protected]>

Author: mtrmac

copy/single.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"reflect"
 	"slices"
 	"strings"
@@ -888,21 +889,33 @@ func updatedBlobInfoFromReuse(inputInfo types.BlobInfo, reusedBlob private.Reuse
 	// Handling of compression, encryption, and the related MIME types and the like are all the responsibility
 	// of the generic code in this package.
 	res := types.BlobInfo{
-		Digest:               reusedBlob.Digest,
-		Size:                 reusedBlob.Size,
-		URLs:                 nil,                   // This _must_ be cleared if Digest changes; clear it in other cases as well, to preserve previous behavior.
-		Annotations:          inputInfo.Annotations, // FIXME: This should remove zstd:chunked annotations (but those annotations being left with incorrect values should not break pulls)
-		MediaType:            inputInfo.MediaType,   // Mostly irrelevant, MediaType is updated based on Compression*/CryptoOperation.
+		Digest: reusedBlob.Digest,
+		Size:   reusedBlob.Size,
+		URLs:   nil, // This _must_ be cleared if Digest changes; clear it in other cases as well, to preserve previous behavior.
+		// FIXME: This should remove zstd:chunked annotations IF the original was chunked and the new one isn’t
+		// (but those annotations being left with incorrect values should not break pulls).
+		Annotations:          maps.Clone(inputInfo.Annotations),
+		MediaType:            inputInfo.MediaType, // Mostly irrelevant, MediaType is updated based on Compression*/CryptoOperation.
 		CompressionOperation: reusedBlob.CompressionOperation,
 		CompressionAlgorithm: reusedBlob.CompressionAlgorithm,
 		CryptoOperation:      inputInfo.CryptoOperation, // Expected to be unset anyway.
 	}
 	// The transport is only expected to fill CompressionOperation and CompressionAlgorithm
-	// if the blob was substituted; otherwise, fill it in based
+	// if the blob was substituted; otherwise, it is optional, and if not set, fill it in based
 	// on what we know from the srcInfos we were given.
 	if reusedBlob.Digest == inputInfo.Digest {
-		res.CompressionOperation = inputInfo.CompressionOperation
-		res.CompressionAlgorithm = inputInfo.CompressionAlgorithm
+		if res.CompressionOperation == types.PreserveOriginal {
+			res.CompressionOperation = inputInfo.CompressionOperation
+		}
+		if res.CompressionAlgorithm == nil {
+			res.CompressionAlgorithm = inputInfo.CompressionAlgorithm
+		}
+	}
+	if len(reusedBlob.CompressionAnnotations) != 0 {
+		if res.Annotations == nil {
+			res.Annotations = map[string]string{}
+		}
+		maps.Copy(res.Annotations, reusedBlob.CompressionAnnotations)
 	}
 	return res
 }

copy/single_test.go

@@ -55,22 +55,42 @@ func TestUpdatedBlobInfoFromReuse(t *testing.T) {
 		},
 		{ // Reuse with substitution
 			reused: private.ReusedBlob{
-				Digest:               "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-				Size:                 513543640,
-				CompressionOperation: types.Decompress,
-				CompressionAlgorithm: nil,
+				Digest:                 "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				Size:                   513543640,
+				CompressionOperation:   types.Decompress,
+				CompressionAlgorithm:   nil,
+				CompressionAnnotations: map[string]string{"decompressed": "value"},
 			},
 			expected: types.BlobInfo{
 				Digest:               "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 				Size:                 513543640,
 				URLs:                 nil,
-				Annotations:          map[string]string{"test-annotation-2": "two"},
+				Annotations:          map[string]string{"test-annotation-2": "two", "decompressed": "value"},
 				MediaType:            imgspecv1.MediaTypeImageLayerGzip,
 				CompressionOperation: types.Decompress,
 				CompressionAlgorithm: nil,
 				// CryptoOperation is set to the zero value
 			},
 		},
+		{ // Reuse turning zstd into zstd:chunked
+			reused: private.ReusedBlob{
+				Digest:                 "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb",
+				Size:                   51354364,
+				CompressionOperation:   types.Compress,
+				CompressionAlgorithm:   &compression.ZstdChunked,
+				CompressionAnnotations: map[string]string{"zstd-toc": "value"},
+			},
+			expected: types.BlobInfo{
+				Digest:               "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb",
+				Size:                 51354364,
+				URLs:                 nil,
+				Annotations:          map[string]string{"test-annotation-2": "two", "zstd-toc": "value"},
+				MediaType:            imgspecv1.MediaTypeImageLayerGzip,
+				CompressionOperation: types.Compress,
+				CompressionAlgorithm: &compression.ZstdChunked,
+				// CryptoOperation is set to the zero value
+			},
+		},
 	} {
 		res := updatedBlobInfoFromReuse(srcInfo, c.reused)
 		assert.Equal(t, c.expected, res, fmt.Sprintf("%#v", c.reused))

internal/imagedestination/wrapper.go

@@ -76,6 +76,9 @@ func (w *wrapped) TryReusingBlobWithOptions(ctx context.Context, info types.Blob
 		Size:                 blob.Size,
 		CompressionOperation: blob.CompressionOperation,
 		CompressionAlgorithm: blob.CompressionAlgorithm,
+		// CompressionAnnotations could be set to blob.Annotations, but that may contain unrelated
+		// annotations, and we didn’t use the blob.Annotations field previously, so we’ll
+		// continue not using it.
 	}, nil
 }
 

internal/private/private.go

@@ -134,9 +134,14 @@ type ReusedBlob struct {
 	Size   int64         // Must be provided
 	// The following compression fields should be set when the reuse substitutes
 	// a differently-compressed blob.
+	// They may be set also to change from a base variant to a specific variant of an algorithm.
 	CompressionOperation types.LayerCompression // Compress/Decompress, matching the reused blob; PreserveOriginal if N/A
 	CompressionAlgorithm *compression.Algorithm // Algorithm if compressed, nil if decompressed or N/A
 
+	// Annotations that should be added, for CompressionAlgorithm. Note that they might need to be
+	// added even if the digest doesn’t change (if we found the annotations in a cache).
+	CompressionAnnotations map[string]string
+
 	MatchedByTOCDigest bool // Whether the layer was reused/matched by TOC digest. Used only for UI purposes.
 }