Skip to content

Commit 5471922

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request #16325 from miminar/encode-dockerimagemetadata-for-schema1
Automatic merge from submit-queue. Correct missing sizes for manifest schema 1 images Instead of filling metadata in the image object in the registry, send the manifest and config blobs to the master and let it do the job. Resolves #16306 Resolves: [bz#1491589](https://bugzilla.redhat.com/show_bug.cgi?id=1491589)
2 parents afd90cd + 5ac2762 commit 5471922

13 files changed

+820
-306
lines changed

‎pkg/dockerregistry/server/manifesthandler.go

+6-8
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,12 @@ import (
1414

1515
// A ManifestHandler defines a common set of operations on all versions of manifest schema.
1616
type ManifestHandler interface {
17-
// FillImageMetadata fills a given image with metadata parsed from manifest. It also corrects layer sizes
18-
// with blob sizes. Newer Docker client versions don't set layer sizes in the manifest schema 1 at all.
19-
// Origin master needs correct layer sizes for proper image quota support. That's why we need to fill the
20-
// metadata in the registry.
21-
FillImageMetadata(ctx context.Context, image *imageapiv1.Image) error
17+
// Config returns a blob with image configuration associated with the manifest. This applies only to
18+
// manifet schema 2.
19+
Config(ctx context.Context) ([]byte, error)
20+
21+
// Digest returns manifest's digest.
22+
Digest() (manifestDigest digest.Digest, err error)
2223

2324
// Manifest returns a deserialized manifest object.
2425
Manifest() distribution.Manifest
@@ -29,9 +30,6 @@ type ManifestHandler interface {
2930

3031
// Verify returns an error if the contained manifest is not valid or has missing dependencies.
3132
Verify(ctx context.Context, skipDependencyVerification bool) error
32-
33-
// Digest returns manifest's digest
34-
Digest() (manifestDigest digest.Digest, err error)
3533
}
3634

3735
// NewManifestHandler creates a manifest handler for the given manifest.

‎pkg/dockerregistry/server/manifestschema1handler.go

+5-59
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,6 @@ import (
1111
"github.com/docker/distribution/manifest/schema1"
1212
"github.com/docker/distribution/reference"
1313
"github.com/docker/libtrust"
14-
15-
"k8s.io/apimachinery/pkg/util/sets"
16-
17-
imageapi "github.com/openshift/origin/pkg/image/apis/image"
18-
imageapiv1 "github.com/openshift/origin/pkg/image/apis/image/v1"
1914
)
2015

2116
func unmarshalManifestSchema1(content []byte, signatures [][]byte) (distribution.Manifest, error) {
@@ -52,57 +47,12 @@ type manifestSchema1Handler struct {
5247

5348
var _ ManifestHandler = &manifestSchema1Handler{}
5449

55-
func (h *manifestSchema1Handler) FillImageMetadata(ctx context.Context, image *imageapiv1.Image) error {
56-
signatures, err := h.manifest.Signatures()
57-
if err != nil {
58-
return err
59-
}
60-
61-
for _, signDigest := range signatures {
62-
image.DockerImageSignatures = append(image.DockerImageSignatures, signDigest)
63-
}
64-
65-
refs := h.manifest.References()
66-
67-
if err := imageMetadataFromManifest(image); err != nil {
68-
return fmt.Errorf("unable to fill image %s metadata: %v", image.Name, err)
69-
}
70-
71-
blobSet := sets.NewString()
72-
meta, ok := image.DockerImageMetadata.Object.(*imageapi.DockerImage)
73-
if !ok {
74-
return fmt.Errorf("image %q does not have metadata", image.Name)
75-
}
76-
meta.Size = int64(0)
77-
78-
blobs := h.repo.Blobs(ctx)
79-
for i := range image.DockerImageLayers {
80-
layer := &image.DockerImageLayers[i]
81-
// DockerImageLayers represents h.manifest.Manifest.FSLayers in reversed order
82-
desc, err := blobs.Stat(ctx, refs[len(image.DockerImageLayers)-i-1].Digest)
83-
if err != nil {
84-
context.GetLogger(ctx).Errorf("failed to stat blob %s of image %s", layer.Name, image.DockerImageReference)
85-
return err
86-
}
87-
// The MediaType appeared in manifest schema v2. We need to fill it
88-
// manually in the old images if it is not already filled.
89-
if len(layer.MediaType) == 0 {
90-
if len(desc.MediaType) > 0 {
91-
layer.MediaType = desc.MediaType
92-
} else {
93-
layer.MediaType = schema1.MediaTypeManifestLayer
94-
}
95-
}
96-
layer.LayerSize = desc.Size
97-
// count empty layer just once (empty layer may actually have non-zero size)
98-
if !blobSet.Has(layer.Name) {
99-
meta.Size += desc.Size
100-
blobSet.Insert(layer.Name)
101-
}
102-
}
103-
image.DockerImageMetadata.Object = meta
50+
func (h *manifestSchema1Handler) Config(ctx context.Context) ([]byte, error) {
51+
return nil, nil
52+
}
10453

105-
return nil
54+
func (h *manifestSchema1Handler) Digest() (digest.Digest, error) {
55+
return digest.FromBytes(h.manifest.Canonical), nil
10656
}
10757

10858
func (h *manifestSchema1Handler) Manifest() distribution.Manifest {
@@ -183,7 +133,3 @@ func (h *manifestSchema1Handler) Verify(ctx context.Context, skipDependencyVerif
183133
}
184134
return nil
185135
}
186-
187-
func (h *manifestSchema1Handler) Digest() (digest.Digest, error) {
188-
return digest.FromBytes(h.manifest.Canonical), nil
189-
}

‎pkg/dockerregistry/server/manifestschema1handler_test.go

+375
Large diffs are not rendered by default.

‎pkg/dockerregistry/server/manifestschema2handler.go

+28-22
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@ package server
33
import (
44
"encoding/json"
55
"errors"
6+
"fmt"
7+
"reflect"
68

79
"github.com/docker/distribution"
810
"github.com/docker/distribution/context"
911
"github.com/docker/distribution/digest"
1012
"github.com/docker/distribution/manifest/schema2"
11-
12-
imageapiv1 "github.com/openshift/origin/pkg/image/apis/image/v1"
1313
)
1414

1515
var (
@@ -23,28 +23,42 @@ func unmarshalManifestSchema2(content []byte) (distribution.Manifest, error) {
2323
return nil, err
2424
}
2525

26+
if !reflect.DeepEqual(deserializedManifest.Versioned, schema2.SchemaVersion) {
27+
return nil, fmt.Errorf("unexpected manifest schema version=%d, mediaType=%q",
28+
deserializedManifest.SchemaVersion,
29+
deserializedManifest.MediaType)
30+
}
31+
2632
return &deserializedManifest, nil
2733
}
2834

2935
type manifestSchema2Handler struct {
30-
repo *repository
31-
manifest *schema2.DeserializedManifest
36+
repo *repository
37+
manifest *schema2.DeserializedManifest
38+
cachedConfig []byte
3239
}
3340

3441
var _ ManifestHandler = &manifestSchema2Handler{}
3542

36-
func (h *manifestSchema2Handler) FillImageMetadata(ctx context.Context, image *imageapiv1.Image) error {
37-
// The manifest.Config references a configuration object for a container by its digest.
38-
// It needs to be fetched in order to fill an image object metadata below.
39-
configBytes, err := h.repo.Blobs(ctx).Get(ctx, h.manifest.Config.Digest)
40-
if err != nil {
41-
context.GetLogger(ctx).Errorf("failed to get image config %s: %v", h.manifest.Config.Digest.String(), err)
42-
return err
43+
func (h *manifestSchema2Handler) Config(ctx context.Context) ([]byte, error) {
44+
if h.cachedConfig == nil {
45+
blob, err := h.repo.Blobs(ctx).Get(ctx, h.manifest.Config.Digest)
46+
if err != nil {
47+
context.GetLogger(ctx).Errorf("failed to get manifest config: %v", err)
48+
return nil, err
49+
}
50+
h.cachedConfig = blob
4351
}
44-
image.DockerImageConfig = string(configBytes)
4552

46-
// We need to populate the image metadata using the manifest.
47-
return imageMetadataFromManifest(image)
53+
return h.cachedConfig, nil
54+
}
55+
56+
func (h *manifestSchema2Handler) Digest() (digest.Digest, error) {
57+
_, p, err := h.manifest.Payload()
58+
if err != nil {
59+
return "", err
60+
}
61+
return digest.FromBytes(p), nil
4862
}
4963

5064
func (h *manifestSchema2Handler) Manifest() distribution.Manifest {
@@ -112,11 +126,3 @@ func (h *manifestSchema2Handler) Verify(ctx context.Context, skipDependencyVerif
112126
}
113127
return nil
114128
}
115-
116-
func (h *manifestSchema2Handler) Digest() (digest.Digest, error) {
117-
_, p, err := h.manifest.Payload()
118-
if err != nil {
119-
return "", err
120-
}
121-
return digest.FromBytes(p), nil
122-
}

‎pkg/dockerregistry/server/manifestschema2handler_test.go

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package server
2+
3+
import (
4+
"reflect"
5+
"strings"
6+
"testing"
7+
8+
"k8s.io/apimachinery/pkg/util/diff"
9+
10+
"github.com/docker/distribution"
11+
"github.com/docker/distribution/digest"
12+
"github.com/docker/distribution/manifest/schema2"
13+
)
14+
15+
func TestUnmarshalManifestSchema2(t *testing.T) {
16+
for _, tc := range []struct {
17+
name string
18+
manifestString string
19+
expectedConfig distribution.Descriptor
20+
expectedReferences []distribution.Descriptor
21+
expectedErrorSubstring string
22+
}{
23+
{
24+
name: "valid nginx image with sizes in manifest",
25+
manifestString: manifestSchema2,
26+
expectedConfig: manifestSchema2ConfigDescriptor,
27+
expectedReferences: []distribution.Descriptor{
28+
manifestSchema2ConfigDescriptor,
29+
manifestSchema2LayerDescriptors[0],
30+
manifestSchema2LayerDescriptors[1],
31+
manifestSchema2LayerDescriptors[2],
32+
},
33+
},
34+
35+
{
36+
name: "invalid schema2 image",
37+
manifestString: manifestSchema2Invalid,
38+
expectedErrorSubstring: "invalid character",
39+
},
40+
41+
{
42+
name: "manifest schema1 image",
43+
manifestString: manifestSchema1,
44+
expectedErrorSubstring: "unexpected manifest schema version",
45+
},
46+
} {
47+
48+
t.Run(tc.name, func(t *testing.T) {
49+
manifest, err := unmarshalManifestSchema2([]byte(tc.manifestString))
50+
if err != nil {
51+
if len(tc.expectedErrorSubstring) == 0 {
52+
t.Fatalf("got unexpected error: (%T) %v", err, err)
53+
}
54+
if !strings.Contains(err.Error(), tc.expectedErrorSubstring) {
55+
t.Fatalf("expected error with string %q, instead got: %v", tc.expectedErrorSubstring, err)
56+
}
57+
return
58+
}
59+
if err == nil && len(tc.expectedErrorSubstring) > 0 {
60+
t.Fatalf("got non-error while expecting: %s", tc.expectedErrorSubstring)
61+
}
62+
63+
dm, ok := manifest.(*schema2.DeserializedManifest)
64+
if !ok {
65+
t.Fatalf("got unexpected manifest schema: %T", manifest)
66+
}
67+
68+
if !reflect.DeepEqual(dm.Config, tc.expectedConfig) {
69+
t.Errorf("got unexpected image config descriptor: %s", diff.ObjectGoPrintDiff(dm.Config, tc.expectedConfig))
70+
}
71+
72+
refs := dm.References()
73+
if !reflect.DeepEqual(refs, tc.expectedReferences) {
74+
t.Errorf("got unexpected image references: %s", diff.ObjectGoPrintDiff(refs, tc.expectedReferences))
75+
}
76+
})
77+
}
78+
}
79+
80+
var manifestSchema2LayerDescriptors = []distribution.Descriptor{
81+
{
82+
MediaType: schema2.MediaTypeLayer,
83+
Digest: digest.Digest("sha256:afeb2bfd31c0760573e7262de6ae67a84da0e0a1c3e8157bbddd41a501b18a5c"),
84+
Size: 22488057,
85+
},
86+
{
87+
MediaType: schema2.MediaTypeLayer,
88+
Digest: digest.Digest("sha256:7ff5d10493db2cdfc1b7238434c503cc0664d48d0f7154ea9472e734b28a72dd"),
89+
Size: 21869700,
90+
},
91+
{
92+
MediaType: schema2.MediaTypeLayer,
93+
Digest: digest.Digest("sha256:d2562f1ae1d0593a26c54006ad0a6211c35fdc8b4067485d7208000d83477de2"),
94+
Size: 201,
95+
},
96+
}
97+
98+
const manifestSchema2ConfigDigest = `sha256:da5939581ac835614e3cf6c765e7489e6d0fc602a44e98c07013f1c938f49675`
99+
100+
var manifestSchema2ConfigDescriptor = distribution.Descriptor{
101+
Digest: digest.Digest(manifestSchema2ConfigDigest),
102+
Size: 5838,
103+
MediaType: schema2.MediaTypeConfig,
104+
}
105+
106+
const manifestSchema2 = `{
107+
"schemaVersion": 2,
108+
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
109+
"config": {
110+
"mediaType": "application/vnd.docker.container.image.v1+json",
111+
"size": 5838,
112+
"digest": "sha256:da5939581ac835614e3cf6c765e7489e6d0fc602a44e98c07013f1c938f49675"
113+
},
114+
"layers": [
115+
{
116+
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
117+
"size": 22488057,
118+
"digest": "sha256:afeb2bfd31c0760573e7262de6ae67a84da0e0a1c3e8157bbddd41a501b18a5c"
119+
},
120+
{
121+
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
122+
"size": 21869700,
123+
"digest": "sha256:7ff5d10493db2cdfc1b7238434c503cc0664d48d0f7154ea9472e734b28a72dd"
124+
},
125+
{
126+
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
127+
"size": 201,
128+
"digest": "sha256:d2562f1ae1d0593a26c54006ad0a6211c35fdc8b4067485d7208000d83477de2"
129+
}
130+
]
131+
}`
132+
133+
const manifestSchema2Invalid = `{
134+
"schemaVersion": 2,
135+
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
136+
"config": {},
137+
[]
138+
}`

0 commit comments

Comments
 (0)