Skip to content

Commit 3f30b36

Browse files
smarterclaytonsmarterclayton
committed
Unify manifest extraction in image commands
1 parent fdf8968 commit 3f30b36

File tree

3 files changed

+195
-161
lines changed

3 files changed

+195
-161
lines changed

‎pkg/oc/cli/image/append/append.go

+10-126
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ import (
99
"io/ioutil"
1010
"net/http"
1111
"os"
12-
"regexp"
13-
"runtime"
1412
"strconv"
1513
"time"
1614

@@ -20,8 +18,6 @@ import (
2018

2119
"github.com/docker/distribution"
2220
distributioncontext "github.com/docker/distribution/context"
23-
"github.com/docker/distribution/manifest/manifestlist"
24-
"github.com/docker/distribution/manifest/schema1"
2521
"github.com/docker/distribution/manifest/schema2"
2622
"github.com/docker/distribution/reference"
2723
"github.com/docker/distribution/registry/client"
@@ -88,10 +84,7 @@ type AppendImageOptions struct {
8884
DropHistory bool
8985
CreatedAt string
9086

91-
OSFilter *regexp.Regexp
92-
DefaultOSFilter bool
93-
94-
FilterByOS string
87+
FilterOptions imagemanifest.FilterOptions
9588

9689
MaxPerRegistry int
9790

@@ -102,12 +95,6 @@ type AppendImageOptions struct {
10295
genericclioptions.IOStreams
10396
}
10497

105-
// schema2ManifestOnly specifically requests a manifest list first
106-
var schema2ManifestOnly = distribution.WithManifestMediaTypes([]string{
107-
manifestlist.MediaTypeManifestList,
108-
schema2.MediaTypeManifest,
109-
})
110-
11198
func NewAppendImageOptions(streams genericclioptions.IOStreams) *AppendImageOptions {
11299
return &AppendImageOptions{
113100
IOStreams: streams,
@@ -131,9 +118,10 @@ func NewCmdAppendImage(name string, streams genericclioptions.IOStreams) *cobra.
131118
}
132119

133120
flag := cmd.Flags()
121+
o.FilterOptions.Bind(flag)
122+
134123
flag.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Print the actions that would be taken and exit without writing to the destination.")
135124
flag.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP")
136-
flag.StringVar(&o.FilterByOS, "filter-by-os", o.FilterByOS, "A regular expression to control which images are mirrored. Images will be passed as '<platform>/<architecture>[/<variant>]'.")
137125

138126
flag.StringVar(&o.From, "from", o.From, "The image to use as a base. If empty, a new scratch image is created.")
139127
flag.StringVar(&o.To, "to", o.To, "The Docker repository tag to upload the appended image to.")
@@ -150,17 +138,8 @@ func NewCmdAppendImage(name string, streams genericclioptions.IOStreams) *cobra.
150138
}
151139

152140
func (o *AppendImageOptions) Complete(cmd *cobra.Command, args []string) error {
153-
pattern := o.FilterByOS
154-
if len(pattern) == 0 && !cmd.Flags().Changed("filter-by-os") {
155-
o.DefaultOSFilter = true
156-
pattern = regexp.QuoteMeta(fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH))
157-
}
158-
if len(pattern) > 0 {
159-
re, err := regexp.Compile(pattern)
160-
if err != nil {
161-
return fmt.Errorf("--filter-by-os was not a valid regular expression: %v", err)
162-
}
163-
o.OSFilter = re
141+
if err := o.FilterOptions.Complete(cmd.Flags()); err != nil {
142+
return err
164143
}
165144

166145
for _, arg := range args {
@@ -177,20 +156,6 @@ func (o *AppendImageOptions) Complete(cmd *cobra.Command, args []string) error {
177156
return nil
178157
}
179158

180-
// includeDescriptor returns true if the provided manifest should be included.
181-
func (o *AppendImageOptions) includeDescriptor(d *manifestlist.ManifestDescriptor, hasMultiple bool) bool {
182-
if o.OSFilter == nil {
183-
return true
184-
}
185-
if o.DefaultOSFilter && !hasMultiple {
186-
return true
187-
}
188-
if len(d.Platform.Variant) > 0 {
189-
return o.OSFilter.MatchString(fmt.Sprintf("%s/%s/%s", d.Platform.OS, d.Platform.Architecture, d.Platform.Variant))
190-
}
191-
return o.OSFilter.MatchString(fmt.Sprintf("%s/%s", d.Platform.OS, d.Platform.Architecture))
192-
}
193-
194159
func (o *AppendImageOptions) Run() error {
195160
var createdAt *time.Time
196161
if len(o.CreatedAt) > 0 {
@@ -258,97 +223,16 @@ func (o *AppendImageOptions) Run() error {
258223
return err
259224
}
260225
fromRepo = repo
261-
var srcDigest digest.Digest
262-
if len(from.Tag) > 0 {
263-
desc, err := repo.Tags(ctx).Get(ctx, from.Tag)
264-
if err != nil {
265-
return err
266-
}
267-
srcDigest = desc.Digest
268-
} else {
269-
srcDigest = digest.Digest(from.ID)
270-
}
271-
manifests, err := repo.Manifests(ctx)
272-
if err != nil {
273-
return err
274-
}
275-
srcManifest, err := manifests.Get(ctx, srcDigest, schema2ManifestOnly)
276-
if err != nil {
277-
return err
278-
}
279226

280-
originalSrcDigest := srcDigest
281-
srcManifests, srcManifest, srcDigest, err := imagemanifest.ProcessManifestList(ctx, srcDigest, srcManifest, manifests, *from, o.includeDescriptor)
227+
srcManifest, _, location, err := imagemanifest.FirstManifest(ctx, *from, repo, o.FilterOptions.Include)
282228
if err != nil {
283-
return err
229+
return fmt.Errorf("unable to read image %s: %v", from, err)
284230
}
285-
if len(srcManifests) == 0 {
286-
return fmt.Errorf("filtered all images from %s", from)
287-
}
288-
289-
var location string
290-
if srcDigest == originalSrcDigest {
291-
location = fmt.Sprintf("manifest %s", srcDigest)
292-
} else {
293-
location = fmt.Sprintf("manifest %s in manifest list %s", srcDigest, originalSrcDigest)
231+
base, layers, err = imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), location)
232+
if err != nil {
233+
return fmt.Errorf("unable to parse image %s: %v", from, err)
294234
}
295235

296-
switch t := srcManifest.(type) {
297-
case *schema2.DeserializedManifest:
298-
if t.Config.MediaType != schema2.MediaTypeImageConfig {
299-
return fmt.Errorf("unable to append layers to images with config %s from %s", t.Config.MediaType, location)
300-
}
301-
configJSON, err := repo.Blobs(ctx).Get(ctx, t.Config.Digest)
302-
if err != nil {
303-
return fmt.Errorf("unable to find manifest for image %s: %v", *from, err)
304-
}
305-
glog.V(4).Infof("Raw image config json:\n%s", string(configJSON))
306-
config := &docker10.DockerImageConfig{}
307-
if err := json.Unmarshal(configJSON, &config); err != nil {
308-
return fmt.Errorf("the source image manifest could not be parsed: %v", err)
309-
}
310-
311-
base = config
312-
layers = t.Layers
313-
base.Size = 0
314-
for _, layer := range t.Layers {
315-
base.Size += layer.Size
316-
}
317-
318-
case *schema1.SignedManifest:
319-
if glog.V(4) {
320-
_, configJSON, _ := srcManifest.Payload()
321-
glog.Infof("Raw image config json:\n%s", string(configJSON))
322-
}
323-
if len(t.History) == 0 {
324-
return fmt.Errorf("input image is in an unknown format: no v1Compatibility history")
325-
}
326-
config := &docker10.DockerV1CompatibilityImage{}
327-
if err := json.Unmarshal([]byte(t.History[0].V1Compatibility), &config); err != nil {
328-
return err
329-
}
330-
331-
base = &docker10.DockerImageConfig{}
332-
if err := docker10.Convert_DockerV1CompatibilityImage_to_DockerImageConfig(config, base); err != nil {
333-
return err
334-
}
335-
336-
// schema1 layers are in reverse order
337-
layers = make([]distribution.Descriptor, 0, len(t.FSLayers))
338-
for i := len(t.FSLayers) - 1; i >= 0; i-- {
339-
layer := distribution.Descriptor{
340-
MediaType: schema2.MediaTypeLayer,
341-
Digest: t.FSLayers[i].BlobSum,
342-
// size must be reconstructed from the blobs
343-
}
344-
// we must reconstruct the tar sum from the blobs
345-
add.AddLayerToConfig(base, layer, "")
346-
layers = append(layers, layer)
347-
}
348-
349-
default:
350-
return fmt.Errorf("unable to append layers to images of type %T from %s", srcManifest, location)
351-
}
352236
} else {
353237
base = add.NewEmptyConfig()
354238
layers = []distribution.Descriptor{add.AddScratchLayerToConfig(base)}

‎pkg/oc/cli/image/manifest/manifest.go

+174-1
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@ package manifest
22

33
import (
44
"context"
5+
"encoding/json"
56
"fmt"
7+
"regexp"
8+
"runtime"
69
"sync"
710

11+
"github.com/spf13/pflag"
12+
813
"github.com/docker/distribution"
914
"github.com/docker/distribution/manifest/manifestlist"
1015
"github.com/docker/distribution/manifest/schema1"
@@ -17,10 +22,178 @@ import (
1722
"github.com/golang/glog"
1823
digest "github.com/opencontainers/go-digest"
1924

25+
"github.com/openshift/origin/pkg/image/apis/image/docker10"
2026
imagereference "github.com/openshift/origin/pkg/image/apis/image/reference"
27+
"github.com/openshift/origin/pkg/image/dockerlayer/add"
2128
)
2229

23-
func ProcessManifestList(ctx context.Context, srcDigest digest.Digest, srcManifest distribution.Manifest, manifests distribution.ManifestService, ref imagereference.DockerImageReference, filterFn func(*manifestlist.ManifestDescriptor, bool) bool) ([]distribution.Manifest, distribution.Manifest, digest.Digest, error) {
30+
// FilterOptions assist in filtering out unneeded manifests from ManifestList objects.
31+
type FilterOptions struct {
32+
FilterByOS string
33+
DefaultOSFilter bool
34+
OSFilter *regexp.Regexp
35+
}
36+
37+
// Bind adds the options to the flag set.
38+
func (o *FilterOptions) Bind(flags *pflag.FlagSet) {
39+
flags.StringVar(&o.FilterByOS, "filter-by-os", o.FilterByOS, "A regular expression to control which images are mirrored. Images will be passed as '<platform>/<architecture>[/<variant>]'.")
40+
}
41+
42+
// Complete checks whether the flags are ready for use.
43+
func (o *FilterOptions) Complete(flags *pflag.FlagSet) error {
44+
pattern := o.FilterByOS
45+
if len(pattern) == 0 && !flags.Changed("filter-by-os") {
46+
o.DefaultOSFilter = true
47+
pattern = regexp.QuoteMeta(fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH))
48+
}
49+
if len(pattern) > 0 {
50+
re, err := regexp.Compile(pattern)
51+
if err != nil {
52+
return fmt.Errorf("--filter-by-os was not a valid regular expression: %v", err)
53+
}
54+
o.OSFilter = re
55+
}
56+
return nil
57+
}
58+
59+
// Include returns true if the provided manifest should be included, or the first image if the user didn't alter the
60+
// default selection and there is only one image.
61+
func (o *FilterOptions) Include(d *manifestlist.ManifestDescriptor, hasMultiple bool) bool {
62+
if o.OSFilter == nil {
63+
return true
64+
}
65+
if o.DefaultOSFilter && !hasMultiple {
66+
return true
67+
}
68+
if len(d.Platform.Variant) > 0 {
69+
return o.OSFilter.MatchString(fmt.Sprintf("%s/%s/%s", d.Platform.OS, d.Platform.Architecture, d.Platform.Variant))
70+
}
71+
return o.OSFilter.MatchString(fmt.Sprintf("%s/%s", d.Platform.OS, d.Platform.Architecture))
72+
}
73+
74+
// IncludeAll returns true if the provided manifest matches the filter, or all if there was no filter.
75+
func (o *FilterOptions) IncludeAll(d *manifestlist.ManifestDescriptor, hasMultiple bool) bool {
76+
if o.OSFilter == nil {
77+
return true
78+
}
79+
if len(d.Platform.Variant) > 0 {
80+
return o.OSFilter.MatchString(fmt.Sprintf("%s/%s/%s", d.Platform.OS, d.Platform.Architecture, d.Platform.Variant))
81+
}
82+
return o.OSFilter.MatchString(fmt.Sprintf("%s/%s", d.Platform.OS, d.Platform.Architecture))
83+
}
84+
85+
type FilterFunc func(*manifestlist.ManifestDescriptor, bool) bool
86+
87+
// PreferManifestList specifically requests a manifest list first
88+
var PreferManifestList = distribution.WithManifestMediaTypes([]string{
89+
manifestlist.MediaTypeManifestList,
90+
schema2.MediaTypeManifest,
91+
})
92+
93+
// FirstManifest returns the first manifest at the request location that matches the filter function.
94+
func FirstManifest(ctx context.Context, from imagereference.DockerImageReference, repo distribution.Repository, filterFn FilterFunc) (distribution.Manifest, digest.Digest, string, error) {
95+
var srcDigest digest.Digest
96+
if len(from.Tag) > 0 {
97+
desc, err := repo.Tags(ctx).Get(ctx, from.Tag)
98+
if err != nil {
99+
return nil, "", "", err
100+
}
101+
srcDigest = desc.Digest
102+
} else {
103+
srcDigest = digest.Digest(from.ID)
104+
}
105+
manifests, err := repo.Manifests(ctx)
106+
if err != nil {
107+
return nil, "", "", err
108+
}
109+
srcManifest, err := manifests.Get(ctx, srcDigest, PreferManifestList)
110+
if err != nil {
111+
return nil, "", "", err
112+
}
113+
114+
originalSrcDigest := srcDigest
115+
srcManifests, srcManifest, srcDigest, err := ProcessManifestList(ctx, srcDigest, srcManifest, manifests, from, filterFn)
116+
if err != nil {
117+
return nil, "", "", err
118+
}
119+
if len(srcManifests) == 0 {
120+
return nil, "", "", fmt.Errorf("filtered all images from %s", from)
121+
}
122+
123+
var location string
124+
if srcDigest == originalSrcDigest {
125+
location = fmt.Sprintf("manifest %s", srcDigest)
126+
} else {
127+
location = fmt.Sprintf("manifest %s in manifest list %s", srcDigest, originalSrcDigest)
128+
}
129+
return srcManifest, srcDigest, location, nil
130+
}
131+
132+
// ManifestToImageConfig takes an image manifest and converts it into a structured object.
133+
func ManifestToImageConfig(ctx context.Context, srcManifest distribution.Manifest, blobs distribution.BlobService, location string) (*docker10.DockerImageConfig, []distribution.Descriptor, error) {
134+
switch t := srcManifest.(type) {
135+
case *schema2.DeserializedManifest:
136+
if t.Config.MediaType != schema2.MediaTypeImageConfig {
137+
return nil, nil, fmt.Errorf("%s does not have the expected image configuration media type: %s", location, t.Config.MediaType)
138+
}
139+
configJSON, err := blobs.Get(ctx, t.Config.Digest)
140+
if err != nil {
141+
return nil, nil, fmt.Errorf("cannot retrieve image configuration for %s: %v", location, err)
142+
}
143+
glog.V(4).Infof("Raw image config json:\n%s", string(configJSON))
144+
config := &docker10.DockerImageConfig{}
145+
if err := json.Unmarshal(configJSON, &config); err != nil {
146+
return nil, nil, fmt.Errorf("unable to parse image configuration: %v", err)
147+
}
148+
149+
base := config
150+
layers := t.Layers
151+
base.Size = 0
152+
for _, layer := range t.Layers {
153+
base.Size += layer.Size
154+
}
155+
156+
return base, layers, nil
157+
158+
case *schema1.SignedManifest:
159+
if glog.V(4) {
160+
_, configJSON, _ := srcManifest.Payload()
161+
glog.Infof("Raw image config json:\n%s", string(configJSON))
162+
}
163+
if len(t.History) == 0 {
164+
return nil, nil, fmt.Errorf("input image is in an unknown format: no v1Compatibility history")
165+
}
166+
config := &docker10.DockerV1CompatibilityImage{}
167+
if err := json.Unmarshal([]byte(t.History[0].V1Compatibility), &config); err != nil {
168+
return nil, nil, err
169+
}
170+
171+
base := &docker10.DockerImageConfig{}
172+
if err := docker10.Convert_DockerV1CompatibilityImage_to_DockerImageConfig(config, base); err != nil {
173+
return nil, nil, err
174+
}
175+
176+
// schema1 layers are in reverse order
177+
layers := make([]distribution.Descriptor, 0, len(t.FSLayers))
178+
for i := len(t.FSLayers) - 1; i >= 0; i-- {
179+
layer := distribution.Descriptor{
180+
MediaType: schema2.MediaTypeLayer,
181+
Digest: t.FSLayers[i].BlobSum,
182+
// size must be reconstructed from the blobs
183+
}
184+
// we must reconstruct the tar sum from the blobs
185+
add.AddLayerToConfig(base, layer, "")
186+
layers = append(layers, layer)
187+
}
188+
189+
return base, layers, nil
190+
191+
default:
192+
return nil, nil, fmt.Errorf("unknown image manifest of type %T from %s", srcManifest, location)
193+
}
194+
}
195+
196+
func ProcessManifestList(ctx context.Context, srcDigest digest.Digest, srcManifest distribution.Manifest, manifests distribution.ManifestService, ref imagereference.DockerImageReference, filterFn FilterFunc) ([]distribution.Manifest, distribution.Manifest, digest.Digest, error) {
24197
var srcManifests []distribution.Manifest
25198
switch t := srcManifest.(type) {
26199
case *manifestlist.DeserializedManifestList:

0 commit comments

Comments
 (0)