Merge pull request #20466 from smarterclayton/extract

Add `oc image extract` to support release tooling

Author: openshift-merge-robot

contrib/completions/bash/oc

@@ -9,8 +9,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"regexp"
-	"runtime"
 	"strconv"
 	"time"
 
@@ -20,8 +18,6 @@ import (
 
 	"github.com/docker/distribution"
 	distributioncontext "github.com/docker/distribution/context"
-	"github.com/docker/distribution/manifest/manifestlist"
-	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/distribution/registry/client"
@@ -38,6 +34,8 @@ import (
 	"github.com/openshift/origin/pkg/image/dockerlayer/add"
 	"github.com/openshift/origin/pkg/image/registryclient"
 	"github.com/openshift/origin/pkg/image/registryclient/dockercredentials"
+	imagemanifest "github.com/openshift/origin/pkg/oc/cli/image/manifest"
+	"github.com/openshift/origin/pkg/oc/cli/image/workqueue"
 )
 
 var (
@@ -86,10 +84,7 @@ type AppendImageOptions struct {
 	DropHistory bool
 	CreatedAt   string
 
-	OSFilter        *regexp.Regexp
-	DefaultOSFilter bool
-
-	FilterByOS string
+	FilterOptions imagemanifest.FilterOptions
 
 	MaxPerRegistry int
 
@@ -100,12 +95,6 @@ type AppendImageOptions struct {
 	genericclioptions.IOStreams
 }
 
-// schema2ManifestOnly specifically requests a manifest list first
-var schema2ManifestOnly = distribution.WithManifestMediaTypes([]string{
-	manifestlist.MediaTypeManifestList,
-	schema2.MediaTypeManifest,
-})
-
 func NewAppendImageOptions(streams genericclioptions.IOStreams) *AppendImageOptions {
 	return &AppendImageOptions{
 		IOStreams:      streams,
@@ -129,9 +118,10 @@ func NewCmdAppendImage(name string, streams genericclioptions.IOStreams) *cobra.
 	}
 
 	flag := cmd.Flags()
+	o.FilterOptions.Bind(flag)
+
 	flag.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Print the actions that would be taken and exit without writing to the destination.")
 	flag.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP")
-	flag.StringVar(&o.FilterByOS, "filter-by-os", o.FilterByOS, "A regular expression to control which images are mirrored. Images will be passed as '<platform>/<architecture>[/<variant>]'.")
 
 	flag.StringVar(&o.From, "from", o.From, "The image to use as a base. If empty, a new scratch image is created.")
 	flag.StringVar(&o.To, "to", o.To, "The Docker repository tag to upload the appended image to.")
@@ -148,17 +138,8 @@ func NewCmdAppendImage(name string, streams genericclioptions.IOStreams) *cobra.
 }
 
 func (o *AppendImageOptions) Complete(cmd *cobra.Command, args []string) error {
-	pattern := o.FilterByOS
-	if len(pattern) == 0 && !cmd.Flags().Changed("filter-by-os") {
-		o.DefaultOSFilter = true
-		pattern = regexp.QuoteMeta(fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH))
-	}
-	if len(pattern) > 0 {
-		re, err := regexp.Compile(pattern)
-		if err != nil {
-			return fmt.Errorf("--filter-by-os was not a valid regular expression: %v", err)
-		}
-		o.OSFilter = re
+	if err := o.FilterOptions.Complete(cmd.Flags()); err != nil {
+		return err
 	}
 
 	for _, arg := range args {
@@ -175,20 +156,6 @@ func (o *AppendImageOptions) Complete(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-// includeDescriptor returns true if the provided manifest should be included.
-func (o *AppendImageOptions) includeDescriptor(d *manifestlist.ManifestDescriptor, hasMultiple bool) bool {
-	if o.OSFilter == nil {
-		return true
-	}
-	if o.DefaultOSFilter && !hasMultiple {
-		return true
-	}
-	if len(d.Platform.Variant) > 0 {
-		return o.OSFilter.MatchString(fmt.Sprintf("%s/%s/%s", d.Platform.OS, d.Platform.Architecture, d.Platform.Variant))
-	}
-	return o.OSFilter.MatchString(fmt.Sprintf("%s/%s", d.Platform.OS, d.Platform.Architecture))
-}
-
 func (o *AppendImageOptions) Run() error {
 	var createdAt *time.Time
 	if len(o.CreatedAt) > 0 {
@@ -256,97 +223,16 @@ func (o *AppendImageOptions) Run() error {
 			return err
 		}
 		fromRepo = repo
-		var srcDigest digest.Digest
-		if len(from.Tag) > 0 {
-			desc, err := repo.Tags(ctx).Get(ctx, from.Tag)
-			if err != nil {
-				return err
-			}
-			srcDigest = desc.Digest
-		} else {
-			srcDigest = digest.Digest(from.ID)
-		}
-		manifests, err := repo.Manifests(ctx)
-		if err != nil {
-			return err
-		}
-		srcManifest, err := manifests.Get(ctx, srcDigest, schema2ManifestOnly)
-		if err != nil {
-			return err
-		}
 
-		originalSrcDigest := srcDigest
-		srcManifests, srcManifest, srcDigest, err := processManifestList(ctx, srcDigest, srcManifest, manifests, *from, o.includeDescriptor)
+		srcManifest, _, location, err := imagemanifest.FirstManifest(ctx, *from, repo, o.FilterOptions.Include)
 		if err != nil {
-			return err
+			return fmt.Errorf("unable to read image %s: %v", from, err)
 		}
-		if len(srcManifests) == 0 {
-			return fmt.Errorf("filtered all images from %s", from)
-		}
-
-		var location string
-		if srcDigest == originalSrcDigest {
-			location = fmt.Sprintf("manifest %s", srcDigest)
-		} else {
-			location = fmt.Sprintf("manifest %s in manifest list %s", srcDigest, originalSrcDigest)
+		base, layers, err = imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), location)
+		if err != nil {
+			return fmt.Errorf("unable to parse image %s: %v", from, err)
 		}
 
-		switch t := srcManifest.(type) {
-		case *schema2.DeserializedManifest:
-			if t.Config.MediaType != schema2.MediaTypeImageConfig {
-				return fmt.Errorf("unable to append layers to images with config %s from %s", t.Config.MediaType, location)
-			}
-			configJSON, err := repo.Blobs(ctx).Get(ctx, t.Config.Digest)
-			if err != nil {
-				return fmt.Errorf("unable to find manifest for image %s: %v", *from, err)
-			}
-			glog.V(4).Infof("Raw image config json:\n%s", string(configJSON))
-			config := &docker10.DockerImageConfig{}
-			if err := json.Unmarshal(configJSON, &config); err != nil {
-				return fmt.Errorf("the source image manifest could not be parsed: %v", err)
-			}
-
-			base = config
-			layers = t.Layers
-			base.Size = 0
-			for _, layer := range t.Layers {
-				base.Size += layer.Size
-			}
-
-		case *schema1.SignedManifest:
-			if glog.V(4) {
-				_, configJSON, _ := srcManifest.Payload()
-				glog.Infof("Raw image config json:\n%s", string(configJSON))
-			}
-			if len(t.History) == 0 {
-				return fmt.Errorf("input image is in an unknown format: no v1Compatibility history")
-			}
-			config := &docker10.DockerV1CompatibilityImage{}
-			if err := json.Unmarshal([]byte(t.History[0].V1Compatibility), &config); err != nil {
-				return err
-			}
-
-			base = &docker10.DockerImageConfig{}
-			if err := docker10.Convert_DockerV1CompatibilityImage_to_DockerImageConfig(config, base); err != nil {
-				return err
-			}
-
-			// schema1 layers are in reverse order
-			layers = make([]distribution.Descriptor, 0, len(t.FSLayers))
-			for i := len(t.FSLayers) - 1; i >= 0; i-- {
-				layer := distribution.Descriptor{
-					MediaType: schema2.MediaTypeLayer,
-					Digest:    t.FSLayers[i].BlobSum,
-					// size must be reconstructed from the blobs
-				}
-				// we must reconstruct the tar sum from the blobs
-				add.AddLayerToConfig(base, layer, "")
-				layers = append(layers, layer)
-			}
-
-		default:
-			return fmt.Errorf("unable to append layers to images of type %T from %s", srcManifest, location)
-		}
 	} else {
 		base = add.NewEmptyConfig()
 		layers = []distribution.Descriptor{add.AddScratchLayerToConfig(base)}
@@ -445,8 +331,8 @@ func (o *AppendImageOptions) Run() error {
 	// upload base layers in parallel
 	stopCh := make(chan struct{})
 	defer close(stopCh)
-	q := newWorkQueue(o.MaxPerRegistry, stopCh)
-	err = q.Try(func(w Try) {
+	q := workqueue.New(o.MaxPerRegistry, stopCh)
+	err = q.Try(func(w workqueue.Try) {
 		for i := range layers[:numLayers] {
 			layer := &layers[i]
 			index := i
@@ -551,7 +437,7 @@ func (o *AppendImageOptions) Run() error {
 	if err != nil {
 		return fmt.Errorf("unable to upload the new image manifest: %v", err)
 	}
-	toDigest, err := putManifestInCompatibleSchema(ctx, manifest, to.Tag, toManifests, fromRepo.Blobs(ctx), toRepo.Named())
+	toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, manifest, to.Tag, toManifests, fromRepo.Blobs(ctx), toRepo.Named())
 	if err != nil {
 		return fmt.Errorf("unable to convert the image to a compatible schema version: %v", err)
 	}

contrib/completions/zsh/oc

@@ -0,0 +1,93 @@
+package archive
+
+import (
+	"archive/tar"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/system"
+	"golang.org/x/sys/unix"
+)
+
+func getWhiteoutConverter(format archive.WhiteoutFormat) tarWhiteoutConverter {
+	if format == archive.OverlayWhiteoutFormat {
+		return overlayWhiteoutConverter{}
+	}
+	return nil
+}
+
+type overlayWhiteoutConverter struct{}
+
+func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) (wo *tar.Header, err error) {
+	// convert whiteouts to AUFS format
+	if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
+		// we just rename the file and make it normal
+		dir, filename := filepath.Split(hdr.Name)
+		hdr.Name = filepath.Join(dir, archive.WhiteoutPrefix+filename)
+		hdr.Mode = 0600
+		hdr.Typeflag = tar.TypeReg
+		hdr.Size = 0
+	}
+
+	if fi.Mode()&os.ModeDir != 0 {
+		// convert opaque dirs to AUFS format by writing an empty file with the prefix
+		opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque")
+		if err != nil {
+			return nil, err
+		}
+		if len(opaque) == 1 && opaque[0] == 'y' {
+			if hdr.Xattrs != nil {
+				delete(hdr.Xattrs, "trusted.overlay.opaque")
+			}
+
+			// create a header for the whiteout file
+			// it should inherit some properties from the parent, but be a regular file
+			wo = &tar.Header{
+				Typeflag:   tar.TypeReg,
+				Mode:       hdr.Mode & int64(os.ModePerm),
+				Name:       filepath.Join(hdr.Name, archive.WhiteoutOpaqueDir),
+				Size:       0,
+				Uid:        hdr.Uid,
+				Uname:      hdr.Uname,
+				Gid:        hdr.Gid,
+				Gname:      hdr.Gname,
+				AccessTime: hdr.AccessTime,
+				ChangeTime: hdr.ChangeTime,
+			}
+		}
+	}
+
+	return
+}
+
+func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) {
+	base := filepath.Base(path)
+	dir := filepath.Dir(path)
+
+	// if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay
+	if base == archive.WhiteoutOpaqueDir {
+		err := unix.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0)
+		// don't write the file itself
+		return false, err
+	}
+
+	// if a file was deleted and we are using overlay, we need to create a character device
+	if strings.HasPrefix(base, archive.WhiteoutPrefix) {
+		originalBase := base[len(archive.WhiteoutPrefix):]
+		originalPath := filepath.Join(dir, originalBase)
+
+		if err := unix.Mknod(originalPath, unix.S_IFCHR, 0); err != nil {
+			return false, err
+		}
+		if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
+			return false, err
+		}
+
+		// don't write the file itself
+		return false, nil
+	}
+
+	return true, nil
+}

docs/man/man1/.files_generated_oc

@@ -0,0 +1,9 @@
+// +build !linux
+
+package archive
+
+import "github.com/docker/docker/pkg/archive"
+
+func getWhiteoutConverter(format archive.WhiteoutFormat) tarWhiteoutConverter {
+	return nil
+}

docs/man/man1/oc-image-extract.1

@@ -0,0 +1,78 @@
+// +build !windows
+
+package archive
+
+import (
+	"archive/tar"
+	"bufio"
+	"fmt"
+	"os"
+
+	"github.com/docker/docker/pkg/system"
+	"golang.org/x/sys/unix"
+)
+
+// runningInUserNS detects whether we are currently running in a user namespace.
+// Copied from github.com/opencontainers/runc/libcontainer/system
+func runningInUserNS() bool {
+	file, err := os.Open("/proc/self/uid_map")
+	if err != nil {
+		// This kernel-provided file only exists if user namespaces are supported
+		return false
+	}
+	defer file.Close()
+
+	buf := bufio.NewReader(file)
+	l, _, err := buf.ReadLine()
+	if err != nil {
+		return false
+	}
+
+	line := string(l)
+	var a, b, c int64
+	fmt.Sscanf(line, "%d %d %d", &a, &b, &c)
+	/*
+	 * We assume we are in the initial user namespace if we have a full
+	 * range - 4294967295 uids starting at uid 0.
+	 */
+	if a == 0 && b == 0 && c == 4294967295 {
+		return false
+	}
+	return true
+}
+
+// handleTarTypeBlockCharFifo is an OS-specific helper function used by
+// createTarFile to handle the following types of header: Block; Char; Fifo
+func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error {
+	if runningInUserNS() {
+		// cannot create a device if running in user namespace
+		return nil
+	}
+
+	mode := uint32(hdr.Mode & 07777)
+	switch hdr.Typeflag {
+	case tar.TypeBlock:
+		mode |= unix.S_IFBLK
+	case tar.TypeChar:
+		mode |= unix.S_IFCHR
+	case tar.TypeFifo:
+		mode |= unix.S_IFIFO
+	}
+
+	return system.Mknod(path, mode, int(system.Mkdev(hdr.Devmajor, hdr.Devminor)))
+}
+
+func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error {
+	if hdr.Typeflag == tar.TypeLink {
+		if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) {
+			if err := os.Chmod(path, hdrInfo.Mode()); err != nil {
+				return err
+			}
+		}
+	} else if hdr.Typeflag != tar.TypeSymlink {
+		if err := os.Chmod(path, hdrInfo.Mode()); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/oc/cli/image/append/append.go

@@ -0,0 +1,18 @@
+// +build windows
+
+package archive
+
+import (
+	"archive/tar"
+	"os"
+)
+
+// handleTarTypeBlockCharFifo is an OS-specific helper function used by
+// createTarFile to handle the following types of header: Block; Char; Fifo
+func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error {
+	return nil
+}
+
+func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error {
+	return nil
+}

pkg/oc/cli/image/append/manifest.go

@@ -0,0 +1,16 @@
+package archive
+
+import (
+	"syscall"
+	"time"
+)
+
+func timeToTimespec(time time.Time) (ts syscall.Timespec) {
+	if time.IsZero() {
+		// Return UTIME_OMIT special value
+		ts.Sec = 0
+		ts.Nsec = ((1 << 30) - 2)
+		return
+	}
+	return syscall.NsecToTimespec(time.UnixNano())
+}

pkg/oc/cli/image/archive/archive.go

@@ -0,0 +1,16 @@
+// +build !linux
+
+package archive
+
+import (
+	"syscall"
+	"time"
+)
+
+func timeToTimespec(time time.Time) (ts syscall.Timespec) {
+	nsec := int64(0)
+	if !time.IsZero() {
+		nsec = time.UnixNano()
+	}
+	return syscall.NsecToTimespec(nsec)
+}

pkg/oc/cli/image/archive/archive_linux.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/openshift/origin/pkg/cmd/templates"
 	"github.com/openshift/origin/pkg/oc/cli/image/append"
+	"github.com/openshift/origin/pkg/oc/cli/image/extract"
 	"github.com/openshift/origin/pkg/oc/cli/image/mirror"
 )
 
@@ -37,6 +38,7 @@ func NewCmdImage(fullName string, f kcmdutil.Factory, streams genericclioptions.
 			Message: "Advanced commands:",
 			Commands: []*cobra.Command{
 				append.NewCmdAppendImage(name, streams),
+				extract.New(name, streams),
 				mirror.NewCmdMirrorImage(name, streams),
 			},
 		},

pkg/oc/cli/image/archive/archive_other.go

@@ -0,0 +1,351 @@
+package manifest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"runtime"
+	"sync"
+
+	"github.com/spf13/pflag"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/api/errcode"
+	"github.com/docker/distribution/registry/api/v2"
+
+	"github.com/docker/libtrust"
+	"github.com/golang/glog"
+	digest "github.com/opencontainers/go-digest"
+
+	"github.com/openshift/origin/pkg/image/apis/image/docker10"
+	imagereference "github.com/openshift/origin/pkg/image/apis/image/reference"
+	"github.com/openshift/origin/pkg/image/dockerlayer/add"
+)
+
+// FilterOptions assist in filtering out unneeded manifests from ManifestList objects.
+type FilterOptions struct {
+	FilterByOS      string
+	DefaultOSFilter bool
+	OSFilter        *regexp.Regexp
+}
+
+// Bind adds the options to the flag set.
+func (o *FilterOptions) Bind(flags *pflag.FlagSet) {
+	flags.StringVar(&o.FilterByOS, "filter-by-os", o.FilterByOS, "A regular expression to control which images are mirrored. Images will be passed as '<platform>/<architecture>[/<variant>]'.")
+}
+
+// Complete checks whether the flags are ready for use.
+func (o *FilterOptions) Complete(flags *pflag.FlagSet) error {
+	pattern := o.FilterByOS
+	if len(pattern) == 0 && !flags.Changed("filter-by-os") {
+		o.DefaultOSFilter = true
+		pattern = regexp.QuoteMeta(fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH))
+	}
+	if len(pattern) > 0 {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return fmt.Errorf("--filter-by-os was not a valid regular expression: %v", err)
+		}
+		o.OSFilter = re
+	}
+	return nil
+}
+
+// Include returns true if the provided manifest should be included, or the first image if the user didn't alter the
+// default selection and there is only one image.
+func (o *FilterOptions) Include(d *manifestlist.ManifestDescriptor, hasMultiple bool) bool {
+	if o.OSFilter == nil {
+		return true
+	}
+	if o.DefaultOSFilter && !hasMultiple {
+		return true
+	}
+	if len(d.Platform.Variant) > 0 {
+		return o.OSFilter.MatchString(fmt.Sprintf("%s/%s/%s", d.Platform.OS, d.Platform.Architecture, d.Platform.Variant))
+	}
+	return o.OSFilter.MatchString(fmt.Sprintf("%s/%s", d.Platform.OS, d.Platform.Architecture))
+}
+
+// IncludeAll returns true if the provided manifest matches the filter, or all if there was no filter.
+func (o *FilterOptions) IncludeAll(d *manifestlist.ManifestDescriptor, hasMultiple bool) bool {
+	if o.OSFilter == nil {
+		return true
+	}
+	if len(d.Platform.Variant) > 0 {
+		return o.OSFilter.MatchString(fmt.Sprintf("%s/%s/%s", d.Platform.OS, d.Platform.Architecture, d.Platform.Variant))
+	}
+	return o.OSFilter.MatchString(fmt.Sprintf("%s/%s", d.Platform.OS, d.Platform.Architecture))
+}
+
+type FilterFunc func(*manifestlist.ManifestDescriptor, bool) bool
+
+// PreferManifestList specifically requests a manifest list first
+var PreferManifestList = distribution.WithManifestMediaTypes([]string{
+	manifestlist.MediaTypeManifestList,
+	schema2.MediaTypeManifest,
+})
+
+// FirstManifest returns the first manifest at the request location that matches the filter function.
+func FirstManifest(ctx context.Context, from imagereference.DockerImageReference, repo distribution.Repository, filterFn FilterFunc) (distribution.Manifest, digest.Digest, string, error) {
+	var srcDigest digest.Digest
+	if len(from.Tag) > 0 {
+		desc, err := repo.Tags(ctx).Get(ctx, from.Tag)
+		if err != nil {
+			return nil, "", "", err
+		}
+		srcDigest = desc.Digest
+	} else {
+		srcDigest = digest.Digest(from.ID)
+	}
+	manifests, err := repo.Manifests(ctx)
+	if err != nil {
+		return nil, "", "", err
+	}
+	srcManifest, err := manifests.Get(ctx, srcDigest, PreferManifestList)
+	if err != nil {
+		return nil, "", "", err
+	}
+
+	originalSrcDigest := srcDigest
+	srcManifests, srcManifest, srcDigest, err := ProcessManifestList(ctx, srcDigest, srcManifest, manifests, from, filterFn)
+	if err != nil {
+		return nil, "", "", err
+	}
+	if len(srcManifests) == 0 {
+		return nil, "", "", fmt.Errorf("filtered all images from %s", from)
+	}
+
+	var location string
+	if srcDigest == originalSrcDigest {
+		location = fmt.Sprintf("manifest %s", srcDigest)
+	} else {
+		location = fmt.Sprintf("manifest %s in manifest list %s", srcDigest, originalSrcDigest)
+	}
+	return srcManifest, srcDigest, location, nil
+}
+
+// ManifestToImageConfig takes an image manifest and converts it into a structured object.
+func ManifestToImageConfig(ctx context.Context, srcManifest distribution.Manifest, blobs distribution.BlobService, location string) (*docker10.DockerImageConfig, []distribution.Descriptor, error) {
+	switch t := srcManifest.(type) {
+	case *schema2.DeserializedManifest:
+		if t.Config.MediaType != schema2.MediaTypeImageConfig {
+			return nil, nil, fmt.Errorf("%s does not have the expected image configuration media type: %s", location, t.Config.MediaType)
+		}
+		configJSON, err := blobs.Get(ctx, t.Config.Digest)
+		if err != nil {
+			return nil, nil, fmt.Errorf("cannot retrieve image configuration for %s: %v", location, err)
+		}
+		glog.V(4).Infof("Raw image config json:\n%s", string(configJSON))
+		config := &docker10.DockerImageConfig{}
+		if err := json.Unmarshal(configJSON, &config); err != nil {
+			return nil, nil, fmt.Errorf("unable to parse image configuration: %v", err)
+		}
+
+		base := config
+		layers := t.Layers
+		base.Size = 0
+		for _, layer := range t.Layers {
+			base.Size += layer.Size
+		}
+
+		return base, layers, nil
+
+	case *schema1.SignedManifest:
+		if glog.V(4) {
+			_, configJSON, _ := srcManifest.Payload()
+			glog.Infof("Raw image config json:\n%s", string(configJSON))
+		}
+		if len(t.History) == 0 {
+			return nil, nil, fmt.Errorf("input image is in an unknown format: no v1Compatibility history")
+		}
+		config := &docker10.DockerV1CompatibilityImage{}
+		if err := json.Unmarshal([]byte(t.History[0].V1Compatibility), &config); err != nil {
+			return nil, nil, err
+		}
+
+		base := &docker10.DockerImageConfig{}
+		if err := docker10.Convert_DockerV1CompatibilityImage_to_DockerImageConfig(config, base); err != nil {
+			return nil, nil, err
+		}
+
+		// schema1 layers are in reverse order
+		layers := make([]distribution.Descriptor, 0, len(t.FSLayers))
+		for i := len(t.FSLayers) - 1; i >= 0; i-- {
+			layer := distribution.Descriptor{
+				MediaType: schema2.MediaTypeLayer,
+				Digest:    t.FSLayers[i].BlobSum,
+				// size must be reconstructed from the blobs
+			}
+			// we must reconstruct the tar sum from the blobs
+			add.AddLayerToConfig(base, layer, "")
+			layers = append(layers, layer)
+		}
+
+		return base, layers, nil
+
+	default:
+		return nil, nil, fmt.Errorf("unknown image manifest of type %T from %s", srcManifest, location)
+	}
+}
+
+func ProcessManifestList(ctx context.Context, srcDigest digest.Digest, srcManifest distribution.Manifest, manifests distribution.ManifestService, ref imagereference.DockerImageReference, filterFn FilterFunc) ([]distribution.Manifest, distribution.Manifest, digest.Digest, error) {
+	var srcManifests []distribution.Manifest
+	switch t := srcManifest.(type) {
+	case *manifestlist.DeserializedManifestList:
+		manifestDigest := srcDigest
+		manifestList := t
+
+		filtered := make([]manifestlist.ManifestDescriptor, 0, len(t.Manifests))
+		for _, manifest := range t.Manifests {
+			if !filterFn(&manifest, len(t.Manifests) > 1) {
+				glog.V(5).Infof("Skipping image for %#v from %s", manifest.Platform, ref)
+				continue
+			}
+			glog.V(5).Infof("Including image for %#v from %s", manifest.Platform, ref)
+			filtered = append(filtered, manifest)
+		}
+
+		if len(filtered) == 0 {
+			return nil, nil, "", nil
+		}
+
+		// if we're filtering the manifest list, update the source manifest and digest
+		if len(filtered) != len(t.Manifests) {
+			var err error
+			t, err = manifestlist.FromDescriptors(filtered)
+			if err != nil {
+				return nil, nil, "", fmt.Errorf("unable to filter source image %s manifest list: %v", ref, err)
+			}
+			_, body, err := t.Payload()
+			if err != nil {
+				return nil, nil, "", fmt.Errorf("unable to filter source image %s manifest list (bad payload): %v", ref, err)
+			}
+			manifestList = t
+			manifestDigest = srcDigest.Algorithm().FromBytes(body)
+			glog.V(5).Infof("Filtered manifest list to new digest %s:\n%s", manifestDigest, body)
+		}
+
+		for i, manifest := range t.Manifests {
+			childManifest, err := manifests.Get(ctx, manifest.Digest, distribution.WithManifestMediaTypes([]string{manifestlist.MediaTypeManifestList, schema2.MediaTypeManifest}))
+			if err != nil {
+				return nil, nil, "", fmt.Errorf("unable to retrieve source image %s manifest #%d from manifest list: %v", ref, i+1, err)
+			}
+			srcManifests = append(srcManifests, childManifest)
+		}
+
+		switch {
+		case len(srcManifests) == 1:
+			_, body, err := srcManifests[0].Payload()
+			if err != nil {
+				return nil, nil, "", fmt.Errorf("unable to convert source image %s manifest list to single manifest: %v", ref, err)
+			}
+			manifestDigest := srcDigest.Algorithm().FromBytes(body)
+			glog.V(5).Infof("Used only one manifest from the list %s", manifestDigest)
+			return srcManifests, srcManifests[0], manifestDigest, nil
+		default:
+			return append(srcManifests, manifestList), manifestList, manifestDigest, nil
+		}
+
+	default:
+		return []distribution.Manifest{srcManifest}, srcManifest, srcDigest, nil
+	}
+}
+
+// TDOO: remove when quay.io switches to v2 schema
+func PutManifestInCompatibleSchema(
+	ctx context.Context,
+	srcManifest distribution.Manifest,
+	tag string,
+	toManifests distribution.ManifestService,
+	// supports schema2 -> schema1 downconversion
+	blobs distribution.BlobService,
+	ref reference.Named,
+) (digest.Digest, error) {
+	var options []distribution.ManifestServiceOption
+	if len(tag) > 0 {
+		glog.V(5).Infof("Put manifest %s:%s", ref, tag)
+		options = []distribution.ManifestServiceOption{distribution.WithTag(tag)}
+	} else {
+		glog.V(5).Infof("Put manifest %s", ref)
+	}
+	toDigest, err := toManifests.Put(ctx, srcManifest, options...)
+	if err == nil {
+		return toDigest, nil
+	}
+	errs, ok := err.(errcode.Errors)
+	if !ok || len(errs) == 0 {
+		return toDigest, err
+	}
+	errcode, ok := errs[0].(errcode.Error)
+	if !ok || errcode.ErrorCode() != v2.ErrorCodeManifestInvalid {
+		return toDigest, err
+	}
+	// try downconverting to v2-schema1
+	schema2Manifest, ok := srcManifest.(*schema2.DeserializedManifest)
+	if !ok {
+		return toDigest, err
+	}
+	tagRef, tagErr := reference.WithTag(ref, tag)
+	if tagErr != nil {
+		return toDigest, err
+	}
+	glog.V(5).Infof("Registry reported invalid manifest error, attempting to convert to v2schema1 as ref %s", tagRef)
+	schema1Manifest, convertErr := convertToSchema1(ctx, blobs, schema2Manifest, tagRef)
+	if convertErr != nil {
+		return toDigest, err
+	}
+	if glog.V(6) {
+		_, data, _ := schema1Manifest.Payload()
+		glog.Infof("Converted to v2schema1\n%s", string(data))
+	}
+	return toManifests.Put(ctx, schema1Manifest, distribution.WithTag(tag))
+}
+
+// TDOO: remove when quay.io switches to v2 schema
+func convertToSchema1(ctx context.Context, blobs distribution.BlobService, schema2Manifest *schema2.DeserializedManifest, ref reference.Named) (distribution.Manifest, error) {
+	targetDescriptor := schema2Manifest.Target()
+	configJSON, err := blobs.Get(ctx, targetDescriptor.Digest)
+	if err != nil {
+		return nil, err
+	}
+	trustKey, err := loadPrivateKey()
+	if err != nil {
+		return nil, err
+	}
+	builder := schema1.NewConfigManifestBuilder(blobs, trustKey, ref, configJSON)
+	for _, d := range schema2Manifest.Layers {
+		if err := builder.AppendReference(d); err != nil {
+			return nil, err
+		}
+	}
+	manifest, err := builder.Build(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return manifest, nil
+}
+
+var (
+	privateKeyLock sync.Mutex
+	privateKey     libtrust.PrivateKey
+)
+
+// TDOO: remove when quay.io switches to v2 schema
+func loadPrivateKey() (libtrust.PrivateKey, error) {
+	privateKeyLock.Lock()
+	defer privateKeyLock.Unlock()
+	if privateKey != nil {
+		return privateKey, nil
+	}
+	trustKey, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		return nil, err
+	}
+	privateKey = trustKey
+	return privateKey, nil
+}

pkg/oc/cli/image/archive/archive_unix.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"regexp"
 	"time"
 
 	"github.com/docker/distribution"
@@ -27,6 +26,8 @@ import (
 	imagereference "github.com/openshift/origin/pkg/image/apis/image/reference"
 	"github.com/openshift/origin/pkg/image/registryclient"
 	"github.com/openshift/origin/pkg/image/registryclient/dockercredentials"
+	imagemanifest "github.com/openshift/origin/pkg/oc/cli/image/manifest"
+	"github.com/openshift/origin/pkg/oc/cli/image/workqueue"
 )
 
 var (
@@ -76,9 +77,8 @@ var (
 
 type MirrorImageOptions struct {
 	Mappings []Mapping
-	OSFilter *regexp.Regexp
 
-	FilterByOS string
+	FilterOptions imagemanifest.FilterOptions
 
 	DryRun             bool
 	Insecure           bool
@@ -102,12 +102,6 @@ func NewMirrorImageOptions(streams genericclioptions.IOStreams) *MirrorImageOpti
 	}
 }
 
-// schema2ManifestOnly specifically requests a manifest list first
-var schema2ManifestOnly = distribution.WithManifestMediaTypes([]string{
-	manifestlist.MediaTypeManifestList,
-	schema2.MediaTypeManifest,
-})
-
 // NewCommandMirrorImage copies images from one location to another.
 func NewCmdMirrorImage(name string, streams genericclioptions.IOStreams) *cobra.Command {
 	o := NewMirrorImageOptions(streams)
@@ -118,17 +112,18 @@ func NewCmdMirrorImage(name string, streams genericclioptions.IOStreams) *cobra.
 		Long:    mirrorDesc,
 		Example: fmt.Sprintf(mirrorExample, name+" mirror"),
 		Run: func(c *cobra.Command, args []string) {
-			kcmdutil.CheckErr(o.Complete(args))
+			kcmdutil.CheckErr(o.Complete(c, args))
 			kcmdutil.CheckErr(o.Run())
 		},
 	}
 
 	flag := cmd.Flags()
+	o.FilterOptions.Bind(flag)
+
 	flag.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Print the actions that would be taken and exit without writing to the destinations.")
 	flag.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP")
 	flag.BoolVar(&o.SkipMount, "skip-mount", o.SkipMount, "Always push layers instead of cross-mounting them")
 	flag.BoolVar(&o.SkipMultipleScopes, "skip-multiple-scopes", o.SkipMultipleScopes, "Some registries do not support multiple scopes passed to the registry login.")
-	flag.StringVar(&o.FilterByOS, "filter-by-os", o.FilterByOS, "A regular expression to control which images are mirrored. Images will be passed as '<platform>/<architecture>[/<variant>]'.")
 	flag.BoolVar(&o.Force, "force", o.Force, "Attempt to write all layers and manifests even if they exist in the remote repository.")
 	flag.IntVar(&o.MaxRegistry, "max-registry", 4, "Number of concurrent registries to connect to at any one time.")
 	flag.IntVar(&o.MaxPerRegistry, "max-per-registry", 6, "Number of concurrent requests allowed per registry.")
@@ -138,7 +133,11 @@ func NewCmdMirrorImage(name string, streams genericclioptions.IOStreams) *cobra.
 	return cmd
 }
 
-func (o *MirrorImageOptions) Complete(args []string) error {
+func (o *MirrorImageOptions) Complete(cmd *cobra.Command, args []string) error {
+	if err := o.FilterOptions.Complete(cmd.Flags()); err != nil {
+		return err
+	}
+
 	overlap := make(map[string]string)
 
 	var err error
@@ -164,15 +163,6 @@ func (o *MirrorImageOptions) Complete(args []string) error {
 		}
 	}
 
-	pattern := o.FilterByOS
-	if len(pattern) > 0 {
-		re, err := regexp.Compile(pattern)
-		if err != nil {
-			return fmt.Errorf("--filter-by-os was not a valid regular expression: %v", err)
-		}
-		o.OSFilter = re
-	}
-
 	return nil
 }
 
@@ -192,17 +182,6 @@ func (o *MirrorImageOptions) Repository(ctx context.Context, context *registrycl
 	}
 }
 
-// includeDescriptor returns true if the provided manifest should be included.
-func (o *MirrorImageOptions) includeDescriptor(d *manifestlist.ManifestDescriptor, hasMultiple bool) bool {
-	if o.OSFilter == nil {
-		return true
-	}
-	if len(d.Platform.Variant) > 0 {
-		return o.OSFilter.MatchString(fmt.Sprintf("%s/%s/%s", d.Platform.OS, d.Platform.Architecture, d.Platform.Variant))
-	}
-	return o.OSFilter.MatchString(fmt.Sprintf("%s/%s", d.Platform.OS, d.Platform.Architecture))
-}
-
 func (o *MirrorImageOptions) Run() error {
 	start := time.Now()
 	p, err := o.plan()
@@ -232,10 +211,10 @@ func (o *MirrorImageOptions) Run() error {
 
 	stopCh := make(chan struct{})
 	defer close(stopCh)
-	q := newWorkQueue(o.MaxRegistry, stopCh)
-	registryWorkers := make(map[string]*workQueue)
+	q := workqueue.New(o.MaxRegistry, stopCh)
+	registryWorkers := make(map[string]workqueue.Interface)
 	for name := range p.RegistryNames() {
-		registryWorkers[name] = newWorkQueue(o.MaxPerRegistry, stopCh)
+		registryWorkers[name] = workqueue.New(o.MaxPerRegistry, stopCh)
 	}
 
 	next := time.Now()
@@ -247,12 +226,12 @@ func (o *MirrorImageOptions) Run() error {
 	ctx := apirequest.NewContext()
 	for j := range work.phases {
 		phase := &work.phases[j]
-		q.Batch(func(w Work) {
+		q.Batch(func(w workqueue.Work) {
 			for i := range phase.independent {
 				unit := phase.independent[i]
 				w.Parallel(func() {
 					// upload blobs
-					registryWorkers[unit.registry.name].Batch(func(w Work) {
+					registryWorkers[unit.registry.name].Batch(func(w workqueue.Work) {
 						for i := range unit.repository.blobs {
 							op := unit.repository.blobs[i]
 							for digestString := range op.blobs {
@@ -318,19 +297,19 @@ func (o *MirrorImageOptions) plan() (*plan, error) {
 
 	stopCh := make(chan struct{})
 	defer close(stopCh)
-	q := newWorkQueue(o.MaxRegistry, stopCh)
-	registryWorkers := make(map[string]*workQueue)
+	q := workqueue.New(o.MaxRegistry, stopCh)
+	registryWorkers := make(map[string]workqueue.Interface)
 	for name := range tree {
 		if _, ok := registryWorkers[name.registry]; !ok {
-			registryWorkers[name.registry] = newWorkQueue(o.MaxPerRegistry, stopCh)
+			registryWorkers[name.registry] = workqueue.New(o.MaxPerRegistry, stopCh)
 		}
 	}
 
 	plan := newPlan()
 
 	for name := range tree {
 		src := tree[name]
-		q.Queue(func(_ Work) {
+		q.Queue(func(_ workqueue.Work) {
 			srcRepo, err := fromContext.Repository(ctx, src.ref.DockerClientDefaults().RegistryURL(), src.ref.RepositoryName(), o.Insecure)
 			if err != nil {
 				plan.AddError(retrieverError{err: fmt.Errorf("unable to connect to %s: %v", src.ref, err), src: src.ref})
@@ -342,7 +321,7 @@ func (o *MirrorImageOptions) plan() (*plan, error) {
 				return
 			}
 			rq := registryWorkers[name.registry]
-			rq.Batch(func(w Work) {
+			rq.Batch(func(w workqueue.Work) {
 				// convert source tags to digests
 				for tag := range src.tags {
 					srcTag, pushTargets := tag, src.tags[tag]
@@ -361,21 +340,21 @@ func (o *MirrorImageOptions) plan() (*plan, error) {
 
 			canonicalFrom := srcRepo.Named()
 
-			rq.Queue(func(w Work) {
+			rq.Queue(func(w workqueue.Work) {
 				for key := range src.digests {
 					srcDigestString, pushTargets := key, src.digests[key]
 					w.Parallel(func() {
 						// load the manifest
 						srcDigest := godigest.Digest(srcDigestString)
-						srcManifest, err := manifests.Get(ctx, godigest.Digest(srcDigest), schema2ManifestOnly)
+						srcManifest, err := manifests.Get(ctx, godigest.Digest(srcDigest), imagemanifest.PreferManifestList)
 						if err != nil {
 							plan.AddError(retrieverError{src: src.ref, err: fmt.Errorf("unable to retrieve source image %s manifest %s: %v", src.ref, srcDigest, err)})
 							return
 						}
 
 						// filter or load manifest list as appropriate
 						originalSrcDigest := srcDigest
-						srcManifests, srcManifest, srcDigest, err := processManifestList(ctx, srcDigest, srcManifest, manifests, src.ref, o.includeDescriptor)
+						srcManifests, srcManifest, srcDigest, err := imagemanifest.ProcessManifestList(ctx, srcDigest, srcManifest, manifests, src.ref, o.FilterOptions.IncludeAll)
 						if err != nil {
 							plan.AddError(retrieverError{src: src.ref, err: err})
 							return
@@ -599,7 +578,7 @@ func copyManifests(
 			panic(fmt.Sprintf("empty source manifest for %s", srcDigest))
 		}
 		for _, tag := range tags.List() {
-			toDigest, err := putManifestInCompatibleSchema(ctx, srcManifest, tag, plan.to, plan.toBlobs, ref)
+			toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, srcManifest, tag, plan.to, plan.toBlobs, ref)
 			if err != nil {
 				errs = append(errs, fmt.Errorf("unable to push manifest to %s: %v", plan.toRef, err))
 				continue
@@ -622,7 +601,7 @@ func copyManifests(
 		if !ok {
 			panic(fmt.Sprintf("empty source manifest for %s", srcDigest))
 		}
-		toDigest, err := putManifestInCompatibleSchema(ctx, srcManifest, "", plan.to, plan.toBlobs, ref)
+		toDigest, err := imagemanifest.PutManifestInCompatibleSchema(ctx, srcManifest, "", plan.to, plan.toBlobs, ref)
 		if err != nil {
 			errs = append(errs, fmt.Errorf("unable to push manifest to %s: %v", plan.toRef, err))
 			continue

pkg/oc/cli/image/archive/archive_windows.go

@@ -1,17 +1,32 @@
-package append
+package workqueue
 
 import (
 	"sync"
 
 	"github.com/golang/glog"
 )
 
+type Work interface {
+	Parallel(fn func())
+}
+
+type Try interface {
+	Try(fn func() error)
+}
+
+type Interface interface {
+	Batch(func(Work))
+	Try(func(Try)) error
+	Queue(func(Work))
+	Done()
+}
+
 type workQueue struct {
 	ch chan workUnit
 	wg *sync.WaitGroup
 }
 
-func newWorkQueue(workers int, stopCh <-chan struct{}) *workQueue {
+func New(workers int, stopCh <-chan struct{}) Interface {
 	q := &workQueue{
 		ch: make(chan workUnit, 100),
 		wg: &sync.WaitGroup{},
@@ -77,14 +92,6 @@ type workUnit struct {
 	wg *sync.WaitGroup
 }
 
-type Work interface {
-	Parallel(fn func())
-}
-
-type Try interface {
-	Try(fn func() error)
-}
-
 type worker struct {
 	wg  *sync.WaitGroup
 	ch  chan workUnit

pkg/oc/cli/image/archive/time_linux.go

@@ -0,0 +1,76 @@
+package images
+
+import (
+	"github.com/MakeNowJust/heredoc"
+	g "github.com/onsi/ginkgo"
+	o "github.com/onsi/gomega"
+
+	kapi "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	imageapi "github.com/openshift/api/image/v1"
+	imageclientset "github.com/openshift/client-go/image/clientset/versioned"
+	exutil "github.com/openshift/origin/test/extended/util"
+)
+
+var _ = g.Describe("[Feature:ImageExtract] Image extract", func() {
+	defer g.GinkgoRecover()
+
+	var oc *exutil.CLI
+	var ns string
+
+	g.AfterEach(func() {
+		if g.CurrentGinkgoTestDescription().Failed && len(ns) > 0 {
+			exutil.DumpPodLogsStartingWithInNamespace("", ns, oc)
+		}
+	})
+
+	oc = exutil.NewCLI("image-extract", exutil.KubeConfigPath())
+
+	g.It("should extract content from an image", func() {
+		ns = oc.Namespace()
+		cli := oc.KubeFramework().PodClient()
+		client := imageclientset.NewForConfigOrDie(oc.UserConfig()).Image()
+
+		_, err := client.ImageStreamImports(ns).Create(&imageapi.ImageStreamImport{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "1",
+			},
+			Spec: imageapi.ImageStreamImportSpec{
+				Import: true,
+				Images: []imageapi.ImageImportSpec{
+					{
+						From: kapi.ObjectReference{Kind: "DockerImage", Name: "busybox:latest"},
+						To:   &kapi.LocalObjectReference{Name: "busybox"},
+					},
+					{
+						From: kapi.ObjectReference{Kind: "DockerImage", Name: "mysql:latest"},
+						To:   &kapi.LocalObjectReference{Name: "mysql"},
+					},
+				},
+			},
+		})
+		o.Expect(err).ToNot(o.HaveOccurred())
+
+		// busyboxLayers := isi.Status.Images[0].Image.DockerImageLayers
+		// busyboxLen := len(busyboxLayers)
+		// mysqlLayers := isi.Status.Images[1].Image.DockerImageLayers
+		// mysqlLen := len(mysqlLayers)
+
+		pod := cli.Create(cliPodWithPullSecret(oc, heredoc.Docf(`
+			set -x
+
+			# command exits if directory doesn't exist
+			! oc image extract --insecure docker-registry.default.svc:5000/%[1]s/1:busybox --path=/:/tmp/doesnotexist
+
+			# extract busybox to a directory, verify the contents
+			mkdir -p /tmp/test
+			oc image extract --insecure docker-registry.default.svc:5000/%[1]s/1:busybox --path=/:/tmp/test
+			[ -d /tmp/test/etc ] && [ -d /tmp/test/bin ]
+			[ -f /tmp/test/bin/ls ] && /tmp/test/bin/ls /tmp/test
+			oc image extract --insecure docker-registry.default.svc:5000/%[1]s/1:busybox --path=/etc/shadow:/tmp --path=/etc/localtime:/tmp
+			[ -f /tmp/shadow ] && [ -f /tmp/localtime ]
+		`, ns)))
+		cli.WaitForSuccess(pod.Name, podStartupTimeout)
+	})
+})