Make registry extended tests dockerless

Author: Oleg Bulatov

test/extended/imageapis/limitrange_admission.go

@@ -16,16 +16,16 @@ import (
 	quotautil "github.com/openshift/origin/pkg/quota/util"
 	imagesutil "github.com/openshift/origin/test/extended/images"
 	exutil "github.com/openshift/origin/test/extended/util"
-	testutil "github.com/openshift/origin/test/util"
 )
 
 const limitRangeName = "limits"
 
-var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/registry/serial][local] Image limit range", func() {
+var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/registry/serial] Image limit range", func() {
 	defer g.GinkgoRecover()
+
 	var oc = exutil.NewCLI("limitrange-admission", exutil.KubeConfigPath())
 
-	g.JustBeforeEach(func() {
+	g.BeforeEach(func() {
 		g.By("waiting for default service account")
 		err := exutil.WaitForServiceAccount(oc.KubeClient().Core().ServiceAccounts(oc.Namespace()), "default")
 		o.Expect(err).NotTo(o.HaveOccurred())
@@ -34,102 +34,77 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis
 		o.Expect(err).NotTo(o.HaveOccurred())
 	})
 
-	// needs to be run at the of of each It; cannot be run in AfterEach which is run after the project
-	// is destroyed
-	tearDown := func(oc *exutil.CLI) {
-		g.By(fmt.Sprintf("Deleting limit range %s", limitRangeName))
-		oc.AdminKubeClient().Core().LimitRanges(oc.Namespace()).Delete(limitRangeName, nil)
-
-		deleteTestImagesAndStreams(oc)
-	}
-
-	g.It(fmt.Sprintf("[Skipped] should deny a push of built image exceeding %s limit", imageapi.LimitTypeImage), func() {
-		g.Skip("FIXME: fill image metadata for schema1 in the registry")
-
-		defer tearDown(oc)
-
-		dClient, err := testutil.NewDockerClient()
-		o.Expect(err).NotTo(o.HaveOccurred())
-
-		_, err = createLimitRangeOfType(oc, imageapi.LimitTypeImage, kapi.ResourceList{
+	g.It(fmt.Sprintf("should deny a push of built image exceeding %s limit", imageapi.LimitTypeImage), func() {
+		_, err := createLimitRangeOfType(oc, imageapi.LimitTypeImage, kapi.ResourceList{
 			kapi.ResourceStorage: resource.MustParse("10Ki"),
 		})
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push an image exceeding size limit with just 1 layer"))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "middle", 16000, 1, false)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "middle", 16000, 1, false)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push an image exceeding size limit in total"))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "middle", 16000, 5, false)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "middle", 16000, 5, false)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push an image with one big layer below size limit"))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "small", 8000, 1, true)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "small", 8000, 1, true)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push an image below size limit"))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "small", 8000, 2, true)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "small", 8000, 2, true)
 		o.Expect(err).NotTo(o.HaveOccurred())
 	})
 
 	g.It(fmt.Sprintf("should deny a push of built image exceeding limit on %s resource", imageapi.ResourceImageStreamImages), func() {
-
-		defer tearDown(oc)
-
 		limits := kapi.ResourceList{
 			imageapi.ResourceImageStreamTags:   resource.MustParse("0"),
 			imageapi.ResourceImageStreamImages: resource.MustParse("0"),
 		}
 		_, err := createLimitRangeOfType(oc, imageapi.LimitTypeImageStream, limits)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
-		dClient, err := testutil.NewDockerClient()
-		o.Expect(err).NotTo(o.HaveOccurred())
-
 		g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "refused", imageSize, 1, false)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "refused", imageSize, 1, false)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		limits, err = bumpLimit(oc, imageapi.ResourceImageStreamImages, "1")
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push image below limits %v", limits))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "first", imageSize, 2, true)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "first", imageSize, 2, true)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "sized", "second", imageSize, 2, false)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "sized", "second", imageSize, 2, false)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push image below limits %v to another image stream", limits))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "second", imageSize, 1, true)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "second", imageSize, 1, true)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		limits, err = bumpLimit(oc, imageapi.ResourceImageStreamImages, "2")
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push image below limits %v", limits))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "third", imageSize, 1, true)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "third", imageSize, 1, true)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push image exceeding limits %v", limits))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "fourth", imageSize, 1, false)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "fourth", imageSize, 1, false)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(`removing tag "second" from "another" image stream`)
 		err = oc.ImageClient().Image().ImageStreamTags(oc.Namespace()).Delete("another:second", nil)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
 		g.By(fmt.Sprintf("trying to push image below limits %v", limits))
-		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, dClient, oc.Namespace(), "another", "replenish", imageSize, 1, true)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), "another", "replenish", imageSize, 1, true)
 		o.Expect(err).NotTo(o.HaveOccurred())
 	})
 
 	g.It(fmt.Sprintf("should deny a docker image reference exceeding limit on %s resource", imageapi.ResourceImageStreamTags), func() {
-
-		defer tearDown(oc)
-
 		tag2Image, err := buildAndPushTestImagesTo(oc, "src", "tag", 2)
 		o.Expect(err).NotTo(o.HaveOccurred())
 
@@ -187,15 +162,12 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis
 	})
 
 	g.It(fmt.Sprintf("should deny an import of a repository exceeding limit on %s resource", imageapi.ResourceImageStreamTags), func() {
-
 		maxBulkImport, err := getMaxImagesBulkImportedPerRepository()
 		if err != nil {
 			g.Skip(err.Error())
 			return
 		}
 
-		defer tearDown(oc)
-
 		s1tag2Image, err := buildAndPushTestImagesTo(oc, "src1st", "tag", maxBulkImport+1)
 		s2tag2Image, err := buildAndPushTestImagesTo(oc, "src2nd", "t", 2)
 		o.Expect(err).NotTo(o.HaveOccurred())
@@ -234,25 +206,18 @@ var _ = g.Describe("[Feature:ImageQuota][registry][Serial][Suite:openshift/regis
 // buildAndPushTestImagesTo builds a given number of test images. The images are pushed to a new image stream
 // of given name under <tagPrefix><X> where X is a number of image starting from 1.
 func buildAndPushTestImagesTo(oc *exutil.CLI, isName string, tagPrefix string, numberOfImages int) (tag2Image map[string]imageapi.Image, err error) {
-	dClient, err := testutil.NewDockerClient()
-	if err != nil {
-		return
-	}
 	tag2Image = make(map[string]imageapi.Image)
 
 	for i := 1; i <= numberOfImages; i++ {
 		tag := fmt.Sprintf("%s%d", tagPrefix, i)
-		dgst, _, err := imagesutil.BuildAndPushImageOfSizeWithDocker(oc, dClient, isName, tag, imageSize, 2, g.GinkgoWriter, true, true)
+		err = imagesutil.BuildAndPushImageOfSizeWithBuilder(oc, nil, oc.Namespace(), isName, tag, imageSize, 2, true)
 		if err != nil {
 			return nil, err
 		}
 		ist, err := oc.ImageClient().Image().ImageStreamTags(oc.Namespace()).Get(isName+":"+tag, metav1.GetOptions{})
 		if err != nil {
 			return nil, err
 		}
-		if dgst != ist.Image.Name {
-			return nil, fmt.Errorf("digest of built image does not match stored: %s != %s", dgst, ist.Image.Name)
-		}
 		tag2Image[tag] = ist.Image
 	}
 
@@ -316,7 +281,7 @@ func bumpLimit(oc *exutil.CLI, resourceName kapi.ResourceName, limit string) (ka
 func getMaxImagesBulkImportedPerRepository() (int, error) {
 	max := os.Getenv("MAX_IMAGES_BULK_IMPORTED_PER_REPOSITORY")
 	if len(max) == 0 {
-		return 0, fmt.Errorf("MAX_IMAGES_BULK_IMAGES_IMPORTED_PER_REPOSITORY is not set")
+		return 0, fmt.Errorf("MAX_IMAGES_BULK_IMPORTED_PER_REPOSITORY is not set")
 	}
 	return strconv.Atoi(max)
 }

test/extended/images/helper.go

@@ -2,40 +2,45 @@ package images
 
 import (
 	"bytes"
+	"context"
 	cryptorand "crypto/rand"
 	"crypto/tls"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	distribution "github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/schema2"
 	dockerclient "github.com/fsouza/go-dockerclient"
 	g "github.com/onsi/ginkgo"
-
 	godigest "github.com/opencontainers/go-digest"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	knet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/retry"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
 
 	imageapi "github.com/openshift/origin/pkg/image/apis/image"
 	imagetypedclientset "github.com/openshift/origin/pkg/image/generated/internalclientset/typed/image/internalversion"
+	"github.com/openshift/origin/pkg/image/registryclient"
 	exutil "github.com/openshift/origin/test/extended/util"
 	testutil "github.com/openshift/origin/test/util"
 )
 
 const (
 	// There are coefficients used to multiply layer data size to get a rough size of uploaded blob.
-	layerSizeMultiplierForDocker18     = 2.0
 	layerSizeMultiplierForLatestDocker = 0.8
 	defaultLayerSize                   = 1024
 	digestSHA256GzippedEmptyTar        = godigest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
@@ -214,8 +219,10 @@ func BuildAndPushImageOfSizeWithBuilder(
 	}
 	buildLog, logsErr := br.Logs()
 
-	if match := reSuccessfulBuild.FindStringSubmatch(buildLog); len(match) > 1 {
-		defer dClient.RemoveImageExtended(match[1], dockerclient.RemoveImageOptions{Force: true})
+	if dClient != nil {
+		if match := reSuccessfulBuild.FindStringSubmatch(buildLog); len(match) > 1 {
+			defer dClient.RemoveImageExtended(match[1], dockerclient.RemoveImageOptions{Force: true})
+		}
 	}
 
 	if !shouldSucceed {
@@ -431,26 +438,35 @@ func pushImageWithDocker(
 	return imageDigest, nil
 }
 
+func newRandomBlob(size uint64) ([]byte, error) {
+	var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+
+	data := make([]byte, size)
+	if _, err := cryptorand.Read(data); err != nil {
+		return nil, err
+	}
+
+	for i := range data {
+		data[i] = letters[uint(data[i])%uint(len(letters))]
+	}
+
+	return data, nil
+}
+
 // createRandomBlob creates a random data with bytes from `letters` in order to let docker take advantage of
 // compression. Resulting layer size will be different due to file metadata overhead and compression.
 func createRandomBlob(dest string, size uint64) error {
-	var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
-
 	f, err := os.Create(dest)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 
-	data := make([]byte, size)
-	if _, err = cryptorand.Read(data); err != nil {
+	data, err := newRandomBlob(size)
+	if err != nil {
 		return err
 	}
 
-	for i := range data {
-		data[i] = letters[uint(data[i])%uint(len(letters))]
-	}
-
 	f.Write(data)
 	return nil
 }
@@ -815,3 +831,96 @@ func (c *CleanUpContainer) Run() {
 		}
 	}
 }
+
+func uploadBlob(ctx context.Context, repo distribution.Repository, blob []byte) (distribution.Descriptor, error) {
+	bs := repo.Blobs(ctx)
+	fmt.Fprintf(g.GinkgoWriter, "uploading blob (%d bytes)...\n", len(blob))
+	return bs.Put(ctx, "", blob)
+}
+
+func uploadManifest(ctx context.Context, repo distribution.Repository, tag string, config distribution.Descriptor, layers []distribution.Descriptor) (godigest.Digest, error) {
+	ms, err := repo.Manifests(ctx)
+	if err != nil {
+		return "", err
+	}
+	manifest, err := schema2.FromStruct(schema2.Manifest{
+		Versioned: schema2.SchemaVersion,
+		Config:    config,
+		Layers:    layers,
+	})
+	if err != nil {
+		return "", err
+	}
+	fmt.Fprintf(g.GinkgoWriter, "uploading manifest...\n")
+	return ms.Put(ctx, manifest, distribution.WithTag(tag))
+}
+
+// BuildAndPushMockImage tries to build a fake image of wanted size and number
+// of layers. Built image is stored as an image stream tag <name>:<tag>. If
+// shouldSucceed is false, a push is expected to fail with a denied error.
+// Returned is an image digest, and an error if any.
+func BuildAndPushMockImage(
+	oc *exutil.CLI,
+	name, tag string,
+	size uint64,
+	numberOfLayers int,
+) (godigest.Digest, error) {
+	e2e.Logf("Uploading mock image to %s:%s (~%d bytes, %d layers)...", name, tag, size, numberOfLayers)
+
+	layerSize := (size + uint64(numberOfLayers) - 1) / uint64(numberOfLayers) // round up
+
+	rt, err := rest.TransportFor(&rest.Config{})
+	if err != nil {
+		return "", err
+	}
+	insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}})
+	if err != nil {
+		return "", err
+	}
+
+	registryHost, err := GetDockerRegistryURL(oc)
+	if err != nil {
+		return "", err
+	}
+	registryURL, err := url.Parse(fmt.Sprintf("http://%s/", registryHost))
+	if err != nil {
+		return "", err
+	}
+
+	out, err := oc.Run("whoami").Args("-t").Output()
+	if err != nil {
+		return "", err
+	}
+	token := strings.TrimSpace(out)
+
+	creds := registryclient.NewBasicCredentials()
+	creds.Add(&url.URL{Host: registryHost}, "unused", token)
+	registryContext := registryclient.NewContext(rt, insecureRT).WithCredentials(creds)
+
+	ctx := context.Background()
+	insecure := true
+	repo, err := registryContext.Repository(ctx, registryURL, name, insecure)
+	if err != nil {
+		return "", err
+	}
+
+	configDesc, err := uploadBlob(ctx, repo, []byte("{}"))
+	if err != nil {
+		return "", fmt.Errorf("unable to upload dummy config: %v", err)
+	}
+
+	var layers []distribution.Descriptor
+	for i := 0; i < numberOfLayers; i++ {
+		blob, err := newRandomBlob(layerSize)
+		if err != nil {
+			return "", err
+		}
+		desc, err := uploadBlob(ctx, repo, blob)
+		if err != nil {
+			return "", fmt.Errorf("unable to upload random blob: %v", err)
+		}
+		layers = append(layers, desc)
+	}
+
+	return uploadManifest(ctx, repo, tag, configDesc, layers)
+}