Testing images: enforce digest and move to standalone list

Signed-off-by: apostasie <[email protected]>

Author: apostasie

cmd/nerdctl/builder/builder_builder_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/containerd/nerdctl/mod/tigron/test"
 
 	"github.com/containerd/nerdctl/v2/pkg/buildkitutil"
+	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
@@ -152,14 +153,19 @@ CMD ["echo", "nerdctl-builder-debug-test-string"]`, testutil.CommonImage)
 					// FIXME: this test should be rewritten to dynamically retrieve the ids, and use images
 					// available on all platforms
 					oldImage := testutil.BusyboxImage
-					oldImageSha := "7b3ccabffc97de872a30dfd234fd972a66d247c8cfc69b0550f276481852627c"
+					parsedOldImage, err := referenceutil.Parse(oldImage)
+					assert.NilError(helpers.T(), err)
+					oldImageSha := parsedOldImage.Digest.String()
+
 					newImage := testutil.AlpineImage
-					newImageSha := "ec14c7992a97fc11425907e908340c6c3d6ff602f5f13d899e6b7027c9b4133a"
+					parsedNewImage, err := referenceutil.Parse(newImage)
+					assert.NilError(helpers.T(), err)
+					newImageSha := parsedNewImage.Digest.String()
 
 					helpers.Ensure("pull", "--quiet", oldImage)
-					helpers.Ensure("tag", oldImage, newImage)
+					helpers.Ensure("tag", oldImage, parsedNewImage.Domain+"/"+parsedNewImage.Path+":"+parsedNewImage.Tag)
 
-					dockerfile := fmt.Sprintf(`FROM %s`, newImage)
+					dockerfile := fmt.Sprintf(`FROM %s`, parsedNewImage.Domain+"/"+parsedNewImage.Path+":"+parsedNewImage.Tag)
 					data.Temp().Save(dockerfile, "Dockerfile")
 					data.Labels().Set("oldImageSha", oldImageSha)
 					data.Labels().Set("newImageSha", newImageSha)

cmd/nerdctl/image/image_history_test.go

@@ -95,8 +95,8 @@ func TestImageHistory(t *testing.T) {
 					assert.NilError(t, err, info)
 					assert.Equal(t, len(history), 2, info)
 
-					localTimeL1, _ := time.Parse(time.RFC3339, "2021-03-31T10:21:23-07:00")
-					localTimeL2, _ := time.Parse(time.RFC3339, "2021-03-31T10:21:21-07:00")
+					localTimeL1, _ := time.Parse(time.RFC3339, "2025-02-14 03:28:36-07:00")
+					localTimeL2, _ := time.Parse(time.RFC3339, "2025-02-14 03:28:36-07:00")
 					compTime1, _ := time.Parse(time.RFC3339, history[0].CreatedAt)
 					compTime2, _ := time.Parse(time.RFC3339, history[1].CreatedAt)
 					assert.Equal(t, compTime1.UTC().String(), localTimeL1.UTC().String(), info)
@@ -123,7 +123,8 @@ func TestImageHistory(t *testing.T) {
 					assert.NilError(t, err, info)
 					assert.Equal(t, history[0].Size, "0", info)
 					assert.Equal(t, history[0].CreatedSince, history[0].CreatedAt, info)
-					assert.Equal(t, history[1].Size, "5947392", info)
+					// FIXME: hardcoded - this is obviously tied to the specific version of the image...
+					assert.Equal(t, history[1].Size, "8843264", info)
 					assert.Equal(t, history[1].CreatedSince, history[1].CreatedAt, info)
 				}),
 			},
@@ -133,22 +134,22 @@ func TestImageHistory(t *testing.T) {
 				Expected: test.Expects(0, nil, func(stdout string, info string, t *testing.T) {
 					history, err := decode(stdout)
 					assert.NilError(t, err, info)
-					assert.Equal(t, history[1].Snapshot, "sha256:56bf55b8eed1f0b4794a30386e4d1d3da949c25bcb5155e898097cd75dc77c2a")
-					assert.Equal(t, history[1].CreatedBy, "/bin/sh -c #(nop) ADD file:3b16ffee2b26d8af5db152fcc582aaccd9e1ec9e3343874e9969a205550fe07d in / ")
+					assert.Equal(t, history[1].Snapshot, "sha256:a16e98724c05975ee8c40d8fe389c3481373d34ab20a1cf52ea2accc43f71f4c")
+					assert.Equal(t, history[1].CreatedBy, "ADD alpine-minirootfs-3.21.3-aarch64.tar.gz / # buildkit")
 				}),
 			},
 			{
 				Description: "Quiet has no effect with format, so, go no-json, no-trunc",
 				Command:     test.Command("image", "history", "--human=false", "--no-trunc", "--quiet", testutil.CommonImage),
 				Expected: test.Expects(0, nil, func(stdout string, info string, t *testing.T) {
-					assert.Equal(t, stdout, "<missing>\nsha256:56bf55b8eed1f0b4794a30386e4d1d3da949c25bcb5155e898097cd75dc77c2a\n")
+					assert.Equal(t, stdout, "<missing>\nsha256:a16e98724c05975ee8c40d8fe389c3481373d34ab20a1cf52ea2accc43f71f4c\n")
 				}),
 			},
 			{
 				Description: "With quiet, trunc has no effect",
 				Command:     test.Command("image", "history", "--human=false", "--no-trunc", "--quiet", testutil.CommonImage),
 				Expected: test.Expects(0, nil, func(stdout string, info string, t *testing.T) {
-					assert.Equal(t, stdout, "<missing>\nsha256:56bf55b8eed1f0b4794a30386e4d1d3da949c25bcb5155e898097cd75dc77c2a\n")
+					assert.Equal(t, stdout, "<missing>\nsha256:a16e98724c05975ee8c40d8fe389c3481373d34ab20a1cf52ea2accc43f71f4c\n")
 				}),
 			},
 		},

cmd/nerdctl/image/image_list_test.go

@@ -19,6 +19,7 @@ package image
 import (
 	"errors"
 	"fmt"
+	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -40,10 +41,12 @@ import (
 func TestImages(t *testing.T) {
 	nerdtest.Setup()
 
+	commonImage, _ := referenceutil.Parse(testutil.CommonImage)
+
 	testCase := &test.Case{
 		Require: require.Not(nerdtest.Docker),
 		Setup: func(data test.Data, helpers test.Helpers) {
-			helpers.Ensure("pull", "--quiet", testutil.CommonImage)
+			helpers.Ensure("pull", "--quiet", commonImage.String())
 			helpers.Ensure("pull", "--quiet", testutil.NginxAlpineImage)
 		},
 		SubTests: []*test.Case{
@@ -66,7 +69,7 @@ func TestImages(t *testing.T) {
 							for _, line := range lines[1:] {
 								repo, _ := tab.ReadRow(line, "REPOSITORY")
 								tag, _ := tab.ReadRow(line, "TAG")
-								if repo+":"+tag == testutil.CommonImage {
+								if repo+":"+tag == commonImage.FamiliarName()+":"+commonImage.Tag {
 									found = true
 									break
 								}
@@ -78,11 +81,11 @@ func TestImages(t *testing.T) {
 			},
 			{
 				Description: "With names",
-				Command:     test.Command("images", "--names", testutil.CommonImage),
+				Command:     test.Command("images", "--names", commonImage.String()),
 				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
 					return &test.Expected{
 						Output: expect.All(
-							expect.Contains(testutil.CommonImage),
+							expect.Contains(commonImage.String()),
 							func(stdout string, info string, t *testing.T) {
 								lines := strings.Split(strings.TrimSpace(stdout), "\n")
 								assert.Assert(t, len(lines) >= 2, info)
@@ -92,7 +95,7 @@ func TestImages(t *testing.T) {
 								found := false
 								for _, line := range lines[1:] {
 									name, _ := tab.ReadRow(line, "NAME")
-									if name == testutil.CommonImage {
+									if name == commonImage.String() {
 										found = true
 										break
 									}
@@ -135,19 +138,21 @@ func TestImages(t *testing.T) {
 func TestImagesFilter(t *testing.T) {
 	nerdtest.Setup()
 
+	commonImage, _ := referenceutil.Parse(testutil.CommonImage)
+
 	testCase := &test.Case{
 		Require: nerdtest.Build,
 		Setup: func(data test.Data, helpers test.Helpers) {
-			helpers.Ensure("pull", "--quiet", testutil.CommonImage)
-			helpers.Ensure("tag", testutil.CommonImage, "taggedimage:one-fragment-one")
-			helpers.Ensure("tag", testutil.CommonImage, "taggedimage:two-fragment-two")
+			helpers.Ensure("pull", "--quiet", commonImage.String())
+			helpers.Ensure("tag", commonImage.String(), "taggedimage:one-fragment-one")
+			helpers.Ensure("tag", commonImage.String(), "taggedimage:two-fragment-two")
 
 			dockerfile := fmt.Sprintf(`FROM %s
 CMD ["echo", "nerdctl-build-test-string"] \n
 LABEL foo=bar
 LABEL version=0.1
 RUN echo "actually creating a layer so that docker sets the createdAt time"
-`, testutil.CommonImage)
+`, commonImage.String())
 			buildCtx := data.Temp().Path()
 			err := os.WriteFile(filepath.Join(buildCtx, "Dockerfile"), []byte(dockerfile), 0o600)
 			assert.NilError(helpers.T(), err)
@@ -237,32 +242,32 @@ RUN echo "actually creating a layer so that docker sets the createdAt time"
 				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
 					return &test.Expected{
 						Output: expect.All(
-							expect.Contains(testutil.ImageRepo(testutil.CommonImage)),
+							expect.Contains(commonImage.FamiliarName(), commonImage.Tag),
 							expect.DoesNotContain(data.Labels().Get("builtImageID")),
 						),
 					}
 				},
 			},
 			{
-				Description: "since=" + testutil.CommonImage,
-				Command:     test.Command("images", "--filter", fmt.Sprintf("since=%s", testutil.CommonImage)),
+				Description: "since=" + commonImage.String(),
+				Command:     test.Command("images", "--filter", fmt.Sprintf("since=%s", commonImage.String())),
 				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
 					return &test.Expected{
 						Output: expect.All(
 							expect.Contains(data.Labels().Get("builtImageID")),
-							expect.DoesNotContain(testutil.ImageRepo(testutil.CommonImage)),
+							expect.DoesNotContain(commonImage.Tag),
 						),
 					}
 				},
 			},
 			{
-				Description: "since=" + testutil.CommonImage + " " + testutil.CommonImage,
-				Command:     test.Command("images", "--filter", fmt.Sprintf("since=%s", testutil.CommonImage), testutil.CommonImage),
+				Description: "since=" + commonImage.String() + " " + commonImage.String(),
+				Command:     test.Command("images", "--filter", fmt.Sprintf("since=%s", commonImage.String()), commonImage.String()),
 				Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
 					return &test.Expected{
 						Output: expect.DoesNotContain(
 							data.Labels().Get("builtImageID"),
-							testutil.ImageRepo(testutil.CommonImage),
+							commonImage.Tag,
 						),
 					}
 				},

cmd/nerdctl/image/image_pull_linux_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
 
+	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest/registry"
@@ -113,6 +114,7 @@ func TestImagePullPlainHttpWithDefaultPort(t *testing.T) {
 	nerdtest.Setup()
 
 	var reg *registry.Server
+	im, _ := referenceutil.Parse(testutil.CommonImage)
 	dockerfile := fmt.Sprintf(`FROM %s
 CMD ["echo", "nerdctl-build-test-string"]
 	`, testutil.CommonImage)
@@ -130,7 +132,7 @@ CMD ["echo", "nerdctl-build-test-string"]
 			reg = nerdtest.RegistryWithNoAuth(data, helpers, 80, false)
 			reg.Setup(data, helpers)
 			testImageRef := fmt.Sprintf("%s/%s:%s",
-				reg.IP.String(), data.Identifier(), strings.Split(testutil.CommonImage, ":")[1])
+				reg.IP.String(), data.Identifier(), im.Tag)
 			buildCtx := data.Temp().Path()
 
 			helpers.Ensure("build", "-t", testImageRef, buildCtx)

cmd/nerdctl/image/image_push_linux_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
 
+	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/testregistry"
@@ -37,6 +38,7 @@ func TestPush(t *testing.T) {
 	nerdtest.Setup()
 
 	var registryNoAuthHTTPRandom, registryNoAuthHTTPDefault, registryTokenAuthHTTPSRandom *testregistry.RegistryServer
+	commonImage, _ := referenceutil.Parse(testutil.CommonImage)
 
 	testCase := &test.Case{
 		Require: require.Linux,
@@ -66,7 +68,7 @@ func TestPush(t *testing.T) {
 				Setup: func(data test.Data, helpers test.Helpers) {
 					helpers.Ensure("pull", "--quiet", testutil.CommonImage)
 					testImageRef := fmt.Sprintf("%s:%d/%s:%s",
-						registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier(), strings.Split(testutil.CommonImage, ":")[1])
+						registryNoAuthHTTPRandom.IP.String(), registryNoAuthHTTPRandom.Port, data.Identifier(), commonImage.Tag)
 					data.Labels().Set("testImageRef", testImageRef)
 					helpers.Ensure("tag", testutil.CommonImage, testImageRef)
 				},

pkg/testutil/images.env

@@ -0,0 +1,19 @@
+ALPINE=alpine:3.21.3@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff4511df3ef88c
+BUSYBOX=busybox:1.37.0@sha256:37f7b378a29ceb4c551b1b5582e27747b855bbfaa73fa11914fe0df028dc581f
+DEBIAN=debian:bookworm-slim@sha256:b1211f6d19afd012477bd34fdcabb6b663d680e0f4b0537da6e6b0fd057a3ec3
+DOCKER_AUTH=cesanta/docker_auth:1.13.0@sha256:c2968eea28ff51b75b88e45147b1d5b8387fd7836400cb97d076c0feb4a451d9
+FLUENTD=fluentd:v1.18.0-debian-1.0@sha256:e7c937425430ad9a23123855379a144f5d079bed8d3b87342bf1e6babb4009fd
+GITLAB=gitlab/gitlab-ee:17.11.0-ee.0@sha256:e0d9d5e0d0068f4b4bac3e15eb48313b5c3bb508425645f421bf2773a964c4ae
+GOLANG=golang:1.23.8-bookworm@sha256:1a73e15c5a17855b58014aab45df0fdf8be778f12a62ed2a48009fe2c0022091
+HARBOR=bitnami/harbor-portal:v2.13.0@sha256:636f39610b359369aeeddd7859cb56274d9a1bc3e467e21d74ea89e1516c1a0c
+KUBO=ipfs/kubo:v0.34.1@sha256:1cf421fbd881d01144c0da39aa1511c05f1ef3fd5eebee6ba7487ecc0f915a44
+MARIADB=mariadb:11.7.2@sha256:81e893032978c4bf8ad43710b7a979774ed90787fa32d199162148ce28fe3b76
+NANOSERVER=mcr.microsoft.com/windows/nanoserver:ltsc2022@sha256:16b61ffac72961551c4b5312a7ab3da7c3dc3258510e40bee309e6e51f006ac6
+NGINX=nginx:alpine3.21@sha256:65645c7bb6a0661892a8b03b89d0743208a18dd2f3f17a54ef4b76fb8e2f2a10
+REGISTRY=registry:3.0.0@sha256:1fc7de654f2ac1247f0b67e8a459e273b0993be7d2beda1f3f56fbf1001ed3e7
+SYSTEMD=ghcr.io/containerd/stargz-snapshotter:0.16.3-kind@sha256:c7c7de2af39f2373a89b6f993a81c9e19b65bf37a65ab14ef2791c97c07e530e
+WORDPRESS=wordpress:6.8.0-php8.4-fpm-alpine@sha256:309b64fa4266d8a3fe6f0973ae3172fec1023c9b18242ccf1dffbff5dc8b81a8
+ALPINE_HISTORY_CREATED="2025-02-14 03:28:36-07:00"
+ALPINE_HISTORY_1_SNAPSHOT=a16e98724c05975ee8c40d8fe389c3481373d34ab20a1cf52ea2accc43f71f4c
+ALPINE_HISTORY_1_SIZE=8843264
+ALPINE_HISTORY_1_CREATED_BY="ADD alpine-minirootfs-3.21.3-aarch64.tar.gz / # buildkit"

pkg/testutil/images_linux.go

@@ -0,0 +1,68 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testutil
+
+import (
+	_ "embed"
+	"fmt"
+	"strings"
+
+	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
+)
+
+//go:embed images.env
+var imagesList string
+
+// getImage retrieves (from the env file) the fully qualified reference of an image, from its short name.
+// No particular error handling effort is done here.
+// If the env file is broken, or if the requested image does not exist, this is fatal.
+func getImage(name string) string {
+	parsed, err := referenceutil.Parse(name)
+
+	if err != nil {
+		panic(fmt.Errorf("malformed image name requested: %w", err))
+	}
+
+	// Ignore tag and digests for now - we currently support only one tag digest per image
+	name = parsed.Domain + "/" + parsed.Path
+
+	for _, k := range strings.Split(imagesList, "\n") {
+		k = strings.TrimSpace(k)
+		if strings.HasPrefix(k, "#") || k == "" {
+			continue
+		}
+
+		spl := strings.Split(k, "=")
+		if len(spl) != 2 {
+			continue
+		}
+
+		item, err := referenceutil.Parse(spl[1])
+		if err != nil {
+			panic(fmt.Errorf("malformed image found in env file: %w (%s)", err, spl[1]))
+		}
+
+		if item.Domain+"/"+item.Path == name {
+			return spl[1]
+		}
+	}
+
+	panic(fmt.Sprintf("no such test image is defined: %s", name))
+
+	//nolint:govet
+	return ""
+}

pkg/testutil/testutil.go

@@ -40,7 +40,6 @@ import (
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/pkg/buildkitutil"
-	"github.com/containerd/nerdctl/v2/pkg/imgutil"
 	"github.com/containerd/nerdctl/v2/pkg/infoutil"
 	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat"
 	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/native"
@@ -749,13 +748,6 @@ func Identifier(t testing.TB) string {
 	return s
 }
 
-// ImageRepo returns the image repo that can be used to, e.g, validate output
-// from `nerdctl images`.
-func ImageRepo(s string) string {
-	repo, _ := imgutil.ParseRepoTag(s)
-	return repo
-}
-
 // RegisterBuildCacheCleanup adds a 'builder prune --all --force' cleanup function
 // to run on test teardown.
 func RegisterBuildCacheCleanup(t *testing.T) {

pkg/testutil/testutil_linux.go

@@ -17,20 +17,24 @@
 package testutil
 
 var (
-	BusyboxImage                = "ghcr.io/containerd/busybox:1.36"
-	AlpineImage                 = mirrorOf("alpine:3.13")
-	NginxAlpineImage            = mirrorOf("nginx:1.19-alpine")
+	AlpineImage         = getImage("alpine")
+	BusyboxImage        = getImage("busybox")
+	DockerAuthImage     = getImage("cesanta/docker_auth")
+	FluentdImage        = getImage("fluentd")
+	GolangImage         = getImage("golang")
+	KuboImage           = getImage("ipfs/kubo")
+	MariaDBImage        = getImage("mariadb")
+	NginxAlpineImage    = getImage("nginx")
+	RegistryImageStable = getImage("registry")
+	RegistryImageNext   = getImage("registry")
+	SystemdImage        = getImage("ghcr.io/containerd/stargz-snapshotter")
+	WordpressImage      = getImage("wordpress")
+
+	// Until we remove the helper
+	_ = mirrorOf("nblblbl")
+
 	NginxAlpineIndexHTMLSnippet = "<title>Welcome to nginx!</title>"
-	RegistryImageStable         = mirrorOf("registry:2")
-	RegistryImageNext           = "ghcr.io/distribution/distribution:"
-	WordpressImage              = mirrorOf("wordpress:5.7")
 	WordpressIndexHTMLSnippet   = "<title>WordPress &rsaquo; Installation</title>"
-	MariaDBImage                = mirrorOf("mariadb:10.5")
-	DockerAuthImage             = mirrorOf("cesanta/docker_auth:1.7")
-	FluentdImage                = "fluent/fluentd:v1.17.0-debian-1.0"
-	KuboImage                   = mirrorOf("ipfs/kubo:v0.16.0")
-	SystemdImage                = "ghcr.io/containerd/stargz-snapshotter:0.15.1-kind"
-	GolangImage                 = mirrorOf("golang:1.18")
 
 	// Source: https://gist.github.com/cpuguy83/fcf3041e5d8fb1bb5c340915aabeebe0
 	NonDistBlobImage = "ghcr.io/cpuguy83/non-dist-blob:latest"