[KOTS]: extract images from the Installer and put in the additionalImages array

.werft/jobs/build/publish-kots.ts

@@ -19,6 +19,9 @@ export async function publishKots(werft: Werft, config: JobConfig) {
     // Set the tag to the current version
     exec(`yq w -i ${REPLICATED_YAML_DIR}/gitpod-installer-job.yaml ${INSTALLER_JOB_IMAGE} ${image}:${config.version}`);
 
+    // Update the additionalImages in the kots-app.yaml
+    exec(`/tmp/installer mirror kots --file ${REPLICATED_YAML_DIR}/kots-app.yaml`);
+
     const app = exec(`kubectl get secret ${REPLICATED_SECRET} --namespace werft -o jsonpath='{.data.app}' | base64 -d`);
     const token = exec(`kubectl get secret ${REPLICATED_SECRET} --namespace werft -o jsonpath='{.data.token}' | base64 -d`);
 

install/installer/cmd/mirror_kots.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+// Licensed under the MIT License. See License-MIT.txt in the project root for license information.
+
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/gitpod-io/gitpod/installer/pkg/config"
+	configv1 "github.com/gitpod-io/gitpod/installer/pkg/config/v1"
+	kots "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/yaml"
+)
+
+var mirrorKotsOpts struct {
+	File string
+}
+
+// mirrorKotsCmd represents the mirror kots command
+var mirrorKotsCmd = &cobra.Command{
+	Use:   "kots",
+	Short: "Renders a list of images used to the KOTS app file",
+	Long: `Renders a list of images used to the KOTS app file
+
+The KOTS application file allows an optional array of strings that
+reference images. These are used to build the air gap bundle and are
+pushed to the local registry during installation.
+
+KOTS documentation:
+https://docs.replicated.com/reference/custom-resource-application#additionalimages`,
+	Example: "gitpod-installer mirror kots --file ../kots/manifests/kots-app.yaml",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// Build a virtual config file
+		rawCfg, cfgVersion, err := config.Load("")
+		if err != nil {
+			return err
+		}
+		cfg := rawCfg.(*configv1.Config)
+
+		if mirrorKotsOpts.File == "" {
+			return fmt.Errorf("kots file must be defined")
+		}
+
+		kotsBytes, err := ioutil.ReadFile(mirrorKotsOpts.File)
+		if err != nil {
+			panic(fmt.Sprintf("couldn't read file %s, %s", mirrorKotsOpts.File, err))
+		}
+
+		var kotsApp kots.Application
+		err = yaml.Unmarshal(kotsBytes, &kotsApp)
+		if err != nil {
+			return err
+		}
+
+		// Fake the required config data
+		cfg.Domain = "gitpod.io"
+		cfg.Repository = "custom-repo-name"
+
+		images, err := generateMirrorList(cfgVersion, cfg)
+		if err != nil {
+			return err
+		}
+
+		// Only append images - this will keep any existing images in the spec
+		for _, img := range images {
+			kotsApp.Spec.AdditionalImages = append(kotsApp.Spec.AdditionalImages, img.Original)
+		}
+
+		fc, err := yaml.Marshal(kotsApp)
+		if err != nil {
+			return err
+		}
+
+		err = ioutil.WriteFile(mirrorKotsOpts.File, fc, 0644)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("Gitpod images written to " + mirrorKotsOpts.File)
+
+		return nil
+	},
+}
+
+func init() {
+	mirrorCmd.AddCommand(mirrorKotsCmd)
+
+	mirrorKotsCmd.Flags().StringVarP(&mirrorKotsOpts.File, "file", "f", "", "path to the kots app file")
+}

install/installer/cmd/mirror_list.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/docker/distribution/reference"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
+	configv1 "github.com/gitpod-io/gitpod/installer/pkg/config/v1"
 	"github.com/spf13/cobra"
 )
 
@@ -61,67 +62,11 @@ image to the "target" repo`,
 			return err
 		}
 
-		// Throw error if set to the default Gitpod repository
-		if cfg.Repository == common.GitpodContainerRegistry {
-			return fmt.Errorf("cannot mirror images to repository %s", common.GitpodContainerRegistry)
-		}
-
-		// Get the target repository from the config
-		targetRepo := strings.TrimRight(cfg.Repository, "/")
-
-		// Use the default Gitpod registry to pull from
-		cfg.Repository = common.GitpodContainerRegistry
-
-		k8s, err := renderKubernetesObjects(cfgVersion, cfg)
+		images, err := generateMirrorList(cfgVersion, cfg)
 		if err != nil {
 			return err
 		}
 
-		// Map of images used for deduping
-		allImages := make(map[string]bool)
-
-		rawImages := make([]string, 0)
-		for _, item := range k8s {
-			rawImages = append(rawImages, getPodImages(item)...)
-			rawImages = append(rawImages, getGenericImages(item)...)
-		}
-
-		images := make([]mirrorListRepo, 0)
-		for _, img := range rawImages {
-			// Dedupe
-			if _, ok := allImages[img]; ok {
-				continue
-			}
-			allImages[img] = true
-
-			// Convert target
-			target := img
-			if strings.Contains(img, cfg.Repository) {
-				// This is the Gitpod registry
-				target = strings.Replace(target, cfg.Repository, targetRepo, 1)
-			} else if !mirrorListOpts.ExcludeThirdParty {
-				// Amend third-party images - remove the first part
-				thirdPartyImg := strings.Join(strings.Split(img, "/")[1:], "/")
-				target = fmt.Sprintf("%s/%s", targetRepo, thirdPartyImg)
-			} else {
-				// Excluding third-party images - just skip this one
-				continue
-			}
-
-			images = append(images, mirrorListRepo{
-				Original: img,
-				Target:   target,
-			})
-		}
-
-		// Sort it by the Original
-		sort.Slice(images, func(i, j int) bool {
-			scoreI := images[i].Original
-			scoreJ := images[j].Original
-
-			return scoreI < scoreJ
-		})
-
 		fc, err := common.ToJSONString(images)
 		if err != nil {
 			return err
@@ -140,6 +85,75 @@ func init() {
 	mirrorListCmd.Flags().StringVarP(&mirrorListOpts.ConfigFN, "config", "c", os.Getenv("GITPOD_INSTALLER_CONFIG"), "path to the config file")
 }
 
+func generateMirrorList(cfgVersion string, cfg *configv1.Config) ([]mirrorListRepo, error) {
+	// Throw error if set to the default Gitpod repository
+	if cfg.Repository == common.GitpodContainerRegistry {
+		return nil, fmt.Errorf("cannot mirror images to repository %s", common.GitpodContainerRegistry)
+	}
+
+	// Get the target repository from the config
+	targetRepo := strings.TrimRight(cfg.Repository, "/")
+
+	// Use the default Gitpod registry to pull from
+	cfg.Repository = common.GitpodContainerRegistry
+
+	k8s, err := renderKubernetesObjects(cfgVersion, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	// Map of images used for deduping
+	allImages := make(map[string]bool)
+
+	rawImages := make([]string, 0)
+	for _, item := range k8s {
+		rawImages = append(rawImages, getPodImages(item)...)
+		rawImages = append(rawImages, getGenericImages(item)...)
+	}
+
+	images := make([]mirrorListRepo, 0)
+	for _, img := range rawImages {
+		// Ignore if the image equals the container registry
+		if img == common.GitpodContainerRegistry {
+			continue
+		}
+		// Dedupe
+		if _, ok := allImages[img]; ok {
+			continue
+		}
+		allImages[img] = true
+
+		// Convert target
+		target := img
+		if strings.Contains(img, cfg.Repository) {
+			// This is the Gitpod registry
+			target = strings.Replace(target, cfg.Repository, targetRepo, 1)
+		} else if !mirrorListOpts.ExcludeThirdParty {
+			// Amend third-party images - remove the first part
+			thirdPartyImg := strings.Join(strings.Split(img, "/")[1:], "/")
+			target = fmt.Sprintf("%s/%s", targetRepo, thirdPartyImg)
+		} else {
+			// Excluding third-party images - just skip this one
+			continue
+		}
+
+		images = append(images, mirrorListRepo{
+			Original: img,
+			Target:   target,
+		})
+	}
+
+	// Sort it by the Original
+	sort.Slice(images, func(i, j int) bool {
+		scoreI := images[i].Original
+		scoreJ := images[j].Original
+
+		return scoreI < scoreJ
+	})
+
+	return images, nil
+}
+
 // getGenericImages this is a bit brute force - anything starting "docker.io" or with Gitpod repo is found
 // this will be in ConfigMaps and could be anything, so will need cleaning up
 func getGenericImages(k8sObj string) []string {