Merge pull request #424 from bpradipt/embed-pause

Embed pause image

Author: bpradipt

config/peerpods/podvm/Dockerfile.podvm-builder

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/ubi:9.3
+FROM registry.access.redhat.com/ubi9/ubi:9.4
 
 # azure-podvm-image-handler.sh script under /scripts/azure-podvm-image-handler.sh
 # aws-podvm-image-handler.sh script under /scripts/aws-podvm-image-handler.sh
@@ -7,8 +7,8 @@ FROM registry.access.redhat.com/ubi9/ubi:9.3
 # Binaries like kubectl, packer and yq under /usr/local/bin will be installed by the scripts
 
 
-LABEL kata_src=https://github.com/kata-containers/kata-containers/tree/CC-0.8.0
-LABEL kata_src_commit=8de1f8e19f858134ba455a7c04edcb21d8bcf6b1
+LABEL kata_src=https://github.com/kata-containers/kata-containers
+LABEL kata_src_commit=stable-3.6
 
 RUN mkdir -p /scripts
 

config/peerpods/podvm/aws-podvm-image-handler.sh

@@ -77,7 +77,6 @@ function verify_vars() {
     [[ -z "${CONFIDENTIAL_COMPUTE_ENABLED}" ]] && error_exit "CONFIDENTIAL_COMPUTE_ENABLED is empty"
     [[ -z "${DISABLE_CLOUD_CONFIG}" ]] && error_exit "DISABLE_CLOUD_CONFIG is empty"
     [[ -z "${ENABLE_NVIDIA_GPU}" ]] && error_exit "ENABLE_NVIDIA_GPU is empty"
-
 }
 
 # function to download and install aws cli
@@ -287,7 +286,6 @@ function delete_ami_id_annotation_from_peer_pods_cm() {
     echo "Ami id annotation deleted from peer-pods-cm configmap successfully"
 }
 
-
 # Function to create the ami in AWS
 
 function create_ami() {
@@ -317,6 +315,9 @@ function create_ami() {
     # Prepare the source code for building the ami
     prepare_source_code
 
+    # Prepare the pause image for embedding into the ami
+    download_and_extract_pause_image "${PAUSE_IMAGE_REPO}" "${PAUSE_IMAGE_VERSION}" "${PAUSE_IMAGE_REPO_AUTH_FILE}"
+
     # Create AWS ami using packer
     create_ami_using_packer
 

config/peerpods/podvm/azure-podvm-image-handler.sh

@@ -498,6 +498,9 @@ function create_image() {
     # Prepare the source code for building the image
     prepare_source_code
 
+    # Prepare the pause image for embedding into the image
+    download_and_extract_pause_image "${PAUSE_IMAGE_REPO}" "${PAUSE_IMAGE_VERSION}" "${PAUSE_IMAGE_REPO_AUTH_FILE}"
+
     # Create Azure image using packer
     create_image_using_packer
 

config/peerpods/podvm/lib.sh

@@ -3,6 +3,12 @@
 
 set -x
 
+# Defaults for pause image
+# This pause image is multi-arch
+PAUSE_IMAGE_REPO_DEFAULT="quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256"
+PAUSE_IMAGE_VERSION_DEFAULT="7f3cb6f9d265291b47a7491c2ba4f4dd0752a18b661eee40584f9a5dbcbe13bb"
+PAUSE_IMAGE_REPO_AUTH_FILE="/tmp/regauth/auth.json"
+
 # function to trap errors and exit
 function error_exit() {
     echo "$1" 1>&2
@@ -22,6 +28,7 @@ function install_rpm_packages() {
         "git"
         "make"
         "unzip"
+        "skopeo"
     )
 
     # Create a new array to store rpm packages that are not installed
@@ -58,16 +65,19 @@ function install_rpm_packages() {
 # are available in the variable REQUIRED_BINARY_PACKAGES
 # the function will download the packages, extract them and install them in /usr/local/bin
 # Following are the packages that are installed:
+# TBD: add multi-arch support for these binaries
 #"packer=https://releases.hashicorp.com/packer/1.9.4/packer_1.9.4_linux_amd64.zip"
 #"kubectl=https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/4.14.9/openshift-client-linux.tar.gz"
 #"yq=https://github.com/mikefarah/yq/releases/download/v4.35.2/yq_linux_amd64.tar.gz"
+#"umoci=https://github.com/opencontainers/umoci/releases/download/v0.4.7/umoci.amd64"
 
 install_binary_packages() {
     # Define the required binary packages
     REQUIRED_BINARY_PACKAGES=(
         "packer=https://releases.hashicorp.com/packer/1.9.4/packer_1.9.4_linux_amd64.zip"
         "kubectl=https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/4.14.9/openshift-client-linux.tar.gz"
         "yq=https://github.com/mikefarah/yq/releases/download/v4.35.2/yq_linux_amd64.tar.gz"
+        "umoci=https://github.com/opencontainers/umoci/releases/download/v0.4.7/umoci.amd64"
     )
 
     # Specify the installation directory
@@ -94,8 +104,9 @@ install_binary_packages() {
                 tar -xf "${download_path}" -C "${install_dir}" ||
                     error_exit "Failed to extract ${package_name}"
             else
-                echo "Unsupported archive format for ${package_name}. Skipping."
-                continue
+                echo "Copying ${download_path} to ${install_dir}/${package_name}"
+                cp "${download_path}" "${install_dir}/${package_name}" ||
+                    error_exit "Failed to move ${package_name} to ${install_dir}"
             fi
 
             echo "Marking  ${install_dir}/${package_name} executable"
@@ -122,20 +133,23 @@ function download_source_code() {
     # Download source code from GitHub
     # If any error occurs, exit the script with an error message
 
-    # Delete the source code directory if it exists
-    [[ -d "${CAA_SRC_DIR}" ]] &&
-        rm -rf "${CAA_SRC_DIR}"
+    # CAA_SRC_DIR is set to CAA_SRC_DOWNLOAD_DIR/src/cloud-api-adaptor
+    # The default value of CAA_SRC_DOWNLOAD_DIR is /src/cloud-api-adaptor
+
+    # Delete the source code download directory if it exists
+    [[ -d "${CAA_SRC_DOWNLOAD_DIR}" ]] &&
+        rm -rf "${CAA_SRC_DOWNLOAD_DIR}"
 
     # Create the root directory for source code
-    mkdir -p "${CAA_SRC_DIR}"
+    mkdir -p "${CAA_SRC_DOWNLOAD_DIR}"
 
     # Download the source code from GitHub
-    git clone "${CAA_SRC}" "${CAA_SRC_DIR}" ||
+    git clone "${CAA_SRC}" "${CAA_SRC_DOWNLOAD_DIR}" ||
         error_exit "Failed to download source code from GitHub"
 
     # Checkout the required commit
-    cd "${CAA_SRC_DIR}" ||
-        error_exit "Failed to change directory to ${CAA_SRC_DIR}"
+    cd "${CAA_SRC_DOWNLOAD_DIR}" ||
+        error_exit "Failed to change directory to ${CAA_SRC_DOWNLOAD_DIR}"
 
     git checkout "${CAA_REF}" ||
         error_exit "Failed to checkout the required commit"
@@ -156,12 +170,12 @@ function prepare_source_code() {
 
     local podvm_dir="${CAA_SRC_DIR}/podvm"
 
+    mkdir -p "${podvm_dir}"/files
+
     # Download the podvm binaries and copy it to the podvm/files directory
     tar xvf /payload/podvm-binaries.tar.gz -C "${podvm_dir}"/files ||
         error_exit "Failed to download podvm binaries"
 
-    mkdir -p "${podvm_dir}"/files/pause_bundle # workaround to avoid pause image requirement
-
     # Set the NVIDIA_DRIVER_VERSION if variable is set
     if [[ "${NVIDIA_DRIVER_VERSION}" ]]; then
         echo "NVIDIA_DRIVER_VERSION is set to ${NVIDIA_DRIVER_VERSION}"
@@ -183,7 +197,53 @@ function prepare_source_code() {
     fi
 }
 
+# Download and extract pause container image
+# Accepts three arguments:
+# 1. pause_image_repo_url: The registry URL of the OCP pause image.
+# 2. pause_image_tag: The tag of the OCP pause image.
+# 2. auth_json_file (optional): Path to the registry secret file to use for downloading the image
+function download_and_extract_pause_image() {
+
+    # Set default values for the OCP pause image
+    pause_image_repo_url="${1:-${PAUSE_IMAGE_REPO_DEFAULT}}"
+    pause_image_tag="${2:-${PAUSE_IMAGE_VERSION_DEFAULT}}"
+    auth_json_file="${3:-${PAUSE_IMAGE_REPO_AUTH_FILE}}"
+
+    # If arguments are not provided, exit the script with an error message
+    [[ $# -lt 2 ]] &&
+        error_exit "Usage: download_and_extract_pause_image <pause_image_repo_url> <pause_image_tag> [registry_secret]"
+
+    # Ensure CAA_SRC_DIR is set
+    [[ -z "${CAA_SRC_DIR}" ]] && error_exit "CAA_SRC_DIR is not set"
+
+    local podvm_dir="${CAA_SRC_DIR}/podvm"
+    local pause_src="/tmp/pause"
+    local pause_bundle="${podvm_dir}/files/pause_bundle"
+
+    mkdir -p "${pause_bundle}" ||
+        error_exit "Failed to create the pause_bundle directory"
+
+    # Form the skopeo CLI. Add authfile if provided
+    if [[ -n "${3}" ]]; then
+        SKOPEO_CLI="skopeo copy --authfile ${auth_json_file}"
+    else
+        SKOPEO_CLI="skopeo copy"
+    fi
+
+    # Download the pause image
+    $SKOPEO_CLI "docker://${pause_image_repo_url}:${pause_image_tag}" "oci:${pause_src}:${pause_image_tag}" ||
+        error_exit "Failed to download the pause image"
+
+    # Extract the pause image using umoci into pause_bundle directory
+    umoci unpack --rootless --image "${pause_src}:${pause_image_tag}" "${pause_bundle}" ||
+        error_exit "Failed to extract the pause image"
+
+}
+
 # Global variables
 
 # Set global variable for the source code directory
-export CAA_SRC_DIR="/src/cloud-api-adaptor"
+# The project layout has changed for the cloud-api-adaptor project
+# https://github.com/confidential-containers/cloud-api-adaptor
+export CAA_SRC_DOWNLOAD_DIR="/src/cloud-api-adaptor"
+export CAA_SRC_DIR="/src/cloud-api-adaptor/src/cloud-api-adaptor"

config/peerpods/podvm/osc-podvm-create-job.yaml

@@ -53,7 +53,12 @@ spec:
           volumeMounts:
             - name: payload
               mountPath: /payload
+            - name: regauth
+              mountPath: /tmp/regauth
       volumes:
         - name: payload
           emptyDir: {}
+        - name: regauth
+          secret:
+            secretName: auth-json-secret
       restartPolicy: Never

config/peerpods/podvm/podvm-handling.md

@@ -19,6 +19,15 @@ don't want to trigger the pod VM image creation process, then you can set the
 respective keys to any dummy value (eg. "****"). Or you can set it to a
 pre-created pod VM image.
 
+The image creation process also embeds the `rootfs` of the `pause` container
+image into the pod VM image. By default OpenShift's `pause` image is embedded.
+If you want to embed a different `pause` container image, then set the following
+variables in the job configuration files (eg,`aws-podvm-image-cm` for AWS):
+
+- **PAUSE_IMAGE_REPO**: eg. quay.io/myimage/image
+- **PAUSE_IMAGE_VERSION**: eg. latest
+- **PAUSE_IMAGE_REPO_AUTH_FILE**: (optional) path to json file with the registry authentication details
+
 Note that the OSC operator controller doesn't watch for changes to the
 `peer-pods-cm` configMap.  However if the OSC operator reconcile loop is
 entered due to the changes in `kataConfig` or node label changes, then the
@@ -42,7 +51,7 @@ The job's pod specification has an init container (**copy**) which uses the
 (`/podvm-binaries.tar.gz`) from its own filesystem to a shared volume
 (`/payload`). This shared volume is of `emptyDir` type.
 
-The main container (create), uses the `osc-podvm-builder-rhel9:latest` image.
+The main container (**create**), uses the `osc-podvm-builder-rhel9:latest` image.
 This image contains scripts and sources for handling pod VM image creation and
 deletion in Azure and AWS. The container runs as root (`runAsUser: 0`). It also
 mounts the shared volume (`/payload`) to access the file copied by the init
@@ -89,7 +98,7 @@ the pod fails, K8s will retry the job once before marking it as failed
 The job uses `peer-pods-secret` for cloud-provider credentials, and three
 configMaps - `peer-pods-cm`, `azure-podvm-image-cm`, `aws-podvm-image-cm`.
 
-The job's pod specification includes a single container (delete-gallery), which
+The job's pod specification includes a single container (**delete-gallery**), which
 uses the `osc-podvm-builder-rhel9:latest` image. The container runs as root
 (`runAsUser: 0`).
 

controllers/image_generator.go

@@ -308,6 +308,15 @@ func (r *ImageGenerator) createJobFromFile(jobFileName string) (*batchv1.Job, er
 		job.Spec.Template.Spec.Containers[0].Image = podvmBuilderImage
 	}
 
+	// If RELATED_PODVM_PAYLOAD_IMAGE environment variable is set, use it
+	// Otherwise, use the default podvm payload image
+	// There is only one initContainer in the job, so we don't need to check the container name
+	podvmPayloadImage := os.Getenv("RELATED_IMAGE_PODVM_PAYLOAD")
+	if podvmPayloadImage != "" {
+		igLogger.Info("Using podvm payload image from environment variable", "image", podvmPayloadImage)
+		job.Spec.Template.Spec.InitContainers[0].Image = podvmPayloadImage
+	}
+
 	return job, nil
 }
 

controllers/openshift_controller.go

@@ -1202,6 +1202,14 @@ func (r *KataConfigOpenShiftReconciler) processKataConfigInstallRequest() (ctrl.
 
 		// create Pod VM image PeerPodConfig CRD and runtimeclass for peerpods
 		if r.kataConfig.Spec.EnablePeerPods {
+			//Get pull-secret from openshift-config ns and save it as auth-json-secret in our ns
+			//This will be used by the podvm image provider to pull the pause image for embedding
+			err = r.createAuthJsonSecret()
+			if err != nil {
+				r.Log.Info("Error in creating auth-json-secret", "err", err)
+				return ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err
+			}
+
 			// Create the podvm image
 			// Since we want to declaratively reach the final state, we need to reconcile when there are errors
 			// as we want the system to give a chance of fixing the error.
@@ -2237,13 +2245,6 @@ func (r *KataConfigOpenShiftReconciler) enablePeerPodsMiscConfigs() error {
 		return err
 	}
 
-	//Get pull-secret from openshift-config ns and save it as auth-json-secret in our ns
-	err = r.createAuthJsonSecret()
-	if err != nil {
-		r.Log.Info("Error in creating auth-json-secret", "err", err)
-		return err
-	}
-
 	// Create the mutating webhook deployment
 	err = r.createMutatingWebhookDeployment()
 	if err != nil {