AGENT-1134: Dev-scripts support to build OVE ISO

- Reuse `make agent` target (instead of introducing a new one)
- Build agent OVE ISO
- Boot up the required VMs
- Set custom rendezvous node (Use AGENT_RENDEZVOUS_NODE_HOSTNAME) and auto-configure rendezvousIP via TUI
- Take TUI screenshots during install flow
- Verify that the assisted install web UI is running and accessible
- Reclaim disk space by cleaning up cache and unnecessary files (Use AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV for local/dev testing)
- Increase VCPU allocation for certain operators requiring more resources
- Install required dependencies
- Temporarily skip common steps not relevant for OVE ISO installs
- Temporarily skip generating unused manifests and config files

To test, set as below

    export AGENT_E2E_TEST_BOOT_MODE=ISO_NO_REGISTRY
    export AGENT_E2E_TEST_SCENARIO=COMPACT_IPV4
    export AGENT_RENDEZVOUS_NODE_HOSTNAME="${CLUSTER_NAME:-ostest}_master_1"
    export AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV=true

    make agent

    ISO will be created at
    ocp/ostest/iso_builder/<OCP_VERSION>/ove/output/agent-ove.x86_64.iso

Author: pawanpinjarkar

Makefile

@@ -46,6 +46,7 @@ agent_remove_extraworker_nodes:
 agent_post_install_validation:
 	./agent/agent_post_install_validation.sh
 
+
 redeploy: ocp_cleanup ironic_cleanup build_installer ironic install_config ocp_run bell
 
 requirements:

agent/01_agent_requirements.sh

@@ -4,13 +4,23 @@ set -o pipefail
 
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
+LOGDIR=${SCRIPTDIR}/logs
+source $SCRIPTDIR/logging.sh
 source $SCRIPTDIR/common.sh
 source $SCRIPTDIR/agent/common.sh
 source $SCRIPTDIR/utils.sh
+source $SCRIPTDIR/ocp_install_env.sh
 source $SCRIPTDIR/validation.sh
 
 early_deploy_validation
 
+function clone_agent_installer_utils() {
+  # Clone repo, if not already present
+  if [[ ! -d $OPENSHIFT_AGENT_INSTALER_UTILS_PATH ]]; then
+    sync_repo_and_patch go/src/github.com/openshift/agent-installer-utils https://github.com/openshift/agent-installer-utils.git
+  fi
+}
+
 if [[ -z ${AGENT_E2E_TEST_SCENARIO} ]]; then
     printf "\nAGENT_E2E_TEST_SCENARIO is missing or empty. Did you forget to set the AGENT_E2E_TEST_SCENARIO env var in the config_<USER>.sh file?"
     invalidAgentValue
@@ -50,3 +60,16 @@ if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISCSI" ]]; then
     # Install shell to administer local storage
     sudo dnf -y install targetcli
 fi
+
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
+   sudo dnf -y install xorriso coreos-installer syslinux skopeo
+    
+   early_deploy_validation
+
+   write_pull_secret
+
+   # Extract an updated client tools from the release image
+   extract_oc "${OPENSHIFT_RELEASE_IMAGE}"
+
+   clone_agent_installer_utils
+fi

agent/03_agent_build_installer.sh

@@ -8,6 +8,11 @@ source $SCRIPTDIR/logging.sh
 source $SCRIPTDIR/common.sh
 source $SCRIPTDIR/agent/common.sh
 
+# Temporarily skip building the agent installer in case of OVE ISO
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
+    exit 0
+fi
+
 # Override build tags
 export OPENSHIFT_INSTALLER_BUILD_TAGS=" "
 

agent/04_agent_prepare_release.sh

@@ -13,6 +13,11 @@ source $SCRIPTDIR/agent/common.sh
 source $SCRIPTDIR/ocp_install_env.sh
 source $SCRIPTDIR/oc_mirror.sh
 
+# Temporarily skip preparing the custom local release in case of OVE ISO
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
+    exit 0
+fi
+
 # To replace an image entry in the openshift release image, set <ENTRYNAME>_LOCAL_REPO so that:
 # - ENTRYNAME matches an uppercase version of the name in the release image with "-" converted to "_" 
 # - The var value must point to an already locally cloned repo

agent/05_agent_configure.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -euxo pipefail
+shopt -s nocasematch
 
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
@@ -14,6 +15,7 @@ source $SCRIPTDIR/agent/common.sh
 source $SCRIPTDIR/oc_mirror.sh
 
 early_deploy_validation
+mkdir -p $OCP_DIR
 
 export CLUSTER_NAMESPACE=${CLUSTER_NAMESPACE:-"cluster0"}
 
@@ -619,10 +621,11 @@ if [[ "${AGENT_PLATFORM_TYPE}" == "external" ]] || [[ "${AGENT_PLATFORM_TYPE}" =
   set_device_mfg worker $NUM_WORKERS ${AGENT_PLATFORM_TYPE} ${AGENT_PLATFORM_NAME}
 fi
 
-generate_cluster_manifests
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" != "ISO_NO_REGISTRY" ]] ; then
+  generate_cluster_manifests
 
-generate_extra_cluster_manifests
-
-write_extra_workers_ips
+  generate_extra_cluster_manifests
 
+  write_extra_workers_ips
+fi
 enable_isolated_baremetal_network

agent/06_agent_create_cluster.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -euxo pipefail
+shopt -s nocasematch
 
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
@@ -12,6 +13,7 @@ source $SCRIPTDIR/validation.sh
 source $SCRIPTDIR/release_info.sh
 source $SCRIPTDIR/agent/common.sh
 source $SCRIPTDIR/agent/iscsi_utils.sh
+source $SCRIPTDIR/agent/e2e/agent-tui/utils.sh
 
 early_deploy_validation
 
@@ -80,6 +82,39 @@ function create_config_image() {
     cp -r ${config_image_dir}/auth ${asset_dir}
 }
 
+function create_agent_iso_no_registry() {
+  # Create agent ISO without registry a.k.a. OVE ISO
+  local asset_dir=${1}
+  pushd .
+  cd $OPENSHIFT_AGENT_INSTALER_UTILS_PATH/tools/iso_builder
+  ./hack/build-ove-image.sh --pull-secret-file "${PULL_SECRET_FILE}" --release-image-url "${OPENSHIFT_RELEASE_IMAGE}" --ssh-key-file "${SSH_KEY_FILE}" --dir "${asset_dir}"
+  popd
+}
+
+# Deletes all files and directories under asset_dir
+# example, ocp/ostest/iso_builder/4.19.* 
+# except the final generated ISO file (agent-ove.x86_64.iso), 
+# to free up disk space while preserving the built artifact.
+# Note: This optional cleanup is relevant only when the
+# AGENT_E2E_TEST_BOOT_MODE is set as as ISO_NO_REGISTRY, 
+# which is supported starting from OCP version 4.19+.
+function cleanup_diskspace_agent_iso_noregistry() {
+ local asset_dir=${1%/}  # Remove trailing slash if present
+
+  # Iterate over all versioned directories matching 4.19.*
+  for dir in "$asset_dir"/4.19.*; do
+    [ -d "$dir" ] || continue
+
+    echo "Cleaning up directory: $dir"
+
+    # Delete all files and symlinks except the agent-ove.x86_64.iso
+    sudo find "$dir" \( -type f -o -type l \) ! -name 'agent-ove.x86_64.iso' -print -delete
+
+    # Remove any empty directories left behind
+    sudo find "$dir" -type d -empty -print -delete
+  done
+}
+
 function set_device_config_image() {
 
     for (( n=0; n<${2}; n++ ))
@@ -152,6 +187,52 @@ function attach_appliance_diskimage() {
     done
 }
 
+function attach_agent_iso_no_registry() {
+    set_file_acl
+
+    local base_dir=$SCRIPTDIR/$OCP_DIR
+    local iso_name="agent-ove.$(uname -p).iso"
+    local agent_iso_no_registry=$(find "$base_dir" -type f -name "$iso_name" 2>/dev/null | head -n 1)
+
+    for (( n=0; n<${2}; n++ ))
+    do
+        name=${CLUSTER_NAME}_${1}_${n}
+        sudo virt-xml ${name} --add-device --disk "${agent_iso_no_registry}",device=cdrom,target.dev=sdc
+        sudo virt-xml ${name} --edit target=sda --disk="boot_order=1"
+        sudo virt-xml ${name} --edit target=sdc --disk="boot_order=2" --start
+    done
+}
+
+function automate_rendezvousIP_selection(){
+  for (( n=0; n<${2}; n++ ))
+    do
+        name=${CLUSTER_NAME}_${1}_${n}
+        # Take screenshots of console before running the automation that configures the rendezvousIP. 
+        # The screenshot may help us see if agent-tui has reached the expected success state.
+        sudo virsh screenshot $name "${OCP_DIR}/${name}_console_screenshot_before_automation_configures_rendezvousIP.ppm"
+
+        ./agent/e2e/agent-tui/automate-no-registry-agent-tui.sh $name
+
+         # Take screenshot of the console after running the automation that configures the rendezvousIP.
+         sudo virsh screenshot $name "${OCP_DIR}/${name}_console_screenshot_after_automation_configures_rendezvousIP.ppm"
+        echo "Finished configuring the rendezvousIP via agent-tui for $name"
+    done
+}
+
+function check_assisted_install_UI(){
+  local rendezvousIP=$(getRendezvousIP)
+  local url="http://$rendezvousIP:3001"
+  while true; do
+    if curl -s -o /dev/null -w "%{http_code}" "$url" | grep -q "^200$"; then
+      echo "Assisted install UI is up: $url"
+      break
+    else
+      echo "Assisted install UI not ready, retrying in 30 seconds..."
+      sleep 30
+    fi
+  done
+}
+
 function get_node0_ip() {
   node0_name=$(printf ${MASTER_HOSTNAME_FORMAT} 0)
   node0_ip=$(sudo virsh net-dumpxml ostestbm | xmllint --xpath "string(//dns[*]/host/hostname[. = '${node0_name}']/../@ip)" -)
@@ -435,9 +516,11 @@ function put_operator_file() {
   ssh "${ssh_opts[@]}" "sudo cp /home/core/operators.yaml /etc/assisted/manifests/."
 }
 
-asset_dir="${1:-${OCP_DIR}}"
-config_image_dir="${1:-${OCP_DIR}/configimage}"
-openshift_install="$(realpath "${OCP_DIR}/openshift-install")"
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" != "ISO_NO_REGISTRY" ]]; then
+  asset_dir="${1:-${OCP_DIR}}"
+  config_image_dir="${1:-${OCP_DIR}/configimage}"
+  openshift_install="$(realpath "${OCP_DIR}/openshift-install")"
+fi
 
 case "${AGENT_E2E_TEST_BOOT_MODE}" in
   "ISO" )
@@ -499,8 +582,38 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
     # (to avoid storage overconsumption on the CI machine)
     sudo rm -f "${OCP_DIR}/appliance.raw"
     ;;
+  "ISO_NO_REGISTRY" )
+    # Build an (OVE) image which does not need registry setup 
+    # Run a script from agent-installer-utils which internally uses openshift-appliance
+    asset_dir=$SCRIPTDIR/$OCP_DIR/iso_builder
+    mkdir -p ${asset_dir}
+    create_agent_iso_no_registry ${asset_dir}
+
+    if [[ "$AGENT_CLEANUP_ISO_BUILDER_CACHE_LOCAL_DEV" == "true" ]]; then
+      # reclaim disk space by deleting unwanted cache, other files
+      cleanup_diskspace_agent_iso_noregistry ${asset_dir}
+    fi
+
+    attach_agent_iso_no_registry master $NUM_MASTERS
+    attach_agent_iso_no_registry worker $NUM_WORKERS
+
+    echo "Waiting for 2 mins to arrive at agent-tui screen"
+    sleep 120
+    automate_rendezvousIP_selection master $NUM_MASTERS
+    automate_rendezvousIP_selection worker $NUM_WORKERS
+
+    check_assisted_install_UI
+    ;;
 esac
 
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
+    # Current goal is to only verify if the nodes are booted fine,
+    # TUI sets the rendezvous IP correctly and UI is accessible.
+    # The next goal is to simulate adding the cluster details via UI
+    # and complete the cluster installation.
+    exit 0
+fi
+
 if [ ! -z "${AGENT_TEST_CASES:-}" ]; then
   run_agent_test_cases
 fi
@@ -528,14 +641,14 @@ if [[ ! -z $AGENT_OPERATORS ]]; then
     put_operator_file
 fi
 
+
 wait_for_cluster_ready
 
 if [ ! -z "${AGENT_DEPLOY_MCE}" ]; then
   mce_complete_deployment
 fi
 
 # e2e test configuration
-
 # Configure storage for the image registry
 oc patch configs.imageregistry.operator.openshift.io \
     cluster --type merge --patch '{"spec":{"storage":{"emptyDir":{}},"managementState":"Managed"}}'

agent/agent_post_install_validation.sh

@@ -5,6 +5,11 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
 source $SCRIPTDIR/common.sh
 
+# Temp code skip the execution flow as cluster is not really installed
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
+  exit 0
+fi
+
 installed_control_plane_nodes=$(oc get nodes --selector=node-role.kubernetes.io/master | grep -v AGE | wc -l)
 
 oc get nodes

agent/cleanup.sh

@@ -29,6 +29,10 @@ if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "DISKIMAGE" ]]; then
     sudo podman rmi -f ${APPLIANCE_IMAGE} || true
 fi
 
+if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
+    sudo rm -rf "${OCP_DIR}/iso_builder"
+fi
+
 if [[ -d ${BOOT_SERVER_DIR} ]]; then
    sudo pkill agentpxeserver || true
    rm -rf ${BOOT_SERVER_DIR}

agent/common.sh

@@ -73,3 +73,9 @@ export AGENT_NODE0_IPSV6=${AGENT_NODE0_IPSV6:-}
 
 # Modifies the baremetal network to be fully isolated.
 export AGENT_ISOLATED_NETWORK=${AGENT_ISOLATED_NETWORK:-"false"}
+
+# Set isolated network to true for truely disconnected OVE env
+# in case of agent ISO with no registry 
+if [ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ] ; then
+    export AGENT_ISOLATED_NETWORK=true
+fi

agent/e2e/agent-tui/automate-no-registry-agent-tui.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../../" && pwd )"
+source $SCRIPTDIR/common.sh
+source $SCRIPTDIR/agent/e2e/agent-tui/utils.sh
+
+set +x
+shopt -s nocasematch
+
+node_name=$1
+
+# The following commands sends key presses through "virsh send-key" to interact
+# with agent-tui
+if [[ "$node_name" == "${AGENT_RENDEZVOUS_NODE_HOSTNAME}" ]]; then
+    pressDown "Save Rendezvous IP" 1 "$node_name"
+    pressDown "This is the rendezvous node" 1 "$node_name"
+    pressEnter "This is the rendezvous node" "" "$node_name"
+    pressEnter "" "" "$node_name"
+    pressEnter "Continue" "" "$node_name"
+else
+    # Retrieves the rendezvousIP and automatically inputs it into the TUI text field
+    # on all other nodes to ensure proper cluster joining.
+    rendezvousIP=$(getRendezvousIP)
+
+    pressKeys "Entering IP address" "$rendezvousIP" "$node_name"
+    pressDown "Save Rendezvous IP" 1 "$node_name"
+    pressEnter "" "" "$node_name"
+    pressEnter "Save and Continue" "" "$node_name"
+fi

agent/e2e/agent-tui/test-fix-wrong-dns.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )"
-
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../../" && pwd )"
 source $SCRIPTDIR/common.sh
 source $SCRIPTDIR/agent/e2e/agent-tui/utils.sh
+
 set +x
 
 GOOD_DNS_IP=$1