[antithesis] Add test setup for xsvm (#2982)

Signed-off-by: marun <[email protected]>
Co-authored-by: Stephen Buttolph <[email protected]>

Author: maru-ava

.github/workflows/ci.yml

@@ -265,8 +265,8 @@ jobs:
       - name: Check image build
         shell: bash
         run: bash -x scripts/tests.build_image.sh
-  test_build_antithesis_avalanchego_image:
-    name: Antithesis avalanchego build
+  test_build_antithesis_avalanchego_images:
+    name: Build Antithesis avalanchego images
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -275,6 +275,16 @@ jobs:
         run: bash -x scripts/tests.build_antithesis_images.sh
         env:
           TEST_SETUP: avalanchego
+  test_build_antithesis_xsvm_images:
+    name: Build Antithesis xsvm images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check image build for xsvm test setup
+        shell: bash
+        run: bash -x scripts/tests.build_antithesis_images.sh
+        env:
+          TEST_SETUP: xsvm
   govulncheck:
     runs-on: ubuntu-latest
     name: govulncheck

.github/workflows/publish_antithesis_images.yml

@@ -31,3 +31,10 @@ jobs:
         IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
         TAG: latest
         TEST_SETUP: avalanchego
+
+    - name: Build and push images for xsvm test setup
+      run: bash -x ./scripts/build_antithesis_images.sh
+      env:
+        IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
+        TAG: latest
+        TEST_SETUP: xsvm

scripts/build_antithesis_images.sh

@@ -5,8 +5,10 @@ set -euo pipefail
 # Builds docker images for antithesis testing.
 
 # e.g.,
-# ./scripts/build_antithesis_images.sh                                           # Build local images
-# IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
+# TEST_SETUP=avalanchego ./scripts/build_antithesis_images.sh                                    # Build local images for avalanchego
+# TEST_SETUP=avalanchego NODE_ONLY=1 ./scripts/build_antithesis_images.sh                        # Build only a local node image for avalanchego
+# TEST_SETUP=xsvm ./scripts/build_antithesis_images.sh                                           # Build local images for xsvm
+# TEST_SETUP=xsvm IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
 
 # Directory above this script
 AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
@@ -28,11 +30,13 @@ GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
 function build_images {
   local test_setup=$1
   local uninstrumented_node_dockerfile=$2
+  local image_prefix=$3
+  local node_only=${4:-}
 
   # Define image names
   local base_image_name="antithesis-${test_setup}"
-  if [[ -n "${IMAGE_PREFIX}" ]]; then
-    base_image_name="${IMAGE_PREFIX}/${base_image_name}"
+  if [[ -n "${image_prefix}" ]]; then
+    base_image_name="${image_prefix}/${base_image_name}"
   fi
   local node_image_name="${base_image_name}-node:${TAG}"
   local workload_image_name="${base_image_name}-workload:${TAG}"
@@ -49,22 +53,65 @@ function build_images {
   fi
 
   # Define default build command
-  local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION}"
-  if [[ -n "${IMAGE_PREFIX}" ]]; then
+  local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION} --build-arg NODE_IMAGE=${node_image_name}"
+
+  if [[ "${test_setup}" == "xsvm" ]]; then
+    # The xsvm node image is built on the avalanchego node image, which is assumed to have already been
+    # built. The image name doesn't include the image prefix because it is not intended to be pushed.
+    docker_cmd="${docker_cmd} --build-arg AVALANCHEGO_NODE_IMAGE=antithesis-avalanchego-node:${TAG}"
+  fi
+
+  # Build node image first to allow the workload image to use it.
+  ${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
+  if [[ -n "${image_prefix}" ]]; then
     # Push images with an image prefix since the prefix defines a registry location
     docker_cmd="${docker_cmd} --push"
   fi
 
-  # Build node image first to allow the config and workload image builds to use it.
-  ${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
-  ${docker_cmd} --build-arg NODE_IMAGE="${node_image_name}" -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
-  ${docker_cmd} --build-arg IMAGE_TAG="${TAG}" -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
+  if [[ -n "${node_only}" ]]; then
+    # Skip building the config and workload images. Supports building the avalanchego
+    # node image as the base image for the xsvm node image.
+    return
+  fi
+
+  TARGET_PATH="${AVALANCHE_PATH}/build/antithesis/${test_setup}"
+  if [[ -d "${TARGET_PATH}" ]]; then
+    # Ensure the target path is empty before generating the compose config
+    rm -r "${TARGET_PATH:?}"
+  fi
+
+  # Define the env vars for the compose config generation
+  COMPOSE_ENV="TARGET_PATH=${TARGET_PATH} IMAGE_TAG=${TAG}"
+
+  if [[ "${test_setup}" == "xsvm" ]]; then
+    # Ensure avalanchego and xsvm binaries are available to create an initial db state that includes subnets.
+    "${AVALANCHE_PATH}"/scripts/build.sh
+    "${AVALANCHE_PATH}"/scripts/build_xsvm.sh
+    COMPOSE_ENV="${COMPOSE_ENV} AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVALANCHEGO_PLUGIN_DIR=${HOME}/.avalanchego/plugins"
+  fi
+
+  # Generate compose config for copying into the config image
+  # shellcheck disable=SC2086
+  env ${COMPOSE_ENV} go run "${AVALANCHE_PATH}/tests/antithesis/${test_setup}/gencomposeconfig"
+
+  # Build the config image
+  ${docker_cmd} -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
+
+  # Build the workload image
+  ${docker_cmd} -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
 }
 
 TEST_SETUP="${TEST_SETUP:-}"
 if [[ "${TEST_SETUP}" == "avalanchego" ]]; then
-  build_images avalanchego "${AVALANCHE_PATH}/Dockerfile"
+  build_images avalanchego "${AVALANCHE_PATH}/Dockerfile" "${IMAGE_PREFIX}" "${NODE_ONLY:-}"
+elif [[ "${TEST_SETUP}" == "xsvm" ]]; then
+  # Only build the node image to use as the base for the xsvm image. Provide an empty
+  # image prefix (the 3rd argument) to prevent the image from being pushed
+  NODE_ONLY=1
+  build_images avalanchego "${AVALANCHE_PATH}/Dockerfile" "" "${NODE_ONLY}"
+
+  build_images xsvm "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile" "${IMAGE_PREFIX}"
 else
-  echo "TEST_SETUP must be set. Valid values are 'avalanchego'"
+  echo "TEST_SETUP must be set. Valid values are 'avalanchego' or 'xsvm'"
   exit 255
 fi

scripts/build_antithesis_xsvm_workload.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Directory above this script
+AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
+# Load the constants
+source "$AVALANCHE_PATH"/scripts/constants.sh
+
+echo "Building Workload..."
+go build -o "$AVALANCHE_PATH/build/antithesis-xsvm-workload" "$AVALANCHE_PATH/tests/antithesis/xsvm/"*.go

scripts/tests.build_antithesis_images.sh

@@ -10,6 +10,10 @@ set -euo pipefail
 #   4. Stopping the workload and its target network
 #
 
+# e.g.,
+# TEST_SETUP=avalanchego ./scripts/tests.build_antithesis_images.sh                 # Test build of images for avalanchego test setup
+# DEBUG=1 TEST_SETUP=avalanchego ./scripts/tests.build_antithesis_images.sh         # Retain the temporary compose path for troubleshooting
+
 AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
 
 # Discover the default tag that will be used for the image
@@ -27,6 +31,8 @@ docker create --name "${CONTAINER_NAME}" "${IMAGE_NAME}:${TAG}" /bin/true
 
 # Create a temporary directory to write the compose configuration to
 TMPDIR="$(mktemp -d)"
+echo "using temporary directory ${TMPDIR} as the docker-compose path"
+
 COMPOSE_FILE="${TMPDIR}/docker-compose.yml"
 COMPOSE_CMD="docker-compose -f ${COMPOSE_FILE}"
 
@@ -36,8 +42,10 @@ function cleanup {
   docker rm "${CONTAINER_NAME}"
   echo "stopping and removing the docker compose project"
   ${COMPOSE_CMD} down --volumes
-  echo "removing temporary dir"
-  rm -rf "${TMPDIR}"
+  if [[ -z "${DEBUG:-}" ]]; then
+    echo "removing temporary dir"
+    rm -rf "${TMPDIR}"
+  fi
 }
 trap cleanup EXIT
 
@@ -47,9 +55,10 @@ docker cp "${CONTAINER_NAME}":/docker-compose.yml "${COMPOSE_FILE}"
 # Copy the volume paths out of the container
 docker cp "${CONTAINER_NAME}":/volumes "${TMPDIR}/"
 
-# Run the docker compose project for one minute without error
+# Run the docker compose project for 30 seconds without error. Local
+# network bootstrap is ~6s, but github workers can be much slower.
 ${COMPOSE_CMD} up -d
-sleep 60
+sleep 30
 if ${COMPOSE_CMD} ps -q | xargs docker inspect -f '{{ .State.Status }}' | grep -v 'running'; then
   echo "An error occurred."
   exit 255

tests/antithesis/README.md

@@ -8,11 +8,14 @@ enables discovery and reproduction of anomalous behavior.
 
 ## Package details
 
-| Filename     | Purpose                                                                           |
-|:-------------|:----------------------------------------------------------------------------------|
-| compose.go   | Enables generation of Docker Compose project files for antithesis testing.        |
-| avalanchego/ | Contains resources supporting antithesis testing of avalanchego's primary chains. |
-
+| Filename       | Purpose                                                                            |
+|:---------------|:-----------------------------------------------------------------------------------|
+| compose.go     | Generates Docker Compose project file and initial database for antithesis testing. |
+| config.go      | Defines common flags for the workload binary.                                      |
+| init_db.go     | Initializes initial db state for subnet testing.                                   |
+| node_health.go | Helper to check node health.                                                       |
+| avalanchego/   | Defines an antithesis test setup for avalanchego's primary chains.                 |
+| xsvm/          | Defines an antithesis test setup for the xsvm VM.                                  |
 
 ## Instrumentation
 
@@ -45,3 +48,55 @@ a test setup:
 In addition, github workflows are suggested to ensure
 `scripts/tests.build_antithesis_images.sh` runs against PRs and
 `scripts/build_antithesis_images.sh` runs against pushes.
+
+## Troubleshooting a test setup
+
+### Running a workload directly
+
+The workload of the 'avalanchego' test setup can be invoked against an
+arbitrary network:
+
+```bash
+$ AVAWL_URIS="http://10.0.20.3:9650 http://10.0.20.4:9650" go run ./tests/antithesis/avalanchego
+```
+
+The workload of a subnet test setup like 'xsvm' additionally requires
+a network with a configured chain for the xsvm VM and the ID for that
+chain needs to be provided to the workload:
+
+```bash
+$ AVAWL_URIS=... CHAIN_IDS="2S9ypz...AzMj9" go run ./tests/antithesis/xsvm
+```
+
+### Running a workload with docker-compose
+
+Running the test script for a given test setup with the `DEBUG` flag
+set will avoid cleaning up the the temporary directory where the
+docker-compose setup is written to. This will allow manual invocation of
+docker-compose to see the log output of the workload.
+
+```bash
+$ DEBUG=1 ./scripts/tests.build_antithesis_images.sh
+```
+
+After the test script has terminated, the name of the temporary
+directory will appear in the output of the script:
+
+```
+...
+using temporary directory /tmp/tmp.E6eHdDr4ln as the docker-compose path"
+...
+```
+
+Running compose from the temporary directory will ensure the workload
+output appears on stdout for inspection:
+
+```bash
+$ cd [temporary directory]
+
+# Start the compose project
+$ docker-compose up
+
+# Cleanup the compose project
+$ docker-compose down --volumes
+```

tests/antithesis/avalanchego/Dockerfile.config

@@ -1,29 +1,5 @@
-# The version is supplied as a build argument rather than hard-coded
-# to minimize the cost of version changes.
-ARG GO_VERSION
-
-# ============= Compilation Stage ================
-FROM golang:$GO_VERSION-bullseye AS builder
-
-WORKDIR /build
-# Copy and download avalanche dependencies using go mod
-COPY go.mod .
-COPY go.sum .
-RUN go mod download
-
-# Copy the code into the container
-COPY . .
-
-# IMAGE_TAG should be set to the tag for the images in the generated
-# docker compose file.
-ARG IMAGE_TAG=latest
-
-# Generate docker compose configuration
-RUN TARGET_PATH=./build IMAGE_TAG="$IMAGE_TAG" go run ./tests/antithesis/avalanchego/gencomposeconfig
-
-# ============= Cleanup Stage ================
 FROM scratch AS execution
 
-# Copy the docker compose file and volumes into the container
-COPY --from=builder /build/build/docker-compose.yml /docker-compose.yml
-COPY --from=builder /build/build/volumes /volumes
+# Copy config artifacts from the build path. For simplicity, artifacts
+# are built outside of the docker image.
+COPY  ./build/antithesis/avalanchego/ /

tests/antithesis/avalanchego/Dockerfile.node

@@ -56,6 +56,9 @@ RUN mkdir -p /symbols
 COPY --from=builder /avalanchego_instrumented/symbols /symbols
 COPY --from=builder /opt/antithesis/lib/libvoidstar.so /usr/lib/libvoidstar.so
 
+# Use the same path as the uninstrumented node image for consistency
+WORKDIR /avalanchego/build
+
 # Copy the executable into the container
 COPY --from=builder /avalanchego_instrumented/customer/build/avalanchego ./avalanchego
 

tests/antithesis/avalanchego/main.go

@@ -11,11 +11,11 @@ import (
 	"os"
 	"time"
 
-	"github.com/ava-labs/avalanchego/api/health"
 	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/genesis"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/snow/choices"
+	"github.com/ava-labs/avalanchego/tests/antithesis"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/crypto/secp256k1"
 	"github.com/ava-labs/avalanchego/utils/set"
@@ -38,13 +38,15 @@ import (
 const NumKeys = 5
 
 func main() {
-	c, err := NewConfig(os.Args)
+	c, err := antithesis.NewConfig(os.Args)
 	if err != nil {
 		log.Fatalf("invalid config: %s", err)
 	}
 
 	ctx := context.Background()
-	awaitHealthyNodes(ctx, c.URIs)
+	if err := antithesis.AwaitHealthyNodes(ctx, c.URIs); err != nil {
+		log.Fatalf("failed to await healthy nodes: %s", err)
+	}
 
 	kc := secp256k1fx.NewKeychain(genesis.EWOQKey)
 	walletSyncStartTime := time.Now()
@@ -99,8 +101,7 @@ func main() {
 			},
 		}})
 		if err != nil {
-			log.Printf("failed to issue initial funding X-chain baseTx: %s", err)
-			return
+			log.Fatalf("failed to issue initial funding X-chain baseTx: %s", err)
 		}
 		log.Printf("issued initial funding X-chain baseTx %s in %s", baseTx.ID(), time.Since(baseStartTime))
 
@@ -133,39 +134,6 @@ func main() {
 	genesisWorkload.run(ctx)
 }
 
-func awaitHealthyNodes(ctx context.Context, uris []string) {
-	for _, uri := range uris {
-		awaitHealthyNode(ctx, uri)
-	}
-	log.Println("all nodes reported healthy")
-}
-
-func awaitHealthyNode(ctx context.Context, uri string) {
-	client := health.NewClient(uri)
-	ticker := time.NewTicker(100 * time.Millisecond)
-	defer ticker.Stop()
-
-	log.Printf("awaiting node health at %s", uri)
-	for {
-		res, err := client.Health(ctx, nil)
-		switch {
-		case err != nil:
-			log.Printf("node couldn't be reached at %s", uri)
-		case res.Healthy:
-			log.Printf("node reported healthy at %s", uri)
-			return
-		default:
-			log.Printf("node reported unhealthy at %s", uri)
-		}
-
-		select {
-		case <-ticker.C:
-		case <-ctx.Done():
-			log.Printf("node health check cancelled at %s", uri)
-		}
-	}
-}
-
 type workload struct {
 	id     int
 	wallet primary.Wallet