fix: verify shasum for finch dependencies (runfinch#969)

Issue #, if available:
Finch dependencies must be verified against known good shasum at pull
time.

*Description of changes:*
This change refactors Finch to use the dependency mechanism in
`finch-core` for pulling and verifying core dependencies such as the OS
image and Lima bundle for macOS and the rootfs archive for Windows
platforms.

As a side effect of this change, dependency updates are now 1-to-1 with
finch-core updates. This is a simplification on the current mechanism
which duplicated the effort for updates.

*Testing done:*
Ran `make` on macOS.

- [x] I've reviewed the guidance in CONTRIBUTING.md

#### License Acceptance

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

Signed-off-by: Austin Vazquez <[email protected]>

Author: austinvazquez

Diff for: .github/bin/update-os-image.sh

@@ -34,18 +34,6 @@ jobs:
         run: |
           git submodule update --remote
 
-      - name: Update OS image
-        run: |
-          ./.github/bin/update-os-image.sh
-
-      - name: Update lima dependencies url
-        run: |
-          ./deps/finch-core/bin/update-deps.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
-
-      - name: Update rootfs
-        run: |
-          ./deps/finch-core/bin/update-rootfs.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
-
       - name: Create PR
         uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
         with:
@@ -55,9 +43,9 @@ jobs:
           body: >
             Bump for Finch's dependencies.
 
-            Currently, this updates values based on artifacts defined in the finch-core repository,
-            such as: operating system images, and the version of Lima (and it's dependencies),
-            which are bundled as `lima-and-qemu.macos-*.*.tar.gz` archives.
+            Updates finch-core submodule which defines the core dependencies for Finch on each respective
+            platform such as: operating system images and Lima bundle (Lima version and it's dependencies)
+            as `lima-and-qemu.macos-*.*.tar.gz` archives.
 
             Since updating the Lima version may also update the default version of the `nerdctl-full`
             archive (which contains nerdctl, containerd, buildkit, etc.), this may require a manual

Diff for: .github/workflows/sync-submodules-and-deps.yaml

@@ -7,20 +7,24 @@ PREFIX ?= $(CURDIR)/_output
 DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::')
 BINDIR ?= /usr/local/bin
 OUTDIR ?= $(CURDIR)/_output
+OS_OUTDIR ?= $(OUTDIR)/os
+
+OUTPUT_DIRECTORIES := $(OUTDIR) $(OS_OUTDIR)
+$(OUTPUT_DIRECTORIES):
+	@mkdir -p $@
+
 PACKAGE := github.com/runfinch/finch
 BINARYNAME := finch
 LIMA_FILENAME := lima
 LIMA_EXTENSION := .tar.gz
 
-LIMA_HOME := $(DEST)/lima/data
 # Created by the CLI after installation, only used in uninstall step
 LIMA_VDE_SUDOERS_FILE := /etc/sudoers.d/finch-lima
 # Final installation prefix for vde created by CLI after installation, only used in uninstall step
 VDE_INSTALL ?= /opt/finch
 UNAME := $(shell uname -m)
 ARCH ?= $(UNAME)
 SUPPORTED_ARCH = false
-CORE_VDE_PREFIX ?= $(OUTDIR)/dependencies/vde/opt/finch
 LICENSEDIR := $(OUTDIR)/license-files
 VERSION := $(shell git describe --match 'v[0-9]*' --dirty='.modified' --always --tags)
 GITCOMMIT := $(shell git rev-parse HEAD)$(shell test -z "$(git status --porcelain)" || echo .m)
@@ -30,9 +34,6 @@ MIN_MACOS_VERSION ?= 11.0
 GOOS ?= $(shell $(GO) env GOOS)
 ifeq ($(GOOS),windows)
 BINARYNAME := $(addsuffix .exe, $(BINARYNAME))
-sha = sha256sum
-else
-sha = shasum -a 256
 endif
 
 .DEFAULT_GOAL := all
@@ -42,129 +43,31 @@ REGISTRY ?= ""
 ifneq (,$(findstring arm64,$(ARCH)))
 	SUPPORTED_ARCH = true
 	LIMA_ARCH = aarch64
-	# From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/aarch64/images/
-	FINCH_OS_BASENAME ?= Fedora-Cloud-Base-40-1.14.aarch64-20240514214641.qcow2
-	LIMA_URL ?= https://deps.runfinch.com/aarch64/lima-and-qemu.macos-aarch64.1715678889.tar.gz
 else ifneq (,$(findstring x86_64,$(ARCH)))
 	SUPPORTED_ARCH = true
 	LIMA_ARCH = x86_64
-	# From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/
-	FINCH_OS_BASENAME ?= Fedora-Cloud-Base-40-1.14.x86_64-20240514214655.qcow2
-	LIMA_URL ?= https://deps.runfinch.com/x86-64/lima-and-qemu.macos-x86_64.1715678889.tar.gz
-	FINCH_ROOTFS_URL ?= https://deps.runfinch.com/common/x86-64/finch-rootfs-production-amd64-1715724303.tar.gz
-	FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_URL))
 endif
 
-FINCH_OS_HASH := `$(sha) $(OUTDIR)/os/$(FINCH_OS_BASENAME) | cut -d ' ' -f 1`
-FINCH_OS_DIGEST := "sha256:$(FINCH_OS_HASH)"
-FINCH_OS_IMAGE_LOCATION_ROOT ?= $(DEST)
-FINCH_OS_IMAGE_LOCATION ?= $(FINCH_OS_IMAGE_LOCATION_ROOT)/os/$(FINCH_OS_BASENAME)
-
-# TODO: Windows PoC extracting rootfs...
-FINCH_ROOTFS_HASH := `$(sha) $(OUTDIR)/os/$(FINCH_ROOTFS_BASENAME) | cut -d ' ' -f 1`
-FINCH_ROOTFS_DIGEST := "sha256:$(FINCH_ROOTFS_HASH)"
-FINCH_ROOTFS_LOCATION_ROOT ?= $(DEST)/
-FINCH_ROOTFS_LOCATION ?= $(FINCH_ROOTFS_LOCATION_ROOT)os/$(FINCH_ROOTFS_BASENAME)
-
 .PHONY: arch-test
 arch-test:
 	@if [ $(SUPPORTED_ARCH) != "true" ]; then echo "Unsupported architecture: $(ARCH)"; exit "1"; fi
 
-.PHONY: all
-ifeq ($(GOOS),windows)
-all: arch-test finch finch-core-local finch.windows.yaml networks.yaml config.yaml
+BUILD_OS ?= $(OS)
+FINCH_CORE_DIR := $(CURDIR)/deps/finch-core
+ifeq ($(BUILD_OS), Windows_NT)
+include Makefile.windows
 else
-all: arch-test finch finch-core finch.yaml networks.yaml config.yaml lima-and-qemu
+include Makefile.darwin
 endif
 
-.PHONY: all-local
-all-local: arch-test networks.yaml config.yaml lima-and-qemu local-core finch.yaml
-
-.PHONY: finch-core
-finch-core:
-	cd deps/finch-core && \
-		FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \
-		VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \
-		"$(MAKE)"
-
-	mkdir -p _output
-	cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
-	rm -rf $(OUTDIR)/lima-template
-
-.PHONY: finch-core-local
-finch-core-local:
-	cd deps/finch-core && \
-		FINCH_OS_x86_URL="$(FINCH_OS_x86_URL)" \
-		FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \
-		VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \
-		"$(MAKE)" all lima
-
-	mkdir -p _output
-	cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
-	rm -rf $(OUTDIR)/lima-template
-
-.PHONY: local-core
-local-core:
-	cd deps/finch-core && \
-		FINCH_OS_x86_URL="$(FINCH_OS_x86_URL)" \
-		FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \
-		VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \
-		"$(MAKE)" lima lima-socket-vmnet
-
-	mkdir -p _output
-	cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
-	cd deps/finch-core/src/lima/_output && tar -cf - * | tar -xvf - -C  $(OUTDIR)/lima
-	cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
-	cd deps/finch-core/src/lima/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)/lima
-	rm -rf $(OUTDIR)/lima-template
-
-.PHONY: lima-and-qemu
-lima-and-qemu: networks.yaml
-	mkdir -p $(OUTDIR)/downloads
-	# download artifacts
-	curl -L $(LIMA_URL) > $(OUTDIR)/downloads/lima-and-qemu.tar.gz
-
-	# Untar LIMA
-	tar -xvf $(OUTDIR)/downloads/lima-and-qemu.tar.gz -C $(OUTDIR)/lima/
-
-	# Delete downloads
-	rm -rf $(OUTDIR)/downloads
-
-
-FINCH_IMAGE_LOCATION ?=
-FINCH_IMAGE_DIGEST ?=
-ifeq ($(GOOS),windows)
-    # Because the path in windows /C:/<some-path> is not an Absolute path, prefix with file:/ which is handled by lima https://github.com/lima-vm/lima/blob/da1260dc87fb30345c3ee7bfb131c29646e26d10/pkg/downloader/downloader.go#L266
-	FINCH_IMAGE_LOCATION := "file:/$(FINCH_ROOTFS_LOCATION)"
-	FINCH_IMAGE_DIGEST := $(FINCH_ROOTFS_DIGEST)
-else
-	FINCH_IMAGE_LOCATION := $(FINCH_OS_IMAGE_LOCATION)
-	FINCH_IMAGE_DIGEST := $(FINCH_OS_DIGEST)
-endif
-.PHONY: finch.yaml
-finch.yaml: finch-core
-	mkdir -p $(OUTDIR)/os
-	# merge the appropriate YAMLs
-	cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' mac.yaml common.yaml > ../finch.yaml
-	cp finch.yaml $(OUTDIR)/os
-	# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
-	# this sed command uses the alternative separator of "|" because the image location uses "/"
-	sed -i.bak -e "s|<finch_image_location>|$(FINCH_IMAGE_LOCATION)|g" $(OUTDIR)/os/finch.yaml
-	sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" $(OUTDIR)/os/finch.yaml
-	sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" $(OUTDIR)/os/finch.yaml
-
-# TODO: Windows PoC - clean this up / consolidate
+all: arch-test finch install.finch-core-dependencies finch.yaml networks.yaml config.yaml
+
+.PHONY: install.finch-core-dependencies
+install.finch-core-dependencies:
+	OUTDIR=$(OUTDIR) "$(MAKE)" -C $(FINCH_CORE_DIR) install.dependencies
+
 .PHONY: finch.yaml
-finch.windows.yaml: finch-core-local
-	mkdir -p $(OUTDIR)/os
-	# merge the appropriate YAMLs
-	cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' windows.yaml common.yaml > ../finch.windows.yaml
-	cp finch.windows.yaml $(OUTDIR)/os/finch.yaml
-	# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
-	# this sed command uses the alternative separator of "|" because the image location uses "/"
-	sed -i.bak -e "s|<finch_image_location>|$(FINCH_IMAGE_LOCATION)|g" $(OUTDIR)/os/finch.yaml
-	sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" $(OUTDIR)/os/finch.yaml
-	sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" $(OUTDIR)/os/finch.yaml
+finch.yaml: $(OS_OUTDIR)/finch.yaml
 
 .PHONY: networks.yaml
 networks.yaml:
@@ -402,7 +305,7 @@ mdlint-ctr:
 ifeq ($(GOOS),windows)
 clean:
 	-@rm -rf $(OUTDIR) 2>/dev/null || true
-	-@rm -rf ./deps/finch-core/_output || true
+	-@"$(MAKE)" -C $(FINCH_CORE_DIR) clean
 	-@rm ./*.tar.gz 2>/dev/null || true
 	-@rm ./*.qcow2 2>/dev/null || true
 	-@rm ./test-coverage.* 2>/dev/null || true
@@ -415,8 +318,7 @@ clean:
 	-sudo rm -rf "/private/var/run/finch-lima"
 	-sudo rm -rf "/private/etc/sudoers.d/finch-lima"
 	-@rm -rf $(OUTDIR) 2>/dev/null || true
-	-@rm -rf ./deps/finch-core/_output || true
-	-@rm -rf ./deps/finch-core/downloads/os/$(FINCH_OS_BASENAME) || true
+	-@$(MAKE) -C $(FINCH_CORE_DIR) clean
 	-@rm ./*.tar.gz 2>/dev/null || true
 	-@rm ./*.qcow2 2>/dev/null || true
 	-@rm ./test-coverage.* 2>/dev/null || true

Diff for: Makefile

@@ -0,0 +1,34 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# For Finch on macOS, the base OS location and digest are set
+# based on the values set in deps/finch-core/deps/full-os.conf
+-include $(FINCH_CORE_DIR)/deps/full-os.conf
+ifeq ($(ARCH),x86_64)
+FINCH_OS_BASENAME=$(X86_64_ARTIFACT)
+FINCH_OS_DIGEST=$(X86_64_512_DIGEST)
+else
+FINCH_OS_BASENAME=$(AARCH64_ARTIFACT)
+FINCH_OS_DIGEST=$(AARCH64_512_DIGEST)
+endif
+
+FINCH_IMAGE_LOCATION:=$(OS_OUTDIR)/$(FINCH_OS_BASENAME)
+FINCH_IMAGE_DIGEST:="sha512:$(FINCH_OS_DIGEST)"
+
+.PHONY: finch.yaml
+finch.yaml: $(OS_OUTDIR)/finch.yaml
+
+$(OS_OUTDIR)/finch.yaml: $(OS_OUTDIR) finch.yaml.d/common.yaml finch.yaml.d/mac.yaml
+	# merge the appropriate YAMLs into a temporary finch.yaml file on the current working directory
+	cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' mac.yaml common.yaml > ../finch.yaml.temp
+
+	# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
+	# this sed command uses the alternative separator of "|" because the image location uses "/"
+	sed -i.bak -e "s|<finch_image_location>|$(FINCH_IMAGE_LOCATION)|g" finch.yaml.temp
+	sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" finch.yaml.temp
+	sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" finch.yaml.temp
+
+	# Replacement was successful, so cleanup .bak
+	@rm finch.yaml.temp.bak
+
+	mv finch.yaml.temp $@

Diff for: Makefile.darwin

@@ -0,0 +1,32 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# For Finch on Windows, the rootfs location and digest are set
+# based on the values in deps/finch-core/deps/rootfs.conf
+-include $(FINCH_CORE_DIR)/deps/rootfs.conf
+ifeq ($(ARCH),x86_64)
+FINCH_ROOTFS_BASENAME ?= $(X86_64_ARTIFACT)
+FINCH_ROOTFS_DIGEST ?= $(X86_64_512_DIGEST)
+else
+$(error Finch on Windows ARM not supported)
+endif
+
+FINCH_IMAGE_LOCATION := $(OS_OUTDIR)/$(FINCH_ROOTFS_BASENAME)
+FINCH_IMAGE_DIGEST := "sha512:$(FINCH_ROOTFS_DIGEST)"
+
+$(OS_OUTDIR)/finch.yaml: $(OS_OUTDIR) finch.yaml.d/common.yaml finch.yaml.d/windows.yaml
+	# merge the appropriate YAMLs into a temporary finch.yaml file on the current working directory
+	cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' windows.yaml common.yaml > ../finch.yaml.temp
+
+	# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
+	# this sed command uses the alternative separator of "|" because the image location uses "/"
+    # Because the path in windows /C:/<some-path> is not an Absolute path, prefix with file:/ which is handled by lima
+	# https://github.com/lima-vm/lima/blob/da1260dc87fb30345c3ee7bfb131c29646e26d10/pkg/downloader/downloader.go#L266
+	sed -i.bak -e "s|<finch_image_location>|file://$(FINCH_IMAGE_LOCATION)|g" finch.yaml.temp
+	sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" finch.yaml.temp
+	sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" finch.yaml.temp
+
+	# Replacement was successful, so cleanup .bak
+	@rm finch.yaml.temp.bak
+
+	mv finch.yaml.temp $@