Skip to content

Commit 985570e

Browse files
pendo324pendo324
authored
build(linux): add Linux packaging and e2e tests (runfinch#1071)
build(linux): add Linux packaging and e2e tests (runfinch#1071) - Add packaging and e2e runners which use the packages to run tests - Before merging this, we need to: - add linux runners to this repository (by merging runfinch/infrastructure#683) - remove the change in this PR which makes it so CI runs on all branches, not just main - uncomment the disabling of the macOS/Windows runners Signed-off-by: Justin Alvarez <[email protected]>
1 parent c382e22 commit 985570e

29 files changed

+920
-169
lines changed

Diff for: ‎.github/workflows/ci-docs.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ on:
2222
paths:
2323
- '**.md'
2424
- 'contrib/**'
25+
- '!contrib/packaging/**'
2526
- '.github/CODEOWNERS'
2627

2728
jobs:

Diff for: ‎.github/workflows/ci.yaml

+49-143
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,18 @@ on:
66
branches:
77
- main
88
paths-ignore:
9-
- '**.md'
10-
- 'contrib/**'
11-
- '.github/CODEOWNERS'
9+
- "**.md"
10+
- "contrib/**"
11+
- ".github/CODEOWNERS"
1212
pull_request:
1313
branches:
1414
- main
15-
paths-ignore:
16-
- '**.md'
17-
- 'contrib/**'
18-
- '.github/CODEOWNERS'
15+
paths:
16+
- "**.go"
17+
- "contrib/packaging/**"
18+
- "!contrib/hello-finch/**"
19+
- "!.github/CODEOWNERS"
20+
workflow_dispatch:
1921
permissions:
2022
id-token: write
2123
contents: write
@@ -141,154 +143,58 @@ jobs:
141143
go-version-file: go.mod
142144
cache: true
143145
- run: make check-licenses
144-
e2e-tests:
146+
macos-e2e-tests:
145147
strategy:
146148
fail-fast: false
147149
matrix:
148-
os:
149-
[
150-
[self-hosted, macos, amd64, 13, test],
151-
[self-hosted, macos, amd64, 14, test],
152-
[self-hosted, macos, arm64, 13, test],
153-
[self-hosted, macos, arm64, 14, test],
154-
]
155-
test-command: ['test-e2e-vm-serial', 'test-e2e-container']
156-
runs-on: ${{ matrix.os }}
157-
steps:
158-
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
159-
with:
160-
# We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
161-
fetch-depth: 0
162-
persist-credentials: false
163-
submodules: recursive
164-
- uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
165-
with:
166-
go-version-file: go.mod
167-
cache: true
168-
- name: Set output variables
169-
id: vars
170-
run: |
171-
has_creds=${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]' }}
172-
echo "has_creds=$has_creds" >> $GITHUB_OUTPUT
173-
- name: configure aws credentials
174-
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
175-
if: ${{ steps.vars.outputs.has_creds == true }}
176-
with:
177-
role-to-assume: ${{ secrets.ROLE }}
178-
role-session-name: credhelper-test
179-
aws-region: ${{ secrets.REGION }}
180-
- name: Clean up previous files
181-
run: |
182-
sudo rm -rf /opt/finch
183-
sudo rm -rf ~/.finch
184-
sudo rm -rf ./_output
185-
if pgrep '^qemu-system'; then
186-
sudo pkill '^qemu-system'
187-
fi
188-
if pgrep '^socket_vmnet'; then
189-
sudo pkill '^socket_vmnet'
190-
fi
191-
- name: Install Rosetta 2
192-
run: echo "A" | softwareupdate --install-rosetta || true
193-
- run: brew install lz4 automake autoconf libtool yq
194-
shell: zsh {0}
195-
- name: Build project
196-
run: |
197-
export PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH"
198-
make
199-
shell: zsh {0}
200-
- run: |
201-
git status
202-
git clean -f -d
203-
REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make ${{ matrix.test-command }}
204-
shell: zsh {0}
150+
version: ["13", "14"]
151+
test-command: ["test-e2e-vm-serial", "test-e2e-container"]
152+
arch: ["X64", "arm64"]
153+
runner-type: ["test"]
154+
uses: ./.github/workflows/e2e-macos.yaml
155+
secrets: inherit
156+
with:
157+
arch: ${{ matrix.arch }}
158+
version: ${{ matrix.version }}
159+
runner-type: ${{ matrix.runner-type }}
160+
test-command: ${{ matrix.test-command }}
205161
windows-e2e-tests:
206162
strategy:
207163
fail-fast: false
208164
matrix:
209-
os: [[self-hosted, windows, amd64, test]]
210-
test-command: ['test-e2e-vm-serial', 'test-e2e-container']
211-
runs-on: ${{ matrix.os }}
212-
timeout-minutes: 180
213-
steps:
214-
- name: Configure git CRLF settings
215-
run: |
216-
git config --global core.autocrlf false
217-
git config --global core.eol lf
218-
- name: Cleanup previous checkouts
219-
run: |
220-
takeown /F C:\actions-runner\_work\finch /R
221-
Remove-Item C:\actions-runner\_work\finch\finch -Recurse -Force -ErrorAction Ignore
222-
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
223-
with:
224-
# We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
225-
fetch-depth: 0
226-
persist-credentials: false
227-
submodules: recursive
228-
- name: Set output variables
229-
id: vars
230-
run: |
231-
$has_creds="${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]'}}"
232-
echo "has_creds=$has_creds" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
233-
exit 0 # if $has_creds is false, powershell will exit with code 1 and this step will fail
234-
- name: configure aws credentials
235-
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
236-
if: env.has_creds == 'true'
237-
with:
238-
role-to-assume: ${{ secrets.ROLE }}
239-
role-session-name: credhelper-test
240-
aws-region: ${{ secrets.REGION }}
241-
- name: Remove Finch VM
242-
run: |
243-
# We want these cleanup commands to always run, ignore errors so the step completes.
244-
$ErrorActionPreference = 'Ignore'
245-
taskkill /f /im wslservice.exe 2> nul || cmd /c "exit /b 0"
246-
wsl --list --verbose
247-
wsl --shutdown
248-
wsl --unregister lima-finch
249-
wsl --list --verbose
250-
- name: Clean up previous files
251-
run: |
252-
Remove-Item C:\Users\Administrator\.finch -Recurse -ErrorAction Ignore
253-
Remove-Item C:\Users\Administrator\AppData\Local\.finch -Recurse -ErrorAction Ignore
254-
make clean
255-
cd deps/finch-core && make clean
256-
- name: Build project
257-
run: |
258-
git status
259-
make
260-
- name: Run e2e tests
261-
run: |
262-
# set path to use newer ssh version
263-
$newPath = (";C:\Program Files\Git\bin\;" + "C:\Program Files\Git\usr\bin\;" + "$env:Path")
264-
$env:Path = $newPath
265-
266-
# set networking config option to allow for VM/container -> host communication
267-
echo "[experimental]`nnetworkingMode=mirrored`nhostAddressLoopback=true" > C:\Users\Administrator\.wslconfig
165+
test-command: ["test-e2e-vm-serial", "test-e2e-container"]
166+
arch: ["amd64"]
167+
runner-type: ["test"]
168+
uses: ./.github/workflows/e2e-windows.yaml
169+
secrets: inherit
170+
with:
171+
arch: ${{ matrix.arch }}
172+
runner-type: ${{ matrix.runner-type }}
173+
test-command: ${{ matrix.test-command }}
174+
linux-e2e-tests:
175+
strategy:
176+
fail-fast: false
177+
matrix:
178+
os: ["amazonlinux"]
179+
arch: ["X64", "arm64"]
180+
version: ["2023", "2"]
181+
test-command: ["test-e2e-container"]
182+
runner-type: ["test"]
183+
uses: ./.github/workflows/e2e-linux.yaml
184+
secrets: inherit
185+
with:
186+
os: ${{ matrix.os }}
187+
arch: ${{ matrix.arch }}
188+
version: ${{ matrix.version }}
189+
runner-type: ${{ matrix.runner-type }}
190+
test-command: ${{ matrix.test-command }}
268191

269-
git status
270-
git clean -f -d
271-
make ${{ matrix.test-command }}
272-
- name: Remove Finch VM and Clean Up Previous Environment
273-
if: ${{ always() }}
274-
run: |
275-
# We want these cleanup commands to always run, ignore errors so the step completes.
276-
$ErrorActionPreference = 'Ignore'
277-
taskkill /f /im wslservice.exe 2> nul || cmd /c "exit /b 0"
278-
wsl --list --verbose
279-
wsl --shutdown
280-
wsl --unregister lima-finch
281-
wsl --list --verbose
282-
Remove-Item C:\Users\Administrator\AppData\Local\.finch -Recurse
283-
make clean
284-
cd deps/finch-core && make clean
285-
exit 0 # Cleanup may set the exit code e.g. if a file doesn't exist; just ignore
286192
mdlint:
287193
runs-on: ubuntu-latest
288194
steps:
289195
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
290196
- uses: avto-dev/markdown-lint@04d43ee9191307b50935a753da3b775ab695eceb # v1.5.0
291197
with:
292-
args: '**/*.md'
198+
args: "**/*.md"
293199
# CHANGELOG.md is only updated by release-please bot.
294-
ignore: 'CHANGELOG.md'
200+
ignore: "CHANGELOG.md"

Diff for: ‎.github/workflows/e2e-linux.yaml

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: e2e-linux
2+
on:
3+
workflow_call:
4+
inputs:
5+
os:
6+
type: string
7+
required: true
8+
arch:
9+
type: string
10+
required: true
11+
version:
12+
type: string
13+
required: true
14+
runner-type:
15+
type: string
16+
required: true
17+
test-command:
18+
type: string
19+
required: true
20+
21+
permissions:
22+
# This is required for configure-aws-credentials to request an OIDC JWT ID token to access AWS resources later on.
23+
# More info: https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
24+
id-token: write
25+
# This is required for actions/checkout
26+
contents: read
27+
28+
jobs:
29+
test:
30+
runs-on:
31+
[
32+
"self-hosted",
33+
"${{ inputs.os }}",
34+
"${{ inputs.arch }}",
35+
"${{ inputs.version }}",
36+
"${{ inputs.runner-type }}",
37+
]
38+
steps:
39+
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
40+
# checkout uses node version 20, which doesn't run on AL2
41+
if: ${{ ! (startsWith(inputs.os, 'amazon') && inputs.version == '2' ) }}
42+
with:
43+
# We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
44+
fetch-depth: 0
45+
persist-credentials: false
46+
submodules: recursive
47+
- name: Check repo out manually
48+
if: ${{ (startsWith(inputs.os, 'amazon') && inputs.version == '2' ) }}
49+
run: |
50+
git clone https://github.com/${GITHUB_REPOSITORY}.git .
51+
git config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/prh/*"
52+
git config --add remote.origin.fetch "+refs/pull/*/merge:refs/remotes/origin/prm/*"
53+
git fetch origin
54+
git checkout ${GITHUB_SHA}
55+
- name: Set output variables
56+
id: vars
57+
run: |
58+
has_creds=${{ (github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name) && github.actor != 'dependabot[bot]' }}
59+
echo "has_creds=$has_creds" >> $GITHUB_OUTPUT
60+
- name: configure aws credentials
61+
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
62+
# this action requires node20, skip on AL2
63+
if: ${{ steps.vars.outputs.has_creds == true && (!(startsWith(inputs.os, 'amazon') && inputs.version == '2' ))}}
64+
with:
65+
role-to-assume: ${{ secrets.ROLE }}
66+
role-session-name: credhelper-test
67+
aws-region: ${{ secrets.REGION }}
68+
- name: Clean up previous files
69+
if: always()
70+
run: |
71+
sudo systemctl stop finch.service || true
72+
sudo systemctl stop finch-buildkit.service || true
73+
sudo systemctl stop finch-soci.service || true
74+
sudo sudo rpm -e runfinch-finch || true
75+
sudo systemctl stop containerd.service
76+
sudo systemctl daemon-reload
77+
sudo rm -rf /etc/finch
78+
sudo rm -rf /var/lib/finch
79+
sudo rm -rf /var/lib/containerd
80+
sudo rm -rf /var/soci-snapshotter
81+
sudo rm -rf ./_output
82+
- name: Build project
83+
run: |
84+
./contrib/packaging/rpm/build.sh --local
85+
- name: Install Finch
86+
run: |
87+
sudo rpm -i ./_output/packages/$(ls -t ./_output/packages/ | grep runfinch-finch | head -1)
88+
sudo systemctl daemon-reload
89+
sudo systemctl start containerd.service
90+
sudo systemctl start finch.service
91+
sudo systemctl start finch-buildkit.service
92+
sudo systemctl start finch-soci.service
93+
- name: Run e2e tests
94+
run: |
95+
git status
96+
git clean -f -d
97+
# required by one of the tests which uses SSH_AUTH_SOCK
98+
eval "$(ssh-agent -s)"
99+
INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} sudo -E make ${{ inputs.test-command }}
100+
- name: Clean up repo AL2
101+
if: ${{ (startsWith(inputs.os, 'amazon') && inputs.version == '2' && always() ) }}
102+
run: |
103+
rm -rf "${GITHUB_WORKSPACE}"

0 commit comments

Comments
 (0)