Skip to content

Commit 6c859e0

Browse files
newhoggynewhoggy
authored
Merge pull request #4717 from input-output-hk/newhoggy/use-cabal-cache
Use cabal-cache
2 parents abeb70b + 0d75329 commit 6c859e0

File tree

2 files changed

+120
-78
lines changed

2 files changed

+120
-78
lines changed

Diff for: ‎.github/workflows/github-page.yml

+43-35
Original file line numberDiff line numberDiff line change
@@ -21,36 +21,41 @@ jobs:
2121
SECP256K1_REF: ac83be33d0956faf6b7f61a60ab524ef7d6a473a
2222

2323
steps:
24-
- uses: actions/checkout@v1
24+
# this seems to break something. It _must_ come after the pacman setup
25+
# above. It appears as if PATHEXT is set _after_ ghcup install ghc/cabal, and
26+
# as such we'd need pacman.exe instead.
27+
- name: Setup Haskell
28+
run: |
29+
# Use GHCUP to manage ghc/cabal
30+
ghcup install ghc --set ${{ matrix.ghc }}
31+
ghcup install cabal --set 3.8.1.0
2532
26-
- name: Set cache version
27-
run: echo "CACHE_VERSION=UN37rUo" >> $GITHUB_ENV
33+
ghc --version
34+
cabal --version
2835
29-
- name: Add build script path
30-
run: echo "$(pwd)/.github/bin" >> $GITHUB_PATH
36+
- name: "Setup cabal-store"
37+
id: cabal-store
38+
shell: bash
39+
run: |
40+
cabal help user-config
41+
cabal_config_file="$(cabal help user-config | grep -A 1 'You can edit the cabal configuration file to set defaults' | tail -n 1 | xargs)"
3142
32-
- uses: haskell/actions/setup@v1
33-
id: setup-haskell
34-
with:
35-
ghc-version: ${{ matrix.ghc }}
36-
cabal-version: 3.6.2.0
43+
echo "cabal-store=$(dirname "$cabal_config_file")/store" | tee -a "$GITHUB_OUTPUT"
3744
38-
- name: Haskell versions
39-
run: |
40-
ghc --version
41-
cabal --version
45+
- uses: actions/checkout@v2
46+
47+
- name: Add build script path
48+
run: echo "$(pwd)/.github/bin" >> $GITHUB_PATH
4249

4350
- name: Install build environment
44-
if: matrix.os == 'ubuntu-latest'
4551
run: |
4652
sudo apt-get update
4753
sudo apt-get -y install libsodium23 libsodium-dev
4854
sudo apt-get -y install libsystemd0 libsystemd-dev
4955
sudo apt-get -y remove --purge software-properties-common
5056
sudo apt-get -y autoremove
5157
52-
- name: "LINUX: Install secp256k1"
53-
if: runner.os != 'Windows'
58+
- name: "Install secp256k1"
5459
shell: bash
5560
env:
5661
CI_SECP_FLAGS: "--prefix=/usr"
@@ -60,13 +65,6 @@ jobs:
6065
- name: Cabal update
6166
run: cabal update
6267

63-
- name: Disable reorder goals on MacOS and Linux
64-
# Avoid reorder goals for platforms that don't need it because re-order goals can take up to 10 minutes
65-
if: matrix.os == 'macos-latest' || matrix.os == 'ubuntu-latest'
66-
run: |
67-
cat cabal.project | sed 's|reorder-goals: True|reorder-goals: False|g' > cabal.project.tmp
68-
mv cabal.project.tmp cabal.project
69-
7068
- name: combine github-pages, and machine local project files.
7169
run: |
7270
cat .github/workflows/cabal.project.local.github-pages \
@@ -79,18 +77,28 @@ jobs:
7977
mkdir ./haddocks
8078
DRY_RUN=1 ./scripts/haddocs.sh ./haddocks true
8179
82-
- name: Record dependencies
83-
run: |
84-
cat ${{ env.PLAN_JSON }} | jq -r '."install-plan"[].id' | sort | uniq > dependencies.txt
85-
86-
- uses: actions/cache@v2
87-
name: Cache cabal store
80+
- name: Cabal cache over S3
81+
uses: action-works/cabal-cache-s3@v1
82+
env:
83+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
84+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
85+
with:
86+
region: us-west-2
87+
dist-dir: dist-newstyle
88+
store-path: ${{ steps.cabal-store.outputs.cabal-store }}
89+
threads: 16
90+
archive-uri: ${{ secrets.BINARY_CACHE_URI }}
91+
skip: "${{ secrets.BINARY_CACHE_URI == '' }}"
92+
93+
- name: Cabal cache over HTTPS
94+
uses: action-works/cabal-cache-s3@v1
8895
with:
89-
path: ${{ steps.setup-haskell.outputs.cabal-store }}
90-
key: cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
91-
restore-keys: |
92-
cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
93-
cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-
96+
dist-dir: dist-newstyle
97+
store-path: ${{ steps.cabal-store.outputs.cabal-store }}
98+
threads: 16
99+
archive-uri: https://iohk.cache.haskellworks.io
100+
skip: "${{ secrets.BINARY_CACHE_URI != '' }}"
101+
enable-save: false
94102

95103
- name: Install dependencies
96104
run: cabal build all

Diff for: ‎.github/workflows/haskell.yml

+77-43
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,6 @@ jobs:
4040
LD_LIBRARY_PATH: ${{ (matrix.os != 'windows-latest' && '/usr/local/lib') || '' }}
4141

4242
steps:
43-
- name: Setup tmate session
44-
if: ${{ failure() }}
45-
uses: mxschmitt/action-tmate@v3
46-
with:
47-
limit-access-to-actor: true
48-
4943
- name: "WIN: Install System Dependencies via pacman (msys2)"
5044
if: runner.os == 'Windows'
5145
run: |
@@ -61,15 +55,15 @@ jobs:
6155
automake `
6256
libtool `
6357
make
64-
58+
6559
# this seems to break something. It _must_ come after the pacman setup
6660
# above. It appears as if PATHEXT is set _after_ ghcup install ghc/cabal, and
6761
# as such we'd need pacman.exe instead.
6862
- name: Setup Haskell
69-
run: |
63+
run: |
7064
# Use GHCUP to manage ghc/cabal
7165
ghcup install ghc --set ${{ matrix.ghc }}
72-
ghcup install cabal --set 3.6.2.0
66+
ghcup install cabal --set 3.8.1.0
7367
7468
ghc --version
7569
cabal --version
@@ -96,19 +90,28 @@ jobs:
9690
-a ("extra-lib-dirs: {0}, C:/msys64/mingw64/lib" -f $ghcMingwDir) `
9791
-f init
9892
99-
- name: "OUTPUT Record cabal-store (Linux)"
100-
id: lin-setup-haskell
101-
if: runner.os != 'Windows'
102-
run: echo "cabal-store=/home/runner/.cabal/store" >> $GITHUB_OUTPUT
103-
104-
- name: "OUTPUT Record cabal-store (Windows)"
105-
id: win-setup-haskell
106-
if: runner.os == 'Windows'
93+
# Unify the computation of the cabal store directory to a single step. This makes referencing the cabal
94+
# store in later steps easier.
95+
#
96+
# We know on cabal store is located in the following locations by OS:
97+
#
98+
# Linux: /home/runner/.cabal/store
99+
# MacOS: /Users/runner/.cabal/store
100+
# Windows: C:\cabal\store
101+
#
102+
# However, we compute these as best be can from information cabal provides in case this changes in the future.
103+
- name: "Setup cabal-store"
104+
id: cabal-store
107105
shell: bash
108-
run: echo "cabal-store=C:\\cabal\\store" >> $GITHUB_OUTPUT
106+
run: |
107+
cabal help user-config
108+
cabal_config_file="$(cabal help user-config | grep -A 1 'You can edit the cabal configuration file to set defaults' | tail -n 1 | xargs)"
109109
110-
- name: Set cache version
111-
run: echo "CACHE_VERSION=grFfw7r" >> $GITHUB_ENV
110+
if [[ '${{ runner.os }}' != 'Windows' ]]; then
111+
echo "cabal-store=$(dirname "$cabal_config_file")/store" | tee -a "$GITHUB_OUTPUT"
112+
else
113+
echo "cabal-store=C:\\cabal\\store" | tee -a "$GITHUB_OUTPUT"
114+
fi
112115
113116
- uses: actions/checkout@v2
114117

@@ -180,36 +183,65 @@ jobs:
180183
echo "# cabal.project.local"
181184
cat cabal.project.local
182185
183-
- name: Record dependencies
184-
id: record-deps
186+
- name: List all pkg-config packages
185187
run: |
186-
cabal build all --dry-run
187-
cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[].id' | sort | uniq > dependencies.txt
188+
# The tests call out to msys2 commands. We generally do not want to mix toolchains, so
189+
# we are very deliberate about only adding msys64 to the path where absolutely necessary.
190+
${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }}
191+
pkg-config --list-all
188192
189-
- name: "OUTPUT Record weeknum"
190-
shell: bash
191-
run: echo "weeknum=$(/usr/bin/date -u "+%W")" >> $GITHUB_OUTPUT
193+
- name: Build dry run
194+
run: |
195+
# The tests call out to msys2 commands. We generally do not want to mix toolchains, so
196+
# we are very deliberate about only adding msys64 to the path where absolutely necessary.
197+
${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }}
198+
cabal build all --dry-run
192199
193-
- name: Cache Cabal store
194-
uses: actions/cache@v2
200+
# For users who fork cardano-node and want to define a writable cache, then can set up their own
201+
# S3 bucket then define in their forked repository settings the following secrets:
202+
#
203+
# AWS_ACCESS_KEY_ID
204+
# AWS_SECRET_ACCESS_KEY
205+
# BINARY_CACHE_URI
206+
# BINARY_CACHE_REGION
207+
- name: Cabal cache over S3
208+
uses: action-works/cabal-cache-s3@v1
209+
env:
210+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
211+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
195212
with:
196-
path: ${{ runner.os == 'Windows' && steps.win-setup-haskell.outputs.cabal-store || steps.lin-setup-haskell.outputs.cabal-store }}
197-
key: cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}-${{ hashFiles('date.txt') }}
198-
restore-keys: |
199-
cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
200-
cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}
201-
202-
- uses: actions/cache@v2
203-
name: "Cache `dist-newstyle`"
213+
region: ${{ secrets.BINARY_CACHE_REGION }}
214+
dist-dir: dist-newstyle
215+
store-path: ${{ steps.cabal-store.outputs.cabal-store }}
216+
threads: 16
217+
archive-uri: ${{ secrets.BINARY_CACHE_URI }}
218+
skip: "${{ secrets.BINARY_CACHE_URI == '' }}"
219+
220+
# It's important to ensure that people who fork this repository can not only successfully build in
221+
# CI by default, but also have meaning cabal store caching.
222+
#
223+
# Because syncing with S3 requires credentials, we cannot rely on S3 for this. For this reason a
224+
# https fallback is used. The https server mirrors the content of the S3 bucket. The https cabal
225+
# store archive is read-only for security reasons.
226+
#
227+
# Users who fork this repository who want to have a writable cabal store archive are encouraged
228+
# to set up their own S3 bucket.
229+
- name: Cabal cache over HTTPS
230+
uses: action-works/cabal-cache-s3@v1
204231
with:
205-
path: |
206-
dist-newstyle
207-
!dist-newstyle/**/.git
208-
key: cache-dist-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ steps.record-deps.outputs.weeknum }}
209-
restore-keys: cache-dist-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}
232+
dist-dir: dist-newstyle
233+
store-path: ${{ steps.cabal-store.outputs.cabal-store }}
234+
threads: 16
235+
archive-uri: https://iohk.cache.haskellworks.io
236+
skip: "${{ secrets.BINARY_CACHE_URI != '' }}"
237+
enable-save: false
210238

211239
- name: Build
212-
run: cabal build cardano-node cardano-cli cardano-node-chairman cardano-submit-api
240+
run: |
241+
# The tests call out to msys2 commands. We generally do not want to mix toolchains, so
242+
# we are very deliberate about only adding msys64 to the path where absolutely necessary.
243+
${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }}
244+
cabal build cardano-node cardano-cli cardano-node-chairman cardano-submit-api
213245
214246
- name: Run tests (all)
215247
if: github.event.inputs.tests == 'all'
@@ -288,6 +320,8 @@ jobs:
288320
# - name: Setup tmate session
289321
# if: ${{ failure() }}
290322
# uses: mxschmitt/action-tmate@v3
323+
# with:
324+
# limit-access-to-actor: true
291325

292326
release:
293327
needs: [build]

0 commit comments

Comments
 (0)