Skip to content

Commit 0b6b5ec

Browse files
Anton-Latukhamergify[bot]
Anton-Latukha
and
mergify[bot]
authored
Caching process update (#2467)
* CI: workflows: disable stack where it is not used Hopefully it would remove it & its files from being cached in the action. & also not includde into environment. * CI: {bench,hackage} add Linux platform config For future caching configuraiton. * CI: {caching,test,bench,hackage}: retrieving index timestamp For 1:1 according cache to it. * CI: {caching,test,bench,hackage}: new caching process Sources are stored separately. Before this cache was storing per Platform/GHC sources & compiled results together. While deps sources does not depend on platform or GHC, they are platform & GHC agnostic. Because sources & compiled results were stored togather & sources took ~1/2-2/3 of that - cache stored multiple duplicates of the same source code. This config stores the source code separately. And at the same time tries to store & share all source code which may be needed. & all compiled deps that would be used in different workflows. * CI: caching: separate source retrieval step & separate build step * CI: {caching,bench}: add executable patching for cache Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
1 parent 07b9310 commit 0b6b5ec

File tree

5 files changed

+156
-45
lines changed

5 files changed

+156
-45
lines changed

Diff for: ‎.github/workflows/bench.yml

+39-8
Original file line numberDiff line numberDiff line change
@@ -47,22 +47,53 @@ jobs:
4747
- run: git fetch origin master # check the master branch for benchmarking
4848

4949
- uses: haskell/actions/setup@v1
50+
id: HaskEnvSetup
5051
with:
5152
ghc-version : ${{ matrix.ghc }}
5253
cabal-version: ${{ matrix.cabal }}
5354
enable-stack: false
5455

55-
- name: Cache Cabal
56+
- name: Linux Platform config
57+
run: |
58+
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
59+
60+
# All workflows which distinquishes cache on `cabal.project` needs this.
61+
- name: Workaround shorten binary names
62+
run: |
63+
sed -i.bak -e 's/haskell-language-server/hls/g' \
64+
-e 's/haskell_language_server/hls/g' \
65+
haskell-language-server.cabal cabal.project
66+
sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
67+
src/**/*.hs exe/*.hs
68+
69+
- name: Retrieving `cabal.project` Hackage timestamp
70+
run: |
71+
# Form: index-state: 2021-11-29T08:11:08Z
72+
INDEX_STATE_ENTRY=$(grep index-state cabal.project)
73+
# Form: 2021-11-29T08-11-08Z
74+
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
75+
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
76+
77+
- name: Hackage sources cache
5678
uses: actions/cache@v2
79+
env:
80+
cache-name: hackage-sources
5781
with:
58-
path: |
59-
~/.cabal/packages
60-
~/.cabal/store
61-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
82+
path: ${{ env.CABAL_PKGS_DIR }}
83+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
84+
restore-keys: ${{ env.cache-name }}-
85+
86+
- name: Compiled deps cache
87+
uses: actions/cache@v2
88+
env:
89+
cache-name: compiled-deps
90+
with:
91+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
92+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
6293
restore-keys: |
63-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
64-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-
65-
v2-${{ runner.os }}-${{ matrix.ghc }}
94+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
95+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
96+
${{ env.cache-name }}-${{ runner.os }}-
6697
6798
- run: cabal update
6899

Diff for: ‎.github/workflows/build.yml

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ jobs:
3333
with:
3434
ghc-version : ${{ matrix.ghc }}
3535
cabal-version: ${{ matrix.cabal }}
36+
enable-stack: false
3637

3738
# some alpines come with integer-simple instead of integer-gmp
3839
- name: Force integer-simple

Diff for: ‎.github/workflows/caching.yml

+59-14
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ on:
3131
branches:
3232
- master
3333

34+
env:
35+
cabalBuild: "v2-build all --enable-tests --enable-benchmarks"
36+
3437
jobs:
3538

3639
pre_job:
@@ -66,19 +69,19 @@ jobs:
6669
- uses: actions/checkout@v2
6770

6871
- uses: haskell/actions/setup@v1
72+
id: HaskEnvSetup
6973
with:
70-
ghc-version: ${{ matrix.ghc }}
74+
ghc-version : ${{ matrix.ghc }}
7175
cabal-version: ${{ matrix.cabal }}
76+
enable-stack: false
7277

7378
- if: runner.os == 'Windows'
7479
name: (Windows) Platform config
7580
run: |
76-
echo "CABAL_STORE_DIR=$SYSTEMDRIVE\\SR" >> $GITHUB_ENV
7781
echo "CABAL_PKGS_DIR=~\\AppData\\cabal\\packages" >> $GITHUB_ENV
7882
- if: ( runner.os == 'Linux' ) || ( runner.os == 'macOS' )
7983
name: (Linux,macOS) Platform config
8084
run: |
81-
echo "CABAL_STORE_DIR=~/.cabal/store" >> $GITHUB_ENV
8285
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
8386
8487
# Needs to be before Cache Cabal so the cache can detect changes to the modified cabal.project file
@@ -92,22 +95,64 @@ jobs:
9295
echo "package floskell" >> cabal.project
9396
echo " ghc-options: -O0" >> cabal.project
9497
95-
- name: Cache Cabal
98+
# Shorten binary names as a workaround for filepath length limits in Windows,
99+
# but since tests are hardcoded on this workaround -
100+
# all platforms (in 2021-12-07) need it.
101+
# All workflows which distinquishes cache on `cabal.project` needs this.
102+
- name: Workaround shorten binary names
103+
run: |
104+
sed -i.bak -e 's/haskell-language-server/hls/g' \
105+
-e 's/haskell_language_server/hls/g' \
106+
haskell-language-server.cabal cabal.project
107+
sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
108+
src/**/*.hs exe/*.hs
109+
110+
- name: Retrieving `cabal.project` Hackage timestamp
111+
run: |
112+
# Form: index-state: 2021-11-29T08:11:08Z
113+
INDEX_STATE_ENTRY=$(grep index-state cabal.project)
114+
# Form: 2021-11-29T08-11-08Z
115+
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
116+
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
117+
118+
# 2021-12-02: NOTE: Cabal Hackage source tree storage does not depend on OS or GHC really,
119+
# but can depend on `base`.
120+
# But this caching is happens only inside `master` for `master` purposes of compiling the deps
121+
# so having a shared pool here that depends only on Hackage pin & does not depend on `base` is "good enough"
122+
# & used such because it preserves 10% of a global cache storage pool.
123+
- name: Hackage sources cache
124+
uses: actions/cache@v2
125+
env:
126+
cache-name: hackage-sources
127+
with:
128+
path: ${{ env.CABAL_PKGS_DIR }}
129+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
130+
restore-keys: ${{ env.cache-name }}-
131+
132+
- name: Compiled deps cache
96133
uses: actions/cache@v2
97134
env:
98-
cache-name: cache-cabal
135+
cache-name: compiled-deps
99136
with:
100-
path: |
101-
${{ env.CABAL_PKGS_DIR }}
102-
${{ env.CABAL_STORE_DIR }}
103-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
137+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
138+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
104139
restore-keys: |
105-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
106-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-
107-
v2-${{ runner.os }}-${{ matrix.ghc }}
140+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
141+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
142+
${{ env.cache-name }}-${{ runner.os }}-
108143
109144
- run: cabal update
110145

146+
- name: Download all sources
147+
run: |
148+
cabal $cabalBuild --only-download
149+
111150
# repeating builds to workaround segfaults in windows and ghc-8.8.4
112-
- name: Build
113-
run: cabal build --only-dependencies || cabal build --only-dependencies || cabal build --only-dependencies
151+
# This build agenda in not to have successful code,
152+
# but to cache what can be cached, so step is fault tolerant & would always succseed.
153+
# 2021-12-11: NOTE: Building all targets, since
154+
# current Cabal does not allow `all --enable-tests --enable-benchmarks --only-dependencies`
155+
- name: Build all targets; try 3 times
156+
continue-on-error: true
157+
run: |
158+
cabal $cabalBuild || cabal $cabalBuild || cabal $cabalBuild

Diff for: ‎.github/workflows/hackage.yml

+30-9
Original file line numberDiff line numberDiff line change
@@ -49,23 +49,44 @@ jobs:
4949
- uses: actions/checkout@v2
5050

5151
- uses: haskell/actions/setup@v1
52+
id: HaskEnvSetup
5253
with:
5354
ghc-version : ${{ matrix.ghc }}
5455
cabal-version: ${{ matrix.cabal }}
56+
enable-stack: false
5557

56-
- name: Cache Cabal
58+
- name: Linux Platform config
59+
run: |
60+
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
61+
62+
- name: Retrieving `cabal.project` Hackage timestamp
63+
run: |
64+
# Form: index-state: 2021-11-29T08:11:08Z
65+
INDEX_STATE_ENTRY=$(grep index-state cabal.project)
66+
# Form: 2021-11-29T08-11-08Z
67+
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
68+
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
69+
70+
- name: Hackage sources cache
71+
uses: actions/cache@v2
72+
env:
73+
cache-name: hackage-sources
74+
with:
75+
path: ${{ env.CABAL_PKGS_DIR }}
76+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
77+
restore-keys: ${{ env.cache-name }}-
78+
79+
- name: Compiled deps cache
5780
uses: actions/cache@v2
5881
env:
59-
cache-name: cache-cabal
82+
cache-name: compiled-deps
6083
with:
61-
path: |
62-
~/.cabal/packages
63-
~/.cabal/store
64-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
84+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
85+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
6586
restore-keys: |
66-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
67-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-
68-
v2-${{ runner.os }}-${{ matrix.ghc }}
87+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
88+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
89+
${{ env.cache-name }}-${{ runner.os }}-
6990
7091
- name: "Run cabal check"
7192
run: |

Diff for: ‎.github/workflows/test.yml

+27-14
Original file line numberDiff line numberDiff line change
@@ -76,19 +76,19 @@ jobs:
7676
- uses: actions/checkout@v2
7777

7878
- uses: haskell/actions/setup@v1
79+
id: HaskEnvSetup
7980
with:
8081
ghc-version : ${{ matrix.ghc }}
8182
cabal-version: ${{ matrix.cabal }}
83+
enable-stack: false
8284

8385
- if: runner.os == 'Windows'
8486
name: (Windows) Platform config
8587
run: |
86-
echo "CABAL_STORE_DIR=$SYSTEMDRIVE\\SR" >> $GITHUB_ENV
8788
echo "CABAL_PKGS_DIR=~\\AppData\\cabal\\packages" >> $GITHUB_ENV
8889
- if: ( runner.os == 'Linux' ) || ( runner.os == 'macOS' )
8990
name: (Linux,macOS) Platform config
9091
run: |
91-
echo "CABAL_STORE_DIR=~/.cabal/store" >> $GITHUB_ENV
9292
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
9393
9494
# Needs to be before Cache Cabal so the cache can detect changes to the modified cabal.project file
@@ -102,9 +102,7 @@ jobs:
102102
echo "package floskell" >> cabal.project
103103
echo " ghc-options: -O0" >> cabal.project
104104
105-
# Shorten binary names as a workaround for filepath length limits in Windows,
106-
# but since tests are hardcoded on this workaround -
107-
# all platforms (in 2021-12-07) need it.
105+
# All workflows which distinquishes cache on `cabal.project` needs this.
108106
- name: Workaround shorten binary names
109107
run: |
110108
sed -i.bak -e 's/haskell-language-server/hls/g' \
@@ -113,19 +111,34 @@ jobs:
113111
sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
114112
src/**/*.hs exe/*.hs
115113
116-
- name: Cache Cabal
114+
- name: Retrieving `cabal.project` Hackage timestamp
115+
run: |
116+
# Form: index-state: 2021-11-29T08:11:08Z
117+
INDEX_STATE_ENTRY=$(grep index-state cabal.project)
118+
# Form: 2021-11-29T08-11-08Z
119+
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
120+
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
121+
122+
- name: Hackage sources cache
123+
uses: actions/cache@v2
124+
env:
125+
cache-name: hackage-sources
126+
with:
127+
path: ${{ env.CABAL_PKGS_DIR }}
128+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
129+
restore-keys: ${{ env.cache-name }}-
130+
131+
- name: Compiled deps cache
117132
uses: actions/cache@v2
118133
env:
119-
cache-name: cache-cabal
134+
cache-name: compiled-deps
120135
with:
121-
path: |
122-
${{ env.CABAL_PKGS_DIR }}
123-
${{ env.CABAL_STORE_DIR }}
124-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
136+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
137+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
125138
restore-keys: |
126-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
127-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-
128-
v2-${{ runner.os }}-${{ matrix.ghc }}
139+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
140+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
141+
${{ env.cache-name }}-${{ runner.os }}-
129142
130143
- run: cabal update
131144

0 commit comments

Comments
 (0)