Boost CI speed (#6842)

cometkim · web-flow · commit 2a0f3bae3698 · 2024-06-30T21:36:35.000+02:00
* poc: cache opam actions * it should be a dirname * debug opam-path * then work? * why * should work * almost there? * let me just sleep please * linux system dependencies * caches * install jsoo with cache * cache rewatch build too * revert some * dedupe deps declaration, this should make new cacahe * use bash instead of powershell * fix Windows * invalidate env cache * fix cache lifecycle * fix windows * fix windows cache * poc: use build image via jobs.container * Revert "poc: use build image via jobs.container" This reverts commit 4e4277e. The custom container feature seems to have some compatibility issues and doesn't provide any additional optimization anyway * remove unnecessary setup-python * fix a wrong path * what was happened * fine
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -54,22 +54,32 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Restore build cache
+        id: build-cache
+        uses: actions/cache@v4
+        with:
+          path: rewatch/target
+          key: rewatch-build-${{ matrix.rust-target }}-${{ hashFiles('rewatch/src/**', 'rewatch/Cargo.lock') }}
+
       - name: Install musl gcc
-        if: runner.os == 'Linux'
-        run: sudo apt-get install -y musl-tools
+        if: steps.build-cache.outputs.cache-hit != 'true' && runner.os == 'Linux'
+        run: sudo apt-get install -y --no-install-recommends musl-tools
 
       - name: Set up sccache
+        if: steps.build-cache.outputs.cache-hit != 'true'
         uses: mozilla-actions/sccache-action@v0.0.4
         with:
           version: "v0.8.0"
 
       - name: Install rust toolchain
+        if: steps.build-cache.outputs.cache-hit != 'true'
         uses: dtolnay/rust-toolchain@master
         with:
           toolchain: stable
           targets: ${{matrix.rust-target}}
 
       - name: Build rewatch
+        if: steps.build-cache.outputs.cache-hit != 'true'
         run: cargo build --manifest-path rewatch/Cargo.toml --target ${{matrix.rust-target}} --release
 
       - name: Get artifact dir name
@@ -106,7 +116,7 @@ jobs:
       - name: Build ninja binary
         uses: docker://ghcr.io/rescript-lang/rescript-ci-build:alpine-3.20-ocaml-5.2.0-01
         with:
-          args: sh -c "cd ninja && LDFLAGS=-static python3 configure.py --bootstrap"
+          args: sh -c "cd ninja && LDFLAGS=-static python configure.py --bootstrap"
 
       - name: "Upload artifacts"
         uses: actions/upload-artifact@v4
@@ -213,17 +223,44 @@ jobs:
           chmod +x rewatch/rewatch
           chmod +x _build/install/default/bin/*
 
+      - name: Install dependencies (Linux)
+        if: runner.os == 'Linux'
+        uses: awalsh128/cache-apt-pkgs-action@v1.4.2
+        with:
+          # See https://github.com/ocaml/setup-ocaml/blob/b2105f9/packages/setup-ocaml/src/unix.ts#L9
+          packages: bubblewrap darcs g++-multilib gcc-multilib mercurial musl-tools rsync
+          version: v1
+
+      - name: Setup Python for ninja build
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Restore OPAM environment
+        id: cache-opam-env
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ runner.tool_cache }}/opam
+            ~/.opam
+            _opam
+            .opam-path
+            .opam-env
+            D:\cygwin
+            D:\.opam
+          key: opam-env-v8-${{ matrix.os }}-${{ hashFiles('dune-project') }}
+
       - name: Use OCaml ${{matrix.ocaml_compiler}}
         uses: ocaml/setup-ocaml@v2
-        if: matrix.os != 'windows-latest'
+        if: steps.cache-opam-env.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
         with:
           ocaml-compiler: ${{matrix.ocaml_compiler}}
           opam-pin: false
           opam-depext: false
 
-      - name: Use OCaml ${{matrix.ocaml_compiler}} (Win)
+      - name: Use OCaml ${{matrix.ocaml_compiler}} (Windows)
         uses: ocaml/setup-ocaml@v2
-        if: matrix.os == 'windows-latest'
+        if: steps.cache-opam-env.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
         with:
           ocaml-compiler: ocaml-variants.5.2.0+options,ocaml-option-mingw
           opam-pin: false
@@ -233,10 +270,74 @@ jobs:
             sunset: https://github.com/ocaml-opam/opam-repository-mingw.git#sunset
             default: https://github.com/ocaml/opam-repository.git
 
-      - name: "Install OPAM dependencies"
+      - name: Get OPAM environment
+        if: steps.cache-opam-env.outputs.cache-hit != 'true'
+        run: |
+          command -v opam | tee .opam-path
+          opam env > .opam-env
+
+      - name: Install OPAM dependencies
+        if: steps.cache-opam-env.outputs.cache-hit != 'true'
         run: opam install . --deps-only
 
-      - name: "Build compiler"
+      - name: Cache OPAM environment
+        if: steps.cache-opam-env.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ${{ runner.tool_cache }}/opam
+            ~/.opam
+            _opam
+            .opam-path
+            .opam-env
+            D:\cygwin
+            D:\.opam
+          key: opam-env-v8-${{ matrix.os }}-${{ hashFiles('dune-project') }}
+
+      - name: Use cached OPAM environment
+        if: steps.cache-opam-env.outputs.cache-hit == 'true'
+        run: |
+          # https://github.com/ocaml/setup-ocaml/blob/b2105f9/packages/setup-ocaml/src/installer.ts#L33
+          echo "OPAMVERBOSE=$RUNNER_DEBUG"   >> "$GITHUB_ENV"
+          echo "OPAMCOLOR=always"            >> "$GITHUB_ENV"
+          echo "OPAMCONFIRMLEVEL=unsafe-yes" >> "$GITHUB_ENV"
+          echo "OPAMERRLOGLEN=0"             >> "$GITHUB_ENV"
+          echo "OPAMPRECISETRACKING=1"       >> "$GITHUB_ENV"
+          echo "OPAMYES=1"                   >> "$GITHUB_ENV"
+
+          if [[ "$RUNNER_OS" != "Windows" ]]; then
+            echo "OPAMROOT=$HOME/.opam"      >> "$GITHUB_ENV"
+          else
+            echo "OPAMROOT=D:\\.opam"         >> "$GITHUB_ENV"
+          fi
+
+          if [[ "$RUNNER_OS" != "Windows" ]]; then
+            OPAM_PATH="$(cat .opam-path)"
+            chmod +x "$OPAM_PATH"
+            dirname "$OPAM_PATH" >> "$GITHUB_PATH"
+
+          else
+            fsutil behavior query SymlinkEvaluation
+            fsutil behavior set symlinkEvaluation R2L:1 R2R:1
+            fsutil behavior query SymlinkEvaluation
+
+            CYGWIN="winsymlinks:native"
+            CYGWIN_ROOT="D:\\cygwin"
+            CYGWIN_ROOT_BIN="D:\\cygwin\\bin"
+            CYGWIN_ROOT_WRAPPERBIN="D:\\cygwin\\wrapperbin"
+
+            echo "HOME=$USERPROFILE" >> "$GITHUB_ENV"
+            echo "MSYS=winsymlinks:native" >> "$GITHUB_ENV"
+            echo "CYGWIN=$CYGWIN" >> "$GITHUB_ENV"
+            echo "CYGWIN_ROOT=$CYGWIN_ROOT" >> "$GITHUB_ENV"
+            echo "CYGWIN_ROOT_BIN=$CYGWIN_ROOT_BIN" >> "$GITHUB_ENV"
+            echo "CYGWIN_ROOT_WRAPPERBIN=$CYGWIN_ROOT_WRAPPERBIN" >> "$GITHUB_ENV"
+
+            echo "$CYGWIN_ROOT_WRAPPERBIN" >> "$GITHUB_PATH"
+          fi
+        shell: bash
+
+      - name: Build compiler
         if: runner.os != 'Linux'
         run: opam exec -- dune build --display quiet --profile release
 
@@ -248,7 +349,7 @@ jobs:
       - name: Install npm packages
         run: npm ci --ignore-scripts
 
-      - name: "Windows: Use MSVC for ninja build"
+      - name: Setup MSVC for ninja build (Windows)
         if: runner.os == 'Windows'
         uses: TheMrMilchmann/setup-msvc-dev@v3
         with:
@@ -304,10 +405,6 @@ jobs:
         run: node scripts/ciTest.js -mocha -theme -format
 
       # Build the playground compiler on the fastest runner (ubuntu-latest)
-      - name: Install JSOO
-        if: matrix.os == 'ubuntu-latest'
-        run: opam install js_of_ocaml.5.8.1
-
       - name: Build playground compiler
         if: matrix.os == 'ubuntu-latest'
         run: |
diff --git a/dune-project b/dune-project
@@ -24,7 +24,7 @@
    (= 0.26.2))
   (cppo
    (= 1.6.9))
-  (js_of_ocaml-compiler
+  (js_of_ocaml
    (= 5.8.1))
   (ounit2
    (= 2.2.7))
diff --git a/ninja/misc/ci.py b/ninja/misc/ci.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 import os
 
diff --git a/ninja/misc/output_test.py b/ninja/misc/output_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 """Runs ./ninja and checks if the output is correct.
 
diff --git a/rescript.opam b/rescript.opam
@@ -10,7 +10,7 @@ depends: [
   "ocaml" {>= "4.10"}
   "ocamlformat" {= "0.26.2"}
   "cppo" {= "1.6.9"}
-  "js_of_ocaml-compiler" {= "5.8.1"}
+  "js_of_ocaml" {= "5.8.1"}
   "ounit2" {= "2.2.7"}
   "reanalyze" {= "2.25.1"}
   "dune"
diff --git a/scripts/buildNinjaBinary.js b/scripts/buildNinjaBinary.js
@@ -5,7 +5,7 @@ const path = require("path");
 
 const platform = process.platform;
 const ninjaDir = path.join(__dirname, "..", "ninja");
-const buildCommand = "python3 configure.py --bootstrap --verbose";
+const buildCommand = "python configure.py --bootstrap --verbose";
 
 if (platform === "win32") {
   // On Windows, the build uses the MSVC compiler which needs to be on the path.

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#!/usr/bin/env python3`
	`1`	`+#!/usr/bin/env python`
`2`	`2`
`3`	`3`	`import os`
`4`	`4`