Refactor reproducibility check

Author: biabbas

.github/workflows/repro_check.yml

@@ -1,79 +1,110 @@
-name: Build and Diff Projects
+# Workflow that runs after a merge to master, builds toolchain in 2 different
+# directories to check for reproducible builds. This check helps track which
+# commits cause reproducibility issues. This check should pass before stable
+# release are made.
+
+name: Reproducibility check
 
 on:
   push:
-    branches:
-      - master
-  pull_request:
     branches:
       - master
       - reproducible
 
 jobs:
   build_and_compare:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
+
+    strategy:
+      matrix:
+        build_dir: [buildA, buildA_extended]
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'  # Adjust to the version you need
+        uses: actions/checkout@v4
 
-      - name: Build and store buildA binaries
-        run: |
-          mkdir ../buildA
-          cp -r "$(pwd)" ../buildA
-          pushd ../buildA
-          SOURCE_DIR=$(dirname $(find . -maxdepth 2  -name x.py))
-          $SOURCE_DIR/configure --set rust.channel=nightly
-          $SOURCE_DIR/x.py build --stage 2 -j$(($(nproc)*2/3))
-          rm -rf $SOURCE_DIR
-          STAGE2_DIR=`find build -name stage2`
-          cp -r "$STAGE2_DIR" .
-          echo "Contents stage 2 dir : `ls stage2`"
-          rm -rf build
-          popd
-
-      - name: Build and store buildA_extended binaries
+      - name: Build and store toolchains
         run: |
-          mkdir ../buildA_extended
-          cp -r "$(pwd)" ../buildA_extended
-          pushd ../buildA_extended
-          SOURCE_DIR=$(dirname $(find . -maxdepth 2  -name x.py))
-          $SOURCE_DIR/configure --set rust.channel=nightly
-          $SOURCE_DIR/x.py build --stage 2 -j$(($(nproc)*2/3))
-          rm -rf $SOURCE_DIR
-          STAGE2_DIR=`find build -name stage2`
-          cp -r "$STAGE2_DIR" .
-          echo "Contents stage 2 dir : `ls stage2`"
-          rm -rf build
-          popd
-
-      - name: Compare builds and archive artifacts
+          # Define build function
+          build_toolchain() {
+            local dir_name="$1"
+            echo "Building in $dir_name..."
+
+            mkdir "../$dir_name"
+            cp -r "$(pwd)" "../$dir_name"
+            pushd "../$dir_name"
+
+            # Find source directory
+            SOURCE_DIR=$(dirname $(find . -maxdepth 2 -name x.py))
+
+            # FIXME: Setting channel to nightly because only nightly builds succeed on CI
+            $SOURCE_DIR/configure --set rust.channel=nightly
+
+            # Build rust till stage 2
+            $SOURCE_DIR/x.py build --stage 2 -j$(($(nproc)*2/3))
+
+            # Remove copy of source directory to save space
+            rm -rf "$SOURCE_DIR"
+
+            # Save stage2 directory
+            STAGE2_DIR=$(find build -name stage2)
+            cp -r "$STAGE2_DIR" .
+            echo "Contents of stage2 dir in $dir_name: $(ls stage2)"
+
+            # Clean up to save space
+            rm -rf build
+            popd
+          }
+
+          # Build both
+          build_toolchain buildA
+          build_toolchain buildA_extended
+
+# Compare the two builds
+      - name: Compare builds
+        id: compare
         run: |
           # Ensure the directories exist
           if [[ ! -d "../buildA" || ! -d "../buildA_extended" ]]; then
             echo "Error: Build directories not found!"
             exit 1
           fi
 
-          # Perform a diff between the two builds
-          diff -r ../buildA/stage2 ../buildA_extended/stage2 || echo "Differences found!"
-
-          tar -czf buildA.tar.gz ../buildA
-          tar -czf buildA_extended.tar.gz ../buildA_extended
+          # Perform a recursive diff between the stage2 directories of both builds
+          # If there are differences, record the result so we can upload artifacts and then fail later
+          # The binaries should be identical, so the cause of difference should be analysed and fixed
+          # appropriately.
+          if diff -r ../buildA/stage2 ../buildA_extended/stage2; then
+            echo "No differences found."
+            echo "has_diff=false" >> $GITHUB_OUTPUT
+          else
+            echo "Differences found!"
+            echo "has_diff=true" >> $GITHUB_OUTPUT
+          fi
 
+# Upload buildA directory as an artifact (for debugging purposes)
+# This artifact contains stage2 folder from buildA. This artifact would
+# be helpful to debug reproducibility issue when this test fails.
       - name: Upload buildA artifact
+        if: steps.compare.outputs.has_diff == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: buildA
-          path: buildA.tar.gz
+          path: ../buildA
 
+# Upload buildA_extended directory as an artifact (for debugging purposes)
+# This artifact contains stage2 folder from buildA_extended. This artifact would
+# be helpful to debug reproducibility issue when this test fails.
       - name: Upload buildA_extended artifact
+        if: steps.compare.outputs.has_diff == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: buildA_extended
-          path: buildA_extended.tar.gz
+          path: ../buildA_extended
+
+# Fail the job if differences were found between the builds
+      - name: Fail the job if there are differences
+        if: steps.compare.outputs.has_diff == 'true'
+        run: |
+          echo "Differences found between buildA and buildA_extended, Reproducibility check failed"
+          exit 1