Merge pull request rust-lang#1104 from bjorn3/build_system_refactor

Build system refactor

Author: bjorn3

.github/workflows/main.yml

@@ -51,4 +51,13 @@ jobs:
         export COMPILE_RUNS=2
         export RUN_RUNS=2
 
-        ./test.sh --release
+        ./test.sh
+
+    - name: Package prebuilt cg_clif
+      run: tar cvfJ cg_clif.tar.xz build
+
+    - name: Upload prebuilt cg_clif
+      uses: actions/upload-artifact@v2
+      with:
+        name: cg_clif-${{ runner.os }}
+        path: cg_clif.tar.xz

.gitignore

@@ -6,7 +6,7 @@ perf.data
 perf.data.old
 *.events
 *.string*
-/build_sysroot/sysroot
+/build
 /build_sysroot/sysroot_src
 /rust
 /rand

Readme.md

@@ -2,48 +2,56 @@
 
 > ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠
 
-The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). This has the potential to improve compilation times in debug mode. If your project doesn't use any of the things listed under "Not yet supported", it should work fine. If not please open an issue.
+The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift).
+This has the potential to improve compilation times in debug mode.
+If your project doesn't use any of the things listed under "Not yet supported", it should work fine.
+If not please open an issue.
 
 ## Building and testing
 
 ```bash
 $ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git
 $ cd rustc_codegen_cranelift
 $ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking
-$ ./test.sh --release
+$ ./build.sh
 ```
 
-If you want to only build but not test you should replace the last command with:
+To run the test suite replace the last command with:
 
 ```bash
-$ cargo build --release
-$ ./build_sysroot/build_sysroot.sh
+$ ./test.sh
 ```
 
+This will implicitly build cg_clif too. Both `build.sh` and `test.sh` accept a `--debug` argument to
+build in debug mode.
+
+Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section
+of workflow runs. Unfortunately due to GHA restrictions you need to be logged in to access it.
+
+[GHA]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
+
 ## Usage
 
 rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
 
-Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `test.sh`).
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
 
 ### Cargo
 
 In the directory with your project (where you can do the usual `cargo build`), run:
 
 ```bash
-$ $cg_clif_dir/cargo.sh run
+$ $cg_clif_dir/build/cargo.sh run
 ```
 
 This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend.
 
-If you compiled cg_clif in debug mode (aka you didn't pass `--release` to `./test.sh`) you should set `CHANNEL="debug"`.
-
 ### Rustc
 
 > You should prefer using the Cargo method.
 
 ```bash
-$ $cg_clif_dir/target/release/cg_clif my_crate.rs
+$ $cg_clif_dir/build/cg_clif my_crate.rs
 ```
 
 ### Jit mode
@@ -54,13 +62,13 @@ In jit mode cg_clif will immediately execute your code without creating an execu
 > The jit mode will probably need cargo integration to make this possible.
 
 ```bash
-$ $cg_clif_dir/cargo.sh jit
+$ $cg_clif_dir/build/cargo.sh jit
 ```
 
 or
 
 ```bash
-$ $cg_clif_dir/target/release/cg_clif --jit my_crate.rs
+$ $cg_clif_dir/build/cg_clif --jit my_crate.rs
 ```
 
 ### Shell
@@ -69,7 +77,7 @@ These are a few functions that allow you to easily run rust code from the shell
 
 ```bash
 function jit_naked() {
-    echo "$@" | $cg_clif_dir/target/release/cg_clif - --jit
+    echo "$@" | $cg_clif_dir/build/cg_clif - --jit
 }
 
 function jit() {

build.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+# Settings
+export CHANNEL="release"
+build_sysroot=1
+target_dir='build'
+while [[ $# != 0 ]]; do
+    case $1 in
+        "--debug")
+            export CHANNEL="debug"
+            ;;
+        "--without-sysroot")
+            build_sysroot=0
+            ;;
+        "--target-dir")
+            target_dir=$2
+            shift
+            ;;
+        *)
+            echo "Unknown flag '$1'"
+            echo "Usage: ./build.sh [--debug] [--without-sysroot] [--target-dir DIR]"
+            ;;
+    esac
+    shift
+done
+
+# Build cg_clif
+export RUSTFLAGS="-Zrun_dsymutil=no"
+if [[ "$CHANNEL" == "release" ]]; then
+    cargo build --release
+else
+    cargo build
+fi
+
+rm -rf $target_dir
+mkdir $target_dir
+cp -a target/$CHANNEL/cg_clif{,_build_sysroot} target/$CHANNEL/*rustc_codegen_cranelift* $target_dir/
+cp -a rust-toolchain scripts/config.sh scripts/cargo.sh $target_dir
+
+if [[ "$build_sysroot" == "1" ]]; then
+    echo "[BUILD] sysroot"
+    export CG_CLIF_INCR_CACHE_DISABLED=1
+    dir=$(pwd)
+    cd $target_dir
+    time $dir/build_sysroot/build_sysroot.sh
+fi

build_sysroot/build_sysroot.sh

@@ -3,28 +3,24 @@
 # Requires the CHANNEL env var to be set to `debug` or `release.`
 
 set -e
-cd $(dirname "$0")
 
-if [ -z $CHANNEL ]; then
-export CHANNEL='release'
-fi
+source ./config.sh
 
-pushd ../ >/dev/null
-source ./scripts/config.sh
-popd >/dev/null
+dir=$(pwd)
 
-# We expect the target dir in the default location. Guard against the user changing it.
-export CARGO_TARGET_DIR=target
+# Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that
+# build scripts are still compiled using cg_llvm.
+export RUSTC=$dir"/cg_clif_build_sysroot"
+export RUSTFLAGS=$RUSTFLAGS" --clif"
+
+cd $(dirname "$0")
 
 # Cleanup for previous run
 #     v Clean target dir except for build scripts and incremental cache
-rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true
-rm -r sysroot/ 2>/dev/null || true
+#rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true
 
-# Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that
-# build scripts are still compiled using cg_llvm.
-export RUSTC=$(pwd)/../"target/"$CHANNEL"/cg_clif_build_sysroot"
-export RUSTFLAGS=$RUSTFLAGS" --clif"
+# We expect the target dir in the default location. Guard against the user changing it.
+export CARGO_TARGET_DIR=target
 
 # Build libs
 export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort"
@@ -39,5 +35,5 @@ else
 fi
 
 # Copy files to sysroot
-mkdir -p sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
-cp -r target/$TARGET_TRIPLE/$sysroot_channel/deps/* sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
+mkdir -p $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
+cp -a target/$TARGET_TRIPLE/$sysroot_channel/deps/* $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/

build_sysroot/prepare_sysroot_src.sh

@@ -12,7 +12,7 @@ fi
 
 rm -rf $DST_DIR
 mkdir -p $DST_DIR/library
-cp -r $SRC_DIR/library $DST_DIR/
+cp -a $SRC_DIR/library $DST_DIR/
 
 pushd $DST_DIR
 echo "[GIT] init"

clean_all.sh

@@ -1,5 +1,5 @@
 #!/bin/bash --verbose
 set -e
 
-rm -rf target/ build_sysroot/{sysroot/,sysroot_src/,target/} perf.data{,.old}
+rm -rf target/ build/ build_sysroot/{sysroot_src/,target/} perf.data{,.old}
 rm -rf rand/ regex/ simple-raytracer/

cargo.sh renamed to scripts/cargo.sh

@@ -1,19 +1,13 @@
 #!/bin/bash
 
-if [ -z $CHANNEL ]; then
-export CHANNEL='release'
-fi
-
-pushd $(dirname "$0") >/dev/null
-source scripts/config.sh
+dir=$(dirname "$0")
+source $dir/config.sh
 
 # read nightly compiler from rust-toolchain file
-TOOLCHAIN=$(cat rust-toolchain)
-
-popd >/dev/null
+TOOLCHAIN=$(cat $dir/rust-toolchain)
 
 cmd=$1
-shift
+shift || true
 
 if [[ "$cmd" = "jit" ]]; then
 cargo +${TOOLCHAIN} rustc $@ -- --jit

scripts/config.sh

@@ -39,18 +39,19 @@ echo
 export RUSTC_WRAPPER=
 fi
 
-export RUSTC=$(pwd)/"target/"$CHANNEL"/cg_clif"
+dir=$(cd $(dirname "$BASH_SOURCE"); pwd)
+
+export RUSTC=$dir"/cg_clif"
 export RUSTFLAGS=$linker
 export RUSTDOCFLAGS=$linker' -Ztrim-diagnostic-paths=no -Cpanic=abort -Zpanic-abort-tests '\
-'-Zcodegen-backend='$(pwd)'/target/'$CHANNEL'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$(pwd)'/build_sysroot/sysroot'
+'-Zcodegen-backend='$dir'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir'/sysroot'
 
 # FIXME remove once the atomic shim is gone
 if [[ `uname` == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
-export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib:\
-$(pwd)/target/"$CHANNEL":$(rustc --print sysroot)/lib"
+export LD_LIBRARY_PATH="$dir:$(rustc --print sysroot)/lib:$dir/target/out:$dir/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib"
 export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
 
 export CG_CLIF_DISPLAY_CG_TIME=1

scripts/filter_profile.rs

@@ -1,9 +1,8 @@
 #!/bin/bash
 #![forbid(unsafe_code)]/* This line is ignored by bash
 # This block is ignored by rustc
-CHANNEL="release"
 pushd $(dirname "$0")/../
-source scripts/config.sh
+source build/config.sh
 popd
 PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0
 #*/

scripts/rustup.sh

@@ -26,6 +26,15 @@ case $1 in
         git add rust-toolchain build_sysroot/Cargo.lock
         git commit -m "Rustup to $(rustc -V)"
         ;;
+    "push")
+	cg_clif=$(pwd)
+	pushd ../rust
+	branch=update_cg_clif-$(date +%Y-%m-%d)
+	git checkout -b $branch
+	git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
+	git push -u my $branch
+	popd
+	;;
     *)
         echo "Unknown command '$1'"
         echo "Usage: ./rustup.sh prepare|commit"