Absolute bare minimum for downloading rustc from CI

- Use the same compiler for stage0 and stage1. This should be fixed at
  some point (so bootstrap isn't constantly rebuilt).
- Make sure `x.py build` and `x.py check` work.
- Use `git merge-base` to determine the most recent commit to download.
- Copy stage0 to the various sysroots in `Sysroot`, and delegate to
  Sysroot in Assemble. Leave all other code unchanged.
- Rename date -> key

  This can also be a commit hash, so 'date' is no longer a good name.

- Add the commented-out option to config.toml.example
- Disable all steps by default when `download-rustc` is enabled

  Most steps don't make sense when downloading a compiler, because they'll
  be pre-built in the sysroot. Only enable the ones that might be useful,
  in particular Rustdoc and all `check` steps.

  At some point, this should probably enable other tools, but rustdoc is
  enough to test out `download-rustc`.

- Don't print 'Skipping' twice in a row

  Bootstrap forcibly enables a dry run if it isn't already set, so
  previously it would print the message twice:

  ```
  Skipping bootstrap::compile::Std because it is not enabled for `download-rustc`
  Skipping bootstrap::compile::Std because it is not enabled for `download-rustc`
  ```

  Now it correctly only prints once.

 ## Future work

- Add FIXME about supporting beta commits
- Debug logging will never work. This should be fixed.

Author: jyn514

config.toml.example

@@ -358,6 +358,12 @@ changelog-seen = 2
 #
 #debug = false
 
+# Whether to download the stage 1 and 2 compilers from CI.
+# This is mostly useful for tools; if you have changes to `compiler/` they will be ignored.
+#
+# FIXME: currently, this also uses the downloaded compiler for stage0, but that causes unnecessary rebuilds.
+#download-rustc = false
+
 # Number of codegen units to use for each compiler invocation. A value of 0
 # means "the number of cores on this machine", and 1+ is passed through to the
 # compiler.

src/bootstrap/bootstrap.py

@@ -378,6 +378,7 @@ def __init__(self):
         self.verbose = False
         self.git_version = None
         self.nix_deps_dir = None
+        self.rustc_commit = None
 
     def download_stage0(self):
         """Fetch the build system for Rust, written in Rust
@@ -394,20 +395,27 @@ def download_stage0(self):
 
         if self.rustc().startswith(self.bin_root()) and \
                 (not os.path.exists(self.rustc()) or
-                 self.program_out_of_date(self.rustc_stamp(), self.date)):
+                 self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
             if os.path.exists(self.bin_root()):
                 shutil.rmtree(self.bin_root())
+            download_rustc = self.rustc_commit is not None
             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
             filename = "rust-std-{}-{}{}".format(
                 rustc_channel, self.build, tarball_suffix)
             pattern = "rust-std-{}".format(self.build)
-            self._download_stage0_helper(filename, pattern, tarball_suffix)
+            self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
                                               tarball_suffix)
-            self._download_stage0_helper(filename, "rustc", tarball_suffix)
+            self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
                                               tarball_suffix)
-            self._download_stage0_helper(filename, "cargo", tarball_suffix)
+            self._download_component_helper(filename, "cargo", tarball_suffix)
+            if self.rustc_commit is not None:
+                filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
+                self._download_component_helper(
+                    filename, "rustc-dev", tarball_suffix, download_rustc
+                )
+
             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
@@ -416,7 +424,7 @@ def download_stage0(self):
                 if lib.endswith(".so"):
                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
             with output(self.rustc_stamp()) as rust_stamp:
-                rust_stamp.write(self.date)
+                rust_stamp.write(self.date + str(self.rustc_commit))
 
         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
             not os.path.exists(self.rustfmt())
@@ -426,7 +434,9 @@ def download_stage0(self):
                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
                 [channel, date] = rustfmt_channel.split('-', 1)
                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
-                self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
+                self._download_component_helper(
+                    filename, "rustfmt-preview", tarball_suffix, key=date
+                )
                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
@@ -482,18 +492,27 @@ def downloading_llvm(self):
         return opt == "true" \
             or (opt == "if-available" and self.build in supported_platforms)
 
-    def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
-        if date is None:
-            date = self.date
+    def _download_component_helper(
+        self, filename, pattern, tarball_suffix, download_rustc=False, key=None
+    ):
+        if key is None:
+            if download_rustc:
+                key = self.rustc_commit
+            else:
+                key = self.date
         cache_dst = os.path.join(self.build_dir, "cache")
-        rustc_cache = os.path.join(cache_dst, date)
+        rustc_cache = os.path.join(cache_dst, key)
         if not os.path.exists(rustc_cache):
             os.makedirs(rustc_cache)
 
-        url = "{}/dist/{}".format(self._download_url, date)
+        if download_rustc:
+            url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
+        else:
+            url = "{}/dist/{}".format(self._download_url, key)
         tarball = os.path.join(rustc_cache, filename)
         if not os.path.exists(tarball):
-            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
+            do_verify = not download_rustc
+            get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 
     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
@@ -613,6 +632,46 @@ def fix_bin_or_dylib(self, fname, rpath_libz=False):
             print("warning: failed to call patchelf:", reason)
             return
 
+    # Return the stage1 compiler to download, if any.
+    def maybe_download_rustc(self):
+        # If `download-rustc` is not set, default to rebuilding.
+        if self.get_toml("download-rustc", section="rust") != "true":
+            return None
+        # Look for a version to compare to based on the current commit.
+        # There are a few different cases to handle.
+        # 1. This commit is a fast-forward from master: `master - * - * - HEAD`
+        # 2. This commit and master have diverged:
+        # ```
+        #   Y - * - HEAD
+        #  /
+        # X - * - master
+        # ```
+        # In this case, we should compare to `X`.
+        # 3. `master` and `HEAD` are radically different (>100 commits, or similar). This probably
+        # means that `master` does *not* correspond to the version we want to compare to, e.g. a
+        # fork. Instead, we want to compare to `rust-lang/rust:master`, which this has to share a
+        # recent merge base with.
+
+        # Find which remote corresponds to `rust-lang/rust`.
+        remotes = subprocess.check_output(["git", "remote", "-v"], universal_newlines=True)
+        # e.g. `origin https://github.com//rust-lang/rust (fetch)`
+        rust_lang_remote = next(line for line in remotes.splitlines() if "rust-lang/rust" in line)
+        rust_lang_remote = rust_lang_remote.split()[0]
+
+        # Find which commit to compare to
+        merge_base = ["git", "merge-base", "HEAD", "{}/master".format(rust_lang_remote)]
+        commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
+
+        # Warn if there were changes to the compiler since the ancestor commit.
+        rev_parse = ["git", "rev-parse", "--show-toplevel"]
+        top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
+        compiler = "{}/compiler/".format(top_level)
+        status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
+        if status != 0:
+            print("warning: `download-rustc` is enabled, but there are changes to compiler/")
+
+        return commit
+
     def rustc_stamp(self):
         """Return the path for .rustc-stamp
 
@@ -1090,6 +1149,13 @@ def bootstrap(help_triggered):
     build.update_submodules()
 
     # Fetch/build the bootstrap
+    build.rustc_commit = build.maybe_download_rustc()
+    if build.rustc_commit is not None:
+        if build.verbose:
+            commit = build.rustc_commit
+            print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
+        # FIXME: support downloading artifacts from the beta channel
+        build.rustc_channel = "nightly"
     build.download_stage0()
     sys.stdout.flush()
     build.ensure_vendored()

src/bootstrap/builder.rs

@@ -57,6 +57,14 @@ pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash {
     /// `true` here can still be overwritten by `should_run` calling `default_condition`.
     const DEFAULT: bool = false;
 
+    /// Whether this step should be run even when `download-rustc` is set.
+    ///
+    /// Most steps are not important when the compiler is downloaded, since they will be included in
+    /// the pre-compiled sysroot. Steps can set this to `true` to be built anyway.
+    ///
+    /// When in doubt, set this to `false`.
+    const ENABLE_DOWNLOAD_RUSTC: bool = false;
+
     /// If true, then this rule should be skipped if --target was specified, but --host was not
     const ONLY_HOSTS: bool = false;
 
@@ -99,6 +107,7 @@ impl RunConfig<'_> {
 
 struct StepDescription {
     default: bool,
+    enable_download_rustc: bool,
     only_hosts: bool,
     should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>,
     make_run: fn(RunConfig<'_>),
@@ -153,6 +162,7 @@ impl StepDescription {
     fn from<S: Step>() -> StepDescription {
         StepDescription {
             default: S::DEFAULT,
+            enable_download_rustc: S::ENABLE_DOWNLOAD_RUSTC,
             only_hosts: S::ONLY_HOSTS,
             should_run: S::should_run,
             make_run: S::make_run,
@@ -169,6 +179,14 @@ impl StepDescription {
                 "{:?} not skipped for {:?} -- not in {:?}",
                 pathset, self.name, builder.config.exclude
             );
+        } else if builder.config.download_rustc && !self.enable_download_rustc {
+            if !builder.config.dry_run {
+                eprintln!(
+                    "Not running {} because its artifacts have been downloaded from CI (`download-rustc` is set)",
+                    self.name
+                );
+            }
+            return;
         }
 
         // Determine the targets participating in this rule.
@@ -629,8 +647,12 @@ impl<'a> Builder<'a> {
                     .join("rustlib")
                     .join(self.target.triple)
                     .join("lib");
-                let _ = fs::remove_dir_all(&sysroot);
-                t!(fs::create_dir_all(&sysroot));
+                // Avoid deleting the rustlib/ directory we just copied
+                // (in `impl Step for Sysroot`).
+                if !builder.config.download_rustc {
+                    let _ = fs::remove_dir_all(&sysroot);
+                    t!(fs::create_dir_all(&sysroot));
+                }
                 INTERNER.intern_path(sysroot)
             }
         }

src/bootstrap/check.rs

@@ -62,6 +62,7 @@ fn cargo_subcommand(kind: Kind) -> &'static str {
 impl Step for Std {
     type Output = ();
     const DEFAULT: bool = true;
+    const ENABLE_DOWNLOAD_RUSTC: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.all_krates("test")
@@ -155,6 +156,7 @@ impl Step for Rustc {
     type Output = ();
     const ONLY_HOSTS: bool = true;
     const DEFAULT: bool = true;
+    const ENABLE_DOWNLOAD_RUSTC: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.all_krates("rustc-main")
@@ -233,6 +235,7 @@ impl Step for CodegenBackend {
     type Output = ();
     const ONLY_HOSTS: bool = true;
     const DEFAULT: bool = true;
+    const ENABLE_DOWNLOAD_RUSTC: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.paths(&["compiler/rustc_codegen_cranelift", "rustc_codegen_cranelift"])
@@ -290,6 +293,7 @@ macro_rules! tool_check_step {
             type Output = ();
             const ONLY_HOSTS: bool = true;
             const DEFAULT: bool = true;
+            const ENABLE_DOWNLOAD_RUSTC: bool = true;
 
             fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
                 run.path($path)

src/bootstrap/compile.rs

@@ -41,7 +41,10 @@ impl Step for Std {
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.all_krates("test")
+        // When downloading stage1, the standard library has already been copied to the sysroot, so
+        // there's no need to rebuild it.
+        let download_rustc = run.builder.config.download_rustc;
+        run.all_krates("test").default_condition(!download_rustc)
     }
 
     fn make_run(run: RunConfig<'_>) {
@@ -904,6 +907,18 @@ impl Step for Sysroot {
         let _ = fs::remove_dir_all(&sysroot);
         t!(fs::create_dir_all(&sysroot));
 
+        // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
+        if builder.config.download_rustc {
+            assert_eq!(
+                builder.config.build, compiler.host,
+                "Cross-compiling is not yet supported with `download-rustc`",
+            );
+            // Copy the compiler into the correct sysroot.
+            let stage0_dir = builder.config.out.join(&*builder.config.build.triple).join("stage0");
+            builder.cp_r(&stage0_dir, &sysroot);
+            return INTERNER.intern_path(sysroot);
+        }
+
         // Symlink the source root into the same location inside the sysroot,
         // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`),
         // so that any tools relying on `rust-src` also work for local builds,
@@ -975,13 +990,16 @@ impl Step for Assemble {
         // produce some other architecture compiler we need to start from
         // `build` to get there.
         //
-        // FIXME: Perhaps we should download those libraries?
-        //        It would make builds faster...
-        //
         // FIXME: It may be faster if we build just a stage 1 compiler and then
         //        use that to bootstrap this compiler forward.
         let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
 
+        // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
+        if builder.config.download_rustc {
+            builder.ensure(Sysroot { compiler: target_compiler });
+            return target_compiler;
+        }
+
         // Build the libraries for this compiler to link to (i.e., the libraries
         // it uses at runtime). NOTE: Crates the target compiler compiles don't
         // link to these. (FIXME: Is that correct? It seems to be correct most

src/bootstrap/config.rs

@@ -80,6 +80,7 @@ pub struct Config {
     pub cmd: Subcommand,
     pub incremental: bool,
     pub dry_run: bool,
+    pub download_rustc: bool,
 
     pub deny_warnings: bool,
     pub backtrace_on_ice: bool,
@@ -503,6 +504,7 @@ struct Rust {
     new_symbol_mangling: Option<bool>,
     profile_generate: Option<String>,
     profile_use: Option<String>,
+    download_rustc: Option<bool>,
 }
 
 /// TOML representation of how each build target is configured.
@@ -885,6 +887,7 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
+            config.download_rustc = rust.download_rustc.unwrap_or(false);
         } else {
             config.rust_profile_use = flags.rust_profile_use;
             config.rust_profile_generate = flags.rust_profile_generate;

src/bootstrap/tool.rs

@@ -477,6 +477,7 @@ pub struct Rustdoc {
 impl Step for Rustdoc {
     type Output = PathBuf;
     const DEFAULT: bool = true;
+    const ENABLE_DOWNLOAD_RUSTC: bool = true;
     const ONLY_HOSTS: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {