rustc: Split Emscripten to a separate codegen backend

This commit introduces a separately compiled backend for Emscripten, avoiding
compiling the `JSBackend` target in the main LLVM codegen backend. This builds
on the foundation provided by rust-lang#47671 to create a new codegen backend dedicated
solely to Emscripten, removing the `JSBackend` of the main codegen backend in
the process.

A new field was added to each target for this commit which specifies the backend
to use for translation, the default being `llvm` which is the main backend that
we use. The Emscripten targets specify an `emscripten` backend instead of the
main `llvm` one.

There's a whole bunch of consequences of this change, but I'll try to enumerate
them here:

* A *second* LLVM submodule was added in this commit. The main LLVM submodule
  will soon start to drift from the Emscripten submodule, but currently they're
  both at the same revision.
* Logic was added to rustbuild to *not* build the Emscripten backend by default.
  This is gated behind a `--enable-emscripten` flag to the configure script. By
  default users should neither check out the emscripten submodule nor compile
  it.
* The `init_repo.sh` script was updated to fetch the Emscripten submodule from
  GitHub the same way we do the main LLVM submodule (a tarball fetch).
* The Emscripten backend, turned off by default, is still turned on for a number
  of targets on CI. We'll only be shipping an Emscripten backend with Tier 1
  platforms, though. All cross-compiled platforms will not be receiving an
  Emscripten backend yet.

This commit means that when you download the `rustc` package in Rustup for Tier
1 platforms you'll be receiving two trans backends, one for Emscripten and one
that's the general LLVM backend. If you never compile for Emscripten you'll
never use the Emscripten backend, so we may update this one day to only download
the Emscripten backend when you add the Emscripten target. For now though it's
just an extra 10MB gzip'd.

Closes rust-lang#46819

Author: alexcrichton

.gitmodules

@@ -51,3 +51,6 @@
 [submodule "src/doc/rust-by-example"]
 	path = src/doc/rust-by-example
 	url = https://github.com/rust-lang/rust-by-example
+[submodule "src/llvm-emscripten"]
+	path = src/llvm-emscripten
+	url = https://github.com/rust-lang/llvm

.travis.yml

@@ -81,7 +81,7 @@ matrix:
     # OSX 10.7 and `xcode7` is the latest Xcode able to compile LLVM for 10.7.
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended --enable-profiler"
+        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-extended --enable-profiler --enable-emscripten"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -95,7 +95,7 @@ matrix:
 
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended --enable-sanitizers --enable-profiler"
+        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended --enable-sanitizers --enable-profiler --enable-emscripten"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1

appveyor.yml

@@ -63,25 +63,27 @@ environment:
       --build=x86_64-pc-windows-msvc
       --enable-extended
       --enable-profiler
+      --enable-emscripten
     SCRIPT: python x.py dist
     DEPLOY: 1
   - RUST_CONFIGURE_ARGS: >
       --build=i686-pc-windows-msvc
       --target=i586-pc-windows-msvc
       --enable-extended
       --enable-profiler
+      --enable-emscripten
     SCRIPT: python x.py dist
     DEPLOY: 1
   - MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-extended
+    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-extended --enable-emscripten
     SCRIPT: python x.py dist
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
     MINGW_DIR: mingw32
     DEPLOY: 1
   - MSYS_BITS: 64
     SCRIPT: python x.py dist
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-extended
+    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-extended --enable-emscripten
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
     MINGW_DIR: mingw64

config.toml.example

@@ -305,6 +305,13 @@
 # result (broken, compiling, testing) into this JSON file.
 #save-toolstates = "/path/to/toolstates.json"
 
+# This is an array of the codegen backends that will be compiled for the rustc
+# that's being compiled. The default is to only build the LLVM codegen backend,
+# but you can also optionally enable the "emscripten" backend for asm.js or
+# make this an empty array (but that probably won't get too far in the
+# bootstrap)
+#codegen-backends = ["llvm"]
+
 # =============================================================================
 # Options for specific targets
 #

src/bootstrap/bootstrap.py

@@ -640,14 +640,23 @@ def update_submodules(self):
              os.path.join(self.rust_root, ".gitmodules"),
              "--get-regexp", "path"]
         ).decode(default_encoding).splitlines()]
-        submodules = [module for module in submodules
-                      if not ((module.endswith("llvm") and
-                               self.get_toml('llvm-config')) or
-                              (module.endswith("jemalloc") and
-                               (self.get_toml('use-jemalloc') == "false" or
-                                self.get_toml('jemalloc'))))]
+        filtered_submodules = []
+        for module in submodules:
+            if module.endswith("llvm"):
+                if self.get_toml('llvm-config'):
+                    continue
+            if module.endswith("llvm-emscripten"):
+                backends = self.get_toml('codegen-backends')
+                if backends is None or not 'emscripten' in backends:
+                    continue
+            if module.endswith("jemalloc"):
+                if self.get_toml('use-jemalloc') == 'false':
+                    continue
+                if self.get_toml('jemalloc'):
+                    continue
+            filtered_submodules.append(module)
         run(["git", "submodule", "update",
-             "--init", "--recursive"] + submodules,
+             "--init", "--recursive"] + filtered_submodules,
             cwd=self.rust_root, verbose=self.verbose)
         run(["git", "submodule", "-q", "foreach", "git",
              "reset", "-q", "--hard"],

src/bootstrap/compile.rs

@@ -581,24 +581,30 @@ impl Step for RustcLink {
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
-pub struct RustcTrans {
+pub struct CodegenBackend {
     pub compiler: Compiler,
     pub target: Interned<String>,
+    pub backend: Interned<String>,
 }
 
-impl Step for RustcTrans {
+impl Step for CodegenBackend {
     type Output = ();
     const ONLY_HOSTS: bool = true;
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun) -> ShouldRun {
-        run.path("src/librustc_trans").krate("rustc_trans")
+        run.path("src/librustc_trans")
     }
 
     fn make_run(run: RunConfig) {
-        run.builder.ensure(RustcTrans {
+        let backend = run.builder.config.rust_codegen_backends.get(0);
+        let backend = backend.cloned().unwrap_or_else(|| {
+            INTERNER.intern_str("llvm")
+        });
+        run.builder.ensure(CodegenBackend {
             compiler: run.builder.compiler(run.builder.top_stage, run.host),
             target: run.target,
+            backend
         });
     }
 
@@ -609,58 +615,92 @@ impl Step for RustcTrans {
 
         builder.ensure(Rustc { compiler, target });
 
-        // Build LLVM for our target. This will implicitly build the host LLVM
-        // if necessary.
-        builder.ensure(native::Llvm { target });
-
         if build.force_use_stage1(compiler, target) {
-            builder.ensure(RustcTrans {
+            builder.ensure(CodegenBackend {
                 compiler: builder.compiler(1, build.build),
                 target,
+                backend: self.backend,
             });
             return;
         }
 
-        let _folder = build.fold_output(|| format!("stage{}-rustc_trans", compiler.stage));
-        println!("Building stage{} trans artifacts ({} -> {})",
-                 compiler.stage, &compiler.host, target);
-
         let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "build");
+        let mut features = build.rustc_features().to_string();
         cargo.arg("--manifest-path")
-            .arg(build.src.join("src/librustc_trans/Cargo.toml"))
-            .arg("--features").arg(build.rustc_features());
+            .arg(build.src.join("src/librustc_trans/Cargo.toml"));
         rustc_cargo_env(build, &mut cargo);
 
-        // Pass down configuration from the LLVM build into the build of
-        // librustc_llvm and librustc_trans.
-        if build.is_rust_llvm(target) {
-            cargo.env("LLVM_RUSTLLVM", "1");
-        }
-        cargo.env("LLVM_CONFIG", build.llvm_config(target));
-        let target_config = build.config.target_config.get(&target);
-        if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
-            cargo.env("CFG_LLVM_ROOT", s);
-        }
-        // Building with a static libstdc++ is only supported on linux right now,
-        // not for MSVC or macOS
-        if build.config.llvm_static_stdcpp &&
-           !target.contains("freebsd") &&
-           !target.contains("windows") &&
-           !target.contains("apple") {
-            let file = compiler_file(build,
-                                     build.cxx(target).unwrap(),
-                                     target,
-                                     "libstdc++.a");
-            cargo.env("LLVM_STATIC_STDCPP", file);
-        }
-        if build.config.llvm_link_shared {
-            cargo.env("LLVM_LINK_SHARED", "1");
+        match &*self.backend {
+            "llvm" | "emscripten" => {
+                // Build LLVM for our target. This will implicitly build the
+                // host LLVM if necessary.
+                let llvm_config = builder.ensure(native::Llvm {
+                    target,
+                    emscripten: self.backend == "emscripten",
+                });
+
+                if self.backend == "emscripten" {
+                    features.push_str(" emscripten");
+                }
+
+                let _folder = build.fold_output(|| format!("stage{}-rustc_trans", compiler.stage));
+                println!("Building stage{} codegen artifacts ({} -> {}, {})",
+                         compiler.stage, &compiler.host, target, self.backend);
+
+                // Pass down configuration from the LLVM build into the build of
+                // librustc_llvm and librustc_trans.
+                if build.is_rust_llvm(target) {
+                    cargo.env("LLVM_RUSTLLVM", "1");
+                }
+                cargo.env("LLVM_CONFIG", &llvm_config);
+                if self.backend != "emscripten" {
+                    let target_config = build.config.target_config.get(&target);
+                    if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
+                        cargo.env("CFG_LLVM_ROOT", s);
+                    }
+                }
+                // Building with a static libstdc++ is only supported on linux right now,
+                // not for MSVC or macOS
+                if build.config.llvm_static_stdcpp &&
+                   !target.contains("freebsd") &&
+                   !target.contains("windows") &&
+                   !target.contains("apple") {
+                    let file = compiler_file(build,
+                                             build.cxx(target).unwrap(),
+                                             target,
+                                             "libstdc++.a");
+                    cargo.env("LLVM_STATIC_STDCPP", file);
+                }
+                if build.config.llvm_link_shared {
+                    cargo.env("LLVM_LINK_SHARED", "1");
+                }
+            }
+            _ => panic!("unknown backend: {}", self.backend),
         }
 
-        run_cargo(build,
-                  &mut cargo,
-                  &librustc_trans_stamp(build, compiler, target),
-                  false);
+        let tmp_stamp = build.cargo_out(compiler, Mode::Librustc, target)
+            .join(".tmp.stamp");
+        let files = run_cargo(build,
+                              cargo.arg("--features").arg(features),
+                              &tmp_stamp,
+                              false);
+        let mut files = files.into_iter()
+            .filter(|f| {
+                let filename = f.file_name().unwrap().to_str().unwrap();
+                is_dylib(filename) && filename.contains("rustc_trans-")
+            });
+        let codegen_backend = match files.next() {
+            Some(f) => f,
+            None => panic!("no dylibs built for codegen backend?"),
+        };
+        if let Some(f) = files.next() {
+            panic!("codegen backend built two dylibs:\n{}\n{}",
+                   codegen_backend.display(),
+                   f.display());
+        }
+        let stamp = codegen_backend_stamp(build, compiler, target, self.backend);
+        let codegen_backend = codegen_backend.to_str().unwrap();
+        t!(t!(File::create(&stamp)).write_all(codegen_backend.as_bytes()));
     }
 }
 
@@ -682,33 +722,29 @@ fn copy_codegen_backends_to_sysroot(builder: &Builder,
     // not linked into the main compiler by default but is rather dynamically
     // selected at runtime for inclusion.
     //
-    // Here we're looking for the output dylib of the `RustcTrans` step and
+    // Here we're looking for the output dylib of the `CodegenBackend` step and
     // we're copying that into the `codegen-backends` folder.
     let libdir = builder.sysroot_libdir(target_compiler, target);
     let dst = libdir.join("codegen-backends");
     t!(fs::create_dir_all(&dst));
-    let stamp = librustc_trans_stamp(build, compiler, target);
 
-    let mut copied = None;
-    for file in read_stamp_file(&stamp) {
-        let filename = match file.file_name().and_then(|s| s.to_str()) {
-            Some(s) => s,
-            None => continue,
+    for backend in builder.config.rust_codegen_backends.iter() {
+        let stamp = codegen_backend_stamp(build, compiler, target, *backend);
+        let mut dylib = String::new();
+        t!(t!(File::open(&stamp)).read_to_string(&mut dylib));
+        let file = Path::new(&dylib);
+        let filename = file.file_name().unwrap().to_str().unwrap();
+        // change `librustc_trans-xxxxxx.so` to `librustc_trans-llvm.so`
+        let target_filename = {
+            let dash = filename.find("-").unwrap();
+            let dot = filename.find(".").unwrap();
+            format!("{}-{}{}",
+                    &filename[..dash],
+                    backend,
+                    &filename[dot..])
         };
-        if !is_dylib(filename) || !filename.contains("rustc_trans-") {
-            continue
-        }
-        match copied {
-            None => copied = Some(file.clone()),
-            Some(ref s) => {
-                panic!("copied two codegen backends:\n{}\n{}",
-                       s.display(),
-                       file.display());
-            }
-        }
-        copy(&file, &dst.join(filename));
+        copy(&file, &dst.join(target_filename));
     }
-    assert!(copied.is_some(), "failed to find a codegen backend to copy");
 }
 
 /// Cargo's output path for the standard library in a given stage, compiled
@@ -729,10 +765,12 @@ pub fn librustc_stamp(build: &Build, compiler: Compiler, target: Interned<String
     build.cargo_out(compiler, Mode::Librustc, target).join(".librustc.stamp")
 }
 
-pub fn librustc_trans_stamp(build: &Build,
-                            compiler: Compiler,
-                            target: Interned<String>) -> PathBuf {
-    build.cargo_out(compiler, Mode::Librustc, target).join(".librustc_trans.stamp")
+fn codegen_backend_stamp(build: &Build,
+                         compiler: Compiler,
+                         target: Interned<String>,
+                         backend: Interned<String>) -> PathBuf {
+    build.cargo_out(compiler, Mode::Librustc, target)
+        .join(format!(".librustc_trans-{}.stamp", backend))
 }
 
 fn compiler_file(build: &Build,
@@ -849,10 +887,13 @@ impl Step for Assemble {
                 compiler: build_compiler,
                 target: target_compiler.host,
             });
-            builder.ensure(RustcTrans {
-                compiler: build_compiler,
-                target: target_compiler.host,
-            });
+            for &backend in build.config.rust_codegen_backends.iter() {
+                builder.ensure(CodegenBackend {
+                    compiler: build_compiler,
+                    target: target_compiler.host,
+                    backend,
+                });
+            }
         }
 
         let stage = target_compiler.stage;
@@ -922,7 +963,9 @@ fn stderr_isatty() -> bool {
     }
 }
 
-pub fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path, is_check: bool) {
+pub fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path, is_check: bool)
+    -> Vec<PathBuf>
+{
     // Instruct Cargo to give us json messages on stdout, critically leaving
     // stderr as piped so we can get those pretty colors.
     cargo.arg("--message-format").arg("json")
@@ -1066,8 +1109,8 @@ pub fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path, is_check: boo
     let mut new_contents = Vec::new();
     let mut max = None;
     let mut max_path = None;
-    for dep in deps {
-        let mtime = mtime(&dep);
+    for dep in deps.iter() {
+        let mtime = mtime(dep);
         if Some(mtime) > max {
             max = Some(mtime);
             max_path = Some(dep.clone());
@@ -1080,12 +1123,13 @@ pub fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path, is_check: boo
     if stamp_contents == new_contents && max <= stamp_mtime {
         build.verbose(&format!("not updating {:?}; contents equal and {} <= {}",
                 stamp, max, stamp_mtime));
-        return
+        return deps
     }
     if max > stamp_mtime {
         build.verbose(&format!("updating {:?} as {:?} changed", stamp, max_path));
     } else {
         build.verbose(&format!("updating {:?} as deps changed", stamp));
     }
     t!(t!(File::create(stamp)).write_all(&new_contents));
+    deps
 }