Add option to include submodules from vendoring

This adds a config option `exclude-submodules-from-vendoring` that
explicitly filters out listed submodules from vendoring. This allows us
to exclude submodules like `rustc-perf`, which vendors a number of
libraries for benchmarking, and has a complicated licensing story.

Closes rust-lang#137272

Author: erickt

config.example.toml

@@ -981,3 +981,6 @@
 
 # Whether to vendor dependencies for the dist tarball.
 #vendor = if "is a tarball source" || "is a git repository" { true } else { false }
+
+# Exclude these submodules from vendoring.
+# exclude-submodules-from-vendoring = []

src/bootstrap/src/core/build_steps/run.rs

@@ -209,7 +209,11 @@ impl Step for GenerateCopyright {
         let dest = builder.out.join("COPYRIGHT.html");
         let dest_libstd = builder.out.join("COPYRIGHT-library.html");
 
-        let paths_to_vendor = default_paths_to_vendor(builder);
+        let paths_to_vendor = default_paths_to_vendor(
+            builder,
+            &builder.config.dist_exclude_submodules_from_vendoring,
+        );
+
         for (_, submodules) in &paths_to_vendor {
             for submodule in submodules {
                 builder.build.require_submodule(submodule, None);

src/bootstrap/src/core/build_steps/vendor.rs

@@ -2,6 +2,7 @@
 //!
 //! This module ensures that all required Cargo dependencies are gathered
 //! and stored in the `<src>/<VENDOR_DIR>` directory.
+use std::collections::BTreeSet;
 use std::path::PathBuf;
 
 use crate::core::build_steps::tool::SUBMODULES_FOR_RUSTBOOK;
@@ -16,21 +17,32 @@ pub const VENDOR_DIR: &str = "vendor";
 /// Returns a `Vec` of `(path_to_manifest, submodules_required)` where
 /// `path_to_manifest` is the cargo workspace, and `submodules_required` is
 /// the set of submodules that must be available.
-pub fn default_paths_to_vendor(builder: &Builder<'_>) -> Vec<(PathBuf, Vec<&'static str>)> {
+pub fn default_paths_to_vendor(
+    builder: &Builder<'_>,
+    excluded_submodules: &BTreeSet<String>,
+) -> Vec<(PathBuf, Vec<&'static str>)> {
     [
-        ("src/tools/cargo/Cargo.toml", vec!["src/tools/cargo"]),
-        ("src/tools/rust-analyzer/Cargo.toml", vec![]),
-        ("compiler/rustc_codegen_cranelift/Cargo.toml", vec![]),
-        ("compiler/rustc_codegen_gcc/Cargo.toml", vec![]),
-        ("library/Cargo.toml", vec![]),
-        ("src/bootstrap/Cargo.toml", vec![]),
-        ("src/tools/rustbook/Cargo.toml", SUBMODULES_FOR_RUSTBOOK.into()),
-        ("src/tools/rustc-perf/Cargo.toml", vec!["src/tools/rustc-perf"]),
-        ("src/tools/opt-dist/Cargo.toml", vec![]),
-        ("src/doc/book/packages/trpl/Cargo.toml", vec![]),
+        ("src/tools/cargo/Cargo.toml", &["src/tools/cargo"][..]),
+        ("src/tools/rust-analyzer/Cargo.toml", &[]),
+        ("compiler/rustc_codegen_cranelift/Cargo.toml", &[]),
+        ("compiler/rustc_codegen_gcc/Cargo.toml", &[]),
+        ("library/Cargo.toml", &[]),
+        ("src/bootstrap/Cargo.toml", &[]),
+        ("src/tools/rustbook/Cargo.toml", SUBMODULES_FOR_RUSTBOOK),
+        ("src/tools/rustc-perf/Cargo.toml", &["src/tools/rustc-perf"]),
+        ("src/tools/opt-dist/Cargo.toml", &[]),
+        ("src/doc/book/packages/trpl/Cargo.toml", &[]),
     ]
     .into_iter()
-    .map(|(path, submodules)| (builder.src.join(path), submodules))
+    .filter_map(|(path, submodules)| {
+        for submodule in submodules {
+            if excluded_submodules.contains(&**submodule) {
+                return None;
+            }
+        }
+
+        Some((builder.src.join(path), submodules.into()))
+    })
     .collect()
 }
 
@@ -82,7 +94,11 @@ impl Step for Vendor {
             cmd.arg("--versioned-dirs");
         }
 
-        let to_vendor = default_paths_to_vendor(builder);
+        let to_vendor = default_paths_to_vendor(
+            builder,
+            &builder.config.dist_exclude_submodules_from_vendoring,
+        );
+
         // These submodules must be present for `x vendor` to work.
         for (_, submodules) in &to_vendor {
             for submodule in submodules {

src/bootstrap/src/core/config/config.rs

@@ -339,6 +339,7 @@ pub struct Config {
     pub dist_compression_profile: String,
     pub dist_include_mingw_linker: bool,
     pub dist_vendor: bool,
+    pub dist_exclude_submodules_from_vendoring: BTreeSet<String>,
 
     // libstd features
     pub backtrace: bool, // support for RUST_BACKTRACE
@@ -1001,6 +1002,7 @@ define_config! {
         compression_profile: Option<String> = "compression-profile",
         include_mingw_linker: Option<bool> = "include-mingw-linker",
         vendor: Option<bool> = "vendor",
+        exclude_submodules_from_vendoring: Option<BTreeSet<String>> = "exclude-submodules-from-vendoring",
     }
 }
 
@@ -2227,10 +2229,13 @@ impl Config {
                 compression_profile,
                 include_mingw_linker,
                 vendor,
+                exclude_submodules_from_vendoring,
             } = dist;
             config.dist_sign_folder = sign_folder.map(PathBuf::from);
             config.dist_upload_addr = upload_addr;
             config.dist_compression_formats = compression_formats;
+            config.dist_exclude_submodules_from_vendoring =
+                exclude_submodules_from_vendoring.unwrap_or_else(BTreeSet::new);
             set(&mut config.dist_compression_profile, compression_profile);
             set(&mut config.rust_dist_src, src_tarball);
             set(&mut config.dist_include_mingw_linker, include_mingw_linker);

src/bootstrap/src/utils/change_tracker.rs

@@ -360,4 +360,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Info,
         summary: "Added `build.test-stage = 2` to 'tools' profile defaults",
     },
+    ChangeInfo {
+        change_id: 137583,
+        severity: ChangeSeverity::Info,
+        summary: "It is now possible to filter out submodules from vendoring with `dist.exclude-submodules-from-vendoring`",
+    },
 ];