From ef9349db86dcbbe705aaf40a2f5dcb1dd9c9063e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 6 Jan 2025 17:23:28 +0100 Subject: [PATCH 1/5] Add test for checking used glibc symbols --- .../rustc-dev-guide/src/tests/directives.md | 2 + src/tools/compiletest/src/directive-list.rs | 1 + src/tools/compiletest/src/header.rs | 2 +- src/tools/compiletest/src/header/cfg.rs | 6 + src/tools/opt-dist/src/tests.rs | 7 +- .../rmake.rs | 108 ++++++++++++++++++ 6 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs diff --git a/src/doc/rustc-dev-guide/src/tests/directives.md b/src/doc/rustc-dev-guide/src/tests/directives.md index e80857b7afac9..426a6ff1da527 100644 --- a/src/doc/rustc-dev-guide/src/tests/directives.md +++ b/src/doc/rustc-dev-guide/src/tests/directives.md @@ -152,6 +152,8 @@ Some examples of `X` in `ignore-X` or `only-X`: `compare-mode-split-dwarf`, `compare-mode-split-dwarf-single` - The two different test modes used by coverage tests: `ignore-coverage-map`, `ignore-coverage-run` +- When testing a dist toolchain: `dist` + - This needs to be enabled with `COMPILETEST_ENABLE_DIST_TESTS=1` The following directives will check rustc build settings and target settings: diff --git a/src/tools/compiletest/src/directive-list.rs b/src/tools/compiletest/src/directive-list.rs index 01068af3e8c24..5784cd831196e 100644 --- a/src/tools/compiletest/src/directive-list.rs +++ b/src/tools/compiletest/src/directive-list.rs @@ -175,6 +175,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[ "only-beta", "only-bpf", "only-cdb", + "only-dist", "only-gnu", "only-i686-pc-windows-gnu", "only-i686-pc-windows-msvc", diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index 8c96554738e68..82925f54a1638 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -1280,7 +1280,7 @@ pub fn llvm_has_libzstd(config: &Config) -> bool { let stderr = String::from_utf8(output.stderr).ok()?; let zstd_available = !stderr.contains("LLVM was not built with LLVM_ENABLE_ZSTD"); - // We don't particularly need to clean the link up (so the previous commands could fail + // We don't partiCOMPILETEST_ENABLE_OPT_DIST_TESTScularly need to clean the link up (so the previous commands could fail // in theory but won't in practice), but we can try. std::fs::remove_file(lld_symlink_path).ok()?; diff --git a/src/tools/compiletest/src/header/cfg.rs b/src/tools/compiletest/src/header/cfg.rs index 3f7225195ce0b..6e5ced17c2084 100644 --- a/src/tools/compiletest/src/header/cfg.rs +++ b/src/tools/compiletest/src/header/cfg.rs @@ -235,6 +235,12 @@ fn parse_cfg_name_directive<'a>( message: "when the test mode is {name}", } + condition! { + name: "dist", + condition: std::env::var("COMPILETEST_ENABLE_DIST_TESTS") == Ok("1".to_string()), + message: "when performing tests on dist toolchain" + } + if prefix == "ignore" && outcome == MatchOutcome::Invalid { // Don't error out for ignore-tidy-* diretives, as those are not handled by compiletest. if name.starts_with("tidy-") { diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 06ed076a86437..2dd116a5f863e 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -108,7 +108,12 @@ llvm-config = "{llvm_config}" for test_path in env.skipped_tests() { args.extend(["--skip", test_path]); } - cmd(&args).env("COMPILETEST_FORCE_STAGE0", "1").run().context("Cannot execute tests") + cmd(&args) + .env("COMPILETEST_FORCE_STAGE0", "1") + // Also run dist-only tests + .env("COMPILETEST_ENABLE_DIST_TESTS", "1") + .run() + .context("Cannot execute tests") } /// Tries to find the version of the dist artifacts (either nightly, beta, or 1.XY.Z). diff --git a/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs new file mode 100644 index 0000000000000..ec693ca793c13 --- /dev/null +++ b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs @@ -0,0 +1,108 @@ +// Check that the compiler toolchain (rustc) that we distribute is not using newer glibc +// symbols than a specified minimum. +// This test should only be executed on an extracted dist archive or in a dist-* CI job. + +//@ only-dist +//@ only-x86_64-unknown-linux-gnu +//@ ignore-cross-compile + +use std::path::{Path, PathBuf}; + +use run_make_support::{cmd, llvm_objdump, regex, rustc_path}; + +fn main() { + // This is the maximum glibc version *supported* by the x86_64-unknown-linux-gnu target. + // All glibc symbols used in the compiler must be lower or equal than this version. + let max_supported = (2, 17, 99); + + let rustc = PathBuf::from(rustc_path()); + // Check symbols directly in rustc + check_symbols(&rustc, max_supported); + + // Find dynamic libraries referenced by rustc that come from our lib directory + let lib_path = rustc.parent().unwrap().parent().unwrap().join("lib"); + let dynamic_libs = find_dynamic_libs(&rustc) + .into_iter() + .filter_map(|path| path.canonicalize().ok()) + .filter(|lib| lib.starts_with(&lib_path)) + .collect::>(); + for lib in dynamic_libs { + check_symbols(&lib, max_supported); + } +} + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq)] +struct GlibcSymbol { + name: String, + version: (u32, u32, u32), +} + +fn find_dynamic_libs(path: &Path) -> Vec { + cmd("ldd") + .arg(path) + .run() + .stdout_utf8() + .lines() + .filter_map(|line| { + let line = line.trim(); + let Some((_, line)) = line.split_once(" => ") else { + return None; + }; + line.split_ascii_whitespace().next().map(|path| PathBuf::from(path)) + }) + .collect() +} + +fn check_symbols(file: &Path, max_supported: (u32, u32, u32)) { + println!("Checking {}", file.display()); + let mut invalid: Vec = get_glibc_symbols(file) + .into_iter() + .filter(|symbol| symbol.version > max_supported) + .collect(); + if !invalid.is_empty() { + invalid.sort(); + panic!( + "Found invalid glibc symbols in {}:\n{}", + file.display(), + invalid + .into_iter() + .map(|symbol| format!( + "{} ({:?} higher than max allowed {:?})", + symbol.name, symbol.version, max_supported + )) + .collect::>() + .join("\n") + ) + } +} + +fn get_glibc_symbols(file: &Path) -> Vec { + let regex = regex::Regex::new(r#"GLIBC_(\d)+\.(\d+)(:?\.(\d+))?"#).unwrap(); + + // Uses llvm-objdump, because implementing this using the `object` crate is quite complicated. + llvm_objdump() + .arg("-T") + .arg(file) + .run() + .stdout_utf8() + .lines() + .filter_map(|line| { + // Example line + // 0000000000000000 DF *UND* 0000000000000000 (GLIBC_2.2.5) sbrk + let mut parts = line.split(" ").collect::>().into_iter().rev(); + let Some(name) = parts.next() else { + return None; + }; + let Some(lib) = parts.next() else { + return None; + }; + let Some(version) = regex.captures(lib) else { + return None; + }; + let major = version.get(1).and_then(|m| m.as_str().parse().ok()).unwrap_or(0); + let minor = version.get(2).and_then(|m| m.as_str().parse().ok()).unwrap_or(0); + let patch = version.get(3).and_then(|m| m.as_str().parse().ok()).unwrap_or(0); + Some(GlibcSymbol { version: (major, minor, patch), name: name.to_string() }) + }) + .collect() +} From 75f8cc6ab651558a37d78b45deddf6ee8f526c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 21 Jan 2025 10:19:04 +0100 Subject: [PATCH 2/5] Enable verbose tests in opt-dist tests --- src/tools/opt-dist/src/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 2dd116a5f863e..00861f0c65941 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -69,6 +69,7 @@ change-id = 115898 [rust] channel = "{channel}" +verbose-tests = true [build] rustc = "{rustc}" From 7877883339e96577701344bd68a80470e1be22aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 21 Jan 2025 12:36:59 +0100 Subject: [PATCH 3/5] Run the glibc run-make test in opt-dist --- src/tools/compiletest/src/header.rs | 2 +- src/tools/opt-dist/src/tests.rs | 1 + tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index 82925f54a1638..8c96554738e68 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -1280,7 +1280,7 @@ pub fn llvm_has_libzstd(config: &Config) -> bool { let stderr = String::from_utf8(output.stderr).ok()?; let zstd_available = !stderr.contains("LLVM was not built with LLVM_ENABLE_ZSTD"); - // We don't partiCOMPILETEST_ENABLE_OPT_DIST_TESTScularly need to clean the link up (so the previous commands could fail + // We don't particularly need to clean the link up (so the previous commands could fail // in theory but won't in practice), but we can try. std::fs::remove_file(lld_symlink_path).ok()?; diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs index 00861f0c65941..8b3bd77141ba8 100644 --- a/src/tools/opt-dist/src/tests.rs +++ b/src/tools/opt-dist/src/tests.rs @@ -103,6 +103,7 @@ llvm-config = "{llvm_config}" "tests/incremental", "tests/mir-opt", "tests/pretty", + "tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu", "tests/ui", "tests/crashes", ]; diff --git a/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs index ec693ca793c13..76acf93c0553c 100644 --- a/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs +++ b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs @@ -81,7 +81,7 @@ fn get_glibc_symbols(file: &Path) -> Vec { // Uses llvm-objdump, because implementing this using the `object` crate is quite complicated. llvm_objdump() - .arg("-T") + .arg("--dynamic-syms") .arg(file) .run() .stdout_utf8() From 46ae7382a43fd8a53d2476208c52ce7255ede79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 22 Jan 2025 09:58:45 +0100 Subject: [PATCH 4/5] Use objdump instead of llvm-objdump --- .../glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs index 76acf93c0553c..3b9963f946591 100644 --- a/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs +++ b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs @@ -79,8 +79,10 @@ fn check_symbols(file: &Path, max_supported: (u32, u32, u32)) { fn get_glibc_symbols(file: &Path) -> Vec { let regex = regex::Regex::new(r#"GLIBC_(\d)+\.(\d+)(:?\.(\d+))?"#).unwrap(); - // Uses llvm-objdump, because implementing this using the `object` crate is quite complicated. - llvm_objdump() + // FIXME(kobzol): llvm-objdump currently chokes on the BOLTed librustc_driver.so file. + // Use objdump instead, since it seems to work, and we only run this test in a specific + // CI environment anyway. + cmd("objdump") .arg("--dynamic-syms") .arg(file) .run() From 5482bbac0f18d45f0e02bbe8153e47b160287fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 22 Jan 2025 17:22:39 +0100 Subject: [PATCH 5/5] Reword comment slightly --- .../run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs index 3b9963f946591..0134457c5c28e 100644 --- a/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs +++ b/tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs @@ -11,8 +11,10 @@ use std::path::{Path, PathBuf}; use run_make_support::{cmd, llvm_objdump, regex, rustc_path}; fn main() { - // This is the maximum glibc version *supported* by the x86_64-unknown-linux-gnu target. + // This is the maximum glibc version that we are *permitted* to use for the + // x86_64-unknown-linux-gnu target. // All glibc symbols used in the compiler must be lower or equal than this version. + // So that if a given machine only has glibc 2.17, it is able to run the compiler. let max_supported = (2, 17, 99); let rustc = PathBuf::from(rustc_path());