From 136bf7f73400137039f3abf009d007501e7b470a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 15 Apr 2025 10:47:10 +0200 Subject: [PATCH 1/4] Fix `llvm-profdata` extension on non Linux OSes --- xtask/src/dist.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/src/dist.rs b/xtask/src/dist.rs index a07244252bf4..8b9872f0777f 100644 --- a/xtask/src/dist.rs +++ b/xtask/src/dist.rs @@ -168,7 +168,7 @@ fn gather_pgo_profile<'a>( .read() .context("cannot resolve target-libdir from rustc")?; let target_bindir = PathBuf::from(target_libdir).parent().unwrap().join("bin"); - let llvm_profdata = target_bindir.join(format!("llvm-profdata{}", EXE_EXTENSION)); + let llvm_profdata = target_bindir.join("llvm-profdata").with_extension(EXE_EXTENSION); // Build RA with PGO instrumentation let cmd_gather = From 813ea83e5f7eb8209a20590b28c3ab8553ee58e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 15 Apr 2025 10:30:13 +0200 Subject: [PATCH 2/4] Allow passing a specific crate for PGO training --- .github/workflows/release.yaml | 4 ++-- xtask/src/dist.rs | 35 ++++++++++++++++++++++++++++------ xtask/src/flags.rs | 23 ++++++++++++++++++++-- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c76d3da5f8d4..a0972dd849c7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,11 +88,11 @@ jobs: - name: Dist (plain) if: ${{ !matrix.zig_target }} - run: cargo xtask dist --client-patch-version ${{ github.run_number }} ${{ matrix.pgo && '--pgo' || ''}} + run: cargo xtask dist --client-patch-version ${{ github.run_number }} ${{ matrix.pgo && format('--pgo {0}', matrix.pgo) || ''}} - name: Dist (using zigbuild) if: ${{ matrix.zig_target }} - run: RA_TARGET=${{ matrix.zig_target}} cargo xtask dist --client-patch-version ${{ github.run_number }} --zig ${{ matrix.pgo && '--pgo' || ''}} + run: RA_TARGET=${{ matrix.zig_target}} cargo xtask dist --client-patch-version ${{ github.run_number }} --zig ${{ matrix.pgo && format('--pgo {0}', matrix.pgo) || ''}} - run: npm ci working-directory: editors/code diff --git a/xtask/src/dist.rs b/xtask/src/dist.rs index 8b9872f0777f..cb15c3a14413 100644 --- a/xtask/src/dist.rs +++ b/xtask/src/dist.rs @@ -12,6 +12,7 @@ use time::OffsetDateTime; use xshell::{Cmd, Shell, cmd}; use zip::{DateTime, ZipWriter, write::SimpleFileOptions}; +use crate::flags::PgoTrainingCrate; use crate::{ date_iso, flags::{self, Malloc}, @@ -93,7 +94,7 @@ fn dist_server( target: &Target, allocator: Malloc, zig: bool, - pgo: bool, + pgo: Option, ) -> anyhow::Result<()> { let _e = sh.push_env("CFG_RELEASE", release); let _e = sh.push_env("CARGO_PROFILE_RELEASE_LTO", "thin"); @@ -111,11 +112,12 @@ fn dist_server( let features = allocator.to_features(); let command = if linux_target && zig { "zigbuild" } else { "build" }; - let pgo_profile = if pgo { + let pgo_profile = if let Some(train_crate) = pgo { Some(gather_pgo_profile( sh, build_command(sh, command, &target_name, features), &target_name, + train_crate, )?) } else { None @@ -155,8 +157,9 @@ fn gather_pgo_profile<'a>( sh: &'a Shell, ra_build_cmd: Cmd<'a>, target: &str, + train_crate: PgoTrainingCrate, ) -> anyhow::Result { - let pgo_dir = std::path::absolute("ra-pgo-profiles")?; + let pgo_dir = std::path::absolute("rust-analyzer-pgo")?; // Clear out any stale profiles if pgo_dir.is_dir() { std::fs::remove_dir_all(&pgo_dir)?; @@ -175,12 +178,21 @@ fn gather_pgo_profile<'a>( ra_build_cmd.env("RUSTFLAGS", format!("-Cprofile-generate={}", pgo_dir.to_str().unwrap())); cmd_gather.run().context("cannot build rust-analyzer with PGO instrumentation")?; - // Run RA on itself to gather profiles - let train_crate = "."; + let (train_path, label) = match &train_crate { + PgoTrainingCrate::RustAnalyzer => (PathBuf::from("."), "itself"), + PgoTrainingCrate::GitHub(url) => { + (download_crate_for_training(sh, &pgo_dir, url)?, url.as_str()) + } + }; + + // Run RA either on itself or on a downloaded crate + eprintln!("Training RA on {label}..."); cmd!( sh, - "target/{target}/release/rust-analyzer analysis-stats {train_crate} --run-all-ide-things" + "target/{target}/release/rust-analyzer analysis-stats --run-all-ide-things {train_path}" ) + // analysis-stats produces an enormous amount of output on stdout + .ignore_stdout() .run() .context("cannot generate PGO profiles")?; @@ -201,6 +213,17 @@ fn gather_pgo_profile<'a>( Ok(merged_profile) } +/// Downloads a crate from GitHub, stores it into `pgo_dir` and returns a path to it. +fn download_crate_for_training(sh: &Shell, pgo_dir: &Path, url: &str) -> anyhow::Result { + let normalized_path = url.replace("/", "-"); + let target_path = pgo_dir.join(normalized_path); + cmd!(sh, "git clone --depth 1 https://github.com/{url} {target_path}") + .run() + .with_context(|| "cannot download PGO training crate from {url}")?; + + Ok(target_path) +} + fn gzip(src_path: &Path, dest_path: &Path) -> anyhow::Result<()> { let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best()); let mut input = io::BufReader::new(File::open(src_path)?); diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs index 81b6a1b7d2ad..700806d178c3 100644 --- a/xtask/src/flags.rs +++ b/xtask/src/flags.rs @@ -60,7 +60,7 @@ xflags::xflags! { /// Use cargo-zigbuild optional --zig /// Apply PGO optimizations - optional --pgo + optional --pgo pgo: PgoTrainingCrate } /// Read a changelog AsciiDoc file and update the GitHub Releases entry in Markdown. cmd publish-release-notes { @@ -143,13 +143,32 @@ pub struct RustcPush { pub branch: Option, } +#[derive(Debug)] +pub enum PgoTrainingCrate { + // Use RA's own sources for PGO training + RustAnalyzer, + // Download a Rust crate from `https://github.com/{0}` and use it for PGO training. + GitHub(String), +} + +impl FromStr for PgoTrainingCrate { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "rust-analyzer" => Ok(Self::RustAnalyzer), + url => Ok(Self::GitHub(url.to_owned())), + } + } +} + #[derive(Debug)] pub struct Dist { pub mimalloc: bool, pub jemalloc: bool, pub client_patch_version: Option, pub zig: bool, - pub pgo: bool, + pub pgo: Option, } #[derive(Debug)] From 07307f5654e1b78c80ab571946e1431266fda0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 15 Apr 2025 12:18:09 +0200 Subject: [PATCH 3/4] Run PGO for Windows on CI --- .github/workflows/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a0972dd849c7..aa0d618f0f48 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -29,8 +29,10 @@ jobs: - os: windows-latest target: x86_64-pc-windows-msvc code-target: win32-x64 + pgo: clap-rs/clap - os: windows-latest target: i686-pc-windows-msvc + pgo: clap-rs/clap - os: windows-latest target: aarch64-pc-windows-msvc code-target: win32-arm64 From e12d15d11980746933aec7df0785a4fd9d0b4e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 15 Apr 2025 12:46:56 +0200 Subject: [PATCH 4/4] Use `-q` to silence RA output --- xtask/src/dist.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xtask/src/dist.rs b/xtask/src/dist.rs index cb15c3a14413..7e26167725f4 100644 --- a/xtask/src/dist.rs +++ b/xtask/src/dist.rs @@ -189,10 +189,8 @@ fn gather_pgo_profile<'a>( eprintln!("Training RA on {label}..."); cmd!( sh, - "target/{target}/release/rust-analyzer analysis-stats --run-all-ide-things {train_path}" + "target/{target}/release/rust-analyzer analysis-stats -q --run-all-ide-things {train_path}" ) - // analysis-stats produces an enormous amount of output on stdout - .ignore_stdout() .run() .context("cannot generate PGO profiles")?;