From 1394b36158ee3e3cd8c7178df096405afe437a8e Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Thu, 23 May 2024 20:36:39 +0000 Subject: [PATCH] feat: vendor crates required by opt-dist to collect profiles These are the default package set required by opt-dist to correctly work, hence for people wanting to build a production grade of rustc in a sandboxed / air-gapped environment, these need to be vendored. The size of `rustc-src-nightly.tar.xz` before and after this change: * Before: 298M * After: 323M (+8%) These crates are the default set of packages required by opt-dist to correctly work, hence for people wanting to build a production grade of rustc in an sandboxed / air-gapped environment, these need to be vendored. The size of `rustc-src-nightly.tar.xz` before and after this change: * Before: 298M * After: 323M (+8%) Size change might or might not be a concern. See the previous discussion: https://github.com/rust-lang/rust/pull/125166#issuecomment-2113626468 Previous efforts on making: * https://github.com/rust-lang/rust/pull/125125 * https://github.com/rust-lang/rust/pull/125166 --- Note that extra works still need to be done to make it fully vendored. * The current pinned rustc-perf uses `tempfile::Tempdir` as the working directory when collecting profiles from some of these packages. This "tmp" working directory usage make it impossible for Cargo to pick up the correct vendor sources setting in `.cargo/config.toml` bundled in the rustc-src tarball. [^1] * opt-dist verifies the final built rustc against a subset of rustc test suite. However it rolls out its own `config.toml` without setting `vendor = true`, and that results in `./vendor/` directory removed. [^2] [^1]: https://github.com/rust-lang/rustc-perf/blob/4f313add609f43e928e98132358e8426ed3969ae/collector/src/compile/benchmark/mod.rs#L164-L173 [^2]: https://github.com/rust-lang/rust/blob/606afbb617a2949a4e35c4b0258ff94c980b9451/src/tools/opt-dist/src/tests.rs#L62-L77 --- src/bootstrap/src/core/build_steps/dist.rs | 15 ++++++++++++++ src/tools/build_helper/src/lib.rs | 23 ++++++++++++++++++++++ src/tools/opt-dist/src/training.rs | 22 +-------------------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 91039d0c8dcc1..60b1ff3e4413e 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1041,6 +1041,21 @@ impl Step for PlainSourceTarball { .env("RUSTC_BOOTSTRAP", "1") .current_dir(plain_dst_src); + // Vendor packages that are required by opt-dist to collect PGO profiles. + let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES + .iter() + .chain(build_helper::RUSTC_PGO_CRATES) + .map(|pkg| { + let mut manifest_path = + builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); + manifest_path.push(pkg); + manifest_path.push("Cargo.toml"); + manifest_path + }); + for manifest_path in pkgs_for_pgo_training { + cmd.arg("--sync").arg(manifest_path); + } + let config = if !builder.config.dry_run() { t!(String::from_utf8(t!(cmd.output()).stdout)) } else { diff --git a/src/tools/build_helper/src/lib.rs b/src/tools/build_helper/src/lib.rs index d831cfa0d79a6..15807d1c0d8f8 100644 --- a/src/tools/build_helper/src/lib.rs +++ b/src/tools/build_helper/src/lib.rs @@ -5,3 +5,26 @@ pub mod git; pub mod metrics; pub mod stage0_parser; pub mod util; + +/// The default set of crates for opt-dist to collect LLVM profiles. +pub const LLVM_PGO_CRATES: &[&str] = &[ + "syn-1.0.89", + "cargo-0.60.0", + "serde-1.0.136", + "ripgrep-13.0.0", + "regex-1.5.5", + "clap-3.1.6", + "hyper-0.14.18", +]; + +/// The default set of crates for opt-dist to collect rustc profiles. +pub const RUSTC_PGO_CRATES: &[&str] = &[ + "externs", + "ctfe-stress-5", + "cargo-0.60.0", + "token-stream-stress", + "match-stress", + "tuple-stress", + "diesel-1.4.8", + "bitmaps-3.1.0", +]; diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs index 09263dc645e2d..89f4d8957c8bf 100644 --- a/src/tools/opt-dist/src/training.rs +++ b/src/tools/opt-dist/src/training.rs @@ -3,30 +3,10 @@ use crate::exec::{cmd, CmdBuilder}; use crate::utils::io::{count_files, delete_directory}; use crate::utils::with_log_group; use anyhow::Context; +use build_helper::{LLVM_PGO_CRATES, RUSTC_PGO_CRATES}; use camino::{Utf8Path, Utf8PathBuf}; use humansize::BINARY; -const LLVM_PGO_CRATES: &[&str] = &[ - "syn-1.0.89", - "cargo-0.60.0", - "serde-1.0.136", - "ripgrep-13.0.0", - "regex-1.5.5", - "clap-3.1.6", - "hyper-0.14.18", -]; - -const RUSTC_PGO_CRATES: &[&str] = &[ - "externs", - "ctfe-stress-5", - "cargo-0.60.0", - "token-stream-stress", - "match-stress", - "tuple-stress", - "diesel-1.4.8", - "bitmaps-3.1.0", -]; - fn init_compiler_benchmarks( env: &Environment, profiles: &[&str],