Auto merge of #8066 - alexcrichton:embed-bitcode-no, r=ehuss

Add support for `-Cembed-bitcode=no`

This commit is the Cargo half of support necessary for
rust-lang/rust#70458. Today the compiler emits embedded bytecode in
rlibs by default, but compresses it. This is both extraneous disk space
and wasted build time for almost all builds, so the PR in question there
is changing rustc to have a `-Cembed-bitcode` flag which, when enabled,
places the bitcode in the object file rather than an auxiliary file (no
extra compression), but also enables `-Cembed-bitcode=no` to disable
bitcode emission entirely.

This Cargo support changes Cargo to pass `-Cembed-bitcode=no` for almost
all compilations. Cargo will keep `lto = true` and such working by not
passing this flag (and thus allowing bitcode to get embedded), but by
default `cargo build` and `cargo build --release` will no longer have
any bitcode in rlibs which should result in speedier builds!

Most of the changes here were around the test suite and various
assertions about the `rustc` command lines we spit out. One test was
hard-disabled until we can get `-Cembed-bitcode=no` into nightly, and
then we can make it a nightly-only test. The test will then be stable
again once `-Cembed-bitcode=no` hits stable.

Note that this is intended to land before the upstream `-Cembed-bitcode`
change. The thinking is that we'll land everything in rust-lang/rust all
at once so there's no build time regressions for anyone. If we were to
land the `-Cembed-bitcode` PR first then there would be a build time
regression until we land Cargo changes because rustc would be emitting
uncompressed bitcode by default and Cargo wouldn't be turning it off.

crates/cargo-test-support/src/lib.rs

@@ -1230,6 +1230,14 @@ impl Execs {
             }
             MatchKind::Unordered => {
                 let mut a = actual.lines().collect::<Vec<_>>();
+                // match more-constrained lines first, although in theory we'll
+                // need some sort of recursive match here. This handles the case
+                // that you expect "a\n[..]b" and two lines are printed out,
+                // "ab\n"a", where technically we do match unordered but a naive
+                // search fails to find this. This simple sort at least gets the
+                // test suite to pass for now, but we may need to get more fancy
+                // if tests start failing again.
+                a.sort_by_key(|s| s.len());
                 let e = out.lines();
 
                 for e_line in e {

src/cargo/core/compiler/build_context/target_info.rs

@@ -41,6 +41,8 @@ pub struct TargetInfo {
     pub rustflags: Vec<String>,
     /// Extra flags to pass to `rustdoc`, see `env_args`.
     pub rustdocflags: Vec<String>,
+    /// REmove this when it hits stable (1.44)
+    pub supports_embed_bitcode: Option<bool>,
 }
 
 /// Kind of each file generated by a Unit, part of `FileType`.
@@ -103,6 +105,13 @@ impl TargetInfo {
             .args(&rustflags)
             .env_remove("RUSTC_LOG");
 
+        let mut embed_bitcode_test = process.clone();
+        embed_bitcode_test.arg("-Cembed-bitcode");
+        let supports_embed_bitcode = match kind {
+            CompileKind::Host => Some(rustc.cached_output(&embed_bitcode_test).is_ok()),
+            _ => None,
+        };
+
         if let CompileKind::Target(target) = kind {
             process.arg("--target").arg(target.rustc_target());
         }
@@ -187,6 +196,7 @@ impl TargetInfo {
                 "RUSTDOCFLAGS",
             )?,
             cfg,
+            supports_embed_bitcode,
         })
     }
 

src/cargo/core/compiler/mod.rs

@@ -801,16 +801,32 @@ fn build_base_args<'a, 'cfg>(
 
     // Disable LTO for host builds as prefer_dynamic and it are mutually
     // exclusive.
-    if unit.target.can_lto() && !unit.target.for_host() {
-        match *lto {
-            Lto::Bool(false) => {}
-            Lto::Bool(true) => {
+    let lto_possible = unit.target.can_lto() && !unit.target.for_host();
+    match lto {
+        Lto::Bool(true) => {
+            if lto_possible {
                 cmd.args(&["-C", "lto"]);
             }
-            Lto::Named(ref s) => {
+        }
+        Lto::Named(s) => {
+            if lto_possible {
                 cmd.arg("-C").arg(format!("lto={}", s));
             }
         }
+        // If LTO isn't being enabled then there's no need for bitcode to be
+        // present in the intermediate artifacts, so shave off some build time
+        // by removing it.
+        Lto::Bool(false) => {
+            if cx
+                .bcx
+                .target_data
+                .info(CompileKind::Host)
+                .supports_embed_bitcode
+                .unwrap()
+            {
+                cmd.arg("-Cembed-bitcode=no");
+            }
+        }
     }
 
     if let Some(n) = codegen_units {

tests/testsuite/build.rs

@@ -1149,13 +1149,13 @@ fn cargo_default_env_metadata_env_var() {
 [COMPILING] bar v0.0.1 ([CWD]/bar)
 [RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type dylib \
         --emit=[..]link \
-        -C prefer-dynamic -C debuginfo=2 \
+        -C prefer-dynamic[..]-C debuginfo=2 \
         -C metadata=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/debug/deps`
 [COMPILING] foo v0.0.1 ([CWD])
 [RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib \
-        --emit=[..]link -C debuginfo=2 \
+        --emit=[..]link[..]-C debuginfo=2 \
         -C metadata=[..] \
         -C extra-filename=[..] \
         --out-dir [..] \
@@ -1177,13 +1177,13 @@ fn cargo_default_env_metadata_env_var() {
 [COMPILING] bar v0.0.1 ([CWD]/bar)
 [RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type dylib \
         --emit=[..]link \
-        -C prefer-dynamic -C debuginfo=2 \
+        -C prefer-dynamic[..]-C debuginfo=2 \
         -C metadata=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/debug/deps`
 [COMPILING] foo v0.0.1 ([CWD])
 [RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib \
-        --emit=[..]link -C debuginfo=2 \
+        --emit=[..]link[..]-C debuginfo=2 \
         -C metadata=[..] \
         -C extra-filename=[..] \
         --out-dir [..] \
@@ -1581,7 +1581,7 @@ fn verbose_build() {
             "\
 [COMPILING] foo v0.0.1 ([CWD])
 [RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib \
-        --emit=[..]link -C debuginfo=2 \
+        --emit=[..]link[..]-C debuginfo=2 \
         -C metadata=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/debug/deps`
@@ -1599,8 +1599,8 @@ fn verbose_release_build() {
             "\
 [COMPILING] foo v0.0.1 ([CWD])
 [RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib \
-        --emit=[..]link \
-        -C opt-level=3 \
+        --emit=[..]link[..]\
+        -C opt-level=3[..]\
         -C metadata=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/release/deps`
@@ -1650,15 +1650,15 @@ fn verbose_release_build_deps() {
 [RUNNING] `rustc --crate-name foo foo/src/lib.rs [..]\
         --crate-type dylib --crate-type rlib \
         --emit=[..]link \
-        -C prefer-dynamic \
-        -C opt-level=3 \
+        -C prefer-dynamic[..]\
+        -C opt-level=3[..]\
         -C metadata=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/release/deps`
 [COMPILING] test v0.0.0 ([CWD])
 [RUNNING] `rustc --crate-name test src/lib.rs [..]--crate-type lib \
-        --emit=[..]link \
-        -C opt-level=3 \
+        --emit=[..]link[..]\
+        -C opt-level=3[..]\
         -C metadata=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/release/deps \
@@ -4205,15 +4205,15 @@ fn build_filter_infer_profile() {
     p.cargo("build -v --test=t1")
         .with_stderr_contains(
             "[RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib \
-             --emit=[..]link -C debuginfo=2 [..]",
+             --emit=[..]link[..]-C debuginfo=2 [..]",
         )
         .with_stderr_contains(
-            "[RUNNING] `rustc --crate-name t1 tests/t1.rs [..]--emit=[..]link \
+            "[RUNNING] `rustc --crate-name t1 tests/t1.rs [..]--emit=[..]link[..]\
              -C debuginfo=2 [..]",
         )
         .with_stderr_contains(
             "[RUNNING] `rustc --crate-name foo src/main.rs [..]--crate-type bin \
-             --emit=[..]link -C debuginfo=2 [..]",
+             --emit=[..]link[..]-C debuginfo=2 [..]",
         )
         .run();
 
@@ -4222,16 +4222,16 @@ fn build_filter_infer_profile() {
     p.cargo("build -v --bench=b1")
         .with_stderr_contains(
             "[RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib \
-             --emit=[..]link -C debuginfo=2 [..]",
+             --emit=[..]link[..]-C debuginfo=2 [..]",
         )
         .with_stderr_contains(
-            "[RUNNING] `rustc --crate-name b1 benches/b1.rs [..]--emit=[..]link \
+            "[RUNNING] `rustc --crate-name b1 benches/b1.rs [..]--emit=[..]link[..]\
              -C debuginfo=2 [..]",
         )
         .with_stderr_does_not_contain("opt-level")
         .with_stderr_contains(
             "[RUNNING] `rustc --crate-name foo src/main.rs [..]--crate-type bin \
-             --emit=[..]link -C debuginfo=2 [..]",
+             --emit=[..]link[..]-C debuginfo=2 [..]",
         )
         .run();
 }
@@ -4252,7 +4252,7 @@ fn targets_selected_default() {
         )
         // Unit tests.
         .with_stderr_does_not_contain(
-            "[RUNNING] `rustc --crate-name foo src/main.rs [..]--emit=[..]link \
+            "[RUNNING] `rustc --crate-name foo src/main.rs [..]--emit=[..]link[..]\
              -C debuginfo=2 --test [..]",
         )
         .run();
@@ -4269,7 +4269,7 @@ fn targets_selected_all() {
         )
         // Unit tests.
         .with_stderr_contains(
-            "[RUNNING] `rustc --crate-name foo src/main.rs [..]--emit=[..]link \
+            "[RUNNING] `rustc --crate-name foo src/main.rs [..]--emit=[..]link[..]\
              -C debuginfo=2 --test [..]",
         )
         .run();
@@ -4286,7 +4286,7 @@ fn all_targets_no_lib() {
         )
         // Unit tests.
         .with_stderr_contains(
-            "[RUNNING] `rustc --crate-name foo src/main.rs [..]--emit=[..]link \
+            "[RUNNING] `rustc --crate-name foo src/main.rs [..]--emit=[..]link[..]\
              -C debuginfo=2 --test [..]",
         )
         .run();
@@ -4726,7 +4726,7 @@ fn build_lib_only() {
             "\
 [COMPILING] foo v0.0.1 ([CWD])
 [RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib \
-        --emit=[..]link -C debuginfo=2 \
+        --emit=[..]link[..]-C debuginfo=2 \
         -C metadata=[..] \
         --out-dir [..] \
         -L dependency=[CWD]/target/debug/deps`

tests/testsuite/build_script.rs

@@ -1139,19 +1139,19 @@ fn build_cmd_with_a_build_cmd() {
 [RUNNING] `rustc [..] a/build.rs [..] --extern b=[..]`
 [RUNNING] `[..]/a-[..]/build-script-build`
 [RUNNING] `rustc --crate-name a [..]lib.rs [..]--crate-type lib \
-    --emit=[..]link -C debuginfo=2 \
+    --emit=[..]link[..]-C debuginfo=2 \
     -C metadata=[..] \
     --out-dir [..]target/debug/deps \
     -L [..]target/debug/deps`
 [COMPILING] foo v0.5.0 ([CWD])
 [RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin \
-    --emit=[..]link \
+    --emit=[..]link[..]\
     -C debuginfo=2 -C metadata=[..] --out-dir [..] \
     -L [..]target/debug/deps \
     --extern a=[..]liba[..].rlib`
 [RUNNING] `[..]/foo-[..]/build-script-build`
 [RUNNING] `rustc --crate-name foo [..]lib.rs [..]--crate-type lib \
-    --emit=[..]link -C debuginfo=2 \
+    --emit=[..]link[..]-C debuginfo=2 \
     -C metadata=[..] \
     --out-dir [..] \
     -L [..]target/debug/deps`

tests/testsuite/cross_compile.rs

@@ -193,7 +193,7 @@ fn linker() {
             "\
 [COMPILING] foo v0.5.0 ([CWD])
 [RUNNING] `rustc --crate-name foo src/foo.rs [..]--crate-type bin \
-    --emit=[..]link -C debuginfo=2 \
+    --emit=[..]link[..]-C debuginfo=2 \
     -C metadata=[..] \
     --out-dir [CWD]/target/{target}/debug/deps \
     --target {target} \

tests/testsuite/freshness.rs

@@ -1427,8 +1427,8 @@ fn reuse_panic_pm() {
         .with_stderr_unordered(
             "\
 [COMPILING] bar [..]
-[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C debuginfo=2 [..]
-[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C debuginfo=2 [..]
+[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link[..]-C debuginfo=2 [..]
+[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort[..]-C debuginfo=2 [..]
 [COMPILING] somepm [..]
 [RUNNING] `rustc --crate-name somepm [..]
 [COMPILING] foo [..]

tests/testsuite/lto.rs

@@ -0,0 +1,26 @@
+use cargo_test_support::project;
+use cargo_test_support::registry::Package;
+
+#[cargo_test]
+fn with_deps() {
+    Package::new("bar", "0.0.1").publish();
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "test"
+                version = "0.0.0"
+
+                [dependencies]
+                bar = "*"
+
+                [profile.release]
+                lto = true
+            "#,
+        )
+        .file("src/main.rs", "extern crate bar; fn main() {}")
+        .build();
+    p.cargo("build -v --release").run();
+}

tests/testsuite/main.rs

@@ -61,6 +61,7 @@ mod local_registry;
 mod locate_project;
 mod lockfile_compat;
 mod login;
+mod lto;
 mod member_errors;
 mod message_format;
 mod metabuild;

tests/testsuite/profile_config.rs

@@ -283,7 +283,7 @@ fn profile_config_override_precedence() {
         .with_stderr(
             "\
 [COMPILING] bar [..]
-[RUNNING] `rustc --crate-name bar [..] -C opt-level=2 -C codegen-units=2 [..]
+[RUNNING] `rustc --crate-name bar [..] -C opt-level=2[..]-C codegen-units=2 [..]
 [COMPILING] foo [..]
 [RUNNING] `rustc --crate-name foo [..]-C codegen-units=2 [..]
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]",

tests/testsuite/profile_overrides.rs

@@ -215,17 +215,17 @@ fn profile_override_hierarchy() {
     p.cargo("build -v").with_stderr_unordered("\
 [COMPILING] m3 [..]
 [COMPILING] dep [..]
-[RUNNING] `rustc --crate-name m3 m3/src/lib.rs [..] --crate-type lib --emit=[..]link -C codegen-units=4 [..]
-[RUNNING] `rustc --crate-name dep [..]dep/src/lib.rs [..] --crate-type lib --emit=[..]link -C codegen-units=3 [..]
-[RUNNING] `rustc --crate-name m3 m3/src/lib.rs [..] --crate-type lib --emit=[..]link -C codegen-units=1 [..]
-[RUNNING] `rustc --crate-name build_script_build m1/build.rs [..] --crate-type bin --emit=[..]link -C codegen-units=4 [..]
+[RUNNING] `rustc --crate-name m3 m3/src/lib.rs [..] --crate-type lib --emit=[..]link[..]-C codegen-units=4 [..]
+[RUNNING] `rustc --crate-name dep [..]dep/src/lib.rs [..] --crate-type lib --emit=[..]link[..]-C codegen-units=3 [..]
+[RUNNING] `rustc --crate-name m3 m3/src/lib.rs [..] --crate-type lib --emit=[..]link[..]-C codegen-units=1 [..]
+[RUNNING] `rustc --crate-name build_script_build m1/build.rs [..] --crate-type bin --emit=[..]link[..]-C codegen-units=4 [..]
 [COMPILING] m2 [..]
-[RUNNING] `rustc --crate-name build_script_build m2/build.rs [..] --crate-type bin --emit=[..]link -C codegen-units=2 [..]
+[RUNNING] `rustc --crate-name build_script_build m2/build.rs [..] --crate-type bin --emit=[..]link[..]-C codegen-units=2 [..]
 [RUNNING] `[..]/m1-[..]/build-script-build`
 [RUNNING] `[..]/m2-[..]/build-script-build`
-[RUNNING] `rustc --crate-name m2 m2/src/lib.rs [..] --crate-type lib --emit=[..]link -C codegen-units=2 [..]
+[RUNNING] `rustc --crate-name m2 m2/src/lib.rs [..] --crate-type lib --emit=[..]link[..]-C codegen-units=2 [..]
 [COMPILING] m1 [..]
-[RUNNING] `rustc --crate-name m1 m1/src/lib.rs [..] --crate-type lib --emit=[..]link -C codegen-units=1 [..]
+[RUNNING] `rustc --crate-name m1 m1/src/lib.rs [..] --crate-type lib --emit=[..]link[..]-C codegen-units=1 [..]
 [FINISHED] dev [unoptimized + debuginfo] [..]
 ",
         )