Add global_cache_tracker stability tests.

.github/workflows/main.yml

@@ -116,6 +116,8 @@ jobs:
       CARGO_PROFILE_TEST_DEBUG: 1
       CARGO_INCREMENTAL: 0
       CARGO_PUBLIC_NETWORK_TESTS: 1
+      # Workaround for https://github.com/rust-lang/rustup/issues/3036
+      RUSTUP_WINDOWS_PATH_ADD_BIN: 0
     strategy:
       matrix:
         include:
@@ -156,6 +158,10 @@ jobs:
     - uses: actions/checkout@v4
     - name: Dump Environment
       run: ci/dump-environment.sh
+    # Some tests require stable. Make sure it is set to the most recent stable
+    # so that we can predictably handle updates if necessary (and not randomly
+    # when GitHub updates its image).
+    - run: rustup update --no-self-update stable
     - run: rustup update --no-self-update ${{ matrix.rust }} && rustup default ${{ matrix.rust }}
     - run: rustup target add ${{ matrix.other }}
     - run: rustup component add rustc-dev llvm-tools-preview rust-docs
@@ -166,7 +172,6 @@ jobs:
     - name: Configure extra test environment
       run: echo CARGO_CONTAINER_TESTS=1 >> $GITHUB_ENV
       if: matrix.os == 'ubuntu-latest'
-
     - run: cargo test -p cargo
     - name: Clear intermediate test output
       run: ci/clean-test-output.sh

crates/cargo-test-macro/src/lib.rs

@@ -1,4 +1,5 @@
 use proc_macro::*;
+use std::path::Path;
 use std::process::Command;
 use std::sync::Once;
 
@@ -74,6 +75,12 @@ pub fn cargo_test(attr: TokenStream, item: TokenStream) -> TokenStream {
                 requires_reason = true;
                 set_ignore!(is_not_nightly, "requires nightly");
             }
+            "requires_rustup_stable" => {
+                set_ignore!(
+                    !has_rustup_stable(),
+                    "rustup or stable toolchain not installed"
+                );
+            }
             s if s.starts_with("requires_") => {
                 let command = &s[9..];
                 set_ignore!(!has_command(command), "{command} not installed");
@@ -204,29 +211,28 @@ fn version() -> (u32, bool) {
     unsafe { VERSION }
 }
 
-fn has_command(command: &str) -> bool {
-    let output = match Command::new(command).arg("--version").output() {
+fn check_command(command_path: &Path, args: &[&str]) -> bool {
+    let mut command = Command::new(command_path);
+    let command_name = command.get_program().to_str().unwrap().to_owned();
+    command.args(args);
+    let output = match command.output() {
         Ok(output) => output,
         Err(e) => {
             // * hg is not installed on GitHub macOS or certain constrained
             //   environments like Docker. Consider installing it if Cargo
             //   gains more hg support, but otherwise it isn't critical.
             // * lldb is not pre-installed on Ubuntu and Windows, so skip.
-            if is_ci() && !["hg", "lldb"].contains(&command) {
-                panic!(
-                    "expected command `{}` to be somewhere in PATH: {}",
-                    command, e
-                );
+            if is_ci() && !matches!(command_name.as_str(), "hg" | "lldb") {
+                panic!("expected command `{command_name}` to be somewhere in PATH: {e}",);
             }
             return false;
         }
     };
     if !output.status.success() {
         panic!(
-            "expected command `{}` to be runnable, got error {}:\n\
+            "expected command `{command_name}` to be runnable, got error {}:\n\
             stderr:{}\n\
             stdout:{}\n",
-            command,
             output.status,
             String::from_utf8_lossy(&output.stderr),
             String::from_utf8_lossy(&output.stdout)
@@ -235,6 +241,34 @@ fn has_command(command: &str) -> bool {
     true
 }
 
+fn has_command(command: &str) -> bool {
+    check_command(Path::new(command), &["--version"])
+}
+
+fn has_rustup_stable() -> bool {
+    if option_env!("CARGO_TEST_DISABLE_NIGHTLY").is_some() {
+        // This cannot run on rust-lang/rust CI due to the lack of rustup.
+        return false;
+    }
+    if cfg!(windows) && !is_ci() && option_env!("RUSTUP_WINDOWS_PATH_ADD_BIN").is_none() {
+        // There is an issue with rustup that doesn't allow recursive cargo
+        // invocations. Disable this on developer machines if the environment
+        // variable is not enabled. This can be removed once
+        // https://github.com/rust-lang/rustup/issues/3036 is resolved.
+        return false;
+    }
+    // Cargo mucks with PATH on Windows, adding sysroot host libdir, which is
+    // "bin", which circumvents the rustup wrapper. Use the path directly from
+    // CARGO_HOME.
+    let home = match option_env!("CARGO_HOME") {
+        Some(home) => home,
+        None if is_ci() => panic!("expected to run under rustup"),
+        None => return false,
+    };
+    let cargo = Path::new(home).join("bin/cargo");
+    check_command(&cargo, &["+stable", "--version"])
+}
+
 /// Whether or not this running in a Continuous Integration environment.
 fn is_ci() -> bool {
     // Consider using `tracked_env` instead of option_env! when it is stabilized.

crates/cargo-test-support/src/lib.rs

@@ -764,6 +764,13 @@ impl Execs {
         self
     }
 
+    pub fn args<T: AsRef<OsStr>>(&mut self, args: &[T]) -> &mut Self {
+        if let Some(ref mut p) = self.process_builder {
+            p.args(args);
+        }
+        self
+    }
+
     pub fn cwd<T: AsRef<OsStr>>(&mut self, path: T) -> &mut Self {
         if let Some(ref mut p) = self.process_builder {
             if let Some(cwd) = p.get_cwd() {

src/cargo/core/global_cache_tracker.rs

@@ -499,7 +499,7 @@ impl GlobalCacheTracker {
         let mut stmt = self.conn.prepare_cached(
             "SELECT git_db.name, git_checkout.name, git_checkout.size, git_checkout.timestamp
              FROM git_db, git_checkout
-             WHERE git_checkout.registry_id = git_db.id",
+             WHERE git_checkout.git_id = git_db.id",
         )?;
         let rows = stmt
             .query_map([], |row| {

tests/testsuite/global_cache_tracker.rs

@@ -14,11 +14,12 @@ use cargo::GlobalContext;
 use cargo_test_support::paths::{self, CargoPathExt};
 use cargo_test_support::registry::{Package, RegistryBuilder};
 use cargo_test_support::{
-    basic_manifest, cargo_process, execs, git, project, retry, sleep_ms, thread_wait_timeout,
-    Project,
+    basic_manifest, cargo_process, execs, git, process, project, retry, sleep_ms,
+    thread_wait_timeout, Execs, Project,
 };
 use itertools::Itertools;
 use std::fmt::Write;
+use std::path::Path;
 use std::path::PathBuf;
 use std::process::Stdio;
 use std::time::{Duration, SystemTime};
@@ -153,6 +154,14 @@ fn populate_cache(
     (cache_dir, src_dir)
 }
 
+fn rustup_cargo() -> Execs {
+    // Get the path to the rustup cargo wrapper. This is necessary because
+    // cargo adds the "deps" directory into PATH on Windows, which points to
+    // the wrong cargo.
+    let rustup_cargo = Path::new(&std::env::var_os("CARGO_HOME").unwrap()).join("bin/cargo");
+    execs().with_process_builder(process(rustup_cargo))
+}
+
 #[cargo_test]
 fn auto_gc_gated() {
     // Requires -Zgc to both track last-use data and to run auto-gc.
@@ -1863,3 +1872,150 @@ fn clean_gc_quiet_is_quiet() {
         )
         .run();
 }
+
+#[cargo_test(requires_rustup_stable)]
+fn compatible_with_older_cargo() {
+    // Ensures that db stays backwards compatible across versions.
+
+    // T-4 months: Current version, build the database.
+    Package::new("old", "1.0.0").publish();
+    Package::new("middle", "1.0.0").publish();
+    Package::new("new", "1.0.0").publish();
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                old = "1.0"
+                middle = "1.0"
+                new = "1.0"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+    // Populate the last-use data.
+    p.cargo("check -Zgc")
+        .masquerade_as_nightly_cargo(&["gc"])
+        .env("__CARGO_TEST_LAST_USE_NOW", months_ago_unix(4))
+        .run();
+    assert_eq!(
+        get_registry_names("src"),
+        ["middle-1.0.0", "new-1.0.0", "old-1.0.0"]
+    );
+    assert_eq!(
+        get_registry_names("cache"),
+        ["middle-1.0.0.crate", "new-1.0.0.crate", "old-1.0.0.crate"]
+    );
+
+    // T-2 months: Stable version, make sure it reads and deletes old src.
+    p.change_file(
+        "Cargo.toml",
+        r#"
+            [package]
+            name = "foo"
+            version = "0.1.0"
+
+            [dependencies]
+            new = "1.0"
+            middle = "1.0"
+        "#,
+    );
+    rustup_cargo()
+        .args(&["+stable", "check", "-Zgc"])
+        .cwd(p.root())
+        .masquerade_as_nightly_cargo(&["gc"])
+        .env("__CARGO_TEST_LAST_USE_NOW", months_ago_unix(2))
+        .run();
+    assert_eq!(get_registry_names("src"), ["middle-1.0.0", "new-1.0.0"]);
+    assert_eq!(
+        get_registry_names("cache"),
+        ["middle-1.0.0.crate", "new-1.0.0.crate", "old-1.0.0.crate"]
+    );
+
+    // T-0 months: Current version, make sure it can read data from stable,
+    // deletes old crate and middle src.
+    p.change_file(
+        "Cargo.toml",
+        r#"
+            [package]
+            name = "foo"
+            version = "0.1.0"
+
+            [dependencies]
+            new = "1.0"
+        "#,
+    );
+    p.cargo("check -Zgc")
+        .masquerade_as_nightly_cargo(&["gc"])
+        .run();
+    assert_eq!(get_registry_names("src"), ["new-1.0.0"]);
+    assert_eq!(
+        get_registry_names("cache"),
+        ["middle-1.0.0.crate", "new-1.0.0.crate"]
+    );
+}
+
+#[cargo_test(requires_rustup_stable)]
+fn forward_compatible() {
+    // Checks that db created in an older version can be read in a newer version.
+    Package::new("bar", "1.0.0").publish();
+    let git_project = git::new("from_git", |p| {
+        p.file("Cargo.toml", &basic_manifest("from_git", "1.0.0"))
+            .file("src/lib.rs", "")
+    });
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            &format!(
+                r#"
+                    [package]
+                    name = "foo"
+
+                    [dependencies]
+                    bar = "1.0.0"
+                    from_git = {{ git = '{}' }}
+                "#,
+                git_project.url()
+            ),
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    rustup_cargo()
+        .args(&["+stable", "check", "-Zgc"])
+        .cwd(p.root())
+        .masquerade_as_nightly_cargo(&["gc"])
+        .run();
+
+    let config = GlobalContextBuilder::new().unstable_flag("gc").build();
+    let lock = config
+        .acquire_package_cache_lock(CacheLockMode::MutateExclusive)
+        .unwrap();
+    let tracker = GlobalCacheTracker::new(&config).unwrap();
+    // Don't want to check the actual index name here, since although the
+    // names are semi-stable, they might change over long periods of time.
+    let indexes = tracker.registry_index_all().unwrap();
+    assert_eq!(indexes.len(), 1);
+    let crates = tracker.registry_crate_all().unwrap();
+    let names: Vec<_> = crates
+        .iter()
+        .map(|(krate, _timestamp)| krate.crate_filename)
+        .collect();
+    assert_eq!(names, &["bar-1.0.0.crate"]);
+    let srcs = tracker.registry_src_all().unwrap();
+    let names: Vec<_> = srcs
+        .iter()
+        .map(|(src, _timestamp)| src.package_dir)
+        .collect();
+    assert_eq!(names, &["bar-1.0.0"]);
+    let dbs: Vec<_> = tracker.git_db_all().unwrap();
+    assert_eq!(dbs.len(), 1);
+    let cos: Vec<_> = tracker.git_checkout_all().unwrap();
+    assert_eq!(cos.len(), 1);
+    drop(lock);
+}