fix: allow gcs:// and oci:// in gateway (#845)

Fixes prefix-dev/pixi#1922

crates/rattler-bin/Cargo.toml

@@ -29,7 +29,7 @@ indicatif = { workspace = true }
 once_cell = { workspace = true }
 rattler = { path="../rattler", version = "0.27.8", default-features = false, features = ["indicatif"] }
 rattler_conda_types = { path="../rattler_conda_types", version = "0.27.4", default-features = false }
-rattler_networking = { path="../rattler_networking", version = "0.21.3", default-features = false }
+rattler_networking = { path="../rattler_networking", version = "0.21.3", default-features = false, features = ["google-cloud-auth"] }
 rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.10", default-features = false, features = ["gateway"] }
 rattler_solve = { path="../rattler_solve", version = "1.0.5", default-features = false, features = ["resolvo", "libsolv_c"] }
 rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.1.1", default-features = false }

crates/rattler-bin/src/commands/create.rs

@@ -140,6 +140,8 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         .with_arc(Arc::new(AuthenticationMiddleware::new(
             authentication_storage,
         )))
+        .with(rattler_networking::OciMiddleware)
+        .with(rattler_networking::GCSMiddleware)
         .build();
 
     // Get the package names from the matchspecs so we can only load the package records that we need.

crates/rattler_cache/src/package_cache/mod.rs

@@ -372,7 +372,10 @@ where
 
         let read_revision = write_lock.read_revision()?;
         if read_revision != cache_revision {
-            tracing::warn!("cache revisions dont match '{}", lock_file_path.display());
+            tracing::debug!(
+                "cache revisions dont match '{}', retrying to acquire lock file.",
+                lock_file_path.display()
+            );
             // The cache has been modified since we last checked. We need to re-validate.
             continue;
         }
@@ -386,8 +389,6 @@ where
             .await
             .map_err(|e| PackageCacheError::FetchError(Arc::new(e)))?;
 
-        tracing::warn!("fetched '{}", lock_file_path.display());
-
         validated_revision = Some(new_revision);
     }
 }

crates/rattler_repodata_gateway/src/gateway/mod.rs

@@ -31,6 +31,7 @@ use reqwest_middleware::ClientWithMiddleware;
 use subdir::{Subdir, SubdirData};
 use tokio::sync::broadcast;
 use tracing::instrument;
+use url::Url;
 
 use crate::{fetch::FetchRepoDataError, gateway::error::SubdirNotFoundError, Reporter};
 
@@ -285,32 +286,32 @@ impl GatewayInner {
                     "unsupported file based url".to_string(),
                 ));
             }
-        } else if url.scheme() == "http" || url.scheme() == "https" {
-            if url.host_str() == Some("fast.prefiks.dev")
-                || url.host_str() == Some("fast.prefix.dev")
-            {
-                sharded_subdir::ShardedSubdir::new(
-                    channel.clone(),
-                    platform.to_string(),
-                    self.client.clone(),
-                    self.cache.clone(),
-                    self.concurrent_requests_semaphore.clone(),
-                    reporter.as_deref(),
-                )
-                .await
-                .map(SubdirData::from_client)
-            } else {
-                remote_subdir::RemoteSubdirClient::new(
-                    channel.clone(),
-                    platform,
-                    self.client.clone(),
-                    self.cache.clone(),
-                    self.channel_config.get(channel).clone(),
-                    reporter,
-                )
-                .await
-                .map(SubdirData::from_client)
-            }
+        } else if supports_sharded_repodata(&url) {
+            sharded_subdir::ShardedSubdir::new(
+                channel.clone(),
+                platform.to_string(),
+                self.client.clone(),
+                self.cache.clone(),
+                self.concurrent_requests_semaphore.clone(),
+                reporter.as_deref(),
+            )
+            .await
+            .map(SubdirData::from_client)
+        } else if url.scheme() == "http"
+            || url.scheme() == "https"
+            || url.scheme() == "gcs"
+            || url.scheme() == "oci"
+        {
+            remote_subdir::RemoteSubdirClient::new(
+                channel.clone(),
+                platform,
+                self.client.clone(),
+                self.cache.clone(),
+                self.channel_config.get(channel).clone(),
+                reporter,
+            )
+            .await
+            .map(SubdirData::from_client)
         } else {
             return Err(GatewayError::UnsupportedUrl(format!(
                 "'{}' is not a supported scheme",
@@ -350,22 +351,27 @@ enum PendingOrFetched<T> {
     Fetched(T),
 }
 
+fn supports_sharded_repodata(url: &Url) -> bool {
+    (url.scheme() == "http" || url.scheme() == "https")
+        && (url.host_str() == Some("fast.prefiks.dev") || url.host_str() == Some("fast.prefix.dev"))
+}
+
 #[cfg(test)]
 mod test {
-    use assert_matches::assert_matches;
     use std::{
         path::{Path, PathBuf},
         str::FromStr,
         sync::Arc,
         time::Instant,
     };
 
+    use assert_matches::assert_matches;
     use dashmap::DashSet;
-    use rattler_cache::default_cache_dir;
-    use rattler_cache::package_cache::PackageCache;
+    use rattler_cache::{default_cache_dir, package_cache::PackageCache};
     use rattler_conda_types::{
-        Channel, ChannelConfig, MatchSpec, PackageName, ParseStrictness::Lenient,
-        ParseStrictness::Strict, Platform, RepoDataRecord,
+        Channel, ChannelConfig, MatchSpec, PackageName,
+        ParseStrictness::{Lenient, Strict},
+        Platform, RepoDataRecord,
     };
     use rstest::rstest;
     use url::Url;
@@ -490,7 +496,8 @@ mod test {
         assert_eq!(non_openssl_total_records, non_openssl_direct_records);
     }
 
-    // Make sure that the direct url version of openssl is used instead of the one from the normal channel.
+    // Make sure that the direct url version of openssl is used instead of the one
+    // from the normal channel.
     #[tokio::test]
     async fn test_select_forced_url_instead_of_deps() {
         let gateway = Gateway::builder()

crates/rattler_virtual_packages/src/lib.rs

@@ -147,7 +147,7 @@ pub enum VirtualPackage {
     /// Available on `Unix` based platforms
     Unix,
 
-    /// Available when running on `Linux``
+    /// Available when running on `Linux`
     Linux(Linux),
 
     /// Available when running on `OSX`