Better parallel testing (#503)

* remove serial tests, connect to random databases and buckets * fix redis * remove unused deps * share postgres and redis * sleep between lock attempts * rename examples
kitsune-soc · Mar 17, 2024 · 7c9e7ac · 7c9e7ac
1 parent b837e3b
commit 7c9e7ac
Show file tree

Hide file tree

Showing 23 changed files with 155 additions and 121 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -38,11 +38,37 @@ jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: test_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@v4
       - uses: DeterminateSystems/magic-nix-cache-action@main
       - uses: Swatinem/rust-cache@v2
       - uses: rui314/setup-mold@v1
       - uses: taiki-e/install-action@nextest
-      - run: nix develop --impure --command bash -c "unset DATABASE_URL LD_LIBRARY_PATH REDIS_URL && cargo nextest run --all-features"
+      - run: nix develop --impure --command bash -c "unset LD_LIBRARY_PATH && cargo nextest run --all-features"
+        env:
+          DATABASE_URL: "postgres://postgres:postgres@localhost/test_db"
+          REDIS_URL: "redis://localhost"
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/kitsune-activitypub/Cargo.toml b/crates/kitsune-activitypub/Cargo.toml
@@ -52,7 +52,6 @@ kitsune-config = { path = "../kitsune-config" }
 kitsune-test = { path = "../kitsune-test" }
 kitsune-webfinger = { path = "../kitsune-webfinger" }
 pretty_assertions = "1.4.0"
-serial_test = "3.0.0"
 tokio = { version = "1.36.0", features = ["macros"] }
 tower = { version = "0.4.13", default-features = false, features = ["util"] }
 

diff --git a/crates/kitsune-activitypub/tests/fetcher/basic.rs b/crates/kitsune-activitypub/tests/fetcher/basic.rs
@@ -20,7 +20,6 @@ use std::sync::Arc;
 use tower::service_fn;
 
 #[tokio::test]
-#[serial_test::serial]
 async fn fetch_actor() {
     database_test(|db_pool| async move {
         let client = Client::builder().service(service_fn(handle));
@@ -63,7 +62,6 @@ async fn fetch_actor() {
 }
 
 #[tokio::test]
-#[serial_test::serial]
 async fn fetch_emoji() {
     database_test(|db_pool| async move {
         let client = Client::builder().service(service_fn(handle));
@@ -116,7 +114,6 @@ async fn fetch_emoji() {
 }
 
 #[tokio::test]
-#[serial_test::serial]
 async fn fetch_note() {
     database_test(|db_pool| async move {
         let client = Client::builder().service(service_fn(handle));

diff --git a/crates/kitsune-activitypub/tests/fetcher/filter.rs b/crates/kitsune-activitypub/tests/fetcher/filter.rs
@@ -14,7 +14,6 @@ use std::{convert::Infallible, sync::Arc};
 use tower::service_fn;
 
 #[tokio::test]
-#[serial_test::serial]
 async fn federation_allow() {
     database_test(|db_pool| async move {
         let builder = Fetcher::builder()
@@ -89,7 +88,6 @@ async fn federation_allow() {
 }
 
 #[tokio::test]
-#[serial_test::serial]
 async fn federation_deny() {
     database_test(|db_pool| async move {
         let client = service_fn(

diff --git a/crates/kitsune-activitypub/tests/fetcher/infinite.rs b/crates/kitsune-activitypub/tests/fetcher/infinite.rs
@@ -24,7 +24,6 @@ use std::{
 use tower::service_fn;
 
 #[tokio::test]
-#[serial_test::serial]
 async fn fetch_infinitely_long_reply_chain() {
     database_test(|db_pool| async move {
             let request_counter = Arc::new(AtomicU32::new(0));

diff --git a/crates/kitsune-activitypub/tests/fetcher/origin.rs b/crates/kitsune-activitypub/tests/fetcher/origin.rs
@@ -14,7 +14,6 @@ use std::{convert::Infallible, sync::Arc};
 use tower::service_fn;
 
 #[tokio::test]
-#[serial_test::serial]
 async fn check_ap_id_authority() {
     database_test(|db_pool| async move {
         let builder = Fetcher::builder()
@@ -80,7 +79,6 @@ async fn check_ap_id_authority() {
 }
 
 #[tokio::test]
-#[serial_test::serial]
 async fn check_ap_content_type() {
     database_test(|db_pool| async move {
         let client = service_fn(|req: Request<_>| async {

diff --git a/crates/kitsune-activitypub/tests/fetcher/webfinger.rs b/crates/kitsune-activitypub/tests/fetcher/webfinger.rs
@@ -16,7 +16,6 @@ use std::{convert::Infallible, sync::Arc};
 use tower::service_fn;
 
 #[tokio::test]
-#[serial_test::serial]
 async fn fetch_actor_with_custom_acct() {
     database_test(|db_pool| async move {
         let mut jrd_base = include_bytes!("../../../../test-fixtures/0x0_jrd.json").to_owned();
@@ -80,7 +79,6 @@ async fn fetch_actor_with_custom_acct() {
 }
 
 #[tokio::test]
-#[serial_test::serial]
 async fn ignore_fake_webfinger_acct() {
     database_test(|db_pool| async move {
         let link = Link {

diff --git a/crates/kitsune-db/Cargo.toml b/crates/kitsune-db/Cargo.toml
@@ -42,7 +42,6 @@ typed-builder = "0.18.1"
 
 [dev-dependencies]
 kitsune-test = { path = "../kitsune-test" }
-serial_test = "3.0.0"
 tokio = { version = "1.36.0", features = ["macros"] }
 
 [lints]

diff --git a/crates/kitsune-db/tests/unicode_collation.rs b/crates/kitsune-db/tests/unicode_collation.rs
@@ -63,7 +63,6 @@ async fn create_user(conn: &mut AsyncPgConnection, username: &str) -> Result<Use
 }
 
 #[tokio::test]
-#[serial_test::serial]
 async fn accounts_username() {
     database_test(|db_pool| async move {
         db_pool
@@ -95,7 +94,6 @@ async fn accounts_username() {
 }
 
 #[tokio::test]
-#[serial_test::serial]
 async fn users_username() {
     database_test(|db_pool| async move {
         db_pool

diff --git a/crates/kitsune-s3/Cargo.toml b/crates/kitsune-s3/Cargo.toml
@@ -17,7 +17,6 @@ typed-builder = "0.18.1"
 
 [dev-dependencies]
 kitsune-test = { path = "../kitsune-test" }
-serial_test = "3.0.0"
 tokio = { version = "1.36.0", features = ["macros", "rt"] }
 
 [lints]

diff --git a/crates/kitsune-s3/src/lib.rs b/crates/kitsune-s3/src/lib.rs
@@ -250,7 +250,6 @@ mod test {
     }
 
     #[tokio::test]
-    #[serial_test::serial]
     async fn full_test() {
         minio_test(|client| async move {
             client
@@ -283,7 +282,6 @@ mod test {
     }
 
     #[tokio::test]
-    #[serial_test::serial]
     async fn abort_request_works() {
         minio_test(|client| async move {
             let result = client

diff --git a/crates/kitsune-service/Cargo.toml b/crates/kitsune-service/Cargo.toml
@@ -80,7 +80,6 @@ kitsune-jobs = { path = "../kitsune-jobs" }
 kitsune-test = { path = "../kitsune-test" }
 kitsune-webfinger = { path = "../kitsune-webfinger" }
 pretty_assertions = "1.4.0"
-serial_test = "3.0.0"
 tempfile = "3.10.1"
 tower = { version = "0.4.13", default-features = false, features = ["util"] }
 

diff --git a/crates/kitsune-service/src/attachment.rs b/crates/kitsune-service/src/attachment.rs
@@ -273,7 +273,6 @@ mod test {
     use tower::service_fn;
 
     #[tokio::test]
-    #[serial_test::serial]
     async fn upload_jpeg() {
         database_test(|db_pool| async move {
             let client = Client::builder().service(service_fn(handle));

diff --git a/crates/kitsune-service/src/post/resolver.rs b/crates/kitsune-service/src/post/resolver.rs
@@ -143,7 +143,7 @@ mod test {
     use tower::service_fn;
 
     #[tokio::test]
-    #[serial_test::serial]
+    #[allow(clippy::too_many_lines)]
     async fn parse_post() {
         redis_test(|redis_pool| async move {
             database_test(|db_pool| async move {

diff --git a/crates/kitsune-test/Cargo.toml b/crates/kitsune-test/Cargo.toml
@@ -7,7 +7,6 @@ license.workspace = true
 
 [dependencies]
 bytes = "1.5.0"
-diesel = "2.1.5"
 diesel-async = "0.4.1"
 futures-util = "0.3.30"
 http = "1.1.0"
@@ -18,18 +17,21 @@ kitsune-db = { path = "../kitsune-db" }
 kitsune-s3 = { path = "../kitsune-s3" }
 multiplex-pool = { path = "../../lib/multiplex-pool" }
 pin-project-lite = "0.2.13"
+rand = "0.8.5"
 redis = { version = "0.25.2", default-features = false, features = [
     "connection-manager",
     "tokio-rustls-comp",
 ] }
 rusty-s3 = { version = "0.5.0", default-features = false }
-scoped-futures = "0.1.3"
 testcontainers = { version = "0.15.0", features = ["watchdog"] }
 testcontainers-modules = { version = "0.3.5", features = [
     "minio",
     "postgres",
     "redis",
 ] }
+tokio = { version = "1.36.0", features = ["time"] }
+url = "2.5.0"
+uuid = { version = "1.7.0", features = ["v4", "fast-rng"] }
 
 [lints]
 workspace = true