Merge #383: Reestablish E2E Tests (reopened)

d92443d fix: [#342] E2E test execution with MySQL (Jose Celano)
31351fa fix: [#342] disable clippy warning (Jose Celano)
f8aa238 fix: [#342] broken E2E tests after renaming Category::category_id field (Jose Celano)
e78607d fix: [#342] SQLite data file path inside the container for E2E tests (Jose Celano)
1cce823 fix: [#342] index container not running for E2E tests (Jose Celano)
03943ef fix: [#342] fix en vars to run E2E tests (Jose Celano)

Pull request description:

  The E2E test suite execution was not reestablished correctly [here](#378).

  Some env vars were renamed (like `TORRUST_INDEX_E2E_SHARED`) so we were not running E2E tests (although they seemed to pass, we were only executing the ones that do not require a shared env).

  This PR only reestablishes the E2E tests. There are a lot of pending refactors that will be done in new PRs like [renaming env vars](#361).

Top commit has no ACKs.

Tree-SHA512: fc8f9f89dab83b8b2cb263befc614c9c8ebec884a060e4e5c6a958827d537b4c1198e0c8cdb4b709902a9869366088dc91ac33cca15a1fdf357a215a8328589e

.github/workflows/testing.yaml

@@ -163,6 +163,10 @@ jobs:
         name: Make Build Clean
         run: cargo clean
 
-      - id: test
-        name: Run Integration Tests
-        run: ./contrib/dev-tools/container/e2e/run-e2e-tests.sh
+      - id: test-sqlite
+        name: Run Integration Tests (SQLite)
+        run: ./contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh
+
+      - id: test-mysql
+        name: Run Integration Tests (MySQL)
+        run: ./contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh

Cargo.toml

@@ -56,6 +56,7 @@ pbkdf2 = { version = "0", features = ["simple"] }
 rand_core = { version = "0", features = ["std"] }
 regex = "1"
 reqwest = { version = "0", features = ["json", "multipart"] }
+rustversion = "1.0.14"
 serde = { version = "1", features = ["rc"] }
 serde_bencode = "0"
 serde_bytes = "0"

compose.yaml

@@ -5,7 +5,7 @@ services:
     build:
       context: .
       dockerfile: ./Containerfile
-      target: debug     
+      target: release
     tty: true
     environment:
       - TORRUST_INDEX_CONFIG=${TORRUST_INDEX_CONFIG}

contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh

@@ -4,13 +4,12 @@ TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.mysql.toml) \
     docker compose build
 
 USER_ID=${USER_ID:-1000} \
-    # Index
     TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.mysql.toml) \
     TORRUST_INDEX_DATABASE_DRIVER="mysql" \
     TORRUST_INDEX_TRACKER_API_TOKEN="MyAccessToken" \
     TORRUST_IDX_BACK_MYSQL_DATABASE="torrust_index_e2e_testing" \
-    # Tracker
-    TORRUST_TRACKER_CONFIG=$(cat ./share/default/config/tracker.container.mysql.toml) \
-    TORRUST_TRACKER_DATABASE_DRIVER="mysql" \
+    TORRUST_TRACKER_CONFIG=$(cat ./share/default/config/tracker.container.sqlite3.toml) \
+    TORRUST_TRACKER_DATABASE_DRIVER="sqlite3" \
     TORRUST_TRACKER_API_ADMIN_TOKEN="MyAccessToken" \
     docker compose up -d
+

contrib/dev-tools/container/e2e/mysql/install.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# This script is only intended to be used for E2E testing environment.
+
+# Database credentials
+MYSQL_USER="root"
+MYSQL_PASSWORD="root_secret_password"
+MYSQL_HOST="127.0.0.1"
+MYSQL_DATABASE="torrust_index_e2e_testing"
+
+# Create the MySQL database for the index. Assumes MySQL client is installed.
+# The docker compose configuration already creates the database the first time
+# the container is created.
+echo "Creating MySQL database $MYSQL_DATABASE for for E2E testing ..."
+MYSQL_PWD=$MYSQL_PASSWORD mysql -h $MYSQL_HOST -u $MYSQL_USER -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE;"

contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+CURRENT_USER_NAME=$(whoami)
+CURRENT_USER_ID=$(id -u)
+echo "User name: $CURRENT_USER_NAME"
+echo "User   id: $CURRENT_USER_ID"
+
+TORRUST_IDX_BACK_USER_UID=$CURRENT_USER_ID
+TORRUST_TRACKER_USER_UID=$CURRENT_USER_ID
+export TORRUST_IDX_BACK_USER_UID
+export TORRUST_TRACKER_USER_UID
+
+# todo: remove duplicate funtion
+wait_for_container_to_be_healthy() {
+    local container_name="$1"
+    local max_retries="$2"
+    local retry_interval="$3"
+    local retry_count=0
+
+    while [ $retry_count -lt "$max_retries" ]; do
+        container_health="$(docker inspect --format='{{json .State.Health}}' "$container_name")"
+        if [ "$container_health" != "{}" ]; then
+            container_status="$(echo "$container_health" | jq -r '.Status')"
+            if [ "$container_status" == "healthy" ]; then
+                echo "Container $container_name is healthy"
+                return 0
+            fi
+        fi
+
+        retry_count=$((retry_count + 1))
+        echo "Waiting for container $container_name to become healthy (attempt $retry_count of $max_retries)..."
+        sleep "$retry_interval"
+    done
+
+    echo "Timeout reached, container $container_name is not healthy"
+    return 1
+}
+
+# Install tool to create torrent files.
+# It's needed by some tests to generate and parse test torrent files.
+cargo install imdl || exit 1
+
+# Install app (no docker) that will run the test suite against the E2E testing 
+# environment (in docker).
+cp .env.local .env || exit 1
+
+# TEST USING MYSQL
+echo "Running E2E tests using MySQL ..."
+
+# Start E2E testing environment
+./contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh || exit 1
+
+wait_for_container_to_be_healthy torrust-mysql-1 10 3
+# todo: implement healthchecks for tracker and index and wait until they are healthy
+#wait_for_container torrust-tracker-1 10 3
+#wait_for_container torrust-idx-back-1 10 3
+sleep 20s
+
+# Just to make sure that everything is up and running
+docker ps
+
+# Install MySQL database for the index
+./contrib/dev-tools/container/e2e/mysql/install.sh || exit 1
+
+# Run E2E tests with shared app instance
+TORRUST_INDEX_E2E_SHARED=true TORRUST_INDEX_E2E_PATH_CONFIG="./share/default/config/index.container.mysql.toml" cargo test || exit 1
+
+# Stop E2E testing environment
+./contrib/dev-tools/container/e2e/mysql/e2e-env-down.sh || exit 1
+

contrib/dev-tools/container/e2e/sqlite/e2e-env-up.sh

@@ -4,11 +4,9 @@ TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.sqlite3.toml)
     docker compose build
 
 USER_ID=${USER_ID:-1000} \
-    # Index
     TORRUST_INDEX_CONFIG=$(cat ./share/default/config/index.container.sqlite3.toml) \
     TORRUST_INDEX_DATABASE_DRIVER="sqlite3" \
     TORRUST_INDEX_TRACKER_API_TOKEN="MyAccessToken" \
-    # Tracker
     TORRUST_TRACKER_CONFIG=$(cat ./share/default/config/tracker.container.sqlite3.toml) \
     TORRUST_TRACKER_DATABASE_DRIVER="sqlite3" \
     TORRUST_TRACKER_API_ADMIN_TOKEN="MyAccessToken" \

contrib/dev-tools/container/e2e/sqlite/install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This script is only intended to be used for E2E testing environment.
+
+# Generate storage directory if it does not exist
+mkdir -p ./storage/index/lib/database
+
+# Generate the sqlite database if it does not exist
+if ! [ -f "./storage/index/lib/database/sqlite3.db" ]; then
+    # todo: it should get the path from tracker.toml and only do it when we use sqlite
+    sqlite3 ./storage/index/lib/database/sqlite3.db "VACUUM;"
+fi
+

contrib/dev-tools/container/e2e/run-e2e-tests.sh → contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh

@@ -10,6 +10,7 @@ TORRUST_TRACKER_USER_UID=$CURRENT_USER_ID
 export TORRUST_IDX_BACK_USER_UID
 export TORRUST_TRACKER_USER_UID
 
+# todo: remove duplicate funtion
 wait_for_container_to_be_healthy() {
     local container_name="$1"
     local max_retries="$2"
@@ -42,7 +43,7 @@ cargo install imdl || exit 1
 # Install app (no docker) that will run the test suite against the E2E testing 
 # environment (in docker).
 cp .env.local .env || exit 1
-./contrib/dev-tools/init/install-local.sh || exit 1
+./contrib/dev-tools/container/e2e/sqlite/install.sh || exit 1
 
 # TEST USING SQLITE
 echo "Running E2E tests using SQLite ..."
@@ -60,39 +61,7 @@ sleep 20s
 docker ps
 
 # Run E2E tests with shared app instance
-TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./share/default/config/index.container.sqlite3.toml" cargo test || exit 1
+TORRUST_INDEX_E2E_SHARED=true TORRUST_INDEX_E2E_PATH_CONFIG="./share/default/config/index.container.sqlite3.toml" cargo test || exit 1
 
 # Stop E2E testing environment
 ./contrib/dev-tools/container/e2e/sqlite/e2e-env-down.sh || exit 1
-
-# TEST USING MYSQL
-echo "Running E2E tests using MySQL ..."
-
-# Start E2E testing environment
-./contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh || exit 1
-
-wait_for_container_to_be_healthy torrust-mysql-1 10 3
-# todo: implement healthchecks for tracker and index and wait until they are healthy
-#wait_for_container torrust-tracker-1 10 3
-#wait_for_container torrust-idx-back-1 10 3
-sleep 20s
-
-# Just to make sure that everything is up and running
-docker ps
-
-# Database credentials
-MYSQL_USER="root"
-MYSQL_PASSWORD="root_secret_password"
-MYSQL_HOST="localhost"
-MYSQL_DATABASE="torrust_index_e2e_testing"
-
-# Create the MySQL database for the index. Assumes MySQL client is installed.
-echo "Creating MySQL database $MYSQL_DATABASE for for E2E testing ..."
-mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PASSWORD -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE;"
-
-# Run E2E tests with shared app instance
-TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./share/default/config/index.container.mysql.toml" cargo test || exit 1
-
-# Stop E2E testing environment
-./contrib/dev-tools/container/e2e/mysql/e2e-env-down.sh || exit 1
-

contrib/dev-tools/init/install-local.sh

@@ -10,3 +10,4 @@ if ! [ -f "./storage/index/lib/database/sqlite3.db" ]; then
     # todo: it should get the path from tracker.toml and only do it when we use sqlite
     sqlite3 ./storage/index/lib/database/sqlite3.db "VACUUM;"
 fi
+

project-words.txt

@@ -68,6 +68,7 @@ rowid
 RUSTDOCFLAGS
 RUSTFLAGS
 rustfmt
+rustversion
 serde
 sgxj
 singlepart

tests/common/contexts/torrent/asserts.rs

@@ -15,7 +15,10 @@ pub fn assert_expected_torrent_details(torrent: &TorrentDetails, expected_torren
         ("info_hash", torrent.info_hash == expected_torrent.info_hash),
         ("title", torrent.title == expected_torrent.title),
         ("description", torrent.description == expected_torrent.description),
-        ("category.category_id", torrent.category.id == expected_torrent.category.id),
+        (
+            "category.category_id",
+            torrent.category.category_id == expected_torrent.category.category_id,
+        ),
         ("category.name", torrent.category.name == expected_torrent.category.name),
         ("file_size", torrent.file_size == expected_torrent.file_size),
         ("seeders", torrent.seeders == expected_torrent.seeders),

tests/common/contexts/torrent/responses.rs

@@ -74,9 +74,19 @@ pub struct TorrentDetails {
     pub encoding: Option<String>,
 }
 
+#[rustversion::stable]
 #[derive(Deserialize, PartialEq, Debug)]
 pub struct Category {
-    pub id: CategoryId,
+    pub category_id: CategoryId, // todo: rename to `id`
+    pub name: String,
+    pub num_torrents: u64,
+}
+
+#[rustversion::nightly]
+#[derive(Deserialize, PartialEq, Debug)]
+#[allow(clippy::struct_field_names)]
+pub struct Category {
+    pub category_id: CategoryId, // todo: rename to `id`
     pub name: String,
     pub num_torrents: u64,
 }

tests/e2e/environment.rs

@@ -114,7 +114,7 @@ impl TestEnv {
 
                     return match maybe_db_driver {
                         Ok(db_driver) => match db_driver {
-                            database::Driver::Sqlite3 => Some(db_path),
+                            database::Driver::Sqlite3 => Some(Self::overwrite_sqlite_path(&db_path, "./storage/index/lib")),
                             database::Driver::Mysql => Some(Self::overwrite_mysql_host(&db_path, "localhost")),
                         },
                         Err(_) => None,
@@ -127,7 +127,26 @@ impl TestEnv {
         }
     }
 
-    /// It overrides the "Host" in a `SQLx` database connection URL. For example:
+    /// It overrides the `SQLite` file path in a `SQLx` database connection URL.
+    /// For example:
+    ///
+    /// For:
+    ///
+    /// `sqlite:///var/lib/torrust/index/database/sqlite3.db?mode=rwc`.
+    ///
+    /// It changes the `mysql` host name to `localhost`:
+    ///
+    /// `sqlite://./storage/index/lib/database/sqlite3.db?mode=rwc`.
+    ///
+    /// For E2E tests, we use docker compose. Inside the container, the
+    /// `SQLite` file path is not the same as the host path.
+    fn overwrite_sqlite_path(db_path: &str, host_path: &str) -> String {
+        // todo: inject value with env var
+        db_path.replace("/var/lib/torrust/index", host_path)
+    }
+
+    /// It overrides the "Host" in a `SQLx` database connection URL.
+    /// For example:
     ///
     /// For:
     ///
@@ -138,10 +157,11 @@ impl TestEnv {
     /// `mysql://root:root_secret_password@localhost:3306/torrust_index_e2e_testing`.
     ///
     /// For E2E tests, we use docker compose, internally the index connects to
-    /// the database using the "mysql" host, which is the docker compose service
-    /// name, but tests connects directly to the localhost since the `MySQL`
-    /// is exposed to the host.
+    /// the `MySQL` database using the "mysql" host, which is the docker compose
+    /// service name, but tests connects directly to the localhost since the
+    /// `MySQL` is exposed to the host.
     fn overwrite_mysql_host(db_path: &str, new_host: &str) -> String {
+        // todo: inject value with env var
         db_path.replace("@mysql:", &format!("@{new_host}:"))
     }
 

tests/e2e/mod.rs

@@ -4,14 +4,14 @@
 //! against an in-process server (isolated).
 //!
 //! If you want to run the tests against an out-of-process server, you need to
-//! set the environment variable `TORRUST_IDX_BACK_E2E_SHARED` to `true`.
+//! set the environment variable `TORRUST_INDEX_E2E_SHARED` to `true`.
 //!
 //! > **NOTICE**: The server must be running before running the tests. The
 //! server url is hardcoded to `http://localhost:3001` for now. We are planning
 //! to make it configurable in the future via a environment variable.
 //!
 //! ```text
-//! TORRUST_IDX_BACK_E2E_SHARED=true cargo test
+//! TORRUST_INDEX_E2E_SHARED=true cargo test
 //! ```
 //!
 //! If you want to run the tests against an isolated server, you need to execute

tests/e2e/web/api/v1/contexts/torrent/contract.rs

@@ -182,7 +182,7 @@ mod for_guests {
             title: test_torrent.index_info.title.clone(),
             description: test_torrent.index_info.description,
             category: Category {
-                id: software_predefined_category_id(),
+                category_id: software_predefined_category_id(),
                 name: test_torrent.index_info.category,
                 num_torrents: 19, // Ignored in assertion
             },