Merge #458: New console command: random seeder

324fac7 fix: [#453] cargo fmt --check for nigthly toolchain (Jose Celano)
935facb feat: [#453] new  console command (Jose Celano)
df3a9be feat: [#453] new console command seeder. Only scaffolding. WIP (Jose Celano)
98fa40f feat: [#453] add cargo dependencies: clap, anyhow (Jose Celano)
ac21c49 feat: [#452] move API client to production code (Jose Celano)
118d6a5 refactor: [#453] reorganize web mod (Jose Celano)
b1df4e8 refactor: [#453] reorganize console mods (Jose Celano)

Pull request description:

  It will upload random torrents to the Index. For testing purposes. It could be used to seed E2E test environments too.

  - [x] Reorganize `console` mod to include more mods.
  - [x] Move API client from testing to production.
  - [x] Create scaffolding for a new console command: `seeder`(with `clap` and `anyhow`)
  - [x] Use the API client to seed a live env with random torrents.

  The console command is:

  ```console
  cargo run --bin seeder -- --api-base-url <API_BASE_URL> --number-of-torrents <NUMBER_OF_TORRENTS> --user <USER> --password <PASSWORD> --interval <INTERVAL>
  ```

  For example:

  ```console
  cargo run --bin seeder -- --api-base-url "localhost:3001" --number-of-torrents 1000 --user admin --password 12345678 --interval 0
  ```

  That command would upload 100o random torrents to the Index using the user account `admin` with password `123456` and wait 1 second between uploads.

  Output:

  ```console
  $ cargo run --bin seeder -- --api-base-url "localhost:3001" --number-of-torrents    3 --user admin --password 12345678 --interval 5
      Finished dev [unoptimized + debuginfo] target(s) in 0.07s
       Running `target/debug/seeder --api-base-url 'localhost:3001' --number-of-torrents 3 --user admin --password 12345678 --interval 5`
  2024-02-06T17:12:23.325055202+00:00 [seeder][INFO] Trying to login with username: admin ...
  2024-02-06T17:12:23.666824220+00:00 [seeder][INFO] Logged as admin with account: admin
  2024-02-06T17:12:23.666870390+00:00 [seeder][INFO] Uploading 3 random torrents to the Torrust Index ...
  2024-02-06T17:12:23.666875190+00:00 [seeder][INFO] Uploading torrent #1 ...
  2024-02-06T17:12:23.666886180+00:00 [seeder][INFO] Uploading torrent with uuid: c6a4752a-d27c-4ba9-a367-cf41c9f22d4a ...
  2024-02-06T17:12:23.735547434+00:00 [seeder][INFO] Uploaded torrent: {"torrent_id":54,"info_hash":"7e0a8e0461bd2fd46b55c95da95721c836ae9349"}
  2024-02-06T17:12:28.735666148+00:00 [seeder][INFO] Uploading torrent #2 ...
  2024-02-06T17:12:28.735738597+00:00 [seeder][INFO] Uploading torrent with uuid: a383519c-0018-43ec-82df-10044c61d46d ...
  2024-02-06T17:12:28.793988941+00:00 [seeder][INFO] Uploaded torrent: {"torrent_id":55,"info_hash":"12f0f9edd2494d56a36c13b833aee37d1a0bb84d"}
  2024-02-06T17:12:33.794118199+00:00 [seeder][INFO] Uploading torrent #3 ...
  2024-02-06T17:12:33.794209398+00:00 [seeder][INFO] Uploading torrent with uuid: bc32812c-aab2-40b7-999e-d58c4070655c ...
  2024-02-06T17:12:33.874166798+00:00 [seeder][INFO] Uploaded torrent: {"torrent_id":56,"info_hash":"4b9f6e1b2e4ddcb25b292ba0cb8561e2daa5b243"}
  ```

ACKs for top commit:
  josecelano:
    ACK 324fac7

Tree-SHA512: c3e9d1e57c45bf720d5dbc9d000260a356d21eeb5fe6749136f60ef80c9f36abe216e9ebf935b7e342adaef01ded2b7bd6c5e66bd16ac3d0a9abbbb7d2d785ca

Cargo.toml

@@ -34,12 +34,14 @@ version = "3.0.0-alpha.3-develop"
 opt-level = 3
 
 [dependencies]
+anyhow = "1.0.79"
 argon2 = "0"
 async-trait = "0"
 axum = { version = "0", features = ["multipart"] }
 binascii = "0"
 bytes = "1"
 chrono = { version = "0", default-features = false, features = ["clock"] }
+clap = { version = "4.4.18", features = ["derive", "env"]}
 config = "0"
 derive_more = "0"
 email_address = "0"
@@ -53,6 +55,7 @@ lazy_static = "1.4.0"
 lettre = { version = "0", features = ["builder", "smtp-transport", "tokio1", "tokio1-native-tls", "tokio1-rustls-tls"] }
 log = "0"
 pbkdf2 = { version = "0", features = ["simple"] }
+rand = "0"
 rand_core = { version = "0", features = ["std"] }
 regex = "1"
 reqwest = { version = "0", features = ["json", "multipart"] }
@@ -76,7 +79,6 @@ urlencoding = "2"
 uuid = { version = "1", features = ["v4"] }
 
 [dev-dependencies]
-rand = "0"
 tempfile = "3"
 uuid = { version = "1", features = ["v4"] }
 which = "5"

src/app.rs

@@ -18,7 +18,7 @@ use crate::services::torrent::{
 use crate::services::user::{self, DbBannedUserList, DbUserProfileRepository, DbUserRepository};
 use crate::services::{proxy, settings, torrent};
 use crate::tracker::statistics_importer::StatisticsImporter;
-use crate::web::api::v1::auth::Authentication;
+use crate::web::api::server::v1::auth::Authentication;
 use crate::web::api::Version;
 use crate::{console, mailer, tracker, web};
 
@@ -159,7 +159,7 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running
 
     // Start cronjob to import tracker torrent data and updating
     // seeders and leechers info.
-    let tracker_statistics_importer_handle = console::tracker_statistics_importer::start(
+    let tracker_statistics_importer_handle = console::cronjobs::tracker_statistics_importer::start(
         importer_port,
         importer_torrent_info_update_interval,
         &tracker_statistics_importer,

src/bin/import_tracker_statistics.rs

@@ -1,11 +1,11 @@
 //! Import Tracker Statistics command.
 //!
-//! It imports the number of seeders and leechers for all torrent from the linked tracker.
+//! It imports the number of seeders and leechers for all torrents from the linked tracker.
 //!
 //! You can execute it with: `cargo run --bin import_tracker_statistics`
-use torrust_index::console::commands::import_tracker_statistics::run_importer;
+use torrust_index::console::commands::tracker_statistics_importer::app::run;
 
 #[tokio::main]
 async fn main() {
-    run_importer().await;
+    run().await;
 }

src/bin/seeder.rs

@@ -0,0 +1,7 @@
+//! Program to upload random torrents to a live Index API.
+use torrust_index::console::commands::seeder::app;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    app::run().await
+}

src/common.rs

@@ -13,7 +13,7 @@ use crate::services::torrent::{
 use crate::services::user::{self, DbBannedUserList, DbUserProfileRepository, DbUserRepository};
 use crate::services::{proxy, settings, torrent};
 use crate::tracker::statistics_importer::StatisticsImporter;
-use crate::web::api::v1::auth::Authentication;
+use crate::web::api::server::v1::auth::Authentication;
 use crate::{mailer, tracker};
 pub type Username = String;
 

src/console/commands/mod.rs

@@ -1 +1,2 @@
-pub mod import_tracker_statistics;
+pub mod seeder;
+pub mod tracker_statistics_importer;

src/console/commands/seeder/api.rs

@@ -0,0 +1,108 @@
+use log::debug;
+use thiserror::Error;
+
+use crate::web::api::client::v1::client::Client;
+use crate::web::api::client::v1::contexts::category::forms::AddCategoryForm;
+use crate::web::api::client::v1::contexts::category::responses::{ListItem, ListResponse};
+use crate::web::api::client::v1::contexts::torrent::forms::UploadTorrentMultipartForm;
+use crate::web::api::client::v1::contexts::torrent::responses::{UploadedTorrent, UploadedTorrentResponse};
+use crate::web::api::client::v1::contexts::user::forms::LoginForm;
+use crate::web::api::client::v1::contexts::user::responses::{LoggedInUserData, SuccessfulLoginResponse};
+use crate::web::api::client::v1::responses::TextResponse;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Torrent with the same info-hash already exist in the database")]
+    TorrentInfoHashAlreadyExists,
+    #[error("Torrent with the same title already exist in the database")]
+    TorrentTitleAlreadyExists,
+}
+
+/// It uploads a torrent file to the Torrust Index.
+///
+/// # Errors
+///
+/// It returns an error if the torrent already exists in the database.
+///
+/// # Panics
+///
+/// Panics if the response body is not a valid JSON.
+pub async fn upload_torrent(client: &Client, upload_torrent_form: UploadTorrentMultipartForm) -> Result<UploadedTorrent, Error> {
+    let categories = get_categories(client).await;
+
+    if !contains_category_with_name(&categories, &upload_torrent_form.category) {
+        add_category(client, &upload_torrent_form.category).await;
+    }
+
+    let response = client.upload_torrent(upload_torrent_form.into()).await;
+
+    debug!(target:"seeder", "response: {}", response.status);
+
+    if response.status == 400 {
+        if response.body.contains("This torrent already exists in our database") {
+            return Err(Error::TorrentInfoHashAlreadyExists);
+        }
+
+        if response.body.contains("This torrent title has already been used") {
+            return Err(Error::TorrentTitleAlreadyExists);
+        }
+    }
+
+    assert!(response.is_json_and_ok(), "Error uploading torrent: {}", response.body);
+
+    let uploaded_torrent_response: UploadedTorrentResponse =
+        serde_json::from_str(&response.body).expect("a valid JSON response should be returned from the Torrust Index API");
+
+    Ok(uploaded_torrent_response.data)
+}
+
+/// It logs in the user and returns the user data.
+///
+/// # Panics
+///
+/// Panics if the response body is not a valid JSON.
+pub async fn login(client: &Client, username: &str, password: &str) -> LoggedInUserData {
+    let response = client
+        .login_user(LoginForm {
+            login: username.to_owned(),
+            password: password.to_owned(),
+        })
+        .await;
+
+    let res: SuccessfulLoginResponse = serde_json::from_str(&response.body).unwrap_or_else(|_| {
+        panic!(
+            "a valid JSON response should be returned after login. Received: {}",
+            response.body
+        )
+    });
+
+    res.data
+}
+
+/// It returns all the index categories.
+///
+/// # Panics
+///
+/// Panics if the response body is not a valid JSON.
+pub async fn get_categories(client: &Client) -> Vec<ListItem> {
+    let response = client.get_categories().await;
+
+    let res: ListResponse = serde_json::from_str(&response.body).unwrap();
+
+    res.data
+}
+
+/// It adds a new category.
+pub async fn add_category(client: &Client, name: &str) -> TextResponse {
+    client
+        .add_category(AddCategoryForm {
+            name: name.to_owned(),
+            icon: None,
+        })
+        .await
+}
+
+/// It checks if the category list contains the given category.
+fn contains_category_with_name(items: &[ListItem], category_name: &str) -> bool {
+    items.iter().any(|item| item.name == category_name)
+}

src/console/commands/seeder/app.rs

@@ -0,0 +1,132 @@
+//! Program to upload random torrent to a live Index API.
+//!
+//! Run with:
+//!
+//! ```text
+//! cargo run --bin seeder -- --api-base-url <API_BASE_URL> --number-of-torrents <NUMBER_OF_TORRENTS> --user <USER> --password <PASSWORD> --interval <INTERVAL>
+//! ```
+//!
+//! For example:
+//!
+//! ```text
+//! cargo run --bin seeder -- --api-base-url "localhost:3001" --number-of-torrents 1000 --user admin --password 12345678 --interval 0
+//! ```
+//!
+//! That command would upload 1000 random torrents to the Index using the user
+//! account admin with password 123456 and waiting 1 second between uploads.
+use std::thread::sleep;
+use std::time::Duration;
+
+use anyhow::Context;
+use clap::Parser;
+use log::{debug, info, LevelFilter};
+use text_colorizer::Colorize;
+use uuid::Uuid;
+
+use super::api::Error;
+use crate::console::commands::seeder::api::{login, upload_torrent};
+use crate::console::commands::seeder::logging;
+use crate::services::torrent_file::generate_random_torrent;
+use crate::utils::parse_torrent;
+use crate::web::api::client::v1::client::Client;
+use crate::web::api::client::v1::contexts::torrent::forms::{BinaryFile, UploadTorrentMultipartForm};
+use crate::web::api::client::v1::contexts::torrent::responses::UploadedTorrent;
+use crate::web::api::client::v1::contexts::user::responses::LoggedInUserData;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    #[arg(short, long)]
+    api_base_url: String,
+
+    #[arg(short, long)]
+    number_of_torrents: i32,
+
+    #[arg(short, long)]
+    user: String,
+
+    #[arg(short, long)]
+    password: String,
+
+    #[arg(short, long)]
+    interval: u64,
+}
+
+/// # Errors
+///
+/// Will not return any errors for the time being.
+pub async fn run() -> anyhow::Result<()> {
+    logging::setup(LevelFilter::Info);
+
+    let args = Args::parse();
+
+    let api_user = login_index_api(&args.api_base_url, &args.user, &args.password).await;
+
+    let api_client = Client::authenticated(&args.api_base_url, &api_user.token);
+
+    info!(target:"seeder", "Uploading { } random torrents to the Torrust Index with a { } seconds interval...", args.number_of_torrents.to_string().yellow(), args.interval.to_string().yellow());
+
+    for i in 1..=args.number_of_torrents {
+        info!(target:"seeder", "Uploading torrent #{} ...", i.to_string().yellow());
+
+        match upload_random_torrent(&api_client).await {
+            Ok(uploaded_torrent) => {
+                debug!(target:"seeder", "Uploaded torrent {uploaded_torrent:?}");
+
+                let json = serde_json::to_string(&uploaded_torrent).context("failed to serialize upload response into JSON")?;
+
+                info!(target:"seeder", "Uploaded torrent: {}", json.yellow());
+            }
+            Err(err) => print!("Error uploading torrent {err:?}"),
+        };
+
+        if i != args.number_of_torrents {
+            sleep(Duration::from_secs(args.interval));
+        }
+    }
+
+    Ok(())
+}
+
+/// It logs in a user in the Index API.
+pub async fn login_index_api(api_url: &str, username: &str, password: &str) -> LoggedInUserData {
+    let unauthenticated_client = Client::unauthenticated(api_url);
+
+    info!(target:"seeder", "Trying to login with username: {} ...", username.yellow());
+
+    let user: LoggedInUserData = login(&unauthenticated_client, username, password).await;
+
+    if user.admin {
+        info!(target:"seeder", "Logged as admin with account: {} ", username.yellow());
+    } else {
+        info!(target:"seeder", "Logged as {} ", username.yellow());
+    }
+
+    user
+}
+
+async fn upload_random_torrent(api_client: &Client) -> Result<UploadedTorrent, Error> {
+    let uuid = Uuid::new_v4();
+
+    info!(target:"seeder", "Uploading torrent with uuid: {} ...", uuid.to_string().yellow());
+
+    let torrent_file = generate_random_torrent_file(uuid);
+
+    let upload_form = UploadTorrentMultipartForm {
+        title: format!("title-{uuid}"),
+        description: format!("description-{uuid}"),
+        category: "test".to_string(),
+        torrent_file,
+    };
+
+    upload_torrent(api_client, upload_form).await
+}
+
+/// It returns the bencoded binary data of the torrent meta file.
+fn generate_random_torrent_file(uuid: Uuid) -> BinaryFile {
+    let torrent = generate_random_torrent(uuid);
+
+    let bytes = parse_torrent::encode_torrent(&torrent).expect("msg:the torrent should be bencoded");
+
+    BinaryFile::from_bytes(torrent.info.name, bytes)
+}

src/console/commands/seeder/logging.rs

@@ -0,0 +1,25 @@
+use log::{debug, LevelFilter};
+
+/// # Panics
+///
+///
+pub fn setup(level: LevelFilter) {
+    if let Err(_err) = fern::Dispatch::new()
+        .format(|out, message, record| {
+            out.finish(format_args!(
+                "{} [{}][{}] {}",
+                chrono::Local::now().format("%+"),
+                record.target(),
+                record.level(),
+                message
+            ));
+        })
+        .level(level)
+        .chain(std::io::stdout())
+        .apply()
+    {
+        panic!("Failed to initialize logging.")
+    }
+
+    debug!("logging initialized.");
+}

src/console/commands/seeder/mod.rs

@@ -0,0 +1,3 @@
+pub mod api;
+pub mod app;
+pub mod logging;

src/console/commands/import_tracker_statistics.rs → src/console/commands/tracker_statistics_importer/app.rs

@@ -75,7 +75,7 @@ fn print_usage() {
 /// # Panics
 ///
 /// Panics if arguments cannot be parsed.
-pub async fn run_importer() {
+pub async fn run() {
     parse_args().expect("unable to parse command arguments");
     import().await;
 }

src/console/commands/tracker_statistics_importer/mod.rs

@@ -0,0 +1 @@
+pub mod app;

src/console/cronjobs/mod.rs

@@ -0,0 +1 @@
+pub mod tracker_statistics_importer;

src/console/tracker_statistics_importer.rs → src/console/cronjobs/tracker_statistics_importer.rs

@@ -33,6 +33,10 @@ struct ImporterState {
     pub torrent_info_update_interval: u64,
 }
 
+/// # Panics
+///
+/// Will panic if it can't start the tracker statistics importer API
+#[must_use]
 pub fn start(
     importer_port: u16,
     torrent_info_update_interval: u64,
@@ -60,7 +64,7 @@ pub fn start(
 
             let addr = format!("{IMPORTER_API_IP}:{importer_port}");
 
-            info!("Tracker statistics importer API server listening on http://{}", addr);
+            info!("Tracker statistics importer API server listening on http://{}", addr); // # DevSkim: ignore DS137138
 
             axum::Server::bind(&addr.parse().unwrap())
                 .serve(app.into_make_service())
@@ -122,7 +126,7 @@ async fn heartbeat_handler(State(state): State<Arc<ImporterState>>) -> Json<Valu
 /// Send a heartbeat from the importer cronjob to the importer API.
 async fn send_heartbeat(importer_port: u16) -> Result<(), reqwest::Error> {
     let client = reqwest::Client::new();
-    let url = format!("http://{IMPORTER_API_IP}:{importer_port}/heartbeat");
+    let url = format!("http://{IMPORTER_API_IP}:{importer_port}/heartbeat"); // # DevSkim: ignore DS137138
 
     client.post(url).send().await?;