From 0e8b32b971f79e89fcf568deacda113f0013f4f6 Mon Sep 17 00:00:00 2001 From: Jose Celano Date: Thu, 16 Nov 2023 17:06:05 +0000 Subject: [PATCH] ci: [#254] add container healthcheck for index service For both services: - The Index API - The Tracker Stattistics Importer (console cronjob) An API was added for the Importer. The Importer API has two endpoints: - GET /health_check -> to get the crobjob status. It only checks that is running periodically. - POST /heartbeat -> used by the cronjob to inform it's alive. And a new endpoint was added the the Index API: - GET /health_check -> to check API responsiveness. --- Containerfile | 9 +- .../container/e2e/mysql/run-e2e-tests.sh | 8 +- .../container/e2e/sqlite/run-e2e-tests.sh | 8 +- .../default/config/index.container.mysql.toml | 1 + .../config/index.container.sqlite3.toml | 1 + .../config/index.development.sqlite3.toml | 1 + src/app.rs | 37 ++--- src/bin/health_check.rs | 37 +++++ src/config.rs | 3 + src/console/mod.rs | 1 + src/console/tracker_statistics_importer.rs | 130 ++++++++++++++++++ src/lib.rs | 1 + src/web/api/v1/contexts/settings/mod.rs | 1 + src/web/api/v1/routes.rs | 11 +- tests/common/contexts/settings/mod.rs | 2 + tests/environments/isolated.rs | 3 + 16 files changed, 220 insertions(+), 34 deletions(-) create mode 100644 src/bin/health_check.rs create mode 100644 src/console/tracker_statistics_importer.rs diff --git a/Containerfile b/Containerfile index 4048d27d..fe7961da 100644 --- a/Containerfile +++ b/Containerfile @@ -85,7 +85,9 @@ COPY --from=build \ RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no-run --archive-file /test/torrust-index.tar.zst RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json -RUN mkdir -p /app/bin/; cp -l /test/src/target/release/torrust-index /app/bin/torrust-index +RUN mkdir -p /app/bin/; \ + cp -l /test/src/target/release/torrust-index /app/bin/torrust-index; \ + cp -l /test/src/target/release/health_check /app/bin/health_check; # RUN mkdir -p /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-index | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1 RUN chown -R root:root /app; chmod -R u=rw,go=r,a+X /app; chmod -R a+x /app/bin @@ -99,11 +101,13 @@ ARG TORRUST_INDEX_PATH_CONFIG="/etc/torrust/index/index.toml" ARG TORRUST_INDEX_DATABASE_DRIVER="sqlite3" ARG USER_ID=1000 ARG API_PORT=3001 +ARG IMPORTER_API_PORT=3002 ENV TORRUST_INDEX_PATH_CONFIG=${TORRUST_INDEX_PATH_CONFIG} ENV TORRUST_INDEX_DATABASE_DRIVER=${TORRUST_INDEX_DATABASE_DRIVER} ENV USER_ID=${USER_ID} ENV API_PORT=${API_PORT} +ENV IMPORTER_API_PORT=${IMPORTER_API_PORT} ENV TZ=Etc/UTC EXPOSE ${API_PORT}/tcp @@ -130,5 +134,6 @@ CMD ["sh"] FROM runtime as release ENV RUNTIME="release" COPY --from=test /app/ /usr/ -# HEALTHCHECK CMD ["/usr/bin/wget", "--no-verbose", "--tries=1", "--spider", "localhost:${API_PORT}/version"] +HEALTHCHECK --interval=5s --timeout=5s --start-period=3s --retries=3 \ + CMD /usr/bin/health_check http://localhost:${API_PORT}/health_check && /usr/bin/health_check http://localhost:${IMPORTER_API_PORT}/health_check || exit 1 CMD ["/usr/bin/torrust-index"] diff --git a/contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh b/contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh index 4e9ab8c7..03f23079 100755 --- a/contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh +++ b/contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh @@ -25,10 +25,10 @@ echo "Running E2E tests using MySQL ..." ./contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh || exit 1 # Wait for conatiners to be healthy -./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3 -# todo: implement healthchecks for tracker and index and wait until they are healthy -#wait_for_container torrust-tracker-1 10 3 -#wait_for_container torrust-idx-back-1 10 3 +./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3 || exit 1 +# todo: implement healthchecks for the tracker and wait until it's healthy +#./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-tracker-1 10 3 +./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-index-1 10 3 || exit 1 sleep 20s # Just to make sure that everything is up and running diff --git a/contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh b/contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh index 60a21c22..afedc5f6 100755 --- a/contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh +++ b/contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh @@ -26,10 +26,10 @@ echo "Running E2E tests using SQLite ..." ./contrib/dev-tools/container/e2e/sqlite/e2e-env-up.sh || exit 1 # Wait for conatiners to be healthy -./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3 -# todo: implement healthchecks for tracker and index and wait until they are healthy -#wait_for_container torrust-tracker-1 10 3 -#wait_for_container torrust-idx-back-1 10 3 +./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3 || exit 1 +# todo: implement healthchecks for the tracker and wait until it's healthy +#./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-tracker-1 10 3 +./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-index-1 10 3 || exit 1 sleep 20s # Just to make sure that everything is up and running diff --git a/share/default/config/index.container.mysql.toml b/share/default/config/index.container.mysql.toml index 1999c4a1..c7b4c33c 100644 --- a/share/default/config/index.container.mysql.toml +++ b/share/default/config/index.container.mysql.toml @@ -48,3 +48,4 @@ max_torrent_page_size = 30 [tracker_statistics_importer] torrent_info_update_interval = 3600 +port = 3002 \ No newline at end of file diff --git a/share/default/config/index.container.sqlite3.toml b/share/default/config/index.container.sqlite3.toml index c0cb6002..ac1dd71d 100644 --- a/share/default/config/index.container.sqlite3.toml +++ b/share/default/config/index.container.sqlite3.toml @@ -48,3 +48,4 @@ max_torrent_page_size = 30 [tracker_statistics_importer] torrent_info_update_interval = 3600 +port = 3002 \ No newline at end of file diff --git a/share/default/config/index.development.sqlite3.toml b/share/default/config/index.development.sqlite3.toml index 06f89a3c..669979af 100644 --- a/share/default/config/index.development.sqlite3.toml +++ b/share/default/config/index.development.sqlite3.toml @@ -44,3 +44,4 @@ max_torrent_page_size = 30 [tracker_statistics_importer] torrent_info_update_interval = 3600 +port = 3002 \ No newline at end of file diff --git a/src/app.rs b/src/app.rs index 353ce274..22ffd4f7 100644 --- a/src/app.rs +++ b/src/app.rs @@ -19,8 +19,8 @@ use crate::services::user::{self, DbBannedUserList, DbUserProfileRepository, DbU use crate::services::{proxy, settings, torrent}; use crate::tracker::statistics_importer::StatisticsImporter; use crate::web::api::v1::auth::Authentication; -use crate::web::api::{start, Version}; -use crate::{mailer, tracker}; +use crate::web::api::Version; +use crate::{console, mailer, tracker, web}; pub struct Running { pub api_socket_addr: SocketAddr, @@ -46,8 +46,12 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running let settings = configuration.settings.read().await; + // From [database] config let database_connect_url = settings.database.connect_url.clone(); - let torrent_info_update_interval = settings.tracker_statistics_importer.torrent_info_update_interval; + // From [importer] config + let importer_torrent_info_update_interval = settings.tracker_statistics_importer.torrent_info_update_interval; + let importer_port = settings.tracker_statistics_importer.port; + // From [net] config let net_ip = "0.0.0.0".to_string(); let net_port = settings.net.port; @@ -153,29 +157,18 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running ban_service, )); - // Start repeating task to import tracker torrent data and updating + // Start cronjob to import tracker torrent data and updating // seeders and leechers info. - - let weak_tracker_statistics_importer = Arc::downgrade(&tracker_statistics_importer); - - let tracker_statistics_importer_handle = tokio::spawn(async move { - let interval = std::time::Duration::from_secs(torrent_info_update_interval); - let mut interval = tokio::time::interval(interval); - interval.tick().await; // first tick is immediate... - loop { - interval.tick().await; - if let Some(tracker) = weak_tracker_statistics_importer.upgrade() { - drop(tracker.import_all_torrents_statistics().await); - } else { - break; - } - } - }); + let tracker_statistics_importer_handle = console::tracker_statistics_importer::start( + importer_port, + importer_torrent_info_update_interval, + &tracker_statistics_importer, + ); // Start API server + let running_api = web::api::start(app_data, &net_ip, net_port, api_version).await; - let running_api = start(app_data, &net_ip, net_port, api_version).await; - + // Full running application Running { api_socket_addr: running_api.socket_addr, api_server: running_api.api_server, diff --git a/src/bin/health_check.rs b/src/bin/health_check.rs new file mode 100644 index 00000000..16e2d874 --- /dev/null +++ b/src/bin/health_check.rs @@ -0,0 +1,37 @@ +//! Minimal `curl` or `wget` to be used for container health checks. +//! +//! It's convenient to avoid using third-party libraries because: +//! +//! - They are harder to maintain. +//! - They introduce new attack vectors. +use std::{env, process}; + +#[tokio::main] +async fn main() { + let args: Vec = env::args().collect(); + if args.len() != 2 { + eprintln!("Usage: cargo run --bin health_check "); + eprintln!("Example: cargo run --bin health_check http://localhost:3002/health_check"); + std::process::exit(1); + } + + println!("Health check ..."); + + let url = &args[1].clone(); + + match reqwest::get(url).await { + Ok(response) => { + if response.status().is_success() { + println!("STATUS: {}", response.status()); + process::exit(0); + } else { + println!("Non-success status received."); + process::exit(1); + } + } + Err(err) => { + println!("ERROR: {err}"); + process::exit(1); + } + } +} diff --git a/src/config.rs b/src/config.rs index 941c3921..4f90aa47 100644 --- a/src/config.rs +++ b/src/config.rs @@ -334,12 +334,15 @@ impl Default for Api { pub struct TrackerStatisticsImporter { /// The interval in seconds to get statistics from the tracker. pub torrent_info_update_interval: u64, + /// The port the Importer API is listening on. Default to `3002`. + pub port: u16, } impl Default for TrackerStatisticsImporter { fn default() -> Self { Self { torrent_info_update_interval: 3600, + port: 3002, } } } diff --git a/src/console/mod.rs b/src/console/mod.rs index 82b6da3c..889c06dc 100644 --- a/src/console/mod.rs +++ b/src/console/mod.rs @@ -1 +1,2 @@ pub mod commands; +pub(crate) mod tracker_statistics_importer; diff --git a/src/console/tracker_statistics_importer.rs b/src/console/tracker_statistics_importer.rs new file mode 100644 index 00000000..c0787ad1 --- /dev/null +++ b/src/console/tracker_statistics_importer.rs @@ -0,0 +1,130 @@ +//! Cronjob to import tracker torrent data and updating seeders and leechers +//! info. +//! +//! It has two services: +//! +//! - The importer which is the cronjob executed at regular intervals. +//! - The importer API. +//! +//! The cronjob sends a heartbeat signal to the API each time it is executed. +//! The last heartbeat signal time is used to determine whether the cronjob was +//! executed successfully or not. The API has a `health_check` endpoint which is +//! used when the application is running in containers. +use std::sync::{Arc, Mutex}; + +use axum::extract::State; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use chrono::{DateTime, Utc}; +use log::{error, info}; +use serde_json::{json, Value}; +use tokio::task::JoinHandle; + +use crate::tracker::statistics_importer::StatisticsImporter; + +const IMPORTER_API_IP: &str = "127.0.0.1"; + +#[derive(Clone)] +struct ImporterState { + /// Shared variable to store the timestamp of the last heartbeat sent + /// by the cronjob. + pub last_heartbeat: Arc>>, + /// Interval between importation executions + pub torrent_info_update_interval: u64, +} + +pub fn start( + importer_port: u16, + torrent_info_update_interval: u64, + tracker_statistics_importer: &Arc, +) -> JoinHandle<()> { + let weak_tracker_statistics_importer = Arc::downgrade(tracker_statistics_importer); + + tokio::spawn(async move { + info!("Tracker statistics importer launcher started"); + + // Start the Importer API + + let _importer_api_handle = tokio::spawn(async move { + let import_state = Arc::new(ImporterState { + last_heartbeat: Arc::new(Mutex::new(Utc::now())), + torrent_info_update_interval, + }); + + let app = Router::new() + .route("/", get(|| async { Json(json!({})) })) + .route("/health_check", get(health_check_handler)) + .with_state(import_state.clone()) + .route("/heartbeat", post(heartbeat_handler)) + .with_state(import_state); + + let addr = format!("{IMPORTER_API_IP}:{importer_port}"); + + info!("Tracker statistics importer API server listening on http://{}", addr); + + axum::Server::bind(&addr.parse().unwrap()) + .serve(app.into_make_service()) + .await + .unwrap(); + }); + + // Start the Importer cronjob + + info!("Tracker statistics importer cronjob starting ..."); + + let interval = std::time::Duration::from_secs(torrent_info_update_interval); + let mut interval = tokio::time::interval(interval); + + interval.tick().await; // first tick is immediate... + + loop { + interval.tick().await; + + info!("Running tracker statistics importer ..."); + + if let Err(e) = send_heartbeat(importer_port).await { + error!("Failed to send heartbeat from importer cronjob: {}", e); + } + + if let Some(tracker) = weak_tracker_statistics_importer.upgrade() { + drop(tracker.import_all_torrents_statistics().await); + } else { + break; + } + } + }) +} + +/// Endpoint for container health check. +async fn health_check_handler(State(state): State>) -> Json { + let margin_in_seconds = 10; + let now = Utc::now(); + let last_heartbeat = state.last_heartbeat.lock().unwrap(); + + if now.signed_duration_since(*last_heartbeat).num_seconds() + <= (state.torrent_info_update_interval + margin_in_seconds).try_into().unwrap() + { + Json(json!({ "status": "Ok" })) + } else { + Json(json!({ "status": "Error" })) + } +} + +/// The tracker statistics importer cronjob sends a heartbeat on each execution +/// to inform that it's alive. This endpoint handles receiving that signal. +async fn heartbeat_handler(State(state): State>) -> Json { + let now = Utc::now(); + let mut last_heartbeat = state.last_heartbeat.lock().unwrap(); + *last_heartbeat = now; + Json(json!({ "status": "Heartbeat received" })) +} + +/// Send a heartbeat from the importer cronjob to the importer API. +async fn send_heartbeat(importer_port: u16) -> Result<(), reqwest::Error> { + let client = reqwest::Client::new(); + let url = format!("http://{IMPORTER_API_IP}:{importer_port}/heartbeat"); + + client.post(url).send().await?; + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 039bdfeb..ba89003c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -200,6 +200,7 @@ //! //! [tracker_statistics_importer] //! torrent_info_update_interval = 3600 +//! port = 3002 //! ``` //! //! For more information about configuration you can visit the documentation for the [`config`]) module. diff --git a/src/web/api/v1/contexts/settings/mod.rs b/src/web/api/v1/contexts/settings/mod.rs index 5bb35151..bd8cd54c 100644 --- a/src/web/api/v1/contexts/settings/mod.rs +++ b/src/web/api/v1/contexts/settings/mod.rs @@ -75,6 +75,7 @@ //! }, //! "tracker_statistics_importer": { //! "torrent_info_update_interval": 3600 +//! "port": 3002 //! } //! } //! } diff --git a/src/web/api/v1/routes.rs b/src/web/api/v1/routes.rs index 44098f4c..c1666a38 100644 --- a/src/web/api/v1/routes.rs +++ b/src/web/api/v1/routes.rs @@ -4,7 +4,8 @@ use std::sync::Arc; use axum::extract::DefaultBodyLimit; use axum::routing::get; -use axum::Router; +use axum::{Json, Router}; +use serde_json::{json, Value}; use tower_http::compression::CompressionLayer; use tower_http::cors::CorsLayer; @@ -34,7 +35,8 @@ pub fn router(app_data: Arc) -> Router { .nest("/proxy", proxy::routes::router(app_data.clone())); let router = Router::new() - .route("/", get(about_page_handler).with_state(app_data)) + .route("/", get(about_page_handler).with_state(app_data.clone())) + .route("/health_check", get(health_check_handler).with_state(app_data)) .nest(&format!("/{API_VERSION_URL_PREFIX}"), v1_api_routes); let router = if env::var(ENV_VAR_CORS_PERMISSIVE).is_ok() { @@ -45,3 +47,8 @@ pub fn router(app_data: Arc) -> Router { router.layer(DefaultBodyLimit::max(10_485_760)).layer(CompressionLayer::new()) } + +/// Endpoint for container health check. +async fn health_check_handler() -> Json { + Json(json!({ "status": "Ok" })) +} diff --git a/tests/common/contexts/settings/mod.rs b/tests/common/contexts/settings/mod.rs index 5ba7a0db..ece13b6d 100644 --- a/tests/common/contexts/settings/mod.rs +++ b/tests/common/contexts/settings/mod.rs @@ -82,6 +82,7 @@ pub struct Api { #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)] pub struct TrackerStatisticsImporter { pub torrent_info_update_interval: u64, + port: u16, } impl From for Settings { @@ -185,6 +186,7 @@ impl From for TrackerStatisticsImporter { fn from(tracker_statistics_importer: DomainTrackerStatisticsImporter) -> Self { Self { torrent_info_update_interval: tracker_statistics_importer.torrent_info_update_interval, + port: tracker_statistics_importer.port, } } } diff --git a/tests/environments/isolated.rs b/tests/environments/isolated.rs index e30c8907..175976f2 100644 --- a/tests/environments/isolated.rs +++ b/tests/environments/isolated.rs @@ -82,6 +82,9 @@ fn ephemeral(temp_dir: &TempDir) -> config::TorrustIndex { // Ephemeral API port configuration.net.port = FREE_PORT; + // Ephemeral Importer API port + configuration.tracker_statistics_importer.port = FREE_PORT; + // Ephemeral SQLite database configuration.database.connect_url = format!("sqlite://{}?mode=rwc", random_database_file_path_in(temp_dir));