diff --git a/.dockerignore b/.dockerignore index b67eebd8..daa3343c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,15 +6,12 @@ /.gitignore /.vscode /bin/ -/config-idx-back.local.toml -/config-tracker.local.toml -/config.local.toml -/config.toml +/config* /cspell.json /data_v2.db* /data.db /data.db* -/docker/ +/docker/bin/ /project-words.txt /README.md /rustfmt.toml diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml new file mode 100644 index 00000000..69a3a01d --- /dev/null +++ b/.github/workflows/container.yaml @@ -0,0 +1,59 @@ +name: Container (Docker) + +on: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + name: Test + runs-on: ubuntu-latest + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v3 + + - id: setup + name: Setup Toolchain + uses: docker/setup-buildx-action@v2 + + - id: build + name: Build + uses: docker/build-push-action@v4 + with: + push: false + load: true + tags: torrust-index-backend:local + cache-from: type=gha + cache-to: type=gha,mode=max + + - id: inspect + name: Inspect + run: docker image inspect torrust-index-backend:local + + - id: compose + name: Compose Applications + run: | + export TORRUST_IDX_BACK_CONFIG=$(cat config-index.mysql.local.toml) + export TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) + + docker compose build \ + --build-arg TORRUST_IDX_BACK_CONFIG="$TORRUST_IDX_BACK_CONFIG" \ + --build-arg TORRUST_TRACKER_CONFIG="$TORRUST_TRACKER_CONFIG" + + - id: run + name: Run Applications + + run: | + export TORRUST_IDX_BACK_CONFIG=$(cat config-index.mysql.local.toml) + export TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) + + docker compose up --detach + + - id: check + name: Check Applications + run: docker ps diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml deleted file mode 100644 index efb54e60..00000000 --- a/.github/workflows/test_docker.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Test Docker Build - -on: - push: - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build docker image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: false - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build docker-compose images - run: docker compose build diff --git a/Dockerfile b/Dockerfile index ffdd7ef0..58df5823 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,72 +1,141 @@ -FROM clux/muslrust:stable AS chef -WORKDIR /app -RUN cargo install cargo-chef - - -FROM chef AS planner -WORKDIR /app -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - - -FROM chef as development -WORKDIR /app -ARG UID=1000 -ARG RUN_AS_USER=appuser -ARG IDX_BACK_API_PORT=3001 -# Add the app user for development -ENV USER=appuser -ENV UID=$UID -RUN adduser --uid "${UID}" "${USER}" -# Build dependencies -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --recipe-path recipe.json -# Build the application -COPY . . -RUN cargo build --bin main -USER $RUN_AS_USER:$RUN_AS_USER -EXPOSE $IDX_BACK_API_PORT/tcp -CMD ["cargo", "run"] - - -FROM chef AS builder -WORKDIR /app -ARG UID=1000 -# Add the app user for production -ENV USER=appuser -ENV UID=$UID -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" -# Build dependencies -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --target x86_64-unknown-linux-musl --recipe-path recipe.json -# Build the application -COPY . . -RUN cargo build --release --target x86_64-unknown-linux-musl --bin main -# Strip the binary -# More info: https://github.com/LukeMathWalker/cargo-chef/issues/149 -RUN strip /app/target/x86_64-unknown-linux-musl/release/main - - -FROM alpine:latest -WORKDIR /app -ARG RUN_AS_USER=appuser -ARG IDX_BACK_API_PORT=3001 -RUN apk --no-cache add ca-certificates +# syntax=docker/dockerfile:latest + +# Torrust Index Backend + +## Builder Image +FROM rust:latest as chef +WORKDIR /tmp +RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash +RUN cargo binstall --no-confirm cargo-chef cargo-nextest + + +## Tester Image +FROM rust:slim as tester +WORKDIR /tmp +### (fixme) https://github.com/cargo-bins/cargo-binstall/issues/1252 +RUN apt-get update; apt-get install -y curl; apt-get autoclean +RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash +RUN cargo binstall --no-confirm cargo-nextest imdl + + +## Chef Prepare (look at project and see wat we need) +FROM chef AS recipe +WORKDIR /build/src +COPY . /build/src +RUN cargo chef prepare --recipe-path /build/recipe.json + + +## Cook (debug) +FROM chef AS dependencies_debug +WORKDIR /build/src +COPY --from=recipe /build/recipe.json /build/recipe.json +RUN cargo chef cook --tests --benches --examples --workspace --all-targets --all-features --recipe-path /build/recipe.json +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/temp.tar.zst ; rm /build/temp.tar.zst + +## Cook (release) +FROM chef AS dependencies +WORKDIR /build/src +COPY --from=recipe /build/recipe.json /build/recipe.json +RUN cargo chef cook --tests --benches --examples --workspace --all-targets --all-features --recipe-path /build/recipe.json --release +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/temp.tar.zst --release ; rm /build/temp.tar.zst + + +## Build Archive (debug) +FROM dependencies_debug AS build_debug +WORKDIR /build/src +COPY . /build/src +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/torrust-index-backend-debug.tar.zst + +## Build Archive (release) +FROM dependencies AS build +WORKDIR /build/src +COPY . /build/src +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/torrust-index-backend.tar.zst --release + + +# Extract and Test (debug) +FROM tester as test_debug +WORKDIR /test +COPY . /test/src +COPY --from=build_debug \ + /build/torrust-index-backend-debug.tar.zst \ + /test/torrust-index-backend-debug.tar.zst +RUN mkdir -p /test/test +RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no-run --archive-file /test/torrust-index-backend-debug.tar.zst +RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json + +RUN mkdir -p /app/bin/; cp -l /test/src/target/debug/main /app/bin/torrust-index-backend +RUN mkdir -p /app/bin/; cp -l /test/src/target/debug/upgrader /app/bin/torrust-index-backend-upgrader +RUN mkdir -p /app/bin/; cp -l /test/src/target/debug/importer /app/bin/torrust-index-backend-importer + +# RUN mkdir /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-index-backend | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1 + +RUN chown -R root:root /app +RUN chmod -R u=rw,go=r,a+X /app +RUN chmod -R a+x /app/bin + +# Extract and Test (release) +FROM tester as test +WORKDIR /test +COPY . /test/src +COPY --from=build \ + /build/torrust-index-backend.tar.zst \ + /test/torrust-index-backend.tar.zst +RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no-run --archive-file /test/torrust-index-backend.tar.zst +RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json + +RUN mkdir -p /app/bin/; cp -l /test/src/target/release/main /app/bin/torrust-index-backend +RUN mkdir -p /app/bin/; cp -l /test/src/target/release/upgrader /app/bin/torrust-index-backend-upgrader +RUN mkdir -p /app/bin/; cp -l /test/src/target/release/importer /app/bin/torrust-index-backend-importer + +# RUN mkdir -p /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-index-backend | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1 + +RUN chown -R root:root /app +RUN chmod -R u=rw,go=r,a+X /app +RUN chmod -R a+x /app/bin + + +## Torrust-Index-Backend (debug) +FROM gcr.io/distroless/cc:debug as index_backend_debug + +RUN ["/busybox/cp", "-sp", "/busybox/sh", "/bin/sh"] +ENV ENV=/etc/profile + +ARG USER_ID=1000 +ARG USER_NAME=appuser +ARG API_PORT=3001 + +ENV USER_ID=${USER_ID} +ENV USER_NAME=${USER_NAME} +ENV API_PORT=${API_PORT} ENV TZ=Etc/UTC -ENV RUN_AS_USER=$RUN_AS_USER -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder --chown=$RUN_AS_USER \ - /app/target/x86_64-unknown-linux-musl/release/main \ - /app/main -RUN chown -R $RUN_AS_USER:$RUN_AS_USER /app -USER $RUN_AS_USER:$RUN_AS_USER -EXPOSE $IDX_BACK_API_PORT/tcp -ENTRYPOINT ["/app/main"] \ No newline at end of file + +EXPOSE ${API_PORT}/tcp + +COPY --from=test_debug /app/ /usr/ + +COPY ./docker/motd.debug /etc/motd + +RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/motd' >> /etc/profile + +WORKDIR /home/${USER_NAME} +RUN adduser --disabled-password --uid "${USER_ID}" "${USER_NAME}" +USER "${USER_NAME}":"${USER_NAME}" + +RUN env + +## Torrust-Index-Backend (release) (default) +FROM gcr.io/distroless/cc:nonroot as index_backend +COPY --from=gcr.io/distroless/cc:debug /busybox/wget /usr/bin/wget +COPY --from=test /app/ /usr/ + +ARG API_PORT=3001 + +ENV API_PORT=${API_PORT} +ENV TZ=Etc/UTC + +EXPOSE ${API_PORT}/tcp + +HEALTHCHECK CMD ["/usr/bin/wget", "--no-verbose", "--tries=1", "--spider", "localhost:${API_PORT}"] + +CMD ["/usr/bin/torrust-index-backend"] diff --git a/bin/install.sh b/bin/install.sh index 30b8a200..e98047eb 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -2,7 +2,7 @@ # Generate the default settings file if it does not exist if ! [ -f "./config.toml" ]; then - cp ./config.local.toml ./config.toml + cp ./config-index.local.toml ./config.toml fi # Generate storage directory if it does not exist diff --git a/compose.yaml b/compose.yaml index 8bf4741e..f2efea3f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,36 +1,19 @@ name: torrust services: - idx-back: - build: - context: . - args: - RUN_AS_USER: appuser - UID: ${TORRUST_IDX_BACK_USER_UID:-1000} - target: development - user: ${TORRUST_IDX_BACK_USER_UID:-1000}:${TORRUST_IDX_BACK_USER_UID:-1000} + index-back: + image: torrust-index-backend:local tty: true environment: - TORRUST_IDX_BACK_CONFIG=${TORRUST_IDX_BACK_CONFIG} + - TORRUST_IDX_BACK_CORS_PERMISSIVE=true - CARGO_HOME=/home/appuser/.cargo networks: - server_side ports: - 3001:3001 - # todo: implement healthcheck - #healthcheck: - # test: - # [ - # "CMD-SHELL", - # "cargo run healthcheck" - # ] - # interval: 10s - # retries: 5 - # start_period: 10s - # timeout: 3s volumes: - - ./:/app - - ~/.cargo:/home/appuser/.cargo + - ./storage:/app/storage depends_on: - tracker - mailcatcher @@ -38,7 +21,6 @@ services: tracker: image: torrust/tracker:develop - user: ${TORRUST_TRACKER_USER_UID:-1000}:${TORRUST_TRACKER_USER_UID:-1000} tty: true environment: - TORRUST_TRACKER_CONFIG=${TORRUST_TRACKER_CONFIG} @@ -48,17 +30,6 @@ services: ports: - 6969:6969/udp - 1212:1212/tcp - # todo: implement healthcheck - #healthcheck: - # test: - # [ - # "CMD-SHELL", - # "/app/main healthcheck" - # ] - # interval: 10s - # retries: 5 - # start_period: 10s - # timeout: 3s volumes: - ./storage:/app/storage depends_on: diff --git a/config.local.toml b/config-index.local.toml similarity index 100% rename from config.local.toml rename to config-index.local.toml diff --git a/config-idx-back.mysql.local.toml b/config-index.mysql.local.toml similarity index 100% rename from config-idx-back.mysql.local.toml rename to config-index.mysql.local.toml diff --git a/config-idx-back.sqlite.local.toml b/config-index.sqlite.local.toml similarity index 100% rename from config-idx-back.sqlite.local.toml rename to config-index.sqlite.local.toml diff --git a/docker/README.md b/docker/README.md index be47bfad..e2732095 100644 --- a/docker/README.md +++ b/docker/README.md @@ -64,7 +64,7 @@ Build and run it locally: ```s TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \ - TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.local.toml) \ + TORRUST_IDX_BACK_CONFIG=$(cat config-index.mysql.local.toml) \ TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) \ TORRUST_TRACKER_API_TOKEN=${TORRUST_TRACKER_API_TOKEN:-MyAccessToken} \ docker compose up -d --build diff --git a/docker/bin/build.sh b/docker/bin/build.sh index 96766624..23678c7d 100755 --- a/docker/bin/build.sh +++ b/docker/bin/build.sh @@ -10,4 +10,4 @@ echo "TORRUST_IDX_BACK_RUN_AS_USER: $TORRUST_IDX_BACK_RUN_AS_USER" docker build \ --build-arg UID="$TORRUST_IDX_BACK_USER_UID" \ --build-arg RUN_AS_USER="$TORRUST_IDX_BACK_RUN_AS_USER" \ - -t torrust-index-backend . + --tag torrust-index-backend:local . diff --git a/docker/bin/e2e/mysql/e2e-env-up.sh b/docker/bin/e2e/mysql/e2e-env-up.sh index 195409b1..16548e74 100755 --- a/docker/bin/e2e/mysql/e2e-env-up.sh +++ b/docker/bin/e2e/mysql/e2e-env-up.sh @@ -4,7 +4,7 @@ TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \ docker compose build TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \ - TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.mysql.local.toml) \ + TORRUST_IDX_BACK_CONFIG=$(cat config-index.mysql.local.toml) \ TORRUST_IDX_BACK_MYSQL_DATABASE="torrust_index_backend_e2e_testing" \ TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) \ TORRUST_TRACKER_API_TOKEN=${TORRUST_TRACKER_API_TOKEN:-MyAccessToken} \ diff --git a/docker/bin/e2e/run-e2e-tests.sh b/docker/bin/e2e/run-e2e-tests.sh index a55b6315..c7a468fe 100755 --- a/docker/bin/e2e/run-e2e-tests.sh +++ b/docker/bin/e2e/run-e2e-tests.sh @@ -42,7 +42,7 @@ cargo install imdl || exit 1 # Install app (no docker) that will run the test suite against the E2E testing # environment (in docker). cp .env.local .env || exit 1 -./bin/install.sh || exit 1 +./docker/bin/install.sh || exit 1 # TEST USING SQLITE echo "Running E2E tests using SQLite ..." @@ -60,7 +60,7 @@ sleep 20s docker ps # Run E2E tests with shared app instance -TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./config-idx-back.sqlite.local.toml" cargo test || exit 1 +TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./config-index.sqlite.local.toml" cargo test || exit 1 # Stop E2E testing environment docker compose down @@ -91,7 +91,7 @@ echo "Creating MySQL database $MYSQL_DATABASE for for E2E testing ..." mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PASSWORD -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE;" # Run E2E tests with shared app instance -TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./config-idx-back.mysql.local.toml" cargo test || exit 1 +TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./config-index.mysql.local.toml" cargo test || exit 1 # Stop E2E testing environment docker compose down diff --git a/docker/bin/e2e/sqlite/e2e-env-up.sh b/docker/bin/e2e/sqlite/e2e-env-up.sh index 25911757..980fc833 100755 --- a/docker/bin/e2e/sqlite/e2e-env-up.sh +++ b/docker/bin/e2e/sqlite/e2e-env-up.sh @@ -4,7 +4,7 @@ TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \ docker compose build TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \ - TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.sqlite.local.toml) \ + TORRUST_IDX_BACK_CONFIG=$(cat config-index.sqlite.local.toml) \ TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) \ TORRUST_TRACKER_API_TOKEN=${TORRUST_TRACKER_API_TOKEN:-MyAccessToken} \ docker compose up -d diff --git a/docker/motd.debug b/docker/motd.debug new file mode 100644 index 00000000..c02cac7b --- /dev/null +++ b/docker/motd.debug @@ -0,0 +1,5 @@ + in debug mode + +run 'torrust-index-backend' to start index +run 'torrust-index-backend-upgrader' to upgrade from v1 index +run 'torrust-index-backend-importer' to import tracker statistics diff --git a/project-words.txt b/project-words.txt index a890960f..ea42d8de 100644 --- a/project-words.txt +++ b/project-words.txt @@ -1,12 +1,15 @@ actix addrs alekitto +appuser AUTOINCREMENT bencode bencoded Benoit binascii btih +buildcache +buildx chrono clippy codecov diff --git a/src/bin/import_tracker_statistics.rs b/src/bin/importer.rs similarity index 79% rename from src/bin/import_tracker_statistics.rs rename to src/bin/importer.rs index 894863fa..28cfed22 100644 --- a/src/bin/import_tracker_statistics.rs +++ b/src/bin/importer.rs @@ -2,7 +2,7 @@ //! //! It imports the number of seeders and leechers for all torrent from the linked tracker. //! -//! You can execute it with: `cargo run --bin import_tracker_statistics` +//! You can execute it with: `cargo run --bin importer` use torrust_index_backend::console::commands::import_tracker_statistics::run_importer; #[tokio::main] diff --git a/src/bin/upgrade.rs b/src/bin/upgrader.rs similarity index 70% rename from src/bin/upgrade.rs rename to src/bin/upgrader.rs index 486bde93..0e21a716 100644 --- a/src/bin/upgrade.rs +++ b/src/bin/upgrader.rs @@ -1,6 +1,6 @@ //! Upgrade command. //! It updates the application from version v1.0.0 to v2.0.0. -//! You can execute it with: `cargo run --bin upgrade ./data.db ./data_v2.db ./uploads` +//! You can execute it with: `cargo run --bin upgrader ./data.db ./data_v2.db ./uploads` use torrust_index_backend::upgrades::from_v1_0_0_to_v2_0_0::upgrader::run; diff --git a/tests/e2e/config.rs b/tests/e2e/config.rs index 60595c79..d9292915 100644 --- a/tests/e2e/config.rs +++ b/tests/e2e/config.rs @@ -19,7 +19,7 @@ pub const ENV_VAR_E2E_CONFIG_PATH: &str = "TORRUST_IDX_BACK_E2E_CONFIG_PATH"; // Default values -pub const ENV_VAR_E2E_DEFAULT_CONFIG_PATH: &str = "./config-idx-back.local.toml"; +pub const ENV_VAR_E2E_DEFAULT_CONFIG_PATH: &str = "./config-index.local.toml"; /// Initialize configuration from file or env var. ///