From c32d46204d9ba80257a188f35fb3c532457cad3a Mon Sep 17 00:00:00 2001 From: Cameron Garnham Date: Fri, 25 Aug 2023 17:41:23 +0200 Subject: [PATCH] dev: container work --- .dockerignore | 2 +- .github/workflows/container.yaml | 59 +++++ .github/workflows/test_docker.yml | 26 --- Dockerfile | 210 ++++++++++++------ compose.yaml | 37 +-- docker/bin/build.sh | 2 +- docker/motd.debug | 5 + project-words.txt | 3 + ...port_tracker_statistics.rs => importer.rs} | 0 src/bin/{upgrade.rs => upgrader.rs} | 0 10 files changed, 212 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/container.yaml delete mode 100644 .github/workflows/test_docker.yml create mode 100644 docker/motd.debug rename src/bin/{import_tracker_statistics.rs => importer.rs} (100%) rename src/bin/{upgrade.rs => upgrader.rs} (100%) diff --git a/.dockerignore b/.dockerignore index b67eebd8..ac6dc8ff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,7 +14,7 @@ /data_v2.db* /data.db /data.db* -/docker/ +/docker/bin/ /project-words.txt /README.md /rustfmt.toml diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml new file mode 100644 index 00000000..c31ca6fe --- /dev/null +++ b/.github/workflows/container.yaml @@ -0,0 +1,59 @@ +name: Container (Docker) + +on: + push: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + name: Test + runs-on: ubuntu-latest + + steps: + - id: checkout + name: Checkout Repository + uses: actions/checkout@v3 + + - id: setup + name: Setup Toolchain + uses: docker/setup-buildx-action@v2 + + - id: build + name: Build + uses: docker/build-push-action@v4 + with: + push: false + load: true + tags: torrust-index-backend:local + cache-from: type=gha + cache-to: type=gha,mode=max + + - id: inspect + name: Inspect + run: docker image inspect torrust-index-backend:local + + - id: compose + name: Compose Applications + run: | + export TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.local.toml) + export TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) + + docker compose build \ + --build-arg TORRUST_IDX_BACK_CONFIG="$TORRUST_IDX_BACK_CONFIG" \ + --build-arg TORRUST_TRACKER_CONFIG="$TORRUST_TRACKER_CONFIG" + + - id: run + name: Run Applications + + run: | + export TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.local.toml) + export TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) + + docker compose up --detach + + - id: check + name: Check Applications + run: docker ps diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml deleted file mode 100644 index efb54e60..00000000 --- a/.github/workflows/test_docker.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Test Docker Build - -on: - push: - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build docker image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: false - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build docker-compose images - run: docker compose build diff --git a/Dockerfile b/Dockerfile index ffdd7ef0..8ba4a1af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,72 +1,140 @@ -FROM clux/muslrust:stable AS chef -WORKDIR /app -RUN cargo install cargo-chef - - -FROM chef AS planner -WORKDIR /app -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - - -FROM chef as development -WORKDIR /app -ARG UID=1000 -ARG RUN_AS_USER=appuser -ARG IDX_BACK_API_PORT=3001 -# Add the app user for development -ENV USER=appuser -ENV UID=$UID -RUN adduser --uid "${UID}" "${USER}" -# Build dependencies -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --recipe-path recipe.json -# Build the application -COPY . . -RUN cargo build --bin main -USER $RUN_AS_USER:$RUN_AS_USER -EXPOSE $IDX_BACK_API_PORT/tcp -CMD ["cargo", "run"] - - -FROM chef AS builder -WORKDIR /app -ARG UID=1000 -# Add the app user for production -ENV USER=appuser -ENV UID=$UID -RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" -# Build dependencies -COPY --from=planner /app/recipe.json recipe.json -RUN cargo chef cook --release --target x86_64-unknown-linux-musl --recipe-path recipe.json -# Build the application -COPY . . -RUN cargo build --release --target x86_64-unknown-linux-musl --bin main -# Strip the binary -# More info: https://github.com/LukeMathWalker/cargo-chef/issues/149 -RUN strip /app/target/x86_64-unknown-linux-musl/release/main - - -FROM alpine:latest -WORKDIR /app -ARG RUN_AS_USER=appuser -ARG IDX_BACK_API_PORT=3001 -RUN apk --no-cache add ca-certificates +# syntax=docker/dockerfile:latest + +# Torrust Index Backend + +## Builder Image +FROM rust:latest as chef +WORKDIR /tmp +RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash +RUN cargo binstall --no-confirm cargo-chef cargo-nextest + + +## Tester Image +FROM rust:slim as tester +WORKDIR /tmp +### (fixme) https://github.com/cargo-bins/cargo-binstall/issues/1252 +RUN apt-get update; apt-get install -y curl; apt-get autoclean +RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash +RUN cargo binstall --no-confirm cargo-nextest imdl + + +## Chef Prepare (look at project and see wat we need) +FROM chef AS recipe +WORKDIR /build/src +COPY . /build/src +RUN cargo chef prepare --recipe-path /build/recipe.json + + +## Cook (debug) +FROM chef AS dependencies_debug +WORKDIR /build/src +COPY --from=recipe /build/recipe.json /build/recipe.json +RUN cargo chef cook --tests --benches --examples --workspace --all-targets --all-features --recipe-path /build/recipe.json +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/temp.tar.zst ; rm /build/temp.tar.zst + +## Cook (release) +FROM chef AS dependencies +WORKDIR /build/src +COPY --from=recipe /build/recipe.json /build/recipe.json +RUN cargo chef cook --tests --benches --examples --workspace --all-targets --all-features --recipe-path /build/recipe.json --release +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/temp.tar.zst --release ; rm /build/temp.tar.zst + + +## Build Archive (debug) +FROM dependencies_debug AS build_debug +WORKDIR /build/src +COPY . /build/src +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/torrust-index-backend-debug.tar.zst + +## Build Archive (release) +FROM dependencies AS build +WORKDIR /build/src +COPY . /build/src +RUN cargo nextest archive --tests --benches --examples --workspace --all-targets --all-features --archive-file /build/torrust-index-backend.tar.zst --release + + +# Extract and Test (debug) +FROM tester as test_debug +WORKDIR /test +COPY . /test/src +COPY --from=build_debug \ + /build/torrust-index-backend-debug.tar.zst \ + /test/torrust-index-backend-debug.tar.zst +RUN mkdir -p /test/test +RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no-run --archive-file /test/torrust-index-backend-debug.tar.zst +RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json + +RUN mkdir -p /app/bin/; cp -l /test/src/target/debug/main /app/bin/torrust-index-backend +RUN mkdir -p /app/bin/; cp -l /test/src/target/debug/upgrader /app/bin/torrust-index-backend-upgrader +RUN mkdir -p /app/bin/; cp -l /test/src/target/debug/importer /app/bin/torrust-index-backend-importer + +# RUN mkdir /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-index-backend | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1 + +RUN chown -R root:root /app +RUN chmod -R u=rw,go=r,a+X /app +RUN chmod -R a+x /app/bin + +# Extract and Test (release) +FROM tester as test +WORKDIR /test +COPY . /test/src +COPY --from=build \ + /build/torrust-index-backend.tar.zst \ + /test/torrust-index-backend.tar.zst +RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no-run --archive-file /test/torrust-index-backend.tar.zst +RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json + +RUN mkdir -p /app/bin/; cp -l /test/src/target/release/main /app/bin/torrust-index-backend +RUN mkdir -p /app/bin/; cp -l /test/src/target/release/upgrader /app/bin/torrust-index-backend-upgrader +RUN mkdir -p /app/bin/; cp -l /test/src/target/release/importer /app/bin/torrust-index-backend-importer + +# RUN mkdir -p /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-index-backend | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1 + +RUN chown -R root:root /app +RUN chmod -R u=rw,go=r,a+X /app +RUN chmod -R a+x /app/bin + + +## Torrust-Index-Backend (debug) +FROM gcr.io/distroless/cc:debug as index_backend_debug + +RUN ["/busybox/cp", "-sp", "/busybox/sh", "/bin/sh"] +ENV ENV=/etc/profile + +ARG USER_ID=1000 +ARG USER_NAME=appuser +ARG API_PORT=3001 + +ENV USER_ID=${USER_ID} +ENV USER_NAME=${USER_NAME} +ENV API_PORT=${API_PORT} ENV TZ=Etc/UTC -ENV RUN_AS_USER=$RUN_AS_USER -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group -COPY --from=builder --chown=$RUN_AS_USER \ - /app/target/x86_64-unknown-linux-musl/release/main \ - /app/main -RUN chown -R $RUN_AS_USER:$RUN_AS_USER /app -USER $RUN_AS_USER:$RUN_AS_USER -EXPOSE $IDX_BACK_API_PORT/tcp -ENTRYPOINT ["/app/main"] \ No newline at end of file + +EXPOSE ${API_PORT}/tcp + +COPY --from=test_debug /app/ /usr/ + +COPY ./docker/motd.debug /etc/motd +RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/motd' >> /etc/profile + +WORKDIR /home/${USER_NAME} +RUN adduser --disabled-password --uid "${USER_ID}" "${USER_NAME}" +USER "${USER_NAME}":"${USER_NAME}" + +RUN env + +## Torrust-Index-Backend (release) (default) +FROM gcr.io/distroless/cc:nonroot as index_backend +COPY --from=gcr.io/distroless/cc:debug /busybox/wget /usr/bin/wget +COPY --from=test /app/ /usr/ + +ARG API_PORT=3001 + +ENV API_PORT=${API_PORT} +ENV TZ=Etc/UTC + +EXPOSE ${API_PORT}/tcp + +HEALTHCHECK CMD ["/usr/bin/wget", "--no-verbose", "--tries=1", "--spider", "localhost:${API_PORT}"] + +CMD ["/usr/bin/torrust-index-backend"] diff --git a/compose.yaml b/compose.yaml index 8bf4741e..f2efea3f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,36 +1,19 @@ name: torrust services: - idx-back: - build: - context: . - args: - RUN_AS_USER: appuser - UID: ${TORRUST_IDX_BACK_USER_UID:-1000} - target: development - user: ${TORRUST_IDX_BACK_USER_UID:-1000}:${TORRUST_IDX_BACK_USER_UID:-1000} + index-back: + image: torrust-index-backend:local tty: true environment: - TORRUST_IDX_BACK_CONFIG=${TORRUST_IDX_BACK_CONFIG} + - TORRUST_IDX_BACK_CORS_PERMISSIVE=true - CARGO_HOME=/home/appuser/.cargo networks: - server_side ports: - 3001:3001 - # todo: implement healthcheck - #healthcheck: - # test: - # [ - # "CMD-SHELL", - # "cargo run healthcheck" - # ] - # interval: 10s - # retries: 5 - # start_period: 10s - # timeout: 3s volumes: - - ./:/app - - ~/.cargo:/home/appuser/.cargo + - ./storage:/app/storage depends_on: - tracker - mailcatcher @@ -38,7 +21,6 @@ services: tracker: image: torrust/tracker:develop - user: ${TORRUST_TRACKER_USER_UID:-1000}:${TORRUST_TRACKER_USER_UID:-1000} tty: true environment: - TORRUST_TRACKER_CONFIG=${TORRUST_TRACKER_CONFIG} @@ -48,17 +30,6 @@ services: ports: - 6969:6969/udp - 1212:1212/tcp - # todo: implement healthcheck - #healthcheck: - # test: - # [ - # "CMD-SHELL", - # "/app/main healthcheck" - # ] - # interval: 10s - # retries: 5 - # start_period: 10s - # timeout: 3s volumes: - ./storage:/app/storage depends_on: diff --git a/docker/bin/build.sh b/docker/bin/build.sh index 96766624..23678c7d 100755 --- a/docker/bin/build.sh +++ b/docker/bin/build.sh @@ -10,4 +10,4 @@ echo "TORRUST_IDX_BACK_RUN_AS_USER: $TORRUST_IDX_BACK_RUN_AS_USER" docker build \ --build-arg UID="$TORRUST_IDX_BACK_USER_UID" \ --build-arg RUN_AS_USER="$TORRUST_IDX_BACK_RUN_AS_USER" \ - -t torrust-index-backend . + --tag torrust-index-backend:local . diff --git a/docker/motd.debug b/docker/motd.debug new file mode 100644 index 00000000..c02cac7b --- /dev/null +++ b/docker/motd.debug @@ -0,0 +1,5 @@ + in debug mode + +run 'torrust-index-backend' to start index +run 'torrust-index-backend-upgrader' to upgrade from v1 index +run 'torrust-index-backend-importer' to import tracker statistics diff --git a/project-words.txt b/project-words.txt index a890960f..ea42d8de 100644 --- a/project-words.txt +++ b/project-words.txt @@ -1,12 +1,15 @@ actix addrs alekitto +appuser AUTOINCREMENT bencode bencoded Benoit binascii btih +buildcache +buildx chrono clippy codecov diff --git a/src/bin/import_tracker_statistics.rs b/src/bin/importer.rs similarity index 100% rename from src/bin/import_tracker_statistics.rs rename to src/bin/importer.rs diff --git a/src/bin/upgrade.rs b/src/bin/upgrader.rs similarity index 100% rename from src/bin/upgrade.rs rename to src/bin/upgrader.rs