feat: Add server that provides k-apps contents (#2680)

.github/workflows/test.yaml

@@ -1,4 +1,4 @@
-name: Go test
+name: Tests
 on:
   push:
     tags:
@@ -32,6 +32,9 @@ jobs:
           devbox run just git-operator-fetch-manifests
           git diff --exit-code
 
+      - name: Check if kommander application server works
+        run: devbox run -- just test-server
+
       - name: Report Coveralls
         uses: coverallsapp/github-action@v2
         with:

devbox.json

@@ -1,8 +1,10 @@
 {
   "packages": {
     "awscli2":         "latest",
+    "curl":            "latest",
     "direnv":          "latest",
     "docker":          "latest",
+    "docker-buildx":   "latest",
     "envsubst":        "latest",
     "fluxcd":          "latest",
     "github-cli":      "latest",

devbox.lock

@@ -65,6 +65,130 @@
         }
       }
     },
+    "curl@latest": {
+      "last_modified": "2024-09-29T13:45:20Z",
+      "resolved": "github:NixOS/nixpkgs/7eee17a8a5868ecf596bbb8c8beb527253ea8f4d#curl",
+      "source": "devbox-search",
+      "version": "8.9.1",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "bin",
+              "path": "/nix/store/b9vcp70yhf5s0qadg2i1pdnzi7mfzkx4-curl-8.9.1-bin",
+              "default": true
+            },
+            {
+              "name": "man",
+              "path": "/nix/store/nlnz9711av6sb5mcx6slmkbngg5kd57r-curl-8.9.1-man",
+              "default": true
+            },
+            {
+              "name": "devdoc",
+              "path": "/nix/store/g3rwsivls1y3b9iyzzgda5s3nb0pnvkg-curl-8.9.1-devdoc"
+            },
+            {
+              "name": "out",
+              "path": "/nix/store/dijbmwxxhizpgva2idb261ic4f0r47cq-curl-8.9.1"
+            },
+            {
+              "name": "dev",
+              "path": "/nix/store/g7iznpcciawm2alm7hn9qf2x2nz9gcld-curl-8.9.1-dev"
+            }
+          ],
+          "store_path": "/nix/store/b9vcp70yhf5s0qadg2i1pdnzi7mfzkx4-curl-8.9.1-bin"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "bin",
+              "path": "/nix/store/jz56y8dn9vyp5qxc2gvs99isis3ql5ml-curl-8.9.1-bin",
+              "default": true
+            },
+            {
+              "name": "man",
+              "path": "/nix/store/cqdyp0jrflbjbr6xcd5dlb704qkzh2sm-curl-8.9.1-man",
+              "default": true
+            },
+            {
+              "name": "debug",
+              "path": "/nix/store/jvjycva14kpdk2yyj3b9hpyi0lgl1rf1-curl-8.9.1-debug"
+            },
+            {
+              "name": "dev",
+              "path": "/nix/store/cgshc8z3alwlyh3yv7bp7lv7hj4ny8mp-curl-8.9.1-dev"
+            },
+            {
+              "name": "devdoc",
+              "path": "/nix/store/b2nzf7rzlrkbw5ha3admbpp2pks3izf2-curl-8.9.1-devdoc"
+            },
+            {
+              "name": "out",
+              "path": "/nix/store/w431gagkp4phpi8dn9i7d02xxgjvmfpx-curl-8.9.1"
+            }
+          ],
+          "store_path": "/nix/store/jz56y8dn9vyp5qxc2gvs99isis3ql5ml-curl-8.9.1-bin"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "bin",
+              "path": "/nix/store/zzz5688jdq269a7mi18al2zbqnmrd7d6-curl-8.9.1-bin",
+              "default": true
+            },
+            {
+              "name": "man",
+              "path": "/nix/store/ap0q2b6i3kx3f3mys9r6rp9kcry17rqj-curl-8.9.1-man",
+              "default": true
+            },
+            {
+              "name": "out",
+              "path": "/nix/store/rl0lg5swlzlsrmhmw7cgi46kbxwxngkh-curl-8.9.1"
+            },
+            {
+              "name": "dev",
+              "path": "/nix/store/iw3jwk48gfg0s0lsz0qpz89paig14fsy-curl-8.9.1-dev"
+            },
+            {
+              "name": "devdoc",
+              "path": "/nix/store/rj4x8krz9n9d6s0ys0x20s6wd2fq29pw-curl-8.9.1-devdoc"
+            }
+          ],
+          "store_path": "/nix/store/zzz5688jdq269a7mi18al2zbqnmrd7d6-curl-8.9.1-bin"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "bin",
+              "path": "/nix/store/6r0bn0dkvlvhicyvair205s07m92dpaz-curl-8.9.1-bin",
+              "default": true
+            },
+            {
+              "name": "man",
+              "path": "/nix/store/4ph78zm36yji16fypdk08bcj3agaqwh5-curl-8.9.1-man",
+              "default": true
+            },
+            {
+              "name": "dev",
+              "path": "/nix/store/a49si5nv16sy1jaabryasl015w3s52rr-curl-8.9.1-dev"
+            },
+            {
+              "name": "devdoc",
+              "path": "/nix/store/3w9135g8l2fqxjcdcqdn400gjigf7w5x-curl-8.9.1-devdoc"
+            },
+            {
+              "name": "out",
+              "path": "/nix/store/x6ssc2mmx1kb52gchksqbzg5c2y0z7lf-curl-8.9.1"
+            },
+            {
+              "name": "debug",
+              "path": "/nix/store/hprnrdjjf4ybw74hf0w852842zzyxq11-curl-8.9.1-debug"
+            }
+          ],
+          "store_path": "/nix/store/6r0bn0dkvlvhicyvair205s07m92dpaz-curl-8.9.1-bin"
+        }
+      }
+    },
     "direnv@latest": {
       "last_modified": "2024-06-12T20:55:33Z",
       "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#direnv",
@@ -113,6 +237,54 @@
         }
       }
     },
+    "docker-buildx@latest": {
+      "last_modified": "2024-09-27T09:34:34Z",
+      "resolved": "github:NixOS/nixpkgs/e0f477a570df7375172a08ddb9199c90853c63f0#docker-buildx",
+      "source": "devbox-search",
+      "version": "0.17.1",
+      "systems": {
+        "aarch64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/v7xirywlpnyxlg36fs7q8dz22m1293f4-docker-buildx-0.17.1",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/v7xirywlpnyxlg36fs7q8dz22m1293f4-docker-buildx-0.17.1"
+        },
+        "aarch64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/3plbx9g03mfzr8baisfhm05fm9cnw63d-docker-buildx-0.17.1",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/3plbx9g03mfzr8baisfhm05fm9cnw63d-docker-buildx-0.17.1"
+        },
+        "x86_64-darwin": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/39ajcbb4irgwy8y1xiqqw6smalkl5l43-docker-buildx-0.17.1",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/39ajcbb4irgwy8y1xiqqw6smalkl5l43-docker-buildx-0.17.1"
+        },
+        "x86_64-linux": {
+          "outputs": [
+            {
+              "name": "out",
+              "path": "/nix/store/6x7xc5zypfnn0yyv1v9zjmjm0hx3snam-docker-buildx-0.17.1",
+              "default": true
+            }
+          ],
+          "store_path": "/nix/store/6x7xc5zypfnn0yyv1v9zjmjm0hx3snam-docker-buildx-0.17.1"
+        }
+      }
+    },
     "docker@latest": {
       "last_modified": "2024-06-22T18:46:48Z",
       "resolved": "github:NixOS/nixpkgs/9693852a2070b398ee123a329e68f0dab5526681#docker",

just/test.just

@@ -0,0 +1,14 @@
+test-server:
+    #!/usr/bin/env bash
+    set -euox pipefail
+    CONTAINER_ID=$(just --justfile {{ justfile() }} --working-directory {{ invocation_directory() }} _run_server 2>&1 | tail -n 1)
+    trap "docker kill ${CONTAINER_ID}" EXIT
+    curl --no-progress-meter --output /dev/null --retry-connrefused --retry 5 --retry-delay 3 http://localhost:5000/{{ archive_name }}
+
+_run_server: (release-server "false")
+    docker run \
+    --env DUFS_TLS_CERT \
+    --env DUFS_TLS_KEY \
+    --network=host \
+    --detach \
+    {{ server_docker_repository }}:{{ git_tag }}

justfile

@@ -8,6 +8,7 @@ repository := org_name / "kommander-applications"
 include_file := justfile_directory() / ".include-airgapped"
 exclude_file := justfile_directory() / ".exclude-airgapped"
 git_operator_version := env("GIT_OPERATOR_VERSION", "latest")
+server_docker_repository := registry / org_name / "kommander-applications-server"
 
 s3_path := "dkp" / git_tag
 s3_bucket := "downloads.mesosphere.io"
@@ -16,12 +17,20 @@ s3_acl := "bucket-owner-full-control"
 archive_name := "kommander-applications-" + git_tag+ ".tar.gz"
 published_url := "https://downloads.d2iq.com" / s3_path / archive_name
 
-release tmp_dir=`mktemp --directory`: (_prepare-archive tmp_dir)
-    aws s3 cp --acl {{ s3_acl }} {{ archive_name }} {{ s3_uri }}
+release publish="true" tmp_dir=`mktemp --directory`: (_prepare-archive tmp_dir) && _cleanup
+    if {{ publish }}; then aws s3 cp --acl {{ s3_acl }} {{ archive_name }} {{ s3_uri }}; fi
     @echo "Published to {{ published_url }}"
 
-release-oci tmp_dir=`mktemp --directory`: (_prepare-files-for-a-bundle tmp_dir)
-    cd {{ tmp_dir }} && echo "${DOCKER_PASSWORD}" | oras push --password-stdin --username "${DOCKER_USERNAME}" --verbose {{ registry }}/{{ repository }}:{{ git_tag }} .
+release-oci publish="true" tmp_dir=`mktemp --directory`: (_prepare-files-for-a-bundle tmp_dir)
+    if {{ publish }}; then \
+      cd {{ tmp_dir }} && echo "${DOCKER_PASSWORD}" | oras push --password-stdin --username "${DOCKER_USERNAME}" --verbose {{ registry }}/{{ repository }}:{{ git_tag }} .;
+    fi
+
+release-server publish="true" tmp_dir=`mktemp --directory`: (_prepare-archive tmp_dir) && _cleanup
+    cp {{ archive_name }} ./server/
+    cd ./server && docker buildx build . --tag {{ server_docker_repository }}:{{ git_tag }} --build-arg ARCHIVE_NAME={{ archive_name }}
+    rm ./server/{{ archive_name }}
+    if {{ publish }}; then docker push {{ server_docker_repository }}:{{ git_tag }}; fi
 
 service_version:=`ls services/git-operator/ | grep -E "v?[[:digit:]]\.[[:digit:]]\.[[:digit:]]"`
 service_dir:=justfile_directory() / "services/git-operator" / service_version
@@ -39,5 +48,7 @@ _cleanup:
     rm {{ archive_name }}
 
 _prepare-files-for-a-bundle output_dir:
-    rsync --archive --recursive --files-from={{ include_file }} --exclude-from={{ exclude_file }} {{ justfile_directory() }} {{ output_dir }}
+    rsync --quiet --archive --recursive --files-from={{ include_file }} --exclude-from={{ exclude_file }} {{ justfile_directory() }} {{ output_dir }}
     yq 'del(.resources[] | select(. == "ai-navigator-repos.yaml"))' --inplace {{ output_dir }}/common/helm-repositories/kustomization.yaml
+
+import 'just/test.just'

server/Dockerfile

@@ -0,0 +1,13 @@
+FROM sigoden/dufs
+
+ARG ARCHIVE_NAME
+
+# Dufs settings:
+ENV DUFS_BIND=127.0.0.1
+ENV DUFS_PORT=5000
+ENV DUFS_TLS_CERT=cert.pem
+ENV DUFS_TLS_KEY=key.pem
+
+COPY ${ARCHIVE_NAME} /data/
+
+CMD ["/data"]