Merge pull request #3531 from magda-io/issue/3337

Issue/3337 Move to Github Container Registry

.gitlab-ci.yml

@@ -30,7 +30,7 @@ stages:
 
 yarn-install:
   stage: builders
-  image: data61/magda-builder-docker:node18
+  image: ghcr.io/magda-io/magda-builder-docker:node18
   retry: 1
   needs: []
   cache:
@@ -49,7 +49,7 @@ yarn-install:
 
 build-builder-image:
   stage: builders
-  image: data61/magda-builder-docker:node18
+  image: ghcr.io/magda-io/magda-builder-docker:node18
   retry: 1
   needs: []
   before_script:
@@ -595,7 +595,7 @@ buildtest:helm-docs-check:
     - docker:dind
   script:
     - code=0
-    - docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.5.0 || code=$?;
+    - docker run --rm -v "$(pwd):/helm-docs" -u $(id -u) jnorwood/helm-docs:v1.13.1 || code=$?;
       if [ "$code" != "0" ]; then 
         echo "Failed to run helm-docs!";
         exit 1;
@@ -605,7 +605,7 @@ buildtest:helm-docs-check:
     - git ls-files -m | grep -i readme.md || code=$?;
       if [ "$code" == "0" ]; then
         echo -e "Some of helm chart docs are required to be updated using the [helm-docs](https://github.com/norwoodj/helm-docs) tool. \n
-        Please run helm-docs (v1.5.0) at project root, review & commit docs changes and push a new commit.";
+        Please run helm-docs (v1.13.1) at project root, review & commit docs changes and push a new commit.";
         exit 1;
       else 
         echo -e "helm docs check passed. helm docs update is not required.";
@@ -810,12 +810,12 @@ dockerize:dockerExtensions:
 (No Data) Run As Preview:
   <<: *runAsPreview
   script:
-    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.combined-db.magda-postgres.postgresql.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.combined-db.magda-postgres.backupRestore.recoveryMode.enabled=false,magda.magda-core.elasticsearch.useGcsSnapshots=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600m --wait
+    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.combined-db.magda-postgres.postgresql.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.combined-db.magda-postgres.backupRestore.recoveryMode.enabled=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600m --wait
     - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"
 (No Data) Run As Multi-tenant Preview:
   <<: *runAsPreview
   script:
-    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview-multi-tenant.yml --set global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.combined-db.magda-postgres.postgresql.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.combined-db.magda-postgres.backupRestore.recoveryMode.enabled=false,magda.magda-core.elasticsearch.useGcsSnapshots=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.magdaAdminPortalName=$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600m --wait
+    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/local-deployment --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview-multi-tenant.yml --set global.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.combined-db.magda-postgres.postgresql.image.tag=$CI_COMMIT_REF_SLUG,magda.magda-core.ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,magda.magda-core.combined-db.magda-postgres.backupRestore.recoveryMode.enabled=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.magdaAdminPortalName=$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600m --wait
     - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"
 
 
@@ -938,22 +938,18 @@ Release-to-Docker-Hub-Github-Container:
     - echo "$GITHUB_ACCESS_TOKEN" | docker login ghcr.io -u magdabot --password-stdin
     - |
       if [ "$RELEASE_VERSION_TAG" == "true" ]; then
-        yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=data61/ --copyFromRegistry=true
         yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=ghcr.io/magda-io/ --copyFromRegistry=true
       fi
     - |
       if [ "$RELEASE_LATEST_TAG" == "true" ]; then
-        yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=data61/ --toVersion=latest --copyFromRegistry=true
         yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=ghcr.io/magda-io/ --toVersion=latest --copyFromRegistry=true
       fi
     - |
       if [ "$RELEASE_MAIN_TAG" == "true" ]; then
-        yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=data61/ --toVersion=main --copyFromRegistry=true
         yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=ghcr.io/magda-io/ --toVersion=main --copyFromRegistry=true
       fi
     - |
       if [ "$RELEASE_NEXT_TAG" == "true" ]; then
-        yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=data61/ --toVersion=next --copyFromRegistry=true
         yarn run retag-and-push -- -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG --toPrefix=ghcr.io/magda-io/ --toVersion=next --copyFromRegistry=true
       fi
 

CHANGES.md

@@ -4,7 +4,9 @@
 
 - Use OpenSearch (v2.14.0) as main search engine
 - Allow supply extra K8s manifests to deploy via `magda-core` `.Values.extraObjects`
-- Stop releasing helm charts to charts.magda.io. Since v2, we started to release charts to Github container OCI registry
+- #3337 Stop releasing helm charts to charts.magda.io. Since v2, we started to release charts to Github container OCI registry
+- #3337 Make all helm charts use github container registry `ghcr.io/magda-io`
+- Upgrade [helm-docs](https://github.com/norwoodj/helm-docs) to 1.13.1
 
 ## v3.0.4
 

README.md

@@ -23,7 +23,7 @@ Over the past 18 months, our focus has been to develop Magda into a more general
 
 ## Features
 
-- Powerful and scalable search based on ElasticSearch
+- Powerful and scalable search based on [OpenSearch](https://opensearch.org/)
 - Quick and reliable aggregation of external sources of datasets
 - An unopinionated central store of metadata, able to cater for most metadata schemas
 - Federated authentication via passport.js - log in via Google, Facebook, WSFed, AAF, CKAN, and easily create new providers.
@@ -65,7 +65,9 @@ Other aspects exist that are written to by many minions - for instance, we have
 
 ### Search
 
-Datasets and distributions in the registry are ingested into an ElasticSearch cluster, which indexes a few core aspects of each and exposes an API.
+Datasets and distributions in the registry are ingested into an [OpenSearch](https://opensearch.org/) cluster, which indexes a few core aspects of each and exposes an API.
+
+> Please note: since v4.0.0, we replaced the internal [ElasticSearch](https://www.elastic.co/elasticsearch) cluster with latest [OpenSearch](https://opensearch.org/) cluster. As OpenSearch is a fork of open source Elasticsearch 7.10. As such, it provides backwards REST APIs for ingest, search, and management. The query syntax and responses are also the same.
 
 ### User Interface
 
@@ -76,16 +78,15 @@ Magda provides a user interface, which is served from its own microservice and c
 If you just want to install a local testing version, installing Magda using [Helm](https://helm.sh/) is relatively easier (you can use [minikube](https://minikube.sigs.k8s.io/docs/) to install a local k8s test cluster):
 
 ```bash
-# Add Magda Helm Chart Repo:
-helm repo add magda-io https://charts.magda.io
-
 # create a namespace "magda" in your cluster
 kubectl create namespace magda
 
 # install Magda version v2.2.0 to namespace "magda", turn off openfass function and expose the service via loadBalancer
-helm upgrade --namespace magda --install --version 2.2.0 --timeout 9999s --set magda-core.gateway.service.type=LoadBalancer magda magda-io/magda
+helm upgrade --namespace magda --install --version 4.0.0 --timeout 9999s --set magda-core.gateway.service.type=LoadBalancer magda oci://ghcr.io/magda-io/charts/magda
 ```
 
+> Since v2, we release our helm charts to Github container registry: `oci://ghcr.io/magda-io/charts`
+
 You can find out the load balancer IP and access it:
 
 ```bash

deploy/helm/internal-charts/admin-api/README.md

@@ -14,7 +14,7 @@ Kubernetes: `>= 1.14.0-0`
 |-----|------|---------|-------------|
 | defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
 | defaultImage.pullSecrets | bool | `false` |  |
-| defaultImage.repository | string | `"docker.io/data61"` |  |
+| defaultImage.repository | string | `"ghcr.io/magda-io"` |  |
 | image.name | string | `"magda-admin-api"` |  |
 | namespace | string | `nil` | the k8s namespace contains job object. When empty, will use `.Release.Namespace` |
 | rbac.create | bool | `true` | Whether or not to create RBAC role & role binding |
@@ -26,4 +26,4 @@ Kubernetes: `>= 1.14.0-0`
 | skipAuth | bool | `false` | when set to true, API will not query policy engine for auth decision but assume it's always permitted.  It's for debugging only. |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

deploy/helm/internal-charts/admin-api/values.yaml

@@ -6,7 +6,7 @@ image:
   # pullSecrets: 
 
 defaultImage:
-  repository: docker.io/data61
+  repository: "ghcr.io/magda-io"
   pullPolicy: IfNotPresent
   pullSecrets: false
 

deploy/helm/internal-charts/apidocs-server/README.md

@@ -14,11 +14,11 @@ Kubernetes: `>= 1.14.0-0`
 |-----|------|---------|-------------|
 | defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
 | defaultImage.pullSecrets | bool | `false` |  |
-| defaultImage.repository | string | `"docker.io/data61"` |  |
+| defaultImage.repository | string | `"ghcr.io/magda-io"` |  |
 | image.name | string | `"magda-apidocs-server"` |  |
 | resources.limits.cpu | string | `"20m"` |  |
 | resources.requests.cpu | string | `"10m"` |  |
 | resources.requests.memory | string | `"10Mi"` |  |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

deploy/helm/internal-charts/apidocs-server/values.yaml

@@ -6,7 +6,7 @@ image:
   # pullSecrets: 
 
 defaultImage:
-  repository: docker.io/data61
+  repository: "ghcr.io/magda-io"
   pullPolicy: IfNotPresent
   pullSecrets: false
 

deploy/helm/internal-charts/authorization-api/README.md

@@ -19,15 +19,15 @@ Kubernetes: `>= 1.14.0-0`
 | debug | bool | `false` | when set to true, auth API will print verbose debug info (e.g. sql statements) to log |
 | defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
 | defaultImage.pullSecrets | bool | `false` |  |
-| defaultImage.repository | string | `"docker.io/data61"` |  |
+| defaultImage.repository | string | `"ghcr.io/magda-io"` |  |
 | failedApiKeyAuthBackOffSeconds | int | `5` | How long time in number of seconds should the auth API fail all API key verification requests immediately without verifying the hash since the last failed API key verification request. |
 | image.name | string | `"magda-authorization-api"` |  |
 | opa.customPolicyConfigMaps | list | `[]` | a list of names of the configMaps that contains custom policy files. the configMap must be created using magda helm chart template: [magda.filesToJson](https://github.com/magda-io/magda/blob/21499b75c7a7ee00d68886338713217d83ccb91f/deploy/helm/magda-core/templates/_helpers.tpl#L244). More info see [here](https://github.com/magda-io/magda-configmap-dir-loader). |
 | opa.image.name | string | `"magda-opa"` |  |
 | opa.loaderImage.name | string | `"magda-configmap-dir-loader"` |  |
 | opa.loaderImage.pullPolicy | string | `"IfNotPresent"` |  |
 | opa.loaderImage.pullSecrets | bool | `false` |  |
-| opa.loaderImage.repository | string | `"docker.io/data61"` |  |
+| opa.loaderImage.repository | string | `"ghcr.io/magda-io"` |  |
 | opa.loaderImage.tag | string | `"1.0.0-alpha.0"` |  |
 | opa.resources.limits.cpu | string | `"500m"` |  |
 | opa.resources.requests.cpu | string | `"20m"` |  |
@@ -38,4 +38,4 @@ Kubernetes: `>= 1.14.0-0`
 | skipAuth | bool | `false` | when set to true, API will not query policy engine for auth decision but assume it's always permitted.  It's for debugging only. |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

deploy/helm/internal-charts/authorization-api/values.yaml

@@ -6,7 +6,7 @@ image:
   # pullSecrets: 
 
 defaultImage:
-  repository: docker.io/data61
+  repository: "ghcr.io/magda-io"
   pullPolicy: IfNotPresent
   pullSecrets: false
 
@@ -41,7 +41,7 @@ opa:
 
   loaderImage: 
     # Github repo: https://github.com/magda-io/magda-configmap-dir-loader
-    repository: docker.io/data61
+    repository: "ghcr.io/magda-io"
     name: "magda-configmap-dir-loader"
     tag: "1.0.0-alpha.0"
     pullPolicy: IfNotPresent

deploy/helm/internal-charts/authorization-db/README.md

@@ -19,7 +19,7 @@ Kubernetes: `>= 1.14.0-0`
 | autoCreateSecret | bool | `true` | whether auto-create secret for client account password. After the migrator runs properly, a non-superuser DB account (named `client`) will be created. Its password will loaded from secret `{{ .Chart.Name }}-password` (e.g. `authorization-db-passwor`) & key: `password`. Set this option to `true` will make helm auto-create the secret with random password, ONLY when the secret not exists. |
 | defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
 | defaultImage.pullSecrets | bool | `false` |  |
-| defaultImage.repository | string | `"docker.io/data61"` |  |
+| defaultImage.repository | string | `"ghcr.io/magda-io"` |  |
 | image | object | `{"name":"magda-migrator-authorization-db"}` | migrator docker image settings |
 | magda-postgres.postgresql.fullnameOverride | string | `"authorization-db-postgresql"` |  |
 | magda-postgres.postgresql.nameOverride | string | `"authorization-db-postgresql"` |  |
@@ -29,4 +29,4 @@ Kubernetes: `>= 1.14.0-0`
 | migratorBackoffLimit | int | `6` | No. of retries before the migrator job is considered as failed. Failed Pods associated with the Job are recreated by the Job controller with an exponential back-off delay (10s, 20s, 40s ...) capped at six minutes. |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

deploy/helm/internal-charts/authorization-db/values.yaml

@@ -7,7 +7,7 @@ image:
   # pullSecrets: 
 
 defaultImage:
-  repository: docker.io/data61
+  repository: "ghcr.io/magda-io"
   pullPolicy: IfNotPresent
   pullSecrets: false
 

deploy/helm/internal-charts/cloud-sql-proxy/README.md

@@ -63,7 +63,7 @@ kubectl -n [Magda Deploy Namespace] create secret generic cloudsql-instance-cred
 | image.pullSecrets | bool | `false` |  |
 | image.repository | string | `"gcr.io/cloudsql-docker"` |  |
 | image.tag | string | `"1.33.2-alpine"` |  |
-| ipAddressTypes | string | `nil` | A comma-delimited list of preferred IP types for connecting to an instance.  For example, setting this to PRIVATE will force the proxy to connect to instances using an instance's associated private IP. Available from docker image 1.23.0 Options: a list of strings separated by ',', e.g. 'PUBLIC,PRIVATE' Default to 'PUBLIC,PRIVATE' |
+| ipAddressTypes | string | PUBLIC,PRIVATE | A comma-delimited list of preferred IP types for connecting to an instance.  For example, setting this to PRIVATE will force the proxy to connect to instances using an instance's associated private IP. Available from docker image 1.23.0 Options: a list of strings separated by ',', e.g. 'PUBLIC,PRIVATE' Default to 'PUBLIC,PRIVATE' |
 | logDebugStdout | bool | true | This is to log non-error output to standard out instead of standard error.  For example, if you don't want connection related messages to log as errors, set this flag to true. Available from docker image 1.23.0 |
 | maxConnections | int | 0 (no limit). | If provided, the maximum number of connections to establish before refusing new connections.  Available from docker image 1.23.0 |
 | replicas | string | `nil` | no. of replicas required for the deployment. If not set, k8s will assume `1` but allows HPA (autoscaler) alters it. @default 1 |

deploy/helm/internal-charts/combined-db/README.md

@@ -26,4 +26,4 @@ Kubernetes: `>= 1.14.0-0`
 | magda-postgres.postgresql.resources.requests.memory | string | `"500Mi"` |  |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

deploy/helm/internal-charts/content-api/README.md

@@ -19,7 +19,7 @@ Kubernetes: `>= 1.14.0-0`
 | debug | bool | `false` | When set to true, print verbose debug info (e.g. SQL statements) to log. |
 | defaultImage.pullPolicy | string | `"IfNotPresent"` |  |
 | defaultImage.pullSecrets | bool | `false` |  |
-| defaultImage.repository | string | `"docker.io/data61"` |  |
+| defaultImage.repository | string | `"ghcr.io/magda-io"` |  |
 | enableScssCompilerJob | bool | `true` | whether or not to enable the SCSS Compiler Job that's triggered on every deployment. This job will make sure latest CSS is compiled from latest SCSS and available from the content API. |
 | image.name | string | `"magda-content-api"` |  |
 | initScssCompilerBackoffLimit | int | `6` | No. of retries before the job is considered as failed. Failed Pods associated with the Job are recreated by the Job controller with an exponential back-off delay (10s, 20s, 40s ...) capped at six minutes. |
@@ -31,4 +31,4 @@ Kubernetes: `>= 1.14.0-0`
 | skipAuth | bool | `false` | when set to true, API will not query policy engine for auth decision but assume it's always permitted.  It's for debugging only. |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0)
+Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)

deploy/helm/internal-charts/content-api/values.yaml

@@ -6,7 +6,7 @@ image:
   # pullSecrets: 
 
 defaultImage:
-  repository: docker.io/data61
+  repository: "ghcr.io/magda-io"
   pullPolicy: IfNotPresent
   pullSecrets: false