ci: Use PostgreSQL 16 and fix database issues in local/sandboxes (#513)

* ci: Copy the new databse dump

* build: Use postgresql dump files

* build: Use PostgreSQL 16

* ci: force remove image build container

---------

Co-authored-by: Nicolas Giard <github@ngpixel.com>

.github/workflows/build.yml

@@ -168,7 +168,7 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.DO_SPACES_DEV_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACES_DEV_ACCESS_KEY }}
       run: |
-        aws s3 cp --quiet --endpoint-url=${{ secrets.DO_SPACES_DEV_ENDPOINT }} s3://wagtail/ietfwww.dump ietfwww.dump
+        aws s3 cp --quiet --endpoint-url=${{ secrets.DO_SPACES_DEV_ENDPOINT }} s3://wagtail/ietfwww.dump ./docker/database/ietfwww.dump
         aws s3 cp --quiet --endpoint-url=${{ secrets.DO_SPACES_DEV_ENDPOINT }} s3://wagtail/media.tgz media.tgz
         
     - name: Deploy to containers

README.md

@@ -36,7 +36,7 @@ So the only requirement to run it locally is a recent version of Docker with doc
 
 ##### How to run (with a database dump)
 
-1. Obtain a recent database dump with name like `ietfa.*.gz` and place in `docker/database/` directory. Otherwise, it will start with a fresh database.
+1. Obtain a recent database dump with name like `ietf*.dump` and place in `docker/database/` directory. Otherwise, it will start with a fresh database.
 2. Obtain and unarchive media files into `media/` folder.
 3. Run `docker compose up`. It will build and start the frontend builder (`yarn run start`) and the backend (`python manage.py runserver` analog), along with a Postgresql database. The first run will take a while because the database dump needs to be restored.
 4. After the frontend compilation finishes, the website should become available at http://localhost:8001

dev/deploy-to-container/cli.js

@@ -47,11 +47,13 @@ async function main () {
     src: [
       'docker/db.Dockerfile',
       'docker/scripts/db-import.sh',
-      'ietfwww.dump'
+      'docker/database/ietfwww.dump'
     ]
   }, {
     dockerfile: 'docker/db.Dockerfile',
-    t: 'ws-db:latest'
+    t: 'ws-db:latest',
+    rm: true,
+    forcerm: true
   })
   await new Promise((resolve, reject) => {
     dock.modem.followProgress(dbImageBuildStream, (err, res) => err ? reject(err) : resolve(res))

docker-compose.yml

@@ -32,7 +32,7 @@ services:
 
     database:
         restart: on-failure
-        image: postgres:14.6
+        image: postgres:16
         volumes:
             - "./docker/database:/docker-entrypoint-initdb.d/"
         environment:

docker/database/02_convert_native_dump_to_sql_and_restore.sh

@@ -5,29 +5,20 @@
 cd /docker-entrypoint-initdb.d
 
 set +e
-FILE=$(ls -1 ietfa.*.gz | head)
+FILE=$(ls -1 *.dump | head)
 
 set -e
 
 restore_dump() {
-	# use docker_process_sql function from docker-entrypoint.sh in the Postgres container
-	# pg_restore -xO means restore no owner, no permissions
-	pg_restore -xO -f - | sed -e '/CREATE SCHEMA public/d' | docker_process_sql
+    # use docker_process_sql function from docker-entrypoint.sh in the Postgres container
+    # pg_restore -xO means restore no owner, no permissions
+    cat "$1" | pg_restore -xO  -f - | sed -e '/CREATE SCHEMA public/d' | docker_process_sql
 }
 
 if [ -s "${FILE}" ]; then
-	echo "Found ${FILE}, converting..."
-	case $FILE in
-		*.gz)
-		echo "Restoring the gzipped archive..."
-		gzip -d -c $FILE | restore_dump
-		;;
-		*)
-		echo "Restoring the archive..."
-		cat $FILE | restore_dump
-		;;
-	esac
+    echo "Restoring the archive..."
+    restore_dump "$FILE"
 else
-	echo "No dump, starting fresh."
+    echo "No dump, starting fresh."
 fi
 

docker/db.Dockerfile

@@ -1,7 +1,7 @@
 # =====================
 # --- Builder Stage ---
 # =====================
-FROM postgres:14.6 AS builder
+FROM postgres:16 AS builder
 
 ENV POSTGRES_PASSWORD=password
 ENV POSTGRES_USER=postgres
@@ -10,15 +10,15 @@ ENV POSTGRES_HOST_AUTH_METHOD=trust
 ENV PGDATA=/data
 
 COPY docker/scripts/db-import.sh /docker-entrypoint-initdb.d/
-COPY ietfa.torchbox.latest.gz /
+COPY docker/database/ietfwww.dump /
 
 RUN ["sed", "-i", "s/exec \"$@\"/echo \"skipping...\"/", "/usr/local/bin/docker-entrypoint.sh"]
 RUN ["/usr/local/bin/docker-entrypoint.sh", "postgres"]
 
 # ===================
 # --- Final Image ---
 # ===================
-FROM postgres:14.6
+FROM postgres:16
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 COPY --from=builder /data $PGDATA