Release v1.4.2

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [1.4.2](https://github.com/opencrvs/opencrvs-countryconfig/compare/v1.4.0...v1.4.2)
+
+- Bugfix: fix critical bug stopping backups from being sent to target backup server
+- Bugfix: add a mechanism for controlling the amount of backups stored on the backup server.
+
 ## [1.3.4](https://github.com/opencrvs/opencrvs-countryconfig/compare/v1.3.3...v1.3.4)
 
 ## Breaking changes

infrastructure/backups/backup.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
@@ -38,10 +40,6 @@ for i in "$@"; do
     SSH_PORT="${i#*=}"
     shift
     ;;
-  --production_ip=*)
-    PRODUCTION_IP="${i#*=}"
-    shift
-    ;;
   --remote_dir=*)
     REMOTE_DIR="${i#*=}"
     shift
@@ -63,7 +61,7 @@ for i in "$@"; do
 done
 
 print_usage_and_exit() {
-  echo 'Usage: ./backup.sh --passphrase=XXX --ssh_user=XXX --ssh_host=XXX --ssh_port=XXX --production_ip=XXX --remote_dir=XXX --replicas=XXX --label=XXX'
+  echo 'Usage: ./backup.sh --passphrase=XXX --ssh_user=XXX --ssh_host=XXX --ssh_port=XXX --remote_dir=XXX --replicas=XXX --label=XXX'
   echo "Script must receive SSH details and a target directory of a remote server to copy backup files to."
   echo "Optionally a LABEL i.e. 'v1.0.1' can be provided to be appended to the backup file labels"
   echo "7 days of backup data will be retained in the manager node"
@@ -96,10 +94,6 @@ if [ "$IS_LOCAL" = false ]; then
     echo "Error: Argument for the --ssh_port is required."
     print_usage_and_exit
   fi
-  if [ -z "$PRODUCTION_IP" ]; then
-    echo "Error: Argument for the --production_ip is required."
-    print_usage_and_exit
-  fi
   if [ -z "$REMOTE_DIR" ]; then
     echo "Error: Argument for the --remote_dir is required."
     print_usage_and_exit
@@ -299,47 +293,41 @@ fi
 
 # Copy the backups to an offsite server in production
 #----------------------------------------------------
-if [[ "$OWN_IP" = "$PRODUCTION_IP" || "$OWN_IP" = "$(dig $PRODUCTION_IP +short)" ]]; then
 
-  # Create a temporary directory to store the backup files before packaging
-  BACKUP_RAW_FILES_DIR=/tmp/backup-${LABEL:-$BACKUP_DATE}/
-  mkdir -p $BACKUP_RAW_FILES_DIR
+# Create a temporary directory to store the backup files before packaging
+BACKUP_RAW_FILES_DIR=/tmp/backup-${LABEL:-$BACKUP_DATE}/
+mkdir -p $BACKUP_RAW_FILES_DIR
 
-  # Copy full directories to the temporary directory
-  cp -r $ROOT_PATH/backups/elasticsearch/ $BACKUP_RAW_FILES_DIR/elasticsearch/
-  cp -r $ROOT_PATH/backups/influxdb/${LABEL:-$BACKUP_DATE} $BACKUP_RAW_FILES_DIR/influxdb/
+# Copy full directories to the temporary directory
+cp -r $ROOT_PATH/backups/elasticsearch/ $BACKUP_RAW_FILES_DIR/elasticsearch/
+cp -r $ROOT_PATH/backups/influxdb/${LABEL:-$BACKUP_DATE} $BACKUP_RAW_FILES_DIR/influxdb/
 
 
-  mkdir -p $BACKUP_RAW_FILES_DIR/minio/ && cp $ROOT_PATH/backups/minio/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/minio/
-  mkdir -p $BACKUP_RAW_FILES_DIR/metabase/ && cp $ROOT_PATH/backups/metabase/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/metabase/
-  mkdir -p $BACKUP_RAW_FILES_DIR/vsexport/ && cp $ROOT_PATH/backups/vsexport/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/vsexport/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/hearth-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/user-mgnt-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/openhim-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/application-config-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/metrics-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/webhooks-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
-  mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/performance-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/minio/ && cp $ROOT_PATH/backups/minio/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/minio/
+mkdir -p $BACKUP_RAW_FILES_DIR/metabase/ && cp $ROOT_PATH/backups/metabase/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/metabase/
+mkdir -p $BACKUP_RAW_FILES_DIR/vsexport/ && cp $ROOT_PATH/backups/vsexport/ocrvs-${LABEL:-$BACKUP_DATE}.tar.gz $BACKUP_RAW_FILES_DIR/vsexport/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/hearth-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/user-mgnt-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/openhim-dev-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/application-config-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/metrics-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/webhooks-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
+mkdir -p $BACKUP_RAW_FILES_DIR/mongo/ && cp $ROOT_PATH/backups/mongo/performance-${LABEL:-$BACKUP_DATE}.gz $BACKUP_RAW_FILES_DIR/mongo/
 
-  tar -czf /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -C "$BACKUP_RAW_FILES_DIR" .
+tar -czf /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -C "$BACKUP_RAW_FILES_DIR" .
 
-  openssl enc -aes-256-cbc -salt -pbkdf2 -in /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -out /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc -pass pass:$PASSPHRASE
+openssl enc -aes-256-cbc -salt -pbkdf2 -in /tmp/${LABEL:-$BACKUP_DATE}.tar.gz -out /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc -pass pass:$PASSPHRASE
 
+if [ "$IS_LOCAL" = false ]; then
+  set +e
   rsync -a -r --rsync-path="mkdir -p $REMOTE_DIR/ && rsync" --progress --rsh="ssh -o StrictHostKeyChecking=no -p $SSH_PORT" /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc $SSH_USER@$SSH_HOST:$REMOTE_DIR/
-
-  echo "Copied backup files to remote server."
-
-  rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc
-  rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz
-  rm -r $BACKUP_RAW_FILES_DIR
+  if [ $? -eq 0 ]; then
+    echo "Copied backup files to remote server."
+  fi
+  set -e
 fi
 
-# Cleanup any old backups from influx or mongo. Keep previous 7 days of data and all elastic data
-# Elastic snapshots require a random selection of files in the data/backups/elasticsearch/indices
-# folder
-#------------------------------------------------------------------------------------------------
-find $ROOT_PATH/backups/influxdb -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/mongo -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/minio -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/metabase -mtime +7 -exec rm {} \;
-find $ROOT_PATH/backups/vsexport -mtime +7 -exec rm {} \;
+rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz.enc
+rm /tmp/${LABEL:-$BACKUP_DATE}.tar.gz
+rm -r $BACKUP_RAW_FILES_DIR
+

infrastructure/backups/download.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
@@ -9,7 +11,7 @@
 
 #------------------------------------------------------------------------------------------------------------------
 # By default OpenCRVS saves a backup of all data on a cron job every day in case of an emergency data loss incident
-# This script clears all data and restores a specific day's data.  It is irreversable, so use with caution.
+# This script downloads all the data based on --label (defaults to current day)
 #------------------------------------------------------------------------------------------------------------------
 
 set -e
@@ -28,10 +30,6 @@ for i in "$@"; do
     SSH_PORT="${i#*=}"
     shift
     ;;
-  --replicas=*)
-    REPLICAS="${i#*=}"
-    shift
-    ;;
   --label=*)
     LABEL="${i#*=}"
     shift
@@ -100,8 +98,8 @@ openssl enc -d -aes-256-cbc -salt -pbkdf2 -in $BACKUP_RAW_FILES_DIR/${LABEL}.tar
 mkdir -p $BACKUP_RAW_FILES_DIR/extract
 tar -xvf $BACKUP_RAW_FILES_DIR/${LABEL}.tar.gz -C $BACKUP_RAW_FILES_DIR/extract
 
-# Move folders
-rm -r /data/backups/elasticsearch
+# Delete previous days restore(s) and move the newly downloaded one in place
+rm -rf /data/backups/*
 mv $BACKUP_RAW_FILES_DIR/extract/elasticsearch /data/backups/elasticsearch
 
 mv $BACKUP_RAW_FILES_DIR/extract/influxdb /data/backups/influxdb/${LABEL}

infrastructure/backups/rotate_backups.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# OpenCRVS is also distributed under the terms of the Civil Registration
+# & Healthcare Disclaimer located at http://opencrvs.org/license.
+#
+# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+
+#------------------------------------------------------------------------------------------------------------------
+# By default OpenCRVS saves a backup of all data on a cron job every day in case of an emergency data loss incident
+# This script downloads all the data based on --label (defaults to current day)
+#------------------------------------------------------------------------------------------------------------------
+
+set -e
+
+print_usage_and_exit() {
+  echo 'Usage: ./rotate_backups.sh --backup_dir=/home/backup/backups --amount_to_keep=7'
+  exit 1
+}
+
+for i in "$@"; do
+  case $i in
+  --backup_dir=*)
+    BACKUP_DIR="${i#*=}"
+    shift
+    ;;
+  --amount_to_keep=*)
+    AMOUNT_TO_KEEP="${i#*=}"
+    shift
+    ;;
+  *) ;;
+  esac
+done
+
+
+if ! [[ "$AMOUNT_TO_KEEP" =~ ^[0-9]+$ ]]; then
+  echo "Script must be passed a positive integer number of backups to keep, got $AMOUNT_TO_KEEP"
+  print_usage_and_exit
+fi
+
+BACKUP_DIR=${BACKUP_DIR:-/home/backup/backups}
+
+if [ ! -d "$BACKUP_DIR" ]; then
+    echo "Error: BACKUP_DIR ($BACKUP_DIR) doesn't exist"
+    print_usage_and_exit
+fi
+
+# Delete subdirectories but keep latest according to AMOUNT_TO_KEEP
+find "$BACKUP_DIR" -mindepth 1 -type d -print | sort -r | tail -n +$(("$AMOUNT_TO_KEEP" + 1)) | xargs rm -rf --

infrastructure/logrotate.conf

@@ -44,7 +44,7 @@ include /etc/logrotate.d
 
 # system-specific logs may be configured here
 
-/var/log/opencrvs-backup.log {
+/var/log/opencrvs-rotate-backups.log {
     missingok
     monthly
     create 0660 root application

infrastructure/server-setup/backups.yml

@@ -86,6 +86,7 @@
   become_method: sudo
   vars:
     manager_hostname: "{{ groups['docker-manager-first'][0] }}"
+    crontab_user: root
   tasks:
     - name: Ensure backup user is present
       user:
@@ -120,3 +121,23 @@
         owner: '{{ external_backup_server_user }}'
       tags:
         - backups
+
+    - name: Copy rotate_backups.sh file to external_backup_server_user's home directory
+      copy:
+        src: ../backups/rotate_backups.sh
+        dest: '{{ external_backup_server_user_home }}/rotate_backups.sh'
+        owner: '{{ external_backup_server_user }}'
+        mode: 0755
+      tags:
+        - backups
+
+    - name: 'Setup backup rotation'
+      cron:
+        user: '{{ crontab_user }}'
+        name: 'rotate backups'
+        minute: '0'
+        hour: '0'
+        job: 'bash {{ external_backup_server_user_home }}/rotate_backups.sh --backup_dir={{ external_backup_server_remote_directory }} --amount_to_keep={{ amount_of_backups_to_keep }} >> /var/log/opencrvs-rotate-backups.log 2>&1'
+        state: "{{ 'present' if (amount_of_backups_to_keep) else 'absent' }}"
+      tags:
+        - backups

infrastructure/server-setup/production.yml

@@ -52,3 +52,5 @@ backups:
   hosts:
     backup: # @todo set this to be the hostname of your backup server
       ansible_host: '66.66.66.66'
+      # @todo how many days to store backups for?
+      amount_of_backups_to_keep: 3

infrastructure/server-setup/staging.yml

@@ -48,3 +48,5 @@ backups:
   hosts:
     backup: # @todo set this to be the hostname of your backup server
       ansible_host: '66.66.66.66' # set this to be the IP address of your backup server
+      # @todo how many days to store backups for?
+      amount_of_backups_to_keep: 3

infrastructure/server-setup/tasks/backups/crontab.yml

@@ -1,10 +1,17 @@
+- name: Copy backups.sh file to external_backup_server_user's home directory
+  copy:
+    src: ../backups/backup.sh
+    dest: '{{ crontab_user_home }}/backup.sh'
+    owner: 'root'
+    mode: 0755
+
 - name: 'Setup crontab to backup the opencrvs data'
   cron:
     user: '{{ crontab_user }}'
     name: 'backup opencrvs'
     minute: '0'
     hour: '0'
-    job: 'cd / && bash /opt/opencrvs/infrastructure/backups/backup.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --production_ip={{ manager_production_server_ip }} --remote_dir={{ external_backup_server_remote_directory }} --replicas=1 >> /var/log/opencrvs-backup.log 2>&1'
+    job: 'bash {{ crontab_user_home }}/backup.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} --replicas=1 >> /var/log/opencrvs-backup.log 2>&1'
     state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and (enable_backups | default(false))) else 'absent' }}"
 
 ##

src/index.ts

@@ -351,6 +351,18 @@ export async function createServer() {
     }
   })
 
+  server.route({
+    method: 'GET',
+    path: '/handlebars.js',
+    handler: handlebarsHandler,
+    options: {
+      auth: false,
+      tags: ['api'],
+      description:
+        'Serves custom handlebar helper functions as JS to be used in certificates'
+    }
+  })
+
   server.route({
     method: 'GET',
     path: '/content/{application}',