Cache submissions while migrations are running

.github/workflows/submit.sh

@@ -19,6 +19,14 @@ echo "database=$database"
 echo "ctest_driver=$ctest_driver"
 echo "submit_type=$submit_type"
 
+# Wait for migrations to finish running by checking for maintenance mode to be lifted
+docker exec cdash bash -c " \
+until [ ! -f /cdash/storage/framework/down ] \
+do \
+  sleep 1 \
+done \
+"
+
 # Suppress any uncommitted changes left after the image build
 docker exec cdash bash -c "cd /cdash && /usr/bin/git checkout ."
 

app/Http/Controllers/SubmissionController.php

@@ -100,7 +100,10 @@ private function submitProcess(): Response
         // Check if we can connect to the database before proceeding any further.
         try {
             DB::connection()->getPdo();
-        } catch (\Exception $e) {
+            if (app()->isDownForMaintenance()) {
+                throw new Exception();
+            }
+        } catch (\Exception) {
             // Write a marker file so we know to process these files when the DB comes back up.
             if (!Storage::exists("DB_WAS_DOWN")) {
                 Storage::put("DB_WAS_DOWN", "");

app/Http/Middleware/CheckForMaintenanceMode.php

@@ -12,6 +12,7 @@ class CheckForMaintenanceMode extends Middleware
      * @var array
      */
     protected $except = [
-        //
+        '/submit.php',
+        '/ping',
     ];
 }

app/Utils/UnparsedSubmissionProcessor.php

@@ -83,7 +83,7 @@ public function postSubmit(): JsonResponse
         // Thus function will throw an exception if invalid data provided
         $this->parseBuildMetadata();
 
-        if ($this->checkDatabaseConnection()) {
+        if ($this->checkDatabaseConnection() && !app()->isDownForMaintenance()) {
             return $this->initializeBuild();
         }
 
@@ -215,7 +215,7 @@ public function putSubmitFile(): JsonResponse
         $this->parseDataFileParameters();
         $ext = pathinfo($this->backupfilename, PATHINFO_EXTENSION);
 
-        $db_up = $this->checkDatabaseConnection();
+        $db_up = $this->checkDatabaseConnection() && !app()->isDownForMaintenance();
         if ($db_up) {
             if (!is_numeric($this->buildid) || $this->buildid < 1) {
                 abort(Response::HTTP_NOT_FOUND, 'Build not found');

docker/docker-entrypoint.sh

@@ -2,26 +2,34 @@
 
 set -e
 
+# Set job control so we can bg/fg processes
+set -m
+
 php artisan key:check || exit 1
 
 # If the "start-website" argument was provided, start the web server
 if [ "$1" = "start-website" ] ; then
-  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
-    bash /cdash/install.sh --dev --initial-docker-install
-  else
-    bash /cdash/install.sh --initial-docker-install
-  fi
-
-  echo "Starting Apache..."
+   echo "Starting Apache..."
 
   # Start Apache under the current user, in case the current user isn't www-data.  Kubernetes-based systems
-  # typically run under a random user.
+  # typically run under a random user.  We start Apache before running the install scripts so the system can
+  # begin collecting submissions while database migrations run.  Apache starts in the background so the
+  # container gets killed if the migrations fail.
   if [ "$BASE_IMAGE" = "debian" ] ; then
     APACHE_RUN_USER=$(id -u -n) /usr/sbin/apache2ctl -D FOREGROUND
   elif [ "$BASE_IMAGE" = "ubi" ]; then
     /usr/libexec/s2i/run
+  fi & # & puts Apache in the background
+
+  if [ "$DEVELOPMENT_BUILD" = "1" ]; then
+    bash /cdash/install.sh --dev --initial-docker-install
+  else
+    bash /cdash/install.sh --initial-docker-install
   fi
 
+  # Bring Apache to the foreground so the container fails if Apache fails after this point.
+  fg
+
 # If the start-worker argument was provided, start a worker process instead
 elif [ "$1" = "start-worker" ] ; then
   php artisan queue:work