Switch to Solid Queue

README.md

@@ -16,11 +16,10 @@ Container_Boundary(repository, "DDBJ Repository") {
     Container_Boundary(api, "API") {
         Container(proxy, "Reverse Proxy", "Varnish")
         Container(app, "Application Server", "Puma (Rails)")
-        ContainerQueue(worker, "Background Job Worker", "Sidekiq")
+        ContainerQueue(worker, "Background Job Worker", "Solid Queue")
         Container_Ext(mb_tools, "ddbj/metabobank_tools")
         ContainerDb(object_storage, "Object Storage", "MinIO")
         ContainerDb(db, "Database", "PostgreSQL")
-        ContainerDb(job_store, "Background Job Store", "Redis")
         Container_Ext(excel2xml, "ddbj/submission-excel2xml")
         Container(noodles-gff, "noodles_gff-rb")
     }
@@ -48,7 +47,6 @@ Rel(app, job_store, "Enqueues jobs in")
 
 Rel(worker, object_storage, "Writes files to")
 Rel(worker, db, "Reads from and writes to")
-Rel(worker, job_store, "Polls for and updates jobs in")
 Rel(worker, ddbj_validator, "Validates BioProject/BioSample files using")
 Rel(worker, mb_tools, "Validates MetaboBank files using")
 Rel(worker, excel2xml, "Validates DRA files using")

ansible/setup.yml

@@ -24,6 +24,5 @@
         paths:
           - ~/repository/STAGE
           - ~/ddbj-repository/data/STAGE/postgres
-          - ~/ddbj-repository/data/STAGE/redis
           - ~/ddbj-repository/data/STAGE/minio
           - ~/ddbj-repository/data/STAGE/virtuoso

api/Gemfile

@@ -20,10 +20,8 @@ gem 'puma'
 gem 'rack-cors'
 gem 'rambulance'
 gem 'sentry-rails'
-gem 'sentry-sidekiq'
 gem 'sequel'
-gem 'sidekiq'
-gem 'sidekiq-scheduler'
+gem 'solid_queue'
 gem 'submission-excel2xml', github: 'ddbj/submission-excel2xml'
 
 group :development do

api/Gemfile.lock

@@ -302,8 +302,6 @@ GEM
       railties
     rdoc (6.7.0)
       psych (>= 4.0.0)
-    redis-client (0.22.2)
-      connection_pool
     regexp_parser (2.9.2)
     reline (0.5.9)
       io-console (~> 0.5)
@@ -332,32 +330,23 @@ GEM
       rspec-support (~> 3.13)
     rspec-support (3.13.1)
     rubyzip (2.3.2)
-    rufus-scheduler (3.9.1)
-      fugit (~> 1.1, >= 1.1.6)
     sentry-rails (5.18.1)
       railties (>= 5.0)
       sentry-ruby (~> 5.18.1)
     sentry-ruby (5.18.1)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
-    sentry-sidekiq (5.18.1)
-      sentry-ruby (~> 5.18.1)
-      sidekiq (>= 3.0)
     sequel (5.82.0)
       bigdecimal
-    sidekiq (7.3.0)
-      concurrent-ruby (< 2)
-      connection_pool (>= 2.3.0)
-      logger
-      rack (>= 2.2.4)
-      redis-client (>= 0.22.2)
-    sidekiq-scheduler (5.0.5)
-      rufus-scheduler (~> 3.2)
-      sidekiq (>= 6, < 8)
-      tilt (>= 1.4.0, < 3)
     skooma (0.3.2)
       json_skooma (~> 0.2.0)
       zeitwerk (~> 2.6)
+    solid_queue (0.3.3)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
     stringio (3.1.1)
     strscan (3.1.0)
     swd (2.0.3)
@@ -367,7 +356,6 @@ GEM
       faraday-follow_redirects
     test-prof (1.3.3.1)
     thor (1.3.1)
-    tilt (2.4.0)
     timeout (0.4.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
@@ -420,11 +408,9 @@ DEPENDENCIES
   rspec-default_http_header
   rspec-rails
   sentry-rails
-  sentry-sidekiq
   sequel
-  sidekiq
-  sidekiq-scheduler
   skooma
+  solid_queue
   submission-excel2xml!
   test-prof
   webmock

api/config/environments/production.rb

@@ -56,7 +56,7 @@
   # config.cache_store = :mem_cache_store
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
-  # config.active_job.queue_adapter     = :resque
+  # config.active_job.queue_adapter = :solid_queue
   # config.active_job.queue_name_prefix = "ddbj_repository_production"
 
   config.action_mailer.perform_caching = false
@@ -84,7 +84,7 @@
   # config.host_authorization = { exclude: ->(request) { request.path == "/up" } }
 
   config.action_mailer.delivery_method = :smtp
-  config.active_job.queue_adapter      = :sidekiq
+  config.active_job.queue_adapter      = :solid_queue
   config.active_storage.service        = :minio
   config.assume_ssl                    = ENV['DISABLE_SSL'] != 'true'
   config.force_ssl                     = ENV['DISABLE_SSL'] != 'true'

api/config/solid_queue.yml

@@ -0,0 +1,18 @@
+# default: &default
+#   dispatchers:
+#     - polling_interval: 1
+#       batch_size: 500
+#   workers:
+#     - queues: "*"
+#       threads: 3
+#       processes: 1
+#       polling_interval: 0.1
+#
+# development:
+#  <<: *default
+#
+# test:
+#  <<: *default
+#
+# production:
+#  <<: *default

api/db/migrate/20240718063750_create_solid_queue_tables.solid_queue.rb

@@ -0,0 +1,101 @@
+# This migration comes from solid_queue (originally 20231211200639)
+class CreateSolidQueueTables < ActiveRecord::Migration[7.0]
+  def change
+    create_table :solid_queue_jobs do |t|
+      t.string :queue_name, null: false
+      t.string :class_name, null: false, index: true
+      t.text :arguments
+      t.integer :priority, default: 0, null: false
+      t.string :active_job_id, index: true
+      t.datetime :scheduled_at
+      t.datetime :finished_at, index: true
+      t.string :concurrency_key
+
+      t.timestamps
+
+      t.index [ :queue_name, :finished_at ], name: "index_solid_queue_jobs_for_filtering"
+      t.index [ :scheduled_at, :finished_at ], name: "index_solid_queue_jobs_for_alerting"
+    end
+
+    create_table :solid_queue_scheduled_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.datetime :scheduled_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :scheduled_at, :priority, :job_id ], name: "index_solid_queue_dispatch_all"
+    end
+
+    create_table :solid_queue_ready_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :priority, :job_id ], name: "index_solid_queue_poll_all"
+      t.index [ :queue_name, :priority, :job_id ], name: "index_solid_queue_poll_by_queue"
+    end
+
+    create_table :solid_queue_claimed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.bigint :process_id
+      t.datetime :created_at, null: false
+
+      t.index [ :process_id, :job_id ]
+    end
+
+    create_table :solid_queue_blocked_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.string :concurrency_key, null: false
+      t.datetime :expires_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :expires_at, :concurrency_key ], name: "index_solid_queue_blocked_executions_for_maintenance"
+    end
+
+    create_table :solid_queue_failed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.text :error
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_pauses do |t|
+      t.string :queue_name, null: false, index: { unique: true }
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_processes do |t|
+      t.string :kind, null: false
+      t.datetime :last_heartbeat_at, null: false, index: true
+      t.bigint :supervisor_id, index: true
+
+      t.integer :pid, null: false
+      t.string :hostname
+      t.text :metadata
+
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_semaphores do |t|
+      t.string :key, null: false, index: { unique: true }
+      t.integer :value, default: 1, null: false
+      t.datetime :expires_at, null: false, index: true
+
+      t.timestamps
+
+      t.index [ :key, :value ], name: "index_solid_queue_semaphores_on_key_and_value"
+    end
+
+    add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end

api/db/migrate/20240718063751_add_missing_index_to_blocked_executions.solid_queue.rb

@@ -0,0 +1,6 @@
+# This migration comes from solid_queue (originally 20240110143450)
+class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1]
+  def change
+    add_index :solid_queue_blocked_executions, [ :concurrency_key, :priority, :job_id ], name: "index_solid_queue_blocked_executions_for_release"
+  end
+end

api/db/migrate/20240718063752_create_recurring_executions.solid_queue.rb

@@ -0,0 +1,15 @@
+# This migration comes from solid_queue (originally 20240218110712)
+class CreateRecurringExecutions < ActiveRecord::Migration[7.1]
+  def change
+    create_table :solid_queue_recurring_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :task_key, null: false
+      t.datetime :run_at, null: false
+      t.datetime :created_at, null: false
+
+      t.index [ :task_key, :run_at ], unique: true
+    end
+
+    add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end