Add Solid Queue alongside Solid Cache (#52804)

* Add Solid Queue alongside Solid Cache

Gemfile

@@ -22,6 +22,7 @@ gem "importmap-rails", ">= 1.2.3"
 gem "tailwindcss-rails"
 gem "dartsass-rails"
 gem "solid_cache"
+gem "solid_queue"
 gem "kamal", require: false
 gem "thruster", require: false
 # require: false so bcrypt is loaded only when has_secure_password is used.

Gemfile.lock

@@ -212,7 +212,7 @@ GEM
     drb (2.2.1)
     ed25519 (1.3.0)
     erubi (1.13.0)
-    et-orbi (1.2.7)
+    et-orbi (1.2.11)
       tzinfo
     event_emitter (0.2.6)
     execjs (2.9.1)
@@ -244,8 +244,8 @@ GEM
     ffi (1.17.0)
     ffi (1.17.0-x86_64-darwin)
     ffi (1.17.0-x86_64-linux-gnu)
-    fugit (1.9.0)
-      et-orbi (~> 1, >= 1.2.7)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
@@ -549,6 +549,13 @@ GEM
       activejob (>= 7.2)
       activerecord (>= 7.2)
       railties (>= 7.2)
+    solid_queue (0.8.1)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
+      thor (~> 1.3.1)
     sorted_set (1.0.3)
       rbtree
       set (~> 1.0)
@@ -585,7 +592,7 @@ GEM
       railties (>= 6.0.0)
     terser (1.1.20)
       execjs (>= 0.3.0, < 3)
-    thor (1.3.0)
+    thor (1.3.2)
     thruster (0.1.7)
     thruster (0.1.7-x86_64-darwin)
     thruster (0.1.7-x86_64-linux)
@@ -696,6 +703,7 @@ DEPENDENCIES
   sidekiq
   sneakers
   solid_cache
+  solid_queue
   sprockets-rails (>= 2.0.0)
   sqlite3 (>= 2.0)
   stackprof

railties/CHANGELOG.md

@@ -1,3 +1,7 @@
+*   Use [Solid Queue](https://github.com/rails/solid_queue) as the default Active Job backend in production, configured as a separate cache database in config/database.yml. In a single-server deployment, it'll run as a Puma plugin. This is configured in `config/deploy.yml` and can easily be changed to use a dedicated jobs machine.
+
+    *DHH*
+
 *   Use [Solid Cache](https://github.com/rails/solid_cache) as the default Rails.cache backend in production, configured as a separate cache database in config/database.yml.
 
     *DHH*

railties/lib/rails/generators/app_base.rb

@@ -756,7 +756,7 @@ module Rails
       def run_solid
         return if skip_solid? || !bundle_install?
 
-        rails_command "solid_cache:install"
+        rails_command "solid_cache:install solid_queue:install"
       end
 
       def add_bundler_platforms

railties/lib/rails/generators/rails/app/templates/Gemfile.tt

@@ -18,6 +18,9 @@ gem "tzinfo-data", platforms: %i[ <%= bundler_windows_platforms %> jruby ]
 
 # Use the database-backed Solid Cache adapter for Rails.cache [https://github.com/rails/solid_cache]
 gem "solid_cache"
+
+# Use the database-backed Solid Queue adapter for Active Job [https://github.com/rails/solid_queue]
+gem "solid_queue"
 <% end -%>
 <% if depend_on_bootsnap? -%>
 

railties/lib/rails/generators/rails/app/templates/config/databases/mysql.yml.tt

@@ -69,4 +69,8 @@ production:
     <<: *primary_production
     database: <%= app_name %>_production_cache
     migrations_paths: db/cache_migrate
+  queue:
+    <<: *primary_production
+    database: <%= app_name %>_production_queue
+    migrations_paths: db/queue_migrate
 <%- end -%>

railties/lib/rails/generators/rails/app/templates/config/databases/postgresql.yml.tt

@@ -101,4 +101,8 @@ production:
     <<: *primary_production
     database: <%= app_name %>_production_cache
     migrations_paths: db/cache_migrate
+  queue:
+    <<: *primary_production
+    database: <%= app_name %>_production_queue
+    migrations_paths: db/queue_migrate
 <%- end -%>

railties/lib/rails/generators/rails/app/templates/config/databases/sqlite3.yml.tt

@@ -58,5 +58,9 @@ production:
     <<: *default
     database: storage/production_cache.sqlite3
     migrations_paths: db/cache_migrate
+  queue:
+    <<: *default
+    database: storage/production_queue.sqlite3
+    migrations_paths: db/queue_migrate
 <%- end -%>
 <%- end -%>

railties/lib/rails/generators/rails/app/templates/config/databases/trilogy.yml.tt

@@ -69,4 +69,8 @@ production:
     <<: *primary_production
     database: <%= app_name %>_production_cache
     migrations_paths: db/cache_migrate
+  queue:
+    <<: *primary_production
+    database: <%= app_name %>_production_queue
+    migrations_paths: db/queue_migrate
 <%- end -%>

railties/lib/rails/generators/rails/app/templates/config/deploy.yml.tt

@@ -11,7 +11,7 @@ servers:
   # job:
   #   hosts:
   #     - 192.168.0.1
-  #   cmd: bin/solid_queue work
+  #   cmd: bin/jobs
 
 # Credentials for your image host.
 registry:
@@ -28,12 +28,28 @@ registry:
 env:
   secret:
     - RAILS_MASTER_KEY
+<% if skip_solid? -%>
   # clear:
   #   # Set number of cores available to the application on each server (default: 1).
   #   WEB_CONCURRENCY: 2
 
-  #   # Match this to the database server to configure Active Record correctly
+  #   # Match this to any external database server to configure Active Record correctly
   #   DB_HOST: 192.168.0.2
+<% else -%>
+  clear:
+    # Run the Solid Queue Supervisor inside the web server's Puma process to do jobs.
+    # When you start using multiple servers, you should split out job processing to a dedicated machine.
+    SOLID_QUEUE_IN_PUMA: true
+
+    # Set number of processes dedicated to Solid Queue (default: 1)
+    # JOB_CONCURRENCY: 3
+
+    # Set number of cores available to the application on each server (default: 1).
+    # WEB_CONCURRENCY: 2
+
+    # Match this to any external database server to configure Active Record correctly
+    # DB_HOST: 192.168.0.2
+<% end -%>
 
 <% unless skip_storage? %>
 # Use a persistent storage volume for sqlite database files and local Active Storage files.

railties/lib/rails/generators/rails/app/templates/config/puma.rb.tt

@@ -33,6 +33,11 @@ port ENV.fetch("PORT", 3000)
 # Allow puma to be restarted by `bin/rails restart` command.
 plugin :tmp_restart
 
+<% unless skip_solid? -%>
+# Run the Solid Queue supervisor inside of Puma for single-server deployments
+plugin :solid_queue if ENV["SOLID_QUEUE_IN_PUMA"]
+
+<% end -%>
 # Specify the PID file. Defaults to tmp/pids/server.pid in development.
 # In other environments, only set the PID file if requested.
 pidfile ENV["PIDFILE"] if ENV["PIDFILE"]

railties/test/generators/app_generator_test.rb

@@ -641,9 +641,13 @@ class AppGeneratorTest < Rails::Generators::TestCase
   def test_configuration_of_solid
     generator [destination_root]
     run_generator_instance
+
     assert_gem "solid_cache"
+    assert_gem "solid_queue"
+
     assert_file "config/database.yml" do |content|
       assert_match(%r{cache:}, content)
+      assert_match(%r{queue:}, content)
     end
   end
 
@@ -808,8 +812,9 @@ class AppGeneratorTest < Rails::Generators::TestCase
     generator([destination_root], skip_solid: true)
     run_generator_instance
 
-    assert_not_includes @rails_commands, "solid_cache:install", "`solid_cache:install` expected to not be called."
+    assert_not_includes @rails_commands, "solid_cache:install solid_queue:install", "`solid_cache:install solid_queue:install` expected to not be called."
     assert_no_gem "solid_cache"
+    assert_no_gem "solid_queue"
   end
 
   def test_skip_javascript_option
@@ -986,7 +991,7 @@ class AppGeneratorTest < Rails::Generators::TestCase
     run_generator_instance
 
     expected_commands = [
-      "credentials:diff --enroll", "importmap:install", "turbo:install stimulus:install", "solid_cache:install"
+      "credentials:diff --enroll", "importmap:install", "turbo:install stimulus:install", "solid_cache:install solid_queue:install"
     ]
     assert_equal expected_commands, @rails_commands
   end