Prevent supervisor from pruning itself

For example, when coming back from being suspended and having its heartbeat expired.
evdevdev · Sep 11, 2024 · 338c1e8 · 338c1e8
1 parent d768187
commit 338c1e8
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 3 deletions.
diff --git a/app/models/solid_queue/process/prunable.rb b/app/models/solid_queue/process/prunable.rb
@@ -10,9 +10,9 @@ module Prunable
       end
 
       class_methods do
-        def prune
+        def prune(excluding: nil)
           SolidQueue.instrument :prune_processes, size: 0 do |payload|
-            prunable.non_blocking_lock.find_in_batches(batch_size: 50) do |batch|
+            prunable.excluding(excluding).non_blocking_lock.find_in_batches(batch_size: 50) do |batch|
               payload[:size] += batch.size
 
               batch.each(&:prune)

diff --git a/lib/solid_queue/supervisor/maintenance.rb b/lib/solid_queue/supervisor/maintenance.rb
@@ -24,7 +24,7 @@ def stop_maintenance_task
       end
 
       def prune_dead_processes
-        wrap_in_app_executor { SolidQueue::Process.prune }
+        wrap_in_app_executor { SolidQueue::Process.prune(excluding: process) }
       end
 
       def fail_orphaned_executions

diff --git a/test/unit/supervisor_test.rb b/test/unit/supervisor_test.rb
@@ -121,6 +121,28 @@ class SupervisorTest < ActiveSupport::TestCase
     end
   end
 
+  test "prune processes with expired heartbeats" do
+    pruned = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-42")
+
+    # Simulate expired heartbeats
+    SolidQueue::Process.update_all(last_heartbeat_at: 10.minutes.ago)
+
+    not_pruned = SolidQueue::Process.register(kind: "Worker", pid: 44, name: "worker-44")
+
+    assert_equal 2, SolidQueue::Process.count
+
+    pid = run_supervisor_as_fork(load_configuration_from: { workers: [ { queues: :background } ] })
+    wait_for_registered_processes(4)
+
+    terminate_process(pid)
+
+    skip_active_record_query_cache do
+      assert_equal 1, SolidQueue::Process.count
+      assert_nil SolidQueue::Process.find_by(id: pruned.id)
+      assert SolidQueue::Process.find_by(id: not_pruned.id).present?
+    end
+  end
+
   private
     def assert_registered_workers(supervisor_pid: nil, count: 1)
       assert_registered_processes(kind: "Worker", count: count, supervisor_pid: supervisor_pid)