explicitly drop connections from async threads

closes FOO-1414
flag=none

TEST PLAN:
  1) run a huge fleet of servers
  2) none of them should accidentally share connections between the
     passenger processes and the live events emitter

Change-Id: Ib121e4c5df2dc4c25867c9636cd035121304ee2c
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/256072
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Cody Cutrer <cody@instructure.com>
QA-Review: Cody Cutrer <cody@instructure.com>
Product-Review: Cody Cutrer <cody@instructure.com>

config/initializers/canvas_panda_pub.rb

@@ -21,4 +21,7 @@ Rails.configuration.to_prepare do
   CanvasPandaPub.plugin_settings = -> { Canvas::Plugin.find(:pandapub) }
   CanvasPandaPub.max_queue_size = -> { Setting.get('pandapub_max_queue_size', 1000).to_i }
   CanvasPandaPub.process_interval = -> { Setting.get('pandapub_process_interval_seconds', 1.0).to_f }
+  # sometimes this async worker thread grabs a connection on a Setting read or similar.
+  # We need it to be released or the main thread can have a real problem.
+  CanvasPandaPub.on_work_unit_end = -> { ActiveRecord::Base.clear_active_connections! }
 end

config/initializers/live_events.rb

@@ -52,4 +52,7 @@ Rails.configuration.to_prepare do
     end
   }
   LiveEvents.stream_client = StubbedClient if ENV['STUB_LIVE_EVENTS_KINESIS']
+  # sometimes this async worker thread grabs a connection on a Setting read or similar.
+  # We need it to be released or the main thread can have a real problem.
+  LiveEvents.on_work_unit_end = -> { ActiveRecord::Base.clear_active_connections! }
 end

gems/canvas_panda_pub/lib/canvas_panda_pub.rb

@@ -22,7 +22,7 @@ require 'canvas_http'
 
 module CanvasPandaPub
   class << self
-    attr_accessor :logger, :cache
+    attr_accessor :logger, :cache, :on_work_unit_end
 
     def plugin_settings=(settings)
       @plugin_settings = settings

gems/canvas_panda_pub/lib/canvas_panda_pub/async_worker.rb

@@ -56,7 +56,7 @@ module CanvasPandaPub
     end
 
     def start!
-        @thread = Thread.new { self.run_thread }
+      @thread = Thread.new { self.run_thread }
     end
 
     def run_thread
@@ -111,8 +111,8 @@ module CanvasPandaPub
           rescue Exception => e
             @logger.error("Exception making PandaPub call to channel #{tag}: #{e}")
           end
+          CanvasPandaPub.on_work_unit_end&.call
         end
-
         break if stop
       end
     end

gems/live_events/lib/live_events.rb

@@ -22,7 +22,7 @@ require 'inst_statsd'
 
 module LiveEvents
   class << self
-    attr_accessor :logger, :cache, :statsd
+    attr_accessor :logger, :cache, :statsd, :on_work_unit_end
     attr_reader :stream_client
 
     # rubocop:disable Style/TrivialAccessors

gems/live_events/lib/live_events/async_worker.rb

@@ -108,6 +108,7 @@ module LiveEvents
         rescue Exception => e
           logger.error("Exception making LiveEvents async call: #{e}")
         end
+        LiveEvents.on_work_unit_end&.call
       end
     end