Add stoplight for object storage failures, return HTTP 503 (mastodon#…

…13043)
hachyderm · Dec 15, 2020 · 1045549 · 1045549
1 parent 75d2762
commit 1045549
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 2 deletions.
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
@@ -40,7 +40,7 @@ class Api::BaseController < ApplicationController
     render json: { error: 'This action is not allowed' }, status: 403
   end
 
-  rescue_from Mastodon::RaceConditionError do
+  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight do
     render json: { error: 'There was a temporary problem serving your request, please try again' }, status: 503
   end
 

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
@@ -28,7 +28,7 @@ class ApplicationController < ActionController::Base
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
   rescue_from Mastodon::NotPermittedError, with: :forbidden
   rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error
-  rescue_from Mastodon::RaceConditionError, with: :service_unavailable
+  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, with: :service_unavailable
   rescue_from Mastodon::RateLimitExceededError, with: :too_many_requests
 
   before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
@@ -228,6 +228,8 @@ def process_emoji(tag)
     emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri)
     emoji.image_remote_url = image_url
     emoji.save
+  rescue Seahorse::Client::NetworkingError
+    nil
   end
 
   def process_attachments
@@ -250,6 +252,8 @@ def process_attachments
         media_attachment.save
       rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError
         RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id)
+      rescue Seahorse::Client::NetworkingError
+        nil
       end
     end
 

diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
@@ -113,3 +113,14 @@
 end
 
 Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES }
+
+# In some places in the code, we rescue this exception, but we don't always
+# load the S3 library, so it may be an undefined constant:
+
+unless defined?(Seahorse)
+  module Seahorse
+    module Client
+      class NetworkingError < StandardError; end
+    end
+  end
+end
diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb
@@ -39,6 +39,23 @@ def variant?(other_filename)
     def default_url(style_name = default_style)
       @url_generator.for_as_default(style_name)
     end
+
+    STOPLIGHT_THRESHOLD = 10
+    STOPLIGHT_COOLDOWN  = 30
+
+    # We overwrite this method to put a circuit breaker around
+    # calls to object storage, to stop hitting APIs that are slow
+    # to respond or don't respond at all and as such minimize the
+    # impact of object storage outages on application throughput
+    def save
+      Stoplight('object-storage') { super }.with_threshold(STOPLIGHT_THRESHOLD).with_cool_off_time(STOPLIGHT_COOLDOWN).with_error_handler do |error, handle|
+        if error.is_a?(Seahorse::Client::NetworkingError)
+          handle.call(error)
+        else
+          raise error
+        end
+      end.run
+    end
   end
 end