[hotfix] updated retries on rate limit upload errors

Author: shlag3n

app/controllers/lcms/engine/admin/curriculums_controller.rb

@@ -17,7 +17,7 @@ def children
               Array.wrap(Lcms::Engine::Resource.tree.find(id)&.children)
             end
 
-          render json: resources.map { |res| CurriculumPresenter.new.parse_jstree_node(res) }
+          render json: resources.tree.map { |res| CurriculumPresenter.new.parse_jstree_node(res) }
         end
 
         def update

docs/env-variables.md

@@ -55,8 +55,11 @@ The project uses several Google products, including analytics, OAuth for allowin
 | GOOGLE_APPLICATION_TEMPLATE_PORTRAIT  | Id of the Google document which is a template for portrait materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `)                          |
 | GOOGLE_APPLICATION_TEMPLATE_LANDSCAPE | Id of the Google document which is a template for landscape materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `)                         |
 | GOOGLE_APPLICATION_PREVIEW_FOLDER_ID  | Folder ID where preview documents should get placed                                                                                                              |
-| GOOGLE_API_CLIENT_UPLOAD_RETRIES      ||     |
-| GOOGLE_API_CLIENT_UPLOAD_TIMEOUT      ||     |
+| GOOGLE_API_CLIENT_UPLOAD_RETRIES      | Number of retries on upload, default is 5 |     |
+| GOOGLE_API_CLIENT_UPLOAD_TIMEOUT      | Upload timeout, default is 60 seconds |     |
+| GOOGLE_API_CLIENT_UPLOAD_RATE_BASE_INTERVAL      | Initial interval in seconds between tries for rate limit errors on upload, default is 2|     |
+| GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_INTERVAL | The maximum interval in seconds that any individual retry can reach, default is 300 | |
+| GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_ELAPSED_TIME      | The maximum amount of total time in seconds that code is allowed to keep being retried, default 900 (15 min)|     |
 
 ### Miscellaneous settings
 | Name                  | Description                                                                                                |

lib/document_exporter/gdoc/base.rb

@@ -12,10 +12,10 @@ class Base < DocumentExporter::Base
         options: {
           open_timeout_sec: GOOGLE_API_CLIENT_UPLOAD_TIMEOUT,
           read_timeout_sec: GOOGLE_API_CLIENT_UPLOAD_TIMEOUT,
-          retries: GOOGLE_API_CLIENT_UPLOAD_RETRIES,
           send_timeout_sec: GOOGLE_API_CLIENT_UPLOAD_TIMEOUT
         }
       }.freeze
+      GOOGLE_API_RATE_RETRIABLE_ERRORS = [Google::Apis::ServerError, Google::Apis::RateLimitError].freeze
       VERSION_RE = /_v\d+$/i.freeze
 
       attr_reader :document, :options
@@ -52,7 +52,18 @@ def export
           upload_source: StringIO.new(content)
         }.merge(GOOGLE_API_UPLOAD_OPTIONS)
 
-        @id = Retriable.retriable(base_interval: 1, tries: GOOGLE_API_CLIENT_UPLOAD_RETRIES) do
+        # use general API settings for inner API call and wrap it with additional retry logic
+        @id = Retriable.retriable(base_interval: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_BASE_INTERVAL', 5).to_i,
+                                  multiplier: ENV.fetch(
+                                    'GOOGLE_API_CLIENT_UPLOAD_RATE_MULTIPLIER', 2
+                                  ).to_f,
+                                  max_interval: ENV.fetch(
+                                    'GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_INTERVAL', 300
+                                  ).to_i,
+                                  max_elapsed_time: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_ELAPSED_TIME',
+                                                              900).to_i,
+                                  on_retry: log_retries(file_id, metadata),
+                                  tries: GOOGLE_API_CLIENT_UPLOAD_RETRIES) do
           if file_id.present?
             drive_service.service.update_file(file_id, metadata, params)
           else
@@ -81,7 +92,17 @@ def export_to(folder_id, file_id: nil)
           upload_source: StringIO.new(content)
         }.merge(GOOGLE_API_UPLOAD_OPTIONS)
 
-        @id = Retriable.retriable(base_interval: 1, tries: GOOGLE_API_CLIENT_UPLOAD_RETRIES) do
+        @id = Retriable.retriable(base_interval: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_BASE_INTERVAL', 5).to_i,
+                                  multiplier: ENV.fetch(
+                                    'GOOGLE_API_CLIENT_UPLOAD_RATE_MULTIPLIER', 2
+                                  ).to_f,
+                                  max_interval: ENV.fetch(
+                                    'GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_INTERVAL', 300
+                                  ).to_i,
+                                  max_elapsed_time: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_ELAPSED_TIME',
+                                                              900).to_i,
+                                  on_retry: log_retries(file_id, metadata),
+                                  tries: GOOGLE_API_CLIENT_UPLOAD_RETRIES) do
           if file_id.present?
             drive_service.service.update_file(file_id, metadata, params)
           else
@@ -140,6 +161,18 @@ def gdoc_folder_tmp(material_ids)
         self
       end
 
+      def log_retries(file_id, metadata)
+        proc do |exception, try, elapsed_time, next_interval|
+          msg = "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds " \
+                "and #{next_interval} seconds until the next try. " \
+                "File ID: #{file_id}, Metadata: #{metadata.inspect}"
+          Rails.logger.error(msg)
+          if defined?(Airbrake) && ENV.fetch('AIRBRAKE_UPLOAD_ERRORS_NOTIFY', 'off') == 'on'
+            ::Airbrake.notify_sync(Airbrake.build_notice(msg))
+          end
+        end
+      end
+
       def post_processing
         Retriable.retriable(base_interval: 5, tries: 10) do
           Lcms::Engine::Google::ScriptService.new(document).execute(@id)