[hotfix] gdoc upload retries

Gemfile

@@ -16,6 +16,9 @@ gemspec
 # To use a debugger
 # gem 'byebug', group: [:development, :test]
 
+gem 'lt-google-api', git: 'https://github.com/learningtapestry/lt-google-api.git', branch: 'google-api-update-0.16'
+gem 'lt-lcms', git: 'https://github.com/learningtapestry/lt-lcms.git', branch: 'google-api-update-0.16'
+
 gem 'wicked_pdf', git: 'https://github.com/learningtapestry/wicked_pdf.git',
                   branch: 'puppeteer-support',
                   ref: '964a090'

Gemfile.lock

@@ -1,3 +1,23 @@
+GIT
+  remote: https://github.com/learningtapestry/lt-google-api.git
+  revision: 15cb66366742649e629ccddd7001cf7bd4ebc672
+  branch: google-api-update-0.16
+  specs:
+    lt-google-api (0.3.1)
+      google-apis-drive_v3 (~> 0.63.0)
+      googleauth (~> 1.11.2)
+
+GIT
+  remote: https://github.com/learningtapestry/lt-lcms.git
+  revision: f4403a3b6befef09baa4c678811c319c05375a88
+  branch: google-api-update-0.16
+  specs:
+    lt-lcms (0.6.1)
+      google-apis-core (~> 0.16.0)
+      httparty (~> 0.18)
+      nokogiri (~> 1.10, >= 1.10.8)
+      rubyzip (~> 2)
+
 GIT
   remote: https://github.com/learningtapestry/wicked_pdf.git
   revision: 964a0906c9284d97927322e4c1e474b1d68373cf
@@ -10,7 +30,7 @@ GIT
 PATH
   remote: .
   specs:
-    lcms-engine (0.5.6)
+    lcms-engine (0.5.7)
       active_model_serializers (~> 0.10.10)
       activejob-retry (~> 0.6.3)
       acts-as-taggable-on (~> 7.0)
@@ -36,15 +56,13 @@ PATH
       fog-aws (~> 3.5, >= 3.5.2)
       font-awesome-sass (~> 5.12)
       foundation-rails (~> 6.6.2, >= 6.6.1)
-      google-apis-drive_v3 (~> 0.46)
-      google-apis-script_v1 (~> 0.21)
+      google-apis-drive_v3 (~> 0.63.0)
+      google-apis-script_v1 (~> 0.27.0)
       hiredis (~> 0.6.3)
       httparty (~> 0.18)
       jbuilder (~> 2.10)
       jquery-rails (~> 4.3, >= 4.3.5)
       js-routes (~> 1.4, >= 1.4.9)
-      lt-google-api (~> 0.3)
-      lt-lcms (~> 0.6)
       migration_data (~> 0.6)
       mini_magick (~> 4.10, >= 4.10.1)
       nested_form (~> 0.3.2)
@@ -348,22 +366,21 @@ GEM
       raabro (~> 1.4)
     globalid (1.0.1)
       activesupport (>= 5.0)
-    google-apis-core (0.11.2)
+    google-apis-core (0.16.0)
       addressable (~> 2.5, >= 2.5.1)
-      googleauth (>= 0.16.2, < 2.a)
-      httpclient (>= 2.8.1, < 3.a)
+      googleauth (~> 1.9)
+      httpclient (>= 2.8.3, < 3.a)
       mini_mime (~> 1.0)
+      mutex_m
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
-      rexml
-      webrick
-    google-apis-drive_v3 (0.46.0)
-      google-apis-core (>= 0.11.0, < 2.a)
-    google-apis-script_v1 (0.21.0)
-      google-apis-core (>= 0.11.0, < 2.a)
+    google-apis-drive_v3 (0.63.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-script_v1 (0.27.0)
+      google-apis-core (>= 0.15.0, < 2.a)
     google-cloud-env (2.1.0)
       faraday (>= 1.0, < 3.a)
-    googleauth (1.9.1)
+    googleauth (1.11.2)
       faraday (>= 1.0, < 3.a)
       google-cloud-env (~> 2.1)
       jwt (>= 1.4, < 3.0)
@@ -405,15 +422,6 @@ GEM
     loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    lt-google-api (0.3.0)
-      google-apis-drive_v3 (~> 0.46)
-      googleauth (~> 1.9)
-    lt-lcms (0.6.0)
-      google-apis-core (~> 0.11, >= 0.11.1)
-      httparty (~> 0.18)
-      lt-google-api (~> 0.3)
-      nokogiri (~> 1.10, >= 1.10.8)
-      rubyzip (~> 2)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (1.0.2)
@@ -432,6 +440,7 @@ GEM
     multi_xml (0.6.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
+    mutex_m (0.3.0)
     nested_form (0.3.2)
     netrc (0.11.0)
     nikkou (0.0.5)
@@ -701,7 +710,6 @@ GEM
       rack-proxy (>= 0.6.1)
       railties (>= 5.2)
       semantic_range (>= 2.3.0)
-    webrick (1.8.1)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -727,6 +735,8 @@ DEPENDENCIES
   factory_bot (~> 5)
   faker (~> 2.1)
   lcms-engine!
+  lt-google-api!
+  lt-lcms!
   overcommit (~> 0.57)
   pry-byebug (~> 3.7)
   pry-rails (~> 0.3.5)
@@ -745,4 +755,4 @@ DEPENDENCIES
   wicked_pdf!
 
 BUNDLED WITH
-   2.4.22
+   2.4.8

app/controllers/lcms/engine/admin/curriculums_controller.rb

@@ -14,7 +14,7 @@ def children
             if id == '#'
               Lcms::Engine::Resource.tree.ordered.roots
             else
-              Array.wrap(Lcms::Engine::Resource.tree.find(id)&.children)
+              Array.wrap(Lcms::Engine::Resource.tree.find(id)&.children&.tree)
             end
 
           render json: resources.map { |res| CurriculumPresenter.new.parse_jstree_node(res) }

app/controllers/lcms/engine/admin/documents_controller.rb

@@ -127,10 +127,10 @@ def reimport_lesson_materials
 
         def set_query_params
           @query_params = params[:query]
-            &.permit(
-              :broken_materials, :course, :grade, :inactive, :locale, :module, :only_failed, :reimport_required,
-              :search_term, :sort_by
-            ) || {}
+                            &.permit(
+                              :broken_materials, :course, :grade, :inactive, :locale, :module, :only_failed,
+                              :reimport_required, :search_term, :sort_by
+                            ) || {}
         end
       end
     end

app/controllers/lcms/engine/admin/materials_controller.rb

@@ -103,10 +103,10 @@ def gdoc_files_from(url)
 
         def set_query_params
           @query_params = params[:query]
-            &.permit(
-              :course, :lesson, :name_date, :orientation, :search_term, :search_file_name, :sort_by, :title, :type,
-              :unit
-            ) || {}
+                            &.permit(
+                              :course, :lesson, :name_date, :orientation, :search_term, :search_file_name, :sort_by,
+                              :title, :type, :unit
+                            ) || {}
         end
       end
     end

app/entities/lcms/engine/external_page.rb

@@ -11,7 +11,7 @@ class ExternalPage
       attribute :description, String
       attribute :permalink, String
       attribute :slug, String
-      attribute :keywords, Array[String], default: []
+      attribute :keywords, [String], default: []
       attribute :teaser, String
       attribute :title, String
 

app/forms/lcms/engine/curriculum_form.rb

@@ -7,11 +7,11 @@ class CurriculumForm
       include Virtus.model
       include ActiveModel::Model
 
-      attribute :change_log, Array[Hash]
+      attribute :change_log, [Hash]
 
       def initialize(params = {})
         parsed_change_log = parse_change_log params
-        super params.merge(change_log: parsed_change_log)
+        super(params.merge(change_log: parsed_change_log))
       end
 
       def save

app/helpers/lcms/engine/resource_helper.rb

@@ -46,7 +46,7 @@ def resource_breadcrumbs_with_links(resource)
 
         [].tap do |result|
           pieces.each_with_index do |piece, idx|
-            ((result << piece) && next) if idx.zero?
+            (result << piece) && next if idx.zero?
 
             slug = Slug.build_from(pieces[0..idx])
 

app/helpers/lcms/engine/view_helper.rb

@@ -105,7 +105,7 @@ def color_code(model, base: false)
 
       def selected_id?(id)
         selected_ids = params[:selected_ids]
-        return unless selected_ids.present?
+        return false unless selected_ids.present?
 
         case selected_ids
         when Array

app/interactors/lcms/engine/explore_curriculum_interactor.rb

@@ -57,7 +57,7 @@ def expanded_props
 
         {
           active: active_branch,
-          expanded: expanded? ? true : nil,
+          expanded: expanded? || nil,
           results: grades.map do |curr|
             CurriculumResourceSerializer.new(
               curr,

app/middleware/remove_session.rb

@@ -16,11 +16,10 @@ def call(env)
     # Don't delete the session cookie if:
     #   - We're in the process of logging in (breaks CSRF for sign in form)
     #   - We're logged in (needed for Devise)
-    skip_delete = (
+    skip_delete =
       path =~ %r{^/users} ||
       user_key.present? ||
       headers[SET_COOKIE].blank?
-    )
 
     signing_out = path == '/users/sign_out'
 

app/models/lcms/engine/search/document.rb

@@ -22,10 +22,10 @@ class Document < ElasticSearchDocument
         attribute :position, String
         attribute :slug, String
         attribute :subject, String
-        attribute :tag_authors, Array[String]
-        attribute :tag_keywords, Array[String]
-        attribute :tag_standards, Array[String]
-        attribute :tag_texts, Array[String]
+        attribute :tag_authors, [String]
+        attribute :tag_keywords, [String]
+        attribute :tag_standards, [String]
+        attribute :tag_texts, [String]
         attribute :teaser, String
         attribute :title, String
 

app/models/lcms/engine/search/repository.rb

@@ -166,11 +166,9 @@ def tags_query(term, tags, options)
             query: {
               bool: {
                 filter: [],
-                should: tags.map { |t| { match: { t => term } } }.concat(
-                  [
-                    { match_phrase: { title: term } },
-                    { match_phrase: { teaser: term } }
-                  ]
+                should: tags.map { |t| { match: { t => term } } }.push(
+                  { match_phrase: { title: term } },
+                  { match_phrase: { teaser: term } }
                 ),
                 minimum_should_match: 1
               }

app/models/lcms/engine/user.rb

@@ -40,7 +40,7 @@ def ready_to_go?
       private
 
       def access_code_valid?
-        return if AccessCode.by_code(access_code).exists?
+        return false if AccessCode.by_code(access_code).exists?
 
         errors.add :access_code, 'not found'
       end

app/serializers/lcms/engine/curriculum_resource_serializer.rb

@@ -7,7 +7,7 @@ class CurriculumResourceSerializer < ActiveModel::Serializer
                  :resource,  :type, :unit_count, :unit_sizes
 
       def initialize(object, options = {})
-        super(object, options)
+        super
         @depth = options[:depth] || 0
         @depth_branch = options[:depth_branch]
       end

app/services/lcms/engine/google/drive_service.rb

@@ -15,15 +15,15 @@ def self.build(document, options = {})
         end
 
         def copy(file_ids, folder_id = parent)
-          super file_ids, folder_id
+          super
         end
 
         def create_folder(folder_name, parent_id = FOLDER_ID)
-          super folder_name, parent_id
+          super
         end
 
         def initialize(document, options)
-          super google_credentials
+          super(google_credentials)
           @document = document
           @options = options
         end

app/services/lcms/engine/html_sanitizer.rb

@@ -332,7 +332,7 @@ def post_processing_images_gdoc(nodes)
 
           css = ':not(.u-ld-not-image-wrap) > img:not([src*=googleapis]):not(.o-ld-icon):not(.o-ld-latex)'
           nodes.css(css).each do |img|
-            img = img.parent.replace(img) if img.parent.name == 'span' || img.parent.name == 'p'
+            img = img.parent.replace(img) if %w(span p).include?(img.parent.name)
             img.replace(%(
           <table class='o-simple-table o-ld-image-wrap--math'>
             <tr>
@@ -380,7 +380,7 @@ def remove_spans_wo_attrs(env)
         def remove_empty_paragraphs(env)
           node = env[:node]
 
-          node.unlink if node.element? && (node.name == 'p' || node.name == 'span') && node.inner_html.squish.blank?
+          node.unlink if node.element? && %w(p span).include?(node.name) && node.inner_html.squish.blank?
         end
 
         # replace inline borders style with width = 0 as they're not processing correct for pdf

app/services/lcms/engine/react_materials_resolver.rb

@@ -23,7 +23,7 @@ def replace_react(node, document) # rubocop:disable Metrics/PerceivedComplexity
                       else
                         JSON.parse(data)
                       end
-          node.remove && return if (raw_props['material_ids']).empty?
+          node.remove && return if raw_props['material_ids'].empty?
 
           props = PreviewsMaterialSerializer.new(raw_props, document)
           node.remove && return if props.data && props.data.empty?

config/initializers/elasticsearch.rb

@@ -6,4 +6,4 @@
   host: ENV.fetch('ELASTICSEARCH_ADDRESS', nil),
   adapter: :net_http
 )
-Hashie.logger = Logger.new('/dev/null')
+Hashie.logger = Logger.new(File::NULL)

docs/env-variables.md

@@ -55,8 +55,11 @@ The project uses several Google products, including analytics, OAuth for allowin
 | GOOGLE_APPLICATION_TEMPLATE_PORTRAIT  | Id of the Google document which is a template for portrait materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `)                          |
 | GOOGLE_APPLICATION_TEMPLATE_LANDSCAPE | Id of the Google document which is a template for landscape materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `)                         |
 | GOOGLE_APPLICATION_PREVIEW_FOLDER_ID  | Folder ID where preview documents should get placed                                                                                                              |
-| GOOGLE_API_CLIENT_UPLOAD_RETRIES      ||     |
-| GOOGLE_API_CLIENT_UPLOAD_TIMEOUT      ||     |
+| GOOGLE_API_CLIENT_UPLOAD_RETRIES      | Number of retries on upload, default is 5 |     |
+| GOOGLE_API_CLIENT_UPLOAD_TIMEOUT      | Upload timeout, default is 60 seconds |     |
+| GOOGLE_API_CLIENT_UPLOAD_RATE_BASE_INTERVAL      | Initial interval in seconds between tries for rate limit errors on upload, default is 2|     |
+| GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_INTERVAL | The maximum interval in seconds that any individual retry can reach, default is 300 | |
+| GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_ELAPSED_TIME      | The maximum amount of total time in seconds that code is allowed to keep being retried, default 900 (15 min)|     |
 
 ### Miscellaneous settings
 | Name                  | Description                                                                                                |

lcms-engine.gemspec

@@ -52,15 +52,15 @@ Gem::Specification.new do |s|
   s.add_dependency 'fog-aws', '~> 3.5', '>= 3.5.2'
   s.add_dependency 'font-awesome-sass', '~> 5.12'
   s.add_dependency 'foundation-rails', '~> 6.6.2', '>= 6.6.1'
-  s.add_dependency 'google-apis-drive_v3', '~> 0.46'
-  s.add_dependency 'google-apis-script_v1', '~> 0.21'
+  s.add_dependency 'google-apis-drive_v3', '~> 0.63.0'
+  s.add_dependency 'google-apis-script_v1', '~> 0.27.0'
   s.add_dependency 'hiredis', '~> 0.6.3'
   s.add_dependency 'httparty', '~> 0.18'
   s.add_dependency 'jbuilder', '~> 2.10'
   s.add_dependency 'jquery-rails', '~> 4.3', '>= 4.3.5'
   s.add_dependency 'js-routes', '~> 1.4', '>= 1.4.9'
-  s.add_dependency 'lt-google-api', '~> 0.3'
-  s.add_dependency 'lt-lcms', '~> 0.6'
+  # s.add_dependency 'lt-google-api', '~> 0.3'
+  # s.add_dependency 'lt-lcms', '~> 0.6'
   s.add_dependency 'migration_data', '~> 0.6'
   s.add_dependency 'mini_magick', '~> 4.10', '>= 4.10.1'
   s.add_dependency 'nested_form', '~> 0.3.2'

lib/doc_template/document.rb

@@ -181,7 +181,7 @@ def parse_node(node)
         context_type: @opts[:context_type],
         data: parsed_tag.tag_data,
         materials: parsed_tag.materials,
-        optional: (parsed_tag.try(:optional?) || false),
+        optional: parsed_tag.try(:optional?) || false,
         placeholder: parsed_tag.placeholder,
         part_type: tag_name.underscore
       }