diff --git a/Gemfile b/Gemfile index 6a86e340..0eefca21 100644 --- a/Gemfile +++ b/Gemfile @@ -16,6 +16,9 @@ gemspec # To use a debugger # gem 'byebug', group: [:development, :test] +gem 'lt-google-api', git: 'https://github.com/learningtapestry/lt-google-api.git', branch: 'google-api-update-0.16' +gem 'lt-lcms', git: 'https://github.com/learningtapestry/lt-lcms.git', branch: 'google-api-update-0.16' + gem 'wicked_pdf', git: 'https://github.com/learningtapestry/wicked_pdf.git', branch: 'puppeteer-support', ref: '964a090' diff --git a/Gemfile.lock b/Gemfile.lock index 98cdd7b4..422300d8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,23 @@ +GIT + remote: https://github.com/learningtapestry/lt-google-api.git + revision: 15cb66366742649e629ccddd7001cf7bd4ebc672 + branch: google-api-update-0.16 + specs: + lt-google-api (0.3.1) + google-apis-drive_v3 (~> 0.63.0) + googleauth (~> 1.11.2) + +GIT + remote: https://github.com/learningtapestry/lt-lcms.git + revision: f4403a3b6befef09baa4c678811c319c05375a88 + branch: google-api-update-0.16 + specs: + lt-lcms (0.6.1) + google-apis-core (~> 0.16.0) + httparty (~> 0.18) + nokogiri (~> 1.10, >= 1.10.8) + rubyzip (~> 2) + GIT remote: https://github.com/learningtapestry/wicked_pdf.git revision: 964a0906c9284d97927322e4c1e474b1d68373cf @@ -10,7 +30,7 @@ GIT PATH remote: . specs: - lcms-engine (0.5.6) + lcms-engine (0.5.7) active_model_serializers (~> 0.10.10) activejob-retry (~> 0.6.3) acts-as-taggable-on (~> 7.0) @@ -36,15 +56,13 @@ PATH fog-aws (~> 3.5, >= 3.5.2) font-awesome-sass (~> 5.12) foundation-rails (~> 6.6.2, >= 6.6.1) - google-apis-drive_v3 (~> 0.46) - google-apis-script_v1 (~> 0.21) + google-apis-drive_v3 (~> 0.63.0) + google-apis-script_v1 (~> 0.27.0) hiredis (~> 0.6.3) httparty (~> 0.18) jbuilder (~> 2.10) jquery-rails (~> 4.3, >= 4.3.5) js-routes (~> 1.4, >= 1.4.9) - lt-google-api (~> 0.3) - lt-lcms (~> 0.6) migration_data (~> 0.6) mini_magick (~> 4.10, >= 4.10.1) nested_form (~> 0.3.2) @@ -348,22 +366,21 @@ GEM raabro (~> 1.4) globalid (1.0.1) activesupport (>= 5.0) - google-apis-core (0.11.2) + google-apis-core (0.16.0) addressable (~> 2.5, >= 2.5.1) - googleauth (>= 0.16.2, < 2.a) - httpclient (>= 2.8.1, < 3.a) + googleauth (~> 1.9) + httpclient (>= 2.8.3, < 3.a) mini_mime (~> 1.0) + mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - rexml - webrick - google-apis-drive_v3 (0.46.0) - google-apis-core (>= 0.11.0, < 2.a) - google-apis-script_v1 (0.21.0) - google-apis-core (>= 0.11.0, < 2.a) + google-apis-drive_v3 (0.63.0) + google-apis-core (>= 0.15.0, < 2.a) + google-apis-script_v1 (0.27.0) + google-apis-core (>= 0.15.0, < 2.a) google-cloud-env (2.1.0) faraday (>= 1.0, < 3.a) - googleauth (1.9.1) + googleauth (1.11.2) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.1) jwt (>= 1.4, < 3.0) @@ -405,15 +422,6 @@ GEM loofah (2.19.1) crass (~> 1.0.2) nokogiri (>= 1.5.9) - lt-google-api (0.3.0) - google-apis-drive_v3 (~> 0.46) - googleauth (~> 1.9) - lt-lcms (0.6.0) - google-apis-core (~> 0.11, >= 0.11.1) - httparty (~> 0.18) - lt-google-api (~> 0.3) - nokogiri (~> 1.10, >= 1.10.8) - rubyzip (~> 2) mail (2.7.1) mini_mime (>= 0.1.1) marcel (1.0.2) @@ -432,6 +440,7 @@ GEM multi_xml (0.6.0) mustermann (3.0.0) ruby2_keywords (~> 0.0.1) + mutex_m (0.3.0) nested_form (0.3.2) netrc (0.11.0) nikkou (0.0.5) @@ -701,7 +710,6 @@ GEM rack-proxy (>= 0.6.1) railties (>= 5.2) semantic_range (>= 2.3.0) - webrick (1.8.1) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -727,6 +735,8 @@ DEPENDENCIES factory_bot (~> 5) faker (~> 2.1) lcms-engine! + lt-google-api! + lt-lcms! overcommit (~> 0.57) pry-byebug (~> 3.7) pry-rails (~> 0.3.5) @@ -745,4 +755,4 @@ DEPENDENCIES wicked_pdf! BUNDLED WITH - 2.4.22 + 2.4.8 diff --git a/app/controllers/lcms/engine/admin/curriculums_controller.rb b/app/controllers/lcms/engine/admin/curriculums_controller.rb index 77425c71..08fd173c 100644 --- a/app/controllers/lcms/engine/admin/curriculums_controller.rb +++ b/app/controllers/lcms/engine/admin/curriculums_controller.rb @@ -14,7 +14,7 @@ def children if id == '#' Lcms::Engine::Resource.tree.ordered.roots else - Array.wrap(Lcms::Engine::Resource.tree.find(id)&.children) + Array.wrap(Lcms::Engine::Resource.tree.find(id)&.children&.tree) end render json: resources.map { |res| CurriculumPresenter.new.parse_jstree_node(res) } diff --git a/app/controllers/lcms/engine/admin/documents_controller.rb b/app/controllers/lcms/engine/admin/documents_controller.rb index 4ca0f797..e22e8e2a 100644 --- a/app/controllers/lcms/engine/admin/documents_controller.rb +++ b/app/controllers/lcms/engine/admin/documents_controller.rb @@ -127,10 +127,10 @@ def reimport_lesson_materials def set_query_params @query_params = params[:query] - &.permit( - :broken_materials, :course, :grade, :inactive, :locale, :module, :only_failed, :reimport_required, - :search_term, :sort_by - ) || {} + &.permit( + :broken_materials, :course, :grade, :inactive, :locale, :module, :only_failed, + :reimport_required, :search_term, :sort_by + ) || {} end end end diff --git a/app/controllers/lcms/engine/admin/materials_controller.rb b/app/controllers/lcms/engine/admin/materials_controller.rb index 4e1b95e2..c74c6ab7 100644 --- a/app/controllers/lcms/engine/admin/materials_controller.rb +++ b/app/controllers/lcms/engine/admin/materials_controller.rb @@ -103,10 +103,10 @@ def gdoc_files_from(url) def set_query_params @query_params = params[:query] - &.permit( - :course, :lesson, :name_date, :orientation, :search_term, :search_file_name, :sort_by, :title, :type, - :unit - ) || {} + &.permit( + :course, :lesson, :name_date, :orientation, :search_term, :search_file_name, :sort_by, + :title, :type, :unit + ) || {} end end end diff --git a/app/entities/lcms/engine/external_page.rb b/app/entities/lcms/engine/external_page.rb index c48c929a..e821d334 100644 --- a/app/entities/lcms/engine/external_page.rb +++ b/app/entities/lcms/engine/external_page.rb @@ -11,7 +11,7 @@ class ExternalPage attribute :description, String attribute :permalink, String attribute :slug, String - attribute :keywords, Array[String], default: [] + attribute :keywords, [String], default: [] attribute :teaser, String attribute :title, String diff --git a/app/forms/lcms/engine/curriculum_form.rb b/app/forms/lcms/engine/curriculum_form.rb index 177e31f2..895b6f96 100644 --- a/app/forms/lcms/engine/curriculum_form.rb +++ b/app/forms/lcms/engine/curriculum_form.rb @@ -7,11 +7,11 @@ class CurriculumForm include Virtus.model include ActiveModel::Model - attribute :change_log, Array[Hash] + attribute :change_log, [Hash] def initialize(params = {}) parsed_change_log = parse_change_log params - super params.merge(change_log: parsed_change_log) + super(params.merge(change_log: parsed_change_log)) end def save diff --git a/app/helpers/lcms/engine/resource_helper.rb b/app/helpers/lcms/engine/resource_helper.rb index 2c901d93..40f4cf83 100644 --- a/app/helpers/lcms/engine/resource_helper.rb +++ b/app/helpers/lcms/engine/resource_helper.rb @@ -46,7 +46,7 @@ def resource_breadcrumbs_with_links(resource) [].tap do |result| pieces.each_with_index do |piece, idx| - ((result << piece) && next) if idx.zero? + (result << piece) && next if idx.zero? slug = Slug.build_from(pieces[0..idx]) diff --git a/app/helpers/lcms/engine/view_helper.rb b/app/helpers/lcms/engine/view_helper.rb index 9622703c..bdb2116f 100644 --- a/app/helpers/lcms/engine/view_helper.rb +++ b/app/helpers/lcms/engine/view_helper.rb @@ -105,7 +105,7 @@ def color_code(model, base: false) def selected_id?(id) selected_ids = params[:selected_ids] - return unless selected_ids.present? + return false unless selected_ids.present? case selected_ids when Array diff --git a/app/interactors/lcms/engine/explore_curriculum_interactor.rb b/app/interactors/lcms/engine/explore_curriculum_interactor.rb index 6c1ca59c..9eadcd4e 100644 --- a/app/interactors/lcms/engine/explore_curriculum_interactor.rb +++ b/app/interactors/lcms/engine/explore_curriculum_interactor.rb @@ -57,7 +57,7 @@ def expanded_props { active: active_branch, - expanded: expanded? ? true : nil, + expanded: expanded? || nil, results: grades.map do |curr| CurriculumResourceSerializer.new( curr, diff --git a/app/middleware/remove_session.rb b/app/middleware/remove_session.rb index c9d88a64..b7409806 100644 --- a/app/middleware/remove_session.rb +++ b/app/middleware/remove_session.rb @@ -16,11 +16,10 @@ def call(env) # Don't delete the session cookie if: # - We're in the process of logging in (breaks CSRF for sign in form) # - We're logged in (needed for Devise) - skip_delete = ( + skip_delete = path =~ %r{^/users} || user_key.present? || headers[SET_COOKIE].blank? - ) signing_out = path == '/users/sign_out' diff --git a/app/models/lcms/engine/search/document.rb b/app/models/lcms/engine/search/document.rb index c888b336..29dcb2fb 100644 --- a/app/models/lcms/engine/search/document.rb +++ b/app/models/lcms/engine/search/document.rb @@ -22,10 +22,10 @@ class Document < ElasticSearchDocument attribute :position, String attribute :slug, String attribute :subject, String - attribute :tag_authors, Array[String] - attribute :tag_keywords, Array[String] - attribute :tag_standards, Array[String] - attribute :tag_texts, Array[String] + attribute :tag_authors, [String] + attribute :tag_keywords, [String] + attribute :tag_standards, [String] + attribute :tag_texts, [String] attribute :teaser, String attribute :title, String diff --git a/app/models/lcms/engine/search/repository.rb b/app/models/lcms/engine/search/repository.rb index ada94315..7b8024de 100644 --- a/app/models/lcms/engine/search/repository.rb +++ b/app/models/lcms/engine/search/repository.rb @@ -166,11 +166,9 @@ def tags_query(term, tags, options) query: { bool: { filter: [], - should: tags.map { |t| { match: { t => term } } }.concat( - [ - { match_phrase: { title: term } }, - { match_phrase: { teaser: term } } - ] + should: tags.map { |t| { match: { t => term } } }.push( + { match_phrase: { title: term } }, + { match_phrase: { teaser: term } } ), minimum_should_match: 1 } diff --git a/app/models/lcms/engine/user.rb b/app/models/lcms/engine/user.rb index 0870a001..39c0db97 100644 --- a/app/models/lcms/engine/user.rb +++ b/app/models/lcms/engine/user.rb @@ -40,7 +40,7 @@ def ready_to_go? private def access_code_valid? - return if AccessCode.by_code(access_code).exists? + return false if AccessCode.by_code(access_code).exists? errors.add :access_code, 'not found' end diff --git a/app/serializers/lcms/engine/curriculum_resource_serializer.rb b/app/serializers/lcms/engine/curriculum_resource_serializer.rb index 1eeae783..03332896 100644 --- a/app/serializers/lcms/engine/curriculum_resource_serializer.rb +++ b/app/serializers/lcms/engine/curriculum_resource_serializer.rb @@ -7,7 +7,7 @@ class CurriculumResourceSerializer < ActiveModel::Serializer :resource, :type, :unit_count, :unit_sizes def initialize(object, options = {}) - super(object, options) + super @depth = options[:depth] || 0 @depth_branch = options[:depth_branch] end diff --git a/app/services/lcms/engine/google/drive_service.rb b/app/services/lcms/engine/google/drive_service.rb index bca75a6c..ad188ddf 100644 --- a/app/services/lcms/engine/google/drive_service.rb +++ b/app/services/lcms/engine/google/drive_service.rb @@ -15,15 +15,15 @@ def self.build(document, options = {}) end def copy(file_ids, folder_id = parent) - super file_ids, folder_id + super end def create_folder(folder_name, parent_id = FOLDER_ID) - super folder_name, parent_id + super end def initialize(document, options) - super google_credentials + super(google_credentials) @document = document @options = options end diff --git a/app/services/lcms/engine/html_sanitizer.rb b/app/services/lcms/engine/html_sanitizer.rb index 62c50bc7..7a062652 100644 --- a/app/services/lcms/engine/html_sanitizer.rb +++ b/app/services/lcms/engine/html_sanitizer.rb @@ -332,7 +332,7 @@ def post_processing_images_gdoc(nodes) css = ':not(.u-ld-not-image-wrap) > img:not([src*=googleapis]):not(.o-ld-icon):not(.o-ld-latex)' nodes.css(css).each do |img| - img = img.parent.replace(img) if img.parent.name == 'span' || img.parent.name == 'p' + img = img.parent.replace(img) if %w(span p).include?(img.parent.name) img.replace(%( @@ -380,7 +380,7 @@ def remove_spans_wo_attrs(env) def remove_empty_paragraphs(env) node = env[:node] - node.unlink if node.element? && (node.name == 'p' || node.name == 'span') && node.inner_html.squish.blank? + node.unlink if node.element? && %w(p span).include?(node.name) && node.inner_html.squish.blank? end # replace inline borders style with width = 0 as they're not processing correct for pdf diff --git a/app/services/lcms/engine/react_materials_resolver.rb b/app/services/lcms/engine/react_materials_resolver.rb index 7724a644..c6b114e5 100644 --- a/app/services/lcms/engine/react_materials_resolver.rb +++ b/app/services/lcms/engine/react_materials_resolver.rb @@ -23,7 +23,7 @@ def replace_react(node, document) # rubocop:disable Metrics/PerceivedComplexity else JSON.parse(data) end - node.remove && return if (raw_props['material_ids']).empty? + node.remove && return if raw_props['material_ids'].empty? props = PreviewsMaterialSerializer.new(raw_props, document) node.remove && return if props.data && props.data.empty? diff --git a/config/initializers/elasticsearch.rb b/config/initializers/elasticsearch.rb index 537c9b4d..94b4a2e1 100644 --- a/config/initializers/elasticsearch.rb +++ b/config/initializers/elasticsearch.rb @@ -6,4 +6,4 @@ host: ENV.fetch('ELASTICSEARCH_ADDRESS', nil), adapter: :net_http ) -Hashie.logger = Logger.new('/dev/null') +Hashie.logger = Logger.new(File::NULL) diff --git a/docs/env-variables.md b/docs/env-variables.md index ffe7c46b..003b60e0 100644 --- a/docs/env-variables.md +++ b/docs/env-variables.md @@ -55,8 +55,11 @@ The project uses several Google products, including analytics, OAuth for allowin | GOOGLE_APPLICATION_TEMPLATE_PORTRAIT | Id of the Google document which is a template for portrait materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `) | | GOOGLE_APPLICATION_TEMPLATE_LANDSCAPE | Id of the Google document which is a template for landscape materials(can be identified the same way as `GOOGLE_APPLICATION_FOLDER_ID `) | | GOOGLE_APPLICATION_PREVIEW_FOLDER_ID | Folder ID where preview documents should get placed | -| GOOGLE_API_CLIENT_UPLOAD_RETRIES || | -| GOOGLE_API_CLIENT_UPLOAD_TIMEOUT || | +| GOOGLE_API_CLIENT_UPLOAD_RETRIES | Number of retries on upload, default is 5 | | +| GOOGLE_API_CLIENT_UPLOAD_TIMEOUT | Upload timeout, default is 60 seconds | | +| GOOGLE_API_CLIENT_UPLOAD_RATE_BASE_INTERVAL | Initial interval in seconds between tries for rate limit errors on upload, default is 2| | +| GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_INTERVAL | The maximum interval in seconds that any individual retry can reach, default is 300 | | +| GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_ELAPSED_TIME | The maximum amount of total time in seconds that code is allowed to keep being retried, default 900 (15 min)| | ### Miscellaneous settings | Name | Description | diff --git a/lcms-engine.gemspec b/lcms-engine.gemspec index 6880d0ae..e5a3bae8 100644 --- a/lcms-engine.gemspec +++ b/lcms-engine.gemspec @@ -52,15 +52,15 @@ Gem::Specification.new do |s| s.add_dependency 'fog-aws', '~> 3.5', '>= 3.5.2' s.add_dependency 'font-awesome-sass', '~> 5.12' s.add_dependency 'foundation-rails', '~> 6.6.2', '>= 6.6.1' - s.add_dependency 'google-apis-drive_v3', '~> 0.46' - s.add_dependency 'google-apis-script_v1', '~> 0.21' + s.add_dependency 'google-apis-drive_v3', '~> 0.63.0' + s.add_dependency 'google-apis-script_v1', '~> 0.27.0' s.add_dependency 'hiredis', '~> 0.6.3' s.add_dependency 'httparty', '~> 0.18' s.add_dependency 'jbuilder', '~> 2.10' s.add_dependency 'jquery-rails', '~> 4.3', '>= 4.3.5' s.add_dependency 'js-routes', '~> 1.4', '>= 1.4.9' - s.add_dependency 'lt-google-api', '~> 0.3' - s.add_dependency 'lt-lcms', '~> 0.6' + # s.add_dependency 'lt-google-api', '~> 0.3' + # s.add_dependency 'lt-lcms', '~> 0.6' s.add_dependency 'migration_data', '~> 0.6' s.add_dependency 'mini_magick', '~> 4.10', '>= 4.10.1' s.add_dependency 'nested_form', '~> 0.3.2' diff --git a/lib/doc_template/document.rb b/lib/doc_template/document.rb index f258335a..e475cd2b 100644 --- a/lib/doc_template/document.rb +++ b/lib/doc_template/document.rb @@ -181,7 +181,7 @@ def parse_node(node) context_type: @opts[:context_type], data: parsed_tag.tag_data, materials: parsed_tag.materials, - optional: (parsed_tag.try(:optional?) || false), + optional: parsed_tag.try(:optional?) || false, placeholder: parsed_tag.placeholder, part_type: tag_name.underscore } diff --git a/lib/doc_template/objects/activity_metadata.rb b/lib/doc_template/objects/activity_metadata.rb index b7d6d2eb..da57ba06 100644 --- a/lib/doc_template/objects/activity_metadata.rb +++ b/lib/doc_template/objects/activity_metadata.rb @@ -35,14 +35,14 @@ class Activity attribute :title, String, default: ->(a, _) { a.activity_title } attribute :time, Integer, default: ->(a, _) { a.activity_time } - attribute :material_ids, Array[Integer], default: [] + attribute :material_ids, [Integer], default: [] def activity_standard_info standard_info [activity_standard, activity_mathematical_practice] end end - attribute :children, Array[Activity] + attribute :children, [Activity] attribute :idx, Integer attribute :task_counter, Hash[String => Integer], default: {} # rubocop:disable Style/HashConversion diff --git a/lib/doc_template/objects/agenda_metadata.rb b/lib/doc_template/objects/agenda_metadata.rb index 3eb20058..c5d46ffa 100644 --- a/lib/doc_template/objects/agenda_metadata.rb +++ b/lib/doc_template/objects/agenda_metadata.rb @@ -23,7 +23,7 @@ class MetaData class Section include Virtus.model - attribute :icons, Array[String], default: [] + attribute :icons, [String], default: [] attribute :materials, String attribute :metacognition, MetaCognition attribute :metadata, MetaData @@ -40,13 +40,13 @@ class Section attribute :time, Integer, default: ->(s, _) { s.metadata.time } attribute :use_color, Boolean, default: false - attribute :material_ids, Array[Integer], default: [] + attribute :material_ids, [Integer], default: [] end class Group include Virtus.model - attribute :children, Array[Section] + attribute :children, [Section] attribute :metadata, MetaData attribute :title, String @@ -57,10 +57,10 @@ class Group attribute :level, Integer, default: 1 attribute :time, Integer, default: ->(g, _) { g.metadata.time } - attribute :material_ids, Array[Integer], default: [] + attribute :material_ids, [Integer], default: [] end - attribute :children, Array[Group] + attribute :children, [Group] def self.build_from(data) # rubocop:disable Metrics/AbcSize copy = Marshal.load Marshal.dump(data) diff --git a/lib/doc_template/objects/sections_metadata.rb b/lib/doc_template/objects/sections_metadata.rb index 97e9458a..f0f67aff 100644 --- a/lib/doc_template/objects/sections_metadata.rb +++ b/lib/doc_template/objects/sections_metadata.rb @@ -10,7 +10,7 @@ class Section include Virtus.model include DocTemplate::Objects::MetadataHelpers - attribute :children, Array[DocTemplate::Objects::ActivityMetadata::Activity] + attribute :children, [DocTemplate::Objects::ActivityMetadata::Activity] attribute :summary, String attribute :time, Integer, default: 0 attribute :title, String @@ -22,7 +22,7 @@ class Section attribute :level, Integer, default: 1 attribute :anchor, String, default: ->(a, _) { DocTemplate::Objects::MetadataHelpers.build_anchor_from(a) } - attribute :material_ids, Array[Integer], default: [] + attribute :material_ids, [Integer], default: [] def add_activity(activity) self.time += activity.time.to_i @@ -35,7 +35,7 @@ def section_standard_info end end - attribute :children, Array[Section] + attribute :children, [Section] attribute :idx, Integer def self.build_from(data, template_type) diff --git a/lib/doc_template/objects/toc_metadata.rb b/lib/doc_template/objects/toc_metadata.rb index 4ed38839..a168248a 100644 --- a/lib/doc_template/objects/toc_metadata.rb +++ b/lib/doc_template/objects/toc_metadata.rb @@ -9,9 +9,9 @@ class Heading include Virtus.model attribute :anchor, String - attribute :children, Array[Heading], default: [] + attribute :children, [Heading], default: [] attribute :level, Integer - attribute :material_ids, Array[Integer], default: [] + attribute :material_ids, [Integer], default: [] attribute :optional, Boolean, default: false attribute :priority, Integer, default: 0 attribute :standard, String, default: '' @@ -45,7 +45,7 @@ def time_with(excludes) end end - attribute :children, Array[Heading], default: [] + attribute :children, [Heading], default: [] attribute :priority, Integer, default: 0 attribute :total_time, Integer, default: ->(t, _) { t.children.sum(&:time) } diff --git a/lib/document_exporter/gdoc/base.rb b/lib/document_exporter/gdoc/base.rb index 3d927322..6f7e6eba 100644 --- a/lib/document_exporter/gdoc/base.rb +++ b/lib/document_exporter/gdoc/base.rb @@ -12,10 +12,10 @@ class Base < DocumentExporter::Base options: { open_timeout_sec: GOOGLE_API_CLIENT_UPLOAD_TIMEOUT, read_timeout_sec: GOOGLE_API_CLIENT_UPLOAD_TIMEOUT, - retries: GOOGLE_API_CLIENT_UPLOAD_RETRIES, send_timeout_sec: GOOGLE_API_CLIENT_UPLOAD_TIMEOUT } }.freeze + GOOGLE_API_RATE_RETRIABLE_ERRORS = [Google::Apis::ServerError, Google::Apis::RateLimitError].freeze VERSION_RE = /_v\d+$/i.freeze attr_reader :document, :options @@ -52,11 +52,24 @@ def export upload_source: StringIO.new(content) }.merge(GOOGLE_API_UPLOAD_OPTIONS) - @id = if file_id.blank? - drive_service.service.create_file(metadata, params) - else - drive_service.service.update_file(file_id, metadata, params) - end.id + # use general API settings for inner API call and wrap it with additional retry logic + @id = Retriable.retriable(base_interval: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_BASE_INTERVAL', 5).to_i, + multiplier: ENV.fetch( + 'GOOGLE_API_CLIENT_UPLOAD_RATE_MULTIPLIER', 2 + ).to_f, + max_interval: ENV.fetch( + 'GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_INTERVAL', 300 + ).to_i, + max_elapsed_time: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_ELAPSED_TIME', + 900).to_i, + on_retry: log_retries(file_id, metadata), + tries: GOOGLE_API_CLIENT_UPLOAD_RETRIES) do + if file_id.present? + drive_service.service.update_file(file_id, metadata, params) + else + drive_service.service.create_file(metadata, params) + end.id + end post_processing @@ -79,11 +92,23 @@ def export_to(folder_id, file_id: nil) upload_source: StringIO.new(content) }.merge(GOOGLE_API_UPLOAD_OPTIONS) - @id = if file_id.present? - drive_service.service.update_file(file_id, metadata, params) - else - drive_service.service.create_file(metadata, params) - end.id + @id = Retriable.retriable(base_interval: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_BASE_INTERVAL', 5).to_i, + multiplier: ENV.fetch( + 'GOOGLE_API_CLIENT_UPLOAD_RATE_MULTIPLIER', 2 + ).to_f, + max_interval: ENV.fetch( + 'GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_INTERVAL', 300 + ).to_i, + max_elapsed_time: ENV.fetch('GOOGLE_API_CLIENT_UPLOAD_RATE_MAX_ELAPSED_TIME', + 900).to_i, + on_retry: log_retries(file_id, metadata), + tries: GOOGLE_API_CLIENT_UPLOAD_RETRIES) do + if file_id.present? + drive_service.service.update_file(file_id, metadata, params) + else + drive_service.service.create_file(metadata, params) + end.id + end post_processing @@ -128,7 +153,7 @@ def gdoc_folder def gdoc_folder_tmp(material_ids) file_ids = material_ids.map do |id| - document.links['materials']&.dig(id.to_s)&.dig('gdoc')&.gsub(/.*id=/, '') + document.links['materials']&.dig(id.to_s, 'gdoc')&.gsub(/.*id=/, '') end @options[:subfolders] = [self.class::FOLDER_NAME] @@ -136,6 +161,20 @@ def gdoc_folder_tmp(material_ids) self end + def log_retries(file_id, metadata) + proc do |exception, try, elapsed_time, next_interval| + msg = "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds " \ + "and #{next_interval} seconds until the next try. " \ + "File ID: #{file_id}, Metadata: #{metadata.inspect}" + Rails.logger.error(msg) + if defined?(Airbrake) && ENV.fetch('AIRBRAKE_UPLOAD_ERRORS_NOTIFY', 'off') == 'on' + ::Airbrake.notify_sync(Airbrake.build_notice(msg)) do |notice| + notice[:context][:severity] = 'warning' + end + end + end + end + def post_processing Retriable.retriable(base_interval: 5, tries: 10) do Lcms::Engine::Google::ScriptService.new(document).execute(@id) diff --git a/lib/lcms/engine/version.rb b/lib/lcms/engine/version.rb index d510935b..eff434c1 100644 --- a/lib/lcms/engine/version.rb +++ b/lib/lcms/engine/version.rb @@ -2,7 +2,7 @@ module Lcms module Engine - VERSION = '0.5.6' + VERSION = '0.5.7' RAILS_5_VERSION = 5.2 end end diff --git a/lib/lti/thin_common_cartridge.rb b/lib/lti/thin_common_cartridge.rb index d6976e60..77a42221 100644 --- a/lib/lti/thin_common_cartridge.rb +++ b/lib/lti/thin_common_cartridge.rb @@ -7,6 +7,10 @@ module Lti class ThinCommonCartridge attr_reader :links + LTI_LINK_RESOURCE_TYPE = 'imsbasiclti_xmlv1p0' + MANIFEST_FILEPATH = ::Lcms::Engine::Engine.root.join 'lib', 'lti', 'xml', 'imsmanifest.xml' + LTI_LINK_FILEPATH = ::Lcms::Engine::Engine.root.join 'lib', 'lti', 'xml', 'lti_link.xml' + def initialize(items) @items = Array.wrap items @xml = File.open(File.expand_path MANIFEST_FILEPATH) { |f| Nokogiri::XML(f) } @@ -28,10 +32,6 @@ def manifest private - LTI_LINK_RESOURCE_TYPE = 'imsbasiclti_xmlv1p0' - MANIFEST_FILEPATH = ::Lcms::Engine::Engine.root.join 'lib', 'lti', 'xml', 'imsmanifest.xml' - LTI_LINK_FILEPATH = ::Lcms::Engine::Engine.root.join 'lib', 'lti', 'xml', 'lti_link.xml' - attr_reader :items, :link_template, :xml def add_item(item, parent) diff --git a/lib/standard_importer.rb b/lib/standard_importer.rb index da4b3f81..557c1080 100644 --- a/lib/standard_importer.rb +++ b/lib/standard_importer.rb @@ -21,6 +21,7 @@ def run private RE_EMPHASIS = /^\(\+\)\s?/.freeze + private_constant :RE_EMPHASIS attr_reader :source_file, :subject diff --git a/lib/tasks/cleanup.rake b/lib/tasks/cleanup.rake index 99e8d167..1a19215d 100644 --- a/lib/tasks/cleanup.rake +++ b/lib/tasks/cleanup.rake @@ -52,7 +52,7 @@ namespace :cleanup do # rubocop:disable Metrics/BlockLength folder_id = ENV.fetch('GOOGLE_APPLICATION_PREVIEW_FOLDER_ID') service = Google::Apis::DriveV3::DriveService.new - service.authorization = ::Lt::Google::Api::Auth::Cli.new.credentials + service.authorization = Lt::Google::Api::Auth::Cli.new.credentials service .list_files(q: "'#{folder_id}' in parents") .files.each { |file| service.delete_file file.id } diff --git a/lib/tasks/cloud66.rake b/lib/tasks/cloud66.rake index 2bdf4f2e..5258a40e 100644 --- a/lib/tasks/cloud66.rake +++ b/lib/tasks/cloud66.rake @@ -2,7 +2,7 @@ namespace :cloud66 do # rubocop:disable Metrics/BlockLength desc 'Post-symlink hook tasks for Cloud66.' - task after_symlink: %i(environment db:migrate db:seed) + task after_symlink: %i(environment db:migrate) namespace :robots do desc 'Add robots.txt to public' diff --git a/lib/tasks/google.rake b/lib/tasks/google.rake index 8cb72295..6af077b1 100644 --- a/lib/tasks/google.rake +++ b/lib/tasks/google.rake @@ -7,7 +7,7 @@ require 'lt/google/api/auth/cli' namespace :google do desc 'Set up google credentials. Specify `domain` argument which will be used as a base for redirect URI' task :setup_auth, [:domain] => [:environment] do |_task, args| - service = ::Lt::Google::Api::Auth::Cli.new + service = Lt::Google::Api::Auth::Cli.new # Check if there is existing auth token if service.credentials.present? @@ -32,7 +32,7 @@ namespace :google do code = $stdin.gets.to_s.strip authorizer.get_and_store_credentials_from_code( - user_id: ::Lt::Google::Api::Auth::Cli::USER_ID, + user_id: Lt::Google::Api::Auth::Cli::USER_ID, code: code, base_url: args[:domain] ) diff --git a/spec/controllers/admin/curriculums_controller_spec.rb b/spec/controllers/admin/curriculums_controller_spec.rb index c36f1b4c..8f94d4c9 100644 --- a/spec/controllers/admin/curriculums_controller_spec.rb +++ b/spec/controllers/admin/curriculums_controller_spec.rb @@ -22,7 +22,7 @@ it 'returns children of the requested resource' do JSON.parse(subject.body).each do |data| - child = ::Lcms::Engine::Resource.find(data['id']) + child = Lcms::Engine::Resource.find(data['id']) expect(child.parent_id).to eq id end end diff --git a/spec/controllers/admin/documents_controller_spec.rb b/spec/controllers/admin/documents_controller_spec.rb index 0d46cdee..f415f6f1 100644 --- a/spec/controllers/admin/documents_controller_spec.rb +++ b/spec/controllers/admin/documents_controller_spec.rb @@ -45,9 +45,9 @@ before do allow(controller).to receive(:google_credentials).and_return(credentials) - allow(::Lt::Google::Api::Drive).to \ + allow(Lt::Google::Api::Drive).to \ receive(:folder_id_for).with(link).and_return(folder_id) - allow(::Lt::Google::Api::Drive).to \ + allow(Lt::Google::Api::Drive).to \ receive_message_chain(:new, :list_file_ids_in, :map).and_return(file_ids) end diff --git a/spec/controllers/admin/materials_controller_spec.rb b/spec/controllers/admin/materials_controller_spec.rb index 1fe8664b..cfd65abf 100644 --- a/spec/controllers/admin/materials_controller_spec.rb +++ b/spec/controllers/admin/materials_controller_spec.rb @@ -45,9 +45,9 @@ before do allow(controller).to receive(:google_credentials).and_return(credentials) - allow(::Lt::Google::Api::Drive).to \ + allow(Lt::Google::Api::Drive).to \ receive(:folder_id_for).with(link).and_return(folder_id) - allow(::Lt::Google::Api::Drive).to \ + allow(Lt::Google::Api::Drive).to \ receive_message_chain(:new, :list_file_ids_in, :map).and_return(file_ids) end diff --git a/spec/dummy/config/environments/production.rb b/spec/dummy/config/environments/production.rb index 83763214..6327c925 100644 --- a/spec/dummy/config/environments/production.rb +++ b/spec/dummy/config/environments/production.rb @@ -80,7 +80,7 @@ config.active_support.deprecation = :notify # Use default logging formatter so that PID and timestamp are not suppressed. - config.log_formatter = ::Logger::Formatter.new + config.log_formatter = Logger::Formatter.new # Use a different logger for distributed setups. # require 'syslog/logger' diff --git a/spec/factories/resources.rb b/spec/factories/resources.rb index 7afb8899..f4b915b6 100644 --- a/spec/factories/resources.rb +++ b/spec/factories/resources.rb @@ -1,14 +1,14 @@ # frozen_string_literal: true FactoryBot.define do - factory :resource, class: ::Lcms::Engine::Resource do - curriculum { ::Lcms::Engine::Curriculum.default || create(:curriculum) } + factory :resource, class: Lcms::Engine::Resource do + curriculum { Lcms::Engine::Curriculum.default || create(:curriculum) } curriculum_type { 'lesson' } metadata do { subject: 'ela', grade: 'grade 2', module: 'module 1', unit: 'unit 1', lesson: 'lesson 1' } end - resource_type { ::Lcms::Engine::Resource.resource_types[:resource] } + resource_type { Lcms::Engine::Resource.resource_types[:resource] } title { 'Test Resource' } tree { true } url { 'Resource URL' } diff --git a/spec/features/admin/materials/add_material_spec.rb b/spec/features/admin/materials/add_material_spec.rb index c7572f4d..0b0d9254 100644 --- a/spec/features/admin/materials/add_material_spec.rb +++ b/spec/features/admin/materials/add_material_spec.rb @@ -35,8 +35,8 @@ # stub GDoc download file_content = File.read File.join(sample_path, data[:file_name]) - base_klass = ::Lt::Lcms::Lesson::Downloader::Base - gdoc_klass = ::Lt::Lcms::Lesson::Downloader::Gdoc + base_klass = Lt::Lcms::Lesson::Downloader::Base + gdoc_klass = Lt::Lcms::Lesson::Downloader::Gdoc allow_any_instance_of(base_klass).to receive(:file).and_return(downloaded_file.new(nil, nil, data[:file_name])) allow_any_instance_of(gdoc_klass).to receive_message_chain(:download, :content).and_return(file_content) @@ -57,7 +57,7 @@ expect(page).to have_field :material_form_link # stub PDF download - base_klass = ::Lt::Lcms::Lesson::Downloader::Base + base_klass = Lt::Lcms::Lesson::Downloader::Base allow_any_instance_of(base_klass).to receive(:file).and_return(downloaded_file.new(nil, nil, data[:file_name])) file_content = File.read File.join(sample_path, data[:file_name]) diff --git a/spec/lib/doc_template/tables/base_spec.rb b/spec/lib/doc_template/tables/base_spec.rb index 42205e26..6d640e30 100644 --- a/spec/lib/doc_template/tables/base_spec.rb +++ b/spec/lib/doc_template/tables/base_spec.rb @@ -17,7 +17,7 @@ it 'prepares the tag to render correctly' do expect(Nokogiri::HTML).to receive(:fragment).with(html).and_return(Nokogiri::HTML.fragment(html)) expect(DocTemplate::Document).to receive_message_chain(:parse, :render).and_return(html_rendered) - expect(::Sanitize).to receive(:fragment).with(html_rendered, elements: opts[:keep_elements]).and_return('') + expect(Sanitize).to receive(:fragment).with(html_rendered, elements: opts[:keep_elements]).and_return('') subject end @@ -27,7 +27,7 @@ it 'skips sanitization' do expect(Nokogiri::HTML).to receive(:fragment).with(html).and_return(Nokogiri::HTML.fragment(html)) expect(DocTemplate::Document).to receive_message_chain(:parse, :render).and_return(html_rendered) - expect(::Sanitize).to_not receive(:fragment) + expect(Sanitize).to_not receive(:fragment) subject end end @@ -78,7 +78,7 @@ describe '#parse_in_context' do let(:contexts) { DocTemplate::DEFAULTS[:context_types] } - let(:metadata) { ::DocTemplate::Objects::BaseMetadata.new } + let(:metadata) { DocTemplate::Objects::BaseMetadata.new } let(:opts) { { explicit_render: false, metadata: metadata } } subject { described_class.new.parse_in_context html, opts } diff --git a/spec/lib/doc_template/tags/base_tag_spec.rb b/spec/lib/doc_template/tags/base_tag_spec.rb index 6680a92b..cfb61cbb 100644 --- a/spec/lib/doc_template/tags/base_tag_spec.rb +++ b/spec/lib/doc_template/tags/base_tag_spec.rb @@ -6,12 +6,12 @@ class BaseSpecTag < DocTemplate::Tags::BaseTag TAG_NAME = 'image' end -describe ::DocTemplate::Tags::BaseTag do +describe DocTemplate::Tags::BaseTag do describe '.tag_with_html_regexp' do let(:content) do <<~HTML -

[image:OP.PT.L13.014, 50, Esta imagen +

[image:OP.PT.L13.014, 50, Esta imagen muestra ruedas de carretas que han sido enterradas por la tierra movida por el viento durante una tormenta de polvo.] Los Estados Unidos tienen una historia trágica con sedimentos en movimiento por el viento Durante la década de 1930, gran parte de los Estados Unidos estaba pasando por una sequía.

diff --git a/spec/lib/doc_template/tags/gls_tag_spec.rb b/spec/lib/doc_template/tags/gls_tag_spec.rb index bae707aa..88375d5a 100644 --- a/spec/lib/doc_template/tags/gls_tag_spec.rb +++ b/spec/lib/doc_template/tags/gls_tag_spec.rb @@ -9,8 +9,8 @@ end let(:original_content) do <<-HTML -

Every year thereafter Hindus in Ayodhya repeated the - custom Define custom as something that is done as a tradition, +

Every year thereafter Hindus in Ayodhya repeated the + custom Define custom as something that is done as a tradition, year after year, over and over again.Define custom as something that is done as a tradition, year after year, over and over again.Define custom as something that is done as a tradition, year after year, over and over again.Define custom as something that is done as a tradition, year after year, over and over diff --git a/spec/lib/lt/lcms/metadata/context_spec.rb b/spec/lib/lt/lcms/metadata/context_spec.rb index 7a021366..e53bf39a 100644 --- a/spec/lib/lt/lcms/metadata/context_spec.rb +++ b/spec/lib/lt/lcms/metadata/context_spec.rb @@ -2,7 +2,7 @@ require 'rails_helper' -describe ::Lt::Lcms::Metadata::Context do +describe Lt::Lcms::Metadata::Context do shared_examples 'reordable' do |curriculum_type| subject { described_class.send("update_#{curriculum_type}s_level_position_for", children) } diff --git a/spec/lib/lt/lcms/metadata/service_spec.rb b/spec/lib/lt/lcms/metadata/service_spec.rb index 7bc82a92..ea5dc52f 100644 --- a/spec/lib/lt/lcms/metadata/service_spec.rb +++ b/spec/lib/lt/lcms/metadata/service_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' # TODO: Need to refactor completely -describe ::Lt::Lcms::Metadata::Service do +describe Lt::Lcms::Metadata::Service do let(:html_document) do <<-HTML diff --git a/spec/models/search/elastic_search_document_spec.rb b/spec/models/search/elastic_search_document_spec.rb index 9f9460ce..69277360 100644 --- a/spec/models/search/elastic_search_document_spec.rb +++ b/spec/models/search/elastic_search_document_spec.rb @@ -15,7 +15,7 @@ describe '.search' do let(:options) { {} } let(:repository) { double Lcms::Engine::Search::Repository } - let(:term) { ::Faker::Lorem.word } + let(:term) { Faker::Lorem.word } before do allow(described_class).to receive(:repository).and_return(repository) diff --git a/spec/services/document_build_service_spec.rb b/spec/services/document_build_service_spec.rb index d58c61fd..d3827824 100644 --- a/spec/services/document_build_service_spec.rb +++ b/spec/services/document_build_service_spec.rb @@ -35,7 +35,7 @@ subject { service.build_for url } before do - allow(::Lt::Lcms::Lesson::Downloader::Gdoc).to receive(:new).with(credentials, url, {}).and_return(downloader) + allow(Lt::Lcms::Lesson::Downloader::Gdoc).to receive(:new).with(credentials, url, {}).and_return(downloader) # NOTE: Think about wrapping all this into instance_double allow(downloader).to receive(:download).and_return(downloader) diff --git a/spec/services/html_sanitizer_spec.rb b/spec/services/html_sanitizer_spec.rb index 63e2362a..18f040ec 100644 --- a/spec/services/html_sanitizer_spec.rb +++ b/spec/services/html_sanitizer_spec.rb @@ -9,10 +9,10 @@ <<-HTML

- @@ -57,9 +57,9 @@ it 'replace sup/sub aligned spans to sup/sub' do expect(subject).to include('80', '79', '24', '23', 'font-style') expect(subject).not_to include('vertical-align') - expect(subject.scan(/span/).size).to eq 10 - expect(subject.scan(/sub/).size).to eq 4 - expect(subject.scan(/sup/).size).to eq 4 + expect(subject.scan('span').size).to eq 10 + expect(subject.scan('sub').size).to eq 4 + expect(subject.scan('sup').size).to eq 4 end end end
+

- d[a][b]ocument-metadata + d[a][b]ocument-metadata [c]