From 170029e3904e5cf19acc239c1dc38a7399b52836 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:03:09 -0700 Subject: [PATCH 01/11] expand timeout for slow fcrepo queries --- config/fedora.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/fedora.yml b/config/fedora.yml index 399e0bf4e..102e80b50 100644 --- a/config/fedora.yml +++ b/config/fedora.yml @@ -18,3 +18,4 @@ production: password: fedoraAdmin url: http://<%= ENV['FCREPO_HOST'] || 'localhost' %>:<%= ENV['FCREPO_PORT'] || 8080 %>/<%= ENV['FCREPO_REST_PATH'] || 'rest' %> base_path: <%= ENV['FCREPO_BASE_PATH'] || '/prod' %> + request: { timeout: 600, open_timeout: 60} From 928cb7ad764f92bff90f243250af7dc1e5cc68c9 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:06:11 -0700 Subject: [PATCH 02/11] pg schema extensions, active fedora timeout increase, solr post instead of get --- config/database.yml | 1 + config/initializers/active_fedora_override.rb | 11 ++++++ config/initializers/apartment.rb | 2 + lib/active_fedora/solr_service_decorator.rb | 20 ++++++++++ lib/tasks/db_enhancements.rake | 24 ++++++++++++ .../find_ids_by_model_decorator.rb | 39 +++++++++++++++++++ 6 files changed, 97 insertions(+) create mode 100644 config/initializers/active_fedora_override.rb create mode 100644 lib/active_fedora/solr_service_decorator.rb create mode 100644 lib/tasks/db_enhancements.rake create mode 100644 lib/wings/services/custom_queries/find_ids_by_model_decorator.rb diff --git a/config/database.yml b/config/database.yml index 865dc1c5b..bc235ae14 100644 --- a/config/database.yml +++ b/config/database.yml @@ -2,6 +2,7 @@ login: &login adapter: <%= ENV['DB_ADAPTER'] || 'postgresql' %> + schema_search_path: "public,shared_extensions" host: <%= ENV['DB_HOST'] %> username: <%= ENV['DB_USER'] %> password: <%= ENV['DB_PASSWORD'] %> diff --git a/config/initializers/active_fedora_override.rb b/config/initializers/active_fedora_override.rb new file mode 100644 index 000000000..58ec7fe04 --- /dev/null +++ b/config/initializers/active_fedora_override.rb @@ -0,0 +1,11 @@ +# Based on https://github.com/samvera/hyrax/issues/4581#issuecomment-843085122 + +# Monkey-patch to short circuit ActiveModel::Dirty which attempts to load the whole master files ordered list when calling nodes_will_change! +# This leads to a stack level too deep exception when attempting to delete a master file from a media object on the manage files step. +# See https://github.com/samvera/active_fedora/pull/1312/commits/7c8bbbefdacefd655a2ca653f5950c991e1dc999#diff-28356c4daa0d55cbaf97e4269869f510R100-R103 +ActiveFedora::Aggregation::ListSource.class_eval do + def attribute_will_change!(attr) + return super unless attr == 'nodes' + attributes_changed_by_setter[:nodes] = true + end +end diff --git a/config/initializers/apartment.rb b/config/initializers/apartment.rb index e523b4cdf..a2647571d 100644 --- a/config/initializers/apartment.rb +++ b/config/initializers/apartment.rb @@ -39,6 +39,8 @@ # Any schemas added here will be available along with your selected Tenant. # # config.persistent_schemas = %w{ hstore } + config.persistent_schemas = ['shared_extensions'] + # <== PostgreSQL only options # diff --git a/lib/active_fedora/solr_service_decorator.rb b/lib/active_fedora/solr_service_decorator.rb new file mode 100644 index 000000000..b50a5749e --- /dev/null +++ b/lib/active_fedora/solr_service_decorator.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# OVERRIDE: class ActiveFedora::SolrService from Fedora 12.1.1 +module ActiveFedora + module SolrServiceDecorator + # Get the count of records that match the query + # @param [String] query a solr query + # @param [Hash] args arguments to pass through to `args' param of SolrService.query + # (note that :rows will be overwritten to 0) + # @return [Integer] number of records matching + # + # OVERRIDE: use `post` rather than `get` to handle larger query sizes + def count(query, args = {}) + args = args.merge(rows: 0) + SolrService.post(query, args)['response']['numFound'].to_i + end + end +end + +ActiveFedora::SolrService.singleton_class.send(:prepend, ActiveFedora::SolrServiceDecorator) diff --git a/lib/tasks/db_enhancements.rake b/lib/tasks/db_enhancements.rake new file mode 100644 index 000000000..551db56c3 --- /dev/null +++ b/lib/tasks/db_enhancements.rake @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +namespace :db do + desc 'Also create shared_extensions Schema' + task extensions: :environment do + # Create Schema + ActiveRecord::Base.connection.execute 'CREATE SCHEMA IF NOT EXISTS shared_extensions;' + # Enable Hstore + ActiveRecord::Base.connection.execute 'CREATE EXTENSION IF NOT EXISTS HSTORE SCHEMA shared_extensions;' + # Enable UUID-OSSP + ActiveRecord::Base.connection.execute 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp" SCHEMA shared_extensions;' + ActiveRecord::Base.connection.execute 'CREATE EXTENSION IF NOT EXISTS "pgcrypto" SCHEMA shared_extensions;' + # Grant usage to public + ActiveRecord::Base.connection.execute 'GRANT usage ON SCHEMA shared_extensions to public;' + end +end + +Rake::Task["db:create"].enhance do + Rake::Task["db:extensions"].invoke +end + +Rake::Task["db:test:purge"].enhance do + Rake::Task["db:extensions"].invoke +end diff --git a/lib/wings/services/custom_queries/find_ids_by_model_decorator.rb b/lib/wings/services/custom_queries/find_ids_by_model_decorator.rb new file mode 100644 index 000000000..7cbaaef29 --- /dev/null +++ b/lib/wings/services/custom_queries/find_ids_by_model_decorator.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# OVERRIDE Hyrax 3.5 to use post instead of get for Solr requests + +module Wings + module CustomQueries + ## + # @see https://github.com/samvera/valkyrie/wiki/Queries#custom-queries + # @see Hyrax::CustomQueries::FindIdsByModel + module FindIdsByModelDecorator + ## + # @note uses solr to do the lookup + # + # @param model [Class] + # @param ids [Enumerable<#to_s>, Symbol] + # + # @return [Enumerable] + def find_ids_by_model(model:, ids: :all) + return enum_for(:find_ids_by_model, model: model, ids: ids) unless block_given? + model_name = ModelRegistry.lookup(model).model_name + + solr_query = "_query_:\"{!raw f=has_model_ssim}#{model_name}\"" + solr_response = ActiveFedora::SolrService.post(solr_query, fl: 'id', rows: @query_rows)['response'] + + loop do + response_docs = solr_response['docs'] + response_docs.select! { |doc| ids.include?(doc['id']) } unless ids == :all + + response_docs.each { |doc| yield doc['id'] } + + break if (solr_response['start'] + solr_response['docs'].count) >= solr_response['numFound'] + solr_response = ActiveFedora::SolrService.post(solr_query, fl: 'id', rows: @query_rows, start: solr_response['start'] + @query_rows)['response'] + end + end + end + end +end + +Wings::CustomQueries::FindIdsByModel.prepend Wings::CustomQueries::FindIdsByModelDecorator From 5ea989565cbff3bc967c5e44ba54e8008f3d2608 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:07:45 -0700 Subject: [PATCH 03/11] make indexing records more resiliant and uniform --- app/jobs/application_job.rb | 4 ++ app/jobs/reindex_collections_job.rb | 3 +- app/jobs/reindex_file_sets_job.rb | 9 +++++ app/jobs/reindex_item_job.rb | 7 ++++ app/jobs/reindex_works_job.rb | 6 ++- lib/tasks/index.rake | 60 +++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 app/jobs/reindex_file_sets_job.rb create mode 100644 app/jobs/reindex_item_job.rb create mode 100644 lib/tasks/index.rake diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index d92ffddcb..70d716899 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,4 +1,8 @@ # frozen_string_literal: true class ApplicationJob < ActiveJob::Base + # limit to 5 attempts + retry_on StandardError, wait: :exponentially_longer, attempts: 5 do |_job, _exception| + # Log error, do nothing, etc. + end end diff --git a/app/jobs/reindex_collections_job.rb b/app/jobs/reindex_collections_job.rb index e1ef281ba..422d1c8de 100644 --- a/app/jobs/reindex_collections_job.rb +++ b/app/jobs/reindex_collections_job.rb @@ -3,8 +3,7 @@ class ReindexCollectionsJob < ApplicationJob def perform Collection.find_each do |collection| - collection.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) - collection.update_index + ReindexItemJob.perform_later(collection) end end end diff --git a/app/jobs/reindex_file_sets_job.rb b/app/jobs/reindex_file_sets_job.rb new file mode 100644 index 000000000..0e2e7931d --- /dev/null +++ b/app/jobs/reindex_file_sets_job.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class ReindexFileSetsJob < ApplicationJob + def perform + FileSet.find_each do |file_set| + ReindexItemJob.perform_later(file_set) + end + end +end diff --git a/app/jobs/reindex_item_job.rb b/app/jobs/reindex_item_job.rb new file mode 100644 index 000000000..d44051be0 --- /dev/null +++ b/app/jobs/reindex_item_job.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class ReindexItemJob < ApplicationJob + def perform(item) + item.update_index + end +end diff --git a/app/jobs/reindex_works_job.rb b/app/jobs/reindex_works_job.rb index 6aee04ba6..56adcf2d0 100644 --- a/app/jobs/reindex_works_job.rb +++ b/app/jobs/reindex_works_job.rb @@ -2,8 +2,10 @@ class ReindexWorksJob < ApplicationJob def perform - Hyrax.config.registered_curation_concern_types.each do |work_type| - work_type.constantize.find_each(&:update_index) + Site.instance.available_works.each do |work_type| + work_type.constantize.find_each do |work| + ReindexItemJob.perform_later(work) + end end end end diff --git a/lib/tasks/index.rake b/lib/tasks/index.rake new file mode 100644 index 000000000..c02885af2 --- /dev/null +++ b/lib/tasks/index.rake @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'ruby-progressbar' + +desc "reindex just the works in the background" +task index_works: :environment do + Account.find_each do |account| + puts "=============== #{account.name}============" + next if account.name == "search" + switch!(account) + in_each_account do + ReindexWorksJob.perform_later + end + end +end + +desc "reindex just the collections in the background" +task index_collections: :environment do + Account.find_each do |account| + puts "=============== #{account.name}============" + next if account.name == "search" + switch!(account) + in_each_account do + ReindexCollectionsJob.perform_later + end + end +end + +desc "reindex just the admin_sets in the background" +task index_admin_sets: :environment do + Account.find_each do |account| + puts "=============== #{account.name}============" + next if account.name == "search" + switch!(account) + in_each_account do + ReindexAdminSetsJob.perform_later + end + end +end + +desc "reindex just the file_sets in the background" +task index_file_sets: :environment do + Account.find_each do |account| + puts "=============== #{account.name}============" + next if account.name == "search" + switch!(account) + in_each_account do + ReindexFileSetsJob.perform_later + end + end +end + +def in_each_account + Account.find_each do |account| + puts "=============== #{account.name}============" + next if account.name == "search" + switch!(account) + yield + end +end From 3d713e384385d740f909c8c94bbe1501ab038b69 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:13:03 -0700 Subject: [PATCH 04/11] ability to mark during importing for actor stack --- app/actors/hyrax/environment.rb | 25 +++++++++++++++++++ .../bulkrax/object_factory_decorator.rb | 14 +++++++++++ 2 files changed, 39 insertions(+) create mode 100644 app/actors/hyrax/environment.rb create mode 100644 app/factories/bulkrax/object_factory_decorator.rb diff --git a/app/actors/hyrax/environment.rb b/app/actors/hyrax/environment.rb new file mode 100644 index 000000000..814895f87 --- /dev/null +++ b/app/actors/hyrax/environment.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# OVERRIDE Hyrax 2.9 to add in import flag +module Hyrax + module Actors + class Environment + # @param [ActiveFedora::Base] curation_concern work to operate on + # @param [Ability] current_ability the authorizations of the acting user + # @param [ActionController::Parameters] attributes user provided form attributes + def initialize(curation_concern, current_ability, attributes, importing = false) + @curation_concern = curation_concern + @current_ability = current_ability + @attributes = attributes.to_h.with_indifferent_access + @importing = importing + end + + attr_reader :curation_concern, :current_ability, :attributes, :importing + + # @return [User] the user from the current_ability + def user + current_ability.current_user + end + end + end +end diff --git a/app/factories/bulkrax/object_factory_decorator.rb b/app/factories/bulkrax/object_factory_decorator.rb new file mode 100644 index 000000000..928cd8384 --- /dev/null +++ b/app/factories/bulkrax/object_factory_decorator.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Add ability to mark environment as from bulk import +module Bulkrax + module ObjectFactoryDecorator + # @param [Hash] attrs the attributes to put in the environment + # @return [Hyrax::Actors::Environment] + def environment(attrs) + Hyrax::Actors::Environment.new(object, Ability.new(@user), attrs, true) + end + end +end + +::Bulkrax::ObjectFactory.prepend(Bulkrax::ObjectFactoryDecorator) From 9ceca2ba1389eef8c06774bba59e49165881355f Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:14:17 -0700 Subject: [PATCH 05/11] make sure layers are flattened to fix pdf tripple issue --- .../derivatives/processors/image_decorator.rb | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lib/hydra/derivatives/processors/image_decorator.rb diff --git a/lib/hydra/derivatives/processors/image_decorator.rb b/lib/hydra/derivatives/processors/image_decorator.rb new file mode 100644 index 000000000..f9f4145af --- /dev/null +++ b/lib/hydra/derivatives/processors/image_decorator.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Fix PDF tripple issue + +module Hydra + module Derivatives + module Processors + module ImageDecorator + protected + + # When resizing images, it is necessary to flatten any layers, otherwise the background + # may be completely black. This happens especially with PDFs. See #110 + def create_resized_image + create_image do |xfrm| + if size + xfrm.combine_options do |i| + i.flatten + i.resize(size) + end + end + end + end + end + end + end +end + +::Hydra::Derivatives::Processors::Image.prepend(Hydra::Derivatives::Processors::ImageDecorator) From a1f0f29a116d2044ca73357549096f0189288e08 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:15:42 -0700 Subject: [PATCH 06/11] use partent title instead of filename in uv --- .../canvas_builder_decorator.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lib/iiif_manifest/manifest_builder/canvas_builder_decorator.rb diff --git a/lib/iiif_manifest/manifest_builder/canvas_builder_decorator.rb b/lib/iiif_manifest/manifest_builder/canvas_builder_decorator.rb new file mode 100644 index 000000000..777d9b03b --- /dev/null +++ b/lib/iiif_manifest/manifest_builder/canvas_builder_decorator.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# OVERRIDE IIIFManifest v0.5.0 to use the parent's title as the label instead of the filename + +module IIIFManifest + module ManifestBuilderDecorator + module CanvasBuilderDecorator + def apply_record_properties + canvas['@id'] = path + canvas.label = record['parent_title_tesim']&.first || record.to_s + end + end + end +end + +IIIFManifest::ManifestBuilder.prepend(IIIFManifest::ManifestBuilderDecorator) +IIIFManifest::ManifestBuilder::CanvasBuilder.prepend(IIIFManifest::ManifestBuilder::CanvasBuilderDecorator) From 235488ec775c5eebb82f8761d7a1dd8357b521e7 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:16:47 -0700 Subject: [PATCH 07/11] account elevator clear especially when on proprieter view --- app/middleware/account_elevator.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/middleware/account_elevator.rb b/app/middleware/account_elevator.rb index 1bdf38e85..6160b9a6e 100644 --- a/app/middleware/account_elevator.rb +++ b/app/middleware/account_elevator.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'apartment/elevators/generic' # Apartment middleware for switching tenants based on the # CNAME entry for an account. class AccountElevator < Apartment::Elevators::Generic @@ -7,7 +8,7 @@ class AccountElevator < Apartment::Elevators::Generic # @return [String] The tenant to switch to def parse_tenant_name(request) account = Account.from_request(request) - + account || Account.new.reset! # reset everything if no account is present account&.tenant end end From 0ebc0507f7eb2933ad6ce25a4941dc0c4f4215e1 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Thu, 21 Sep 2023 00:09:30 -0700 Subject: [PATCH 08/11] view updates for devise from adventist --- app/views/devise/passwords/new.html.erb | 20 ++++++++++++ .../registrations/registrations/new.html.erb | 26 ++++++++++++++++ app/views/devise/sessions/new.html.erb | 31 +++++++++++++++++++ app/views/devise/shared/_links.html.erb | 2 +- 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 app/views/devise/passwords/new.html.erb create mode 100644 app/views/devise/registrations/registrations/new.html.erb create mode 100644 app/views/devise/sessions/new.html.erb diff --git a/app/views/devise/passwords/new.html.erb b/app/views/devise/passwords/new.html.erb new file mode 100644 index 000000000..d03d87c17 --- /dev/null +++ b/app/views/devise/passwords/new.html.erb @@ -0,0 +1,20 @@ +

Forgot your password?

+
+
+ <%= form_for(resource, as: resource_name, url: password_path(resource_name), html: { method: :post }) do |f| %> + <%= render "devise/shared/error_messages", resource: resource %> + +
+ <%= f.label :email %>
+ <%= f.email_field :email, autofocus: true, autocomplete: "email", class: "form-control" %> +
+ +
+ <%= f.submit "Send me reset password instructions", class: 'btn btn-primary' %> +
+ <% end %> +
+ <%= render "devise/shared/links" %> +
+
+
\ No newline at end of file diff --git a/app/views/devise/registrations/registrations/new.html.erb b/app/views/devise/registrations/registrations/new.html.erb new file mode 100644 index 000000000..4ed469233 --- /dev/null +++ b/app/views/devise/registrations/registrations/new.html.erb @@ -0,0 +1,26 @@ +

<%= t(".sign_up_header") %>

+
+
+ <%= simple_form_for(resource, as: resource_name, url: registration_path(resource_name), html: { class: 'form-horizontal' }) do |f| %> + + <%= f.error_notification %> + + +
+ <%= f.input :display_name, required: true, wrapper: :inline %> + <%= f.input :email, required: true, autofocus: true, wrapper: :inline %> + <%= f.input :password, required: true, wrapper: :inline %> + <%= f.input :password_confirmation, required: true, wrapper: :inline %> +
+ +
+ <%= render 'devise/shared/links' %> +
+
+ <%= f.button :submit, t(".sign_up") %> +
+
+ <% end %> +
+ + diff --git a/app/views/devise/sessions/new.html.erb b/app/views/devise/sessions/new.html.erb new file mode 100644 index 000000000..090e2cbd8 --- /dev/null +++ b/app/views/devise/sessions/new.html.erb @@ -0,0 +1,31 @@ + +

Log in

+
+
+ <%= form_for(resource, as: resource_name, url: session_path(resource_name), class: "form-horizontal") do |f| %> +
+ <%= f.label :email %>
+ <%= f.email_field :email, autofocus: true, class: "form-control" %> +
+ +
+ <%= f.label :password, class: "control-label" %>
+ <%= f.password_field :password, autocomplete: "off", class: "form-control" %> +
+ + <% if devise_mapping.rememberable? -%> +
+ <%= f.check_box :remember_me %> + <%= f.label :remember_me, class: "control-label" %> +
+ <% end -%> + +
+ <%= f.submit "Log in", class: 'btn btn-primary' %> +
+ <% end %> +
+ <%= render "devise/shared/links" %> +
+
+
\ No newline at end of file diff --git a/app/views/devise/shared/_links.html.erb b/app/views/devise/shared/_links.html.erb index caeb8b523..65e56d9bc 100644 --- a/app/views/devise/shared/_links.html.erb +++ b/app/views/devise/shared/_links.html.erb @@ -7,7 +7,7 @@ <% end -%> <%- if devise_mapping.recoverable? && controller_name != 'passwords' %> - <%= link_to t(".forgot_your_password"), new_password_path(resource_name) %>
+ <%= link_to t(".forgot_your_password"), new_password_path(resource_name), class: 'lh-40' %>
<% end -%> <%- if devise_mapping.confirmable? && controller_name != 'confirmations' %> From 485dcec8e93ba9f3c526bc4cb938d2adc682f158 Mon Sep 17 00:00:00 2001 From: Kirk Wang Date: Thu, 21 Sep 2023 08:52:40 -0700 Subject: [PATCH 09/11] Update lib/hydra/derivatives/processors/image_decorator.rb Co-authored-by: Alisha Evans --- lib/hydra/derivatives/processors/image_decorator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hydra/derivatives/processors/image_decorator.rb b/lib/hydra/derivatives/processors/image_decorator.rb index f9f4145af..6c4d610af 100644 --- a/lib/hydra/derivatives/processors/image_decorator.rb +++ b/lib/hydra/derivatives/processors/image_decorator.rb @@ -9,7 +9,7 @@ module ImageDecorator protected # When resizing images, it is necessary to flatten any layers, otherwise the background - # may be completely black. This happens especially with PDFs. See #110 + # may be completely black. This happens especially with PDFs. See https://github.com/samvera/hydra-derivatives/issues/110 def create_resized_image create_image do |xfrm| if size From 0158bd8b70cebac08d78702bb657f1d3c515b28f Mon Sep 17 00:00:00 2001 From: Kirk Wang Date: Fri, 22 Sep 2023 12:54:46 -0700 Subject: [PATCH 10/11] lint fix --- .../services/custom_queries/find_ids_by_model_decorator.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/wings/services/custom_queries/find_ids_by_model_decorator.rb b/lib/wings/services/custom_queries/find_ids_by_model_decorator.rb index 7cbaaef29..2edd1e108 100644 --- a/lib/wings/services/custom_queries/find_ids_by_model_decorator.rb +++ b/lib/wings/services/custom_queries/find_ids_by_model_decorator.rb @@ -29,7 +29,10 @@ def find_ids_by_model(model:, ids: :all) response_docs.each { |doc| yield doc['id'] } break if (solr_response['start'] + solr_response['docs'].count) >= solr_response['numFound'] - solr_response = ActiveFedora::SolrService.post(solr_query, fl: 'id', rows: @query_rows, start: solr_response['start'] + @query_rows)['response'] + solr_response = ActiveFedora::SolrService.post(solr_query, + fl: 'id', + rows: @query_rows, + start: solr_response['start'] + @query_rows)['response'] end end end From 4574ecc10d6b432044b7dcb35c0aedf658521261 Mon Sep 17 00:00:00 2001 From: Rob Kaufman Date: Mon, 25 Sep 2023 14:18:32 -0400 Subject: [PATCH 11/11] iiif rodeo includes (#1985) * iiif rodeo includes * Remove Adventist specific code from application.rb --------- Co-authored-by: Kirk Wang --- config/application.rb | 56 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/config/application.rb b/config/application.rb index 850a2b515..c221d0090 100644 --- a/config/application.rb +++ b/config/application.rb @@ -9,7 +9,34 @@ Bundler.require(*groups) module Hyku + # Providing a common method to ensure consistent UTF-8 encoding. Also removing the tricksy Byte + # Order Marker character which is an invisible 0 space character. + # + # @note In testing, we encountered errors with the file's character encoding + # (e.g. `Encoding::UndefinedConversionError`). The following will force the encoding to + # UTF-8 and replace any invalid or undefined characters from the original encoding with a + # "?". + # + # Given that we still have the original, and this is a derivative, the forced encoding + # should be acceptable. + # + # @param [String] + # @return [String] + # + # @see https://sentry.io/organizations/scientist-inc/issues/3773392603/?project=6745020&query=is%3Aunresolved&referrer=issue-stream + # @see https://github.com/samvera-labs/bulkrax/pull/689 + # @see https://github.com/samvera-labs/bulkrax/issues/688 + # @see https://github.com/scientist-softserv/adventist-dl/issues/179 + def self.utf_8_encode(string) + string + .encode(Encoding.find('UTF-8'), invalid: :replace, undef: :replace, replace: "?") + .delete("\xEF\xBB\xBF") + end + class Application < Rails::Application + # Add this line to load the lib folder first because we need + config.autoload_paths.unshift("#{Rails.root}/lib") + # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. @@ -33,22 +60,33 @@ class Application < Rails::Application end config.to_prepare do - # Allows us to use decorator files in the app directory + + # Add any extra services before IiifPrint::PluggableDerivativeService to enable processing + Hyrax::DerivativeService.services = [IiifPrint::PluggableDerivativeService] + + # When you are ready to use the derivative rodeo instead of the pluggable uncomment the + # following and comment out the preceding Hyrax::DerivativeService.service + # + # Hyrax::DerivativeService.services = [ + # Adventist::TextFileTextExtractionService, + # IiifPrint::DerivativeRodeoService, + # Hyrax::FileSetDerivativesService] + + DerivativeRodeo::Generators::HocrGenerator.additional_tessearct_options = "-l eng_best" + + # Allows us to use decorator files Dir.glob(File.join(File.dirname(__FILE__), "../app/**/*_decorator*.rb")).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) end - end - config.to_prepare do - # Allows us to use decorator files in the app directory Dir.glob(File.join(File.dirname(__FILE__), "../lib/**/*_decorator*.rb")).sort.each do |c| Rails.configuration.cache_classes ? require(c) : load(c) end - end - # OAI additions - Dir.glob(File.join(File.dirname(__FILE__), "../lib/oai/**/*.rb")).sort.each do |c| - Rails.configuration.cache_classes ? require(c) : load(c) + # OAI additions + Dir.glob(File.join(File.dirname(__FILE__), "../lib/oai/**/*.rb")).sort.each do |c| + Rails.configuration.cache_classes ? require(c) : load(c) + end end # resolve reloading issue in dev mode @@ -67,6 +105,8 @@ class Application < Rails::Application Object.include(AccountSwitch) end + # copies tinymce assets directly into public/assets + config.tinymce.install = :copy ## # Psych Allow YAML Classes #