Bulkrax importing improvements / Knapsack / Allow DB Prepared Stateme…

…nts Off (#1970)

* include for the optional gem knapsack, to make developement and deployment easier

* work on caching, get gem version correct

* more db options

* update bulkrax to handle blanks

* docker build effenencies, try to get arm builds back

.env

@@ -1,12 +1,14 @@
 CHROME_HOSTNAME=chrome
+COMPOSE_DOCKER_CLI_BUILD=1
 DB_ADAPTER=postgresql
 DB_HOST=db
+DB_HOST=db
 DB_NAME=hyku
 DB_PASSWORD=DatabaseFTW
+DB_PORT=5432
 DB_TEST_NAME=hyku_test
 DB_USER=postgres
-DB_HOST=db
-DB_PORT=5432
+DOCKER_BUILDKIT=1
 FCREPO_BASE_PATH=/hykudemo
 FCREPO_HOST=fcrepo
 FCREPO_PORT=8080
@@ -15,6 +17,7 @@ [email protected]
 INITIAL_ADMIN_PASSWORD=testing123
 JAVA_OPTS=-Xmx4g -Xms1g
 IN_DOCKER=true
+JAVA_OPTS=
 LD_LIBRARY_PATH=/opt/fits/tools/mediainfo/linux
 PASSENGER_APP_ENV=development
 RAILS_LOG_TO_STDOUT=true

.github/workflows/base.yaml

@@ -14,6 +14,6 @@ jobs:
     uses: scientist-softserv/actions/.github/workflows/[email protected]
     secrets: inherit
     with:
-      platforms: "linux/amd64" # "linux/amd64,linux/arm64"
+      platforms: "linux/amd64,linux/arm64"
       target: hyku-base
       image_name: samvera/hyku/base

Dockerfile

@@ -1,4 +1,5 @@
 ARG HYRAX_IMAGE_VERSION=hyrax-v4.0.0.rc1
+ARG RUBY_VERSION=2.7.7
 FROM ghcr.io/samvera/hyrax/hyrax-base:$HYRAX_IMAGE_VERSION as hyku-base
 
 USER root
@@ -84,7 +85,8 @@ RUN ln -sf /usr/lib/libmediainfo.so.0 /app/fits/tools/mediainfo/linux/libmediain
 ONBUILD COPY --chown=1001:101 $APP_PATH/bin/db-migrate-seed.sh /app/samvera/
 
 ONBUILD COPY --chown=1001:101 $APP_PATH/Gemfile* /app/samvera/hyrax-webapp/
-ONBUILD RUN bundle install --jobs "$(nproc)"
+ONBUILD RUN git config --global --add safe.directory /app/samvera && \
+  bundle install --jobs "$(nproc)"
 
 ONBUILD COPY --chown=1001:101 $APP_PATH /app/samvera/hyrax-webapp
 

Gemfile

@@ -16,7 +16,7 @@ gem 'blacklight', '~> 6.7'
 gem 'blacklight_oai_provider', '~> 6.1', '>= 6.1.1'
 gem 'bolognese', '>= 1.9.10'
 gem 'bootstrap-datepicker-rails'
-gem 'bulkrax', '~> 5.0'
+gem 'bulkrax', '~> 5.3'
 gem 'byebug', group: %i[development test]
 gem 'capybara', group: %i[test]
 gem 'capybara-screenshot', '~> 1.0', group: %i[test]
@@ -90,4 +90,9 @@ gem 'turbolinks', '~> 5'
 gem 'web-console', '>= 3.3.0', group: %i[development] # <%= console %> in views
 gem 'webdrivers', '~> 4.7.0', group: %i[test]
 gem 'webmock', group: %i[test]
+
+# This gem does nothing by default, but is instead a tool to ease developer flow
+# and place overrides, themes and deployment code.
+gem 'hyku_knapsack', github: 'samvera-labs/hyku_knapsack', branch: 'upstream_main'
+
 # rubocop:enable Metrics/LineLength

Gemfile.lock

@@ -7,6 +7,14 @@ GIT
       aws-sdk-sqs (~> 1)
       rails (>= 4.2)
 
+GIT
+  remote: https://github.com/samvera-labs/hyku_knapsack.git
+  revision: 47bbb4a95ff7b94c06e6ee1700c47fac1e728a0f
+  branch: upstream_main
+  specs:
+    hyku_knapsack (0.0.1)
+      rails (>= 5.2.0)
+
 GIT
   remote: https://github.com/samvera-labs/hyrax-doi.git
   revision: d494a50ef8ce3eae594c7ed7148c33b3c977d4a7
@@ -164,7 +172,7 @@ GEM
     babel-transpiler (0.7.0)
       babel-source (>= 4.0, < 6)
       execjs (~> 2.0)
-    bagit (0.4.5)
+    bagit (0.4.6)
       docopt (~> 0.5.0)
       validatable (~> 1.6)
     bcp47 (0.3.3)
@@ -250,9 +258,10 @@ GEM
       signet (~> 0.8)
       typhoeus
     builder (3.2.4)
-    bulkrax (5.2.1)
+    bulkrax (5.3.0)
       bagit (~> 0.4)
       coderay
+      dry-monads (~> 1.4.0)
       iso8601 (~> 0.9.0)
       kaminari
       language_list (~> 1.2, >= 1.2.1)
@@ -1290,7 +1299,7 @@ DEPENDENCIES
   blacklight_oai_provider (~> 6.1, >= 6.1.1)
   bolognese (>= 1.9.10)
   bootstrap-datepicker-rails
-  bulkrax (~> 5.0)
+  bulkrax (~> 5.3)
   byebug
   capybara
   capybara-screenshot (~> 1.0)
@@ -1309,6 +1318,7 @@ DEPENDENCIES
   fcrepo_wrapper (~> 0.4)
   flipflop (~> 2.6.0)
   flutie
+  hyku_knapsack!
   hyrax (~> 3.5.0)
   hyrax-doi!
   hyrax-iiif_av!
@@ -1367,4 +1377,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   2.1.4
+   2.4.14

config/database.yml

@@ -8,7 +8,8 @@ login: &login
   database: <%= ENV['DB_NAME'] || 'hyku' %>
   pool: 50
   timeout: 5000
-
+  prepared_statements: <%= ENV.fetch('DB_PREPARED_STATEMENTS', true) %>
+  advisory_locks: <%= ENV.fetch('DB_ADVISORY_LOCKS', true) %>
 
 development:
   <<: *login

config/initializers/bulkrax.rb

@@ -53,9 +53,34 @@
     #   e.g. to exclude date
     #   config.field_mappings["Bulkrax::OaiDcParser"]["date"] = { from: ["date"], excluded: true  }
 
+
     default_field_mapping = {
+      'abstract' => { from: ['abstract'], split: true },
+      'alternative_title' => { from: ['alternative_title'], split: /\s*[|]\s*/ },
+      'based_near' => { from: ['based_near'], split: true },
+      'bibliographic_citation' => { from: ['bibliographic_citation'], split: true },
+      'contributor' => { from: ['contributor'], split: true },
+      'create_date' => { from: ['create_date'], split: true },
+      'children' => { from: ['children'], related_children_field_mapping: true },
+      'creator' => { from: ['creator'], split: true },
+      'date_created' => { from: ['date_created'], split: true },
+      'description' => { from: ['description'], split: true },
+      'extent' => { from: ['extent'], split: true },
+      'file' => { from: ['file'], split: /\s*[|]\s*/ },
+      'identifier' => { from: ['identifier'], split: true },
+      'keyword' => { from: ['keyword'], split: true },
+      'language' => { from: ['language'], split: true },
+      'license' => { from: ['license'], split: /\s*[|]\s*/ },
+      'modified_date' => { from: ['modified_date'], split: true },
       'parents' => { from: ['parents'], related_parents_field_mapping: true },
-      'children' => { from: ['children'], related_children_field_mapping: true }
+      'publisher' => { from: ['publisher'], split: true },
+      'related_url' => { from: ['related_url'], split: /\s* [|]\s*/ },
+      'remote_files' => { from: ['remote_files'], split: /\s*[|]\s*/},
+      'resource_type' => { from: ['resource_type'], split: true },
+      'rights_notes' => { from: ['rights_notes'], split: true },
+      'source' => { from: ['source'], split: true },
+      'subject' => { from: ['subject'], split: true },
+      'title' => { from: ['title'], split: /\s*[|]\s*/ }
     }
 
     config.field_mappings["Bulkrax::BagitParser"] = default_field_mapping.merge({

docker-compose.yml

@@ -1,12 +1,13 @@
 version: '3.8'
-
 x-app: &app
   build:
     context: .
     target: hyku-web
+    args:
+      BUILDKIT_INLINE_CACHE: 1
     cache_from:
-      - ghcr.io/samvera/hyku/base:${TAG:-latest}
-      - ghcr.io/samvera/hyku:${TAG:-latest}
+      - ghcr.io/samvera/hyku/base:latest
+      - ghcr.io/samvera/hyku:latest
   image: ghcr.io/samvera/hyku:${TAG:-latest}
   env_file:
     - .env
@@ -126,6 +127,10 @@ services:
     build:
       context: .
       target: hyku-base
+      cache_from:
+        - ghcr.io/samvera/hyku/base:latest
+      args:
+        BUILDKIT_INLINE_CACHE: 1
 
   web:
     <<: *app
@@ -164,9 +169,12 @@ services:
     build:
       context: .
       target: hyku-worker
+      args:
+        BUILDKIT_INLINE_CACHE: 1
       cache_from:
-        - ghcr.io/samvera/hyku:${TAG:-latest}
-        - ghcr.io/samvera/hyku/worker:${TAG:-latest}
+        - ghcr.io/samvera/hyku/base:latest
+        - ghcr.io/samvera/hyku:latest
+        - ghcr.io/samvera/hyku/worker:latest
     command: bundle exec sidekiq
     depends_on:
       check_volumes: