diff --git a/Gemfile b/Gemfile index 547a71cb7..968af7b84 100644 --- a/Gemfile +++ b/Gemfile @@ -14,15 +14,8 @@ gem 'puma', ">= 4.3.0" gem 'shakapacker', '~> 7.0.2' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder gem 'jbuilder', '~> 2.5' -# Use Redis adapter to run Action Cable in production -gem 'redis', '~> 4.5.0' -# Resque pool -gem 'resque-pool', '~> 0.7.0' # SideKiq for queueing jobs -gem 'sidekiq', '~> 5.2.10' -# When downgrading Sidekiq, rack needed to be downgraded as well. -# This can be removed for Sidekiq 6 or greater. -gem 'rack', '~> 2.2.6.3' +gem 'sidekiq', '~> 7.1.2' # Use SCSS for stylesheets gem 'sassc-rails', '~> 2.1.0' # Jquery for rails diff --git a/Gemfile.lock b/Gemfile.lock index 87bab37cd..41f31069e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -101,7 +101,7 @@ GEM clamby (1.6.8) cocoon (1.2.15) concurrent-ruby (1.2.2) - connection_pool (2.3.0) + connection_pool (2.4.1) countries (5.3.1) unaccent (~> 0.3) crack (0.4.5) @@ -195,12 +195,8 @@ GEM mini_mime (1.1.2) mini_portile2 (2.8.2) minitest (5.18.0) - mono_logger (1.1.1) msgpack (1.6.1) - multi_json (1.15.0) multi_xml (0.6.0) - mustermann (3.0.0) - ruby2_keywords (~> 0.0.1) mysql2 (0.5.5) net-imap (0.3.4) date @@ -243,8 +239,6 @@ GEM nio4r (~> 2.0) racc (1.7.1) rack (2.2.6.4) - rack-protection (3.0.5) - rack rack-proxy (0.7.6) rack rack-test (2.1.0) @@ -284,23 +278,14 @@ GEM rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) - redis (4.5.1) - redis-namespace (1.10.0) - redis (>= 4) + redis-client (0.14.1) + connection_pool regexp_parser (2.8.1) request_store (1.5.1) rack (>= 1.4) responders (3.1.0) actionpack (>= 5.2) railties (>= 5.2) - resque (2.5.0) - mono_logger (~> 1.0) - multi_json (~> 1.0) - redis-namespace (~> 1.6) - sinatra (>= 0.9.2) - resque-pool (0.7.1) - rake (>= 10.0, < 14.0) - resque (>= 1.22, < 3) rexml (3.2.5) rsolr (2.5.0) builder (>= 2.1.2) @@ -370,11 +355,11 @@ GEM semantic_range (>= 2.3.0) shoulda-matchers (4.3.0) activesupport (>= 4.2.0) - sidekiq (5.2.10) - connection_pool (~> 2.2, >= 2.2.2) - rack (~> 2.0) - rack-protection (>= 1.5.0) - redis (~> 4.5, < 4.6.0) + sidekiq (7.1.2) + concurrent-ruby (< 2) + connection_pool (>= 2.3.0) + rack (>= 2.2.4) + redis-client (>= 0.14.0) simple_form (5.2.0) actionpack (>= 5.2) activemodel (>= 5.2) @@ -383,11 +368,6 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) - sinatra (3.0.5) - mustermann (~> 3.0) - rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.0.5) - tilt (~> 2.0) snaky_hash (2.0.1) hashie version_gem (~> 1.1, >= 1.1.1) @@ -479,11 +459,8 @@ DEPENDENCIES prawn (~> 2.4.0) psych (< 4) puma (>= 4.3.0) - rack (~> 2.2.6.3) rails (~> 6.1) rails-controller-testing (~> 1.0.0) - redis (~> 4.5.0) - resque-pool (~> 0.7.0) rsolr (~> 2.5.0) rspec-rails (~> 4.0.0) rspec-retry (~> 0.6.0) @@ -495,7 +472,7 @@ DEPENDENCIES selenium-webdriver (~> 4.0) shakapacker (~> 7.0.2) shoulda-matchers (~> 4.3.0) - sidekiq (~> 5.2.10) + sidekiq (~> 7.1.2) simple_form (>= 5.0.0) simplecov (~> 0.17.0) spring (~> 2.1.0) diff --git a/README.md b/README.md index 0b2fbde7b..5fd9a0a2b 100755 --- a/README.md +++ b/README.md @@ -7,7 +7,8 @@ * Ruby version: 3.1 * Node version 16 (using yarn as npm) * Rails 6.1 -* Sidekiq 5 +* Sidekiq 7 +* Redis 6.2+ ## Setup diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 73e95ab15..72515c924 100755 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -20,7 +20,7 @@ class ApplicationController < ActionController::Base rescue_from ActiveRecord::StatementInvalid, with: :render_500 rescue_from Mysql2::Error, with: :render_500 rescue_from Net::LDAP::LdapError, with: :render_500 - rescue_from Redis::CannotConnectError, with: :render_500 + rescue_from RedisClient::CannotConnectError, with: :render_500 rescue_from Errno::ECONNREFUSED, with: :render_500 rescue_from ActionDispatch::Cookies::CookieOverflow, with: :render_500 rescue_from RuntimeError, with: :render_500 diff --git a/config/initializers/redis_config.rb b/config/initializers/redis_config.rb deleted file mode 100644 index a59874da5..000000000 --- a/config/initializers/redis_config.rb +++ /dev/null @@ -1,13 +0,0 @@ -require 'redis' -config = YAML.load(ERB.new(IO.read(File.join(Rails.root, 'config', 'redis.yml'))).result)[Rails.env].with_indifferent_access - -if defined?(PhusionPassenger) - PhusionPassenger.on_event(:starting_worker_process) do |forked| - next unless forked - # Re-establish redis connection - Redis.current.disconnect! - Redis.current = Redis.new(config.merge(thread_safe: true)) - end -else - Redis.current = Redis.new(config.merge(thread_safe: true)) -end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 2b296c9ea..fca06da4a 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -4,7 +4,6 @@ sidekiq_config = Hash.new sidekiq_config['password'] = redis_config['password'] if redis_config['password'] -sidekiq_config['namespace'] = "etda_workflow_#{current_partner.id}" sidekiq_url = "redis://#{redis_config.fetch(:host, 'localhost')}:#{redis_config.fetch(:port, 6379)}" sidekiq_config['url'] = sidekiq_url diff --git a/docker-compose.yml b/docker-compose.yml index 44fbab1ee..ff43b0da5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "4444:4444" redis: - image: redis:5.0.5 + image: redis:7.0.12 volumes: - redis-data:/data ports: diff --git a/lib/tasks/redis_migrate.rake b/lib/tasks/redis_migrate.rake index df718998b..b404edae2 100644 --- a/lib/tasks/redis_migrate.rake +++ b/lib/tasks/redis_migrate.rake @@ -1,39 +1,22 @@ namespace :redis_migrate do - desc 'Migrate redis db to keys with namespaces identified' - task 'to_namespaces' => :environment do - start = Time.now - graduate_ns = "etda_workflow_graduate:" - honors_ns = "etda_workflow_honors:" - redis = Redis.current + desc 'Migrate redis db to remove namespaces' + task 'remove_namespaces' => :environment do + # This script migrates data from a single namespaced redis + # instance to multiple distinct redis instances for each partner - schedule_key = redis.keys("schedule") - graduate_schedule_key_contents = [] - honors_schedule_key_contents = [] - old_schedule_key_contents = Hash.new - redis.zscan_each(schedule_key) do |value, score| - old_schedule_key_contents[score] = value - end + # Setup redis connection + redis_config = RedisClient.config(**Rails.application.config_for(:redis)) + redis = redis_config.new_pool(timeout: 0.5, size: 5) - count = 0 - grad_count = 0 - honors_count = 0 - old_schedule_key_contents.each do |score, value| - if eval(value)[:args][0] > 10000 - graduate_schedule_key_contents << [score, value] - grad_count += 1 - else - honors_schedule_key_contents << [score, value] - honors_count += 1 - end - count += 1 + # Rename all keys including this partner's namespace to the original key with namespace removed + redis.call("KEYS", "etda_workflow_#{current_partner.id}*").each do |key| + redis.call("RENAME", key, key.gsub("etda_workflow_#{current_partner.id}:", '')) end - redis.del(schedule_key) - - redis.zadd([graduate_ns, schedule_key].join(""), graduate_schedule_key_contents) - redis.zadd([honors_ns, schedule_key].join(""), honors_schedule_key_contents) - - puts "Process completed in #{(Time.now - start)} sec. #{count} records migrated. #{grad_count} records migrated to graduate namespace. #{honors_count} records migrated to honors namespace." + # Delete all the keys still containing a namespace, hence deleting other partner's keys + redis.call("KEYS", "*").each do |key| + redis.call("DEL", key) if key.include?('etda_workflow') + end end end