Switch from websocket to http in Atc::Aws::FixityCheck class; Also re…

…factor code to simplify swapping between WEBSOCKET and HTTP fixity check methods

Gemfile

@@ -75,7 +75,7 @@ group :development, :test do
   # Use sqlite3 as the database for Active Record
   gem 'sqlite3', '~> 1.4'
   # Rubocul for linting
-  gem 'rubocul', '~> 4.0.11'
+  gem 'rubocul', '~> 4.0.12'
   # gem 'rubocul', path: '../rubocul'
   gem 'webmock', '~> 3.23.1'
 end

Gemfile.lock

@@ -230,7 +230,7 @@ GEM
       actionview (>= 5.0.0)
       activesupport (>= 5.0.0)
     jmespath (1.6.2)
-    json (2.7.1)
+    json (2.7.2)
     jwt (2.8.1)
       base64
     language_server-protocol (3.17.0.3)
@@ -286,8 +286,8 @@ GEM
       omniauth (>= 2.0)
     orm_adapter (0.5.0)
     os (1.1.4)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    parallel (1.25.1)
+    parser (3.3.4.0)
       ast (~> 2.4.1)
       racc
     psych (5.1.2)
@@ -388,38 +388,38 @@ GEM
       rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
-    rubocop-capybara (2.20.0)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
+    rubocop-capybara (2.21.0)
       rubocop (~> 1.41)
-    rubocop-factory_bot (2.25.1)
-      rubocop (~> 1.41)
-    rubocop-graphql (1.5.1)
-      rubocop (>= 0.90, < 2)
-    rubocop-performance (1.21.0)
+    rubocop-factory_bot (2.26.1)
+      rubocop (~> 1.61)
+    rubocop-graphql (1.5.3)
+      rubocop (>= 1.50, < 2)
+    rubocop-performance (1.21.1)
       rubocop (>= 1.48.1, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-rails (2.24.1)
+    rubocop-rails (2.25.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.33.0, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
-    rubocop-rspec (2.28.0)
+    rubocop-rspec (2.31.0)
       rubocop (~> 1.40)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
       rubocop-rspec_rails (~> 2.28)
-    rubocop-rspec_rails (2.28.2)
-      rubocop (~> 1.40)
-    rubocul (4.0.11)
-      rubocop (~> 1.26)
-      rubocop-ast
-      rubocop-capybara
-      rubocop-factory_bot
-      rubocop-graphql
-      rubocop-performance
-      rubocop-rails
-      rubocop-rspec
+    rubocop-rspec_rails (2.29.1)
+      rubocop (~> 1.61)
+    rubocul (4.0.12)
+      rubocop (~> 1.62.1)
+      rubocop-ast (~> 1.31)
+      rubocop-capybara (~> 2.21)
+      rubocop-factory_bot (~> 2.26)
+      rubocop-graphql (~> 1.5)
+      rubocop-performance (~> 1.21)
+      rubocop-rails (~> 2.25)
+      rubocop-rspec (~> 2.28)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
@@ -531,7 +531,7 @@ DEPENDENCIES
   resque (~> 2.6)
   retriable (~> 3.1)
   rspec-rails
-  rubocul (~> 4.0.11)
+  rubocul (~> 4.0.12)
   selenium-webdriver
   simplecov (~> 0.22)
   sinatra (~> 3.0)

app/jobs/verify_fixity_job.rb

@@ -18,6 +18,15 @@ def perform(stored_object_id)
     fixity_verification_record = create_pending_fixity_verification stored_object
     provider_fixity_check = instantiate_provider_fixity_check fixity_verification_record
     verify_fixity(fixity_verification_record, provider_fixity_check)
+  rescue StandardError => e
+    handle_unexpected_error(fixity_verification_record, e) unless fixity_verification_record.nil?
+  end
+
+  def handle_unexpected_error(fixity_verification_record, err)
+    fixity_verification_record.update!(
+      status: :failure,
+      error_message: "An unexpected error occurred: #{err.message}"
+    )
   end
 
   def process_existing_fixity_verification_record(existing_fixity_verification_record)
@@ -49,6 +58,7 @@ def instantiate_provider_fixity_check(fixity_verification_record)
 
   def verify_fixity(fixity_verification_record, provider_fixity_check)
     object_checksum, object_size, fixity_check_error = provider_fixity_check.fixity_checksum_object_size
+
     if fixity_check_error.present?
       fixity_verification_record.error_message = fixity_check_error
       fixity_verification_record.failure!

lib/atc/aws/fixity_check.rb

@@ -1,41 +1,25 @@
 # frozen_string_literal: true
 
 class Atc::Aws::FixityCheck
-  def initialize(stored_object, stream_id)
+  def initialize(stored_object, fixity_check_identifier)
     @bucket_name = stored_object.storage_provider.container_name
     @object_path = stored_object.path
     @fixity_checksum_algorithm = stored_object.source_object.fixity_checksum_algorithm
-    @stream_id = stream_id
+    @fixity_check_identifier = fixity_check_identifier
   end
 
+  # Returns an array with the checksum, object size, and (if something went wrong) an error error_message.
+  # If there is an error, checksum and object size will be nil.  If there is not an error,
+  # checksum and object size will be non-nil and error will be nil.
+  # @return [Array(String, Integer, String)] A 3-element array containing: [checksum, object_size, error_message]
   def fixity_checksum_object_size
-    aws_fixity_check_response =
-      aws_fixity_websocket_channel_stream(@bucket_name,
-                                          @object_path,
-                                          @fixity_checksum_algorithm.name.downcase,
-                                          @stream_id)
-    case aws_fixity_check_response['type']
-    when 'fixity_check_complete'
-      [aws_fixity_check_response['data']['checksum_hexdigest'], aws_fixity_check_response['data']['object_size'], nil]
-    when 'fixity_check_error'
-      # if only want to return the error from the AWS fixity response, without data,
-      # use the following commented-out line instead of the last line
-      # [nil, nil, aws_fixity_check_response['data']['error_message']]
-      [nil, nil, response_data_as_string(aws_fixity_check_response)]
-    end
-  end
-
-  def response_data_as_string(aws_fixity_check_response)
-    # AWS response data contains, among other things, the error message
-    "AWS error response with the following data: #{aws_fixity_check_response['data']} "
-  end
-
-  def aws_fixity_websocket_channel_stream(bucket_name,
-                                          object_path,
-                                          checksum_algorithm_name,
-                                          job_identifier)
-    # Response received (Hash) is returned as-is.
-    remote_fixity_check = Atc::Aws::RemoteFixityCheck.new(CHECK_PLEASE['ws_url'], CHECK_PLEASE['auth_token'])
-    remote_fixity_check.perform(job_identifier, bucket_name, object_path, checksum_algorithm_name)
+    response = Atc::Aws::RemoteFixityCheck.new(
+      CHECK_PLEASE['http_base_url'], CHECK_PLEASE['ws_url'], CHECK_PLEASE['auth_token']
+    ).perform(
+      @fixity_check_identifier, @bucket_name,
+      @object_path, @fixity_checksum_algorithm.name.downcase,
+      Atc::Aws::RemoteFixityCheck::HTTP
+    )
+    [response['checksum_hexdigest'], response['object_size'], response['error_message']]
   end
 end

lib/atc/aws/remote_fixity_check.rb

@@ -18,7 +18,7 @@ def initialize(http_base_url, ws_url, auth_token)
   end
 
   def http_client
-    @http_conn = ::Faraday.new(url: @http_base_url) do |f|
+    @http_client ||= ::Faraday.new(url: @http_base_url) do |f|
       f.request :authorization, 'Bearer', @auth_token
       f.response :json # decode response bodies as JSON
       f.response :raise_error # raise 4xx and 5xx responses as errors
@@ -38,7 +38,7 @@ def create_websocket_connection
   def perform(job_identifier, bucket_name, object_path, checksum_algorithm_name, method = WEBSOCKET)
     case method
     when WEBSOCKET
-      perform_websocket(job_identifier, bucket_name, object_path, checksum_algorithm_name)
+      perform_websocket(job_identifier, bucket_name, object_path, checksum_algorithm_name)['data']
     when HTTP
       perform_http(bucket_name, object_path, checksum_algorithm_name)
     else
@@ -54,12 +54,15 @@ def perform_http(bucket_name, object_path, checksum_algorithm_name)
         'checksum_algorithm_name' => checksum_algorithm_name
       }
     }.to_json
-
     response = http_client.post('/fixity_checks/run_fixity_check_for_s3_object', payload) do |request|
       request.headers['Content-Type'] = 'application/json'
     end
 
     JSON.parse(response.body)
+  rescue StandardError => e
+    {
+      'checksum_hexdigest' => nil, 'object_size' => nil, 'error_message' => "An unexpected error occurred: #{e.message}"
+    }
   end
 
   def perform_websocket(job_identifier, bucket_name, object_path, checksum_algorithm_name)

lib/tasks/comparison.rake

@@ -41,5 +41,23 @@ namespace :atc do
 
       puts "Done!"
     end
+
+    desc 'List multipart info for an S3 object (including part size)'
+    task list_multipart_info: :environment do
+      bucket_name = ENV['bucket_name']
+      path = ENV['path']
+
+      attributes = S3_CLIENT.get_object_attributes(
+        bucket: 'cul-dlstor-digital-working',
+        key: 'test/ave_biggert_00001r.tif',
+        #object_attributes: ['ETag']
+        #object_attributes: ['ETag', 'Checksum' 'ObjectParts' ]
+      )
+
+      puts attributes.inspect
+
+    end
   end
+
+
 end

spec/atc/aws/fixity_check_spec.rb

@@ -18,50 +18,55 @@
                       source_object: source_object,
                       storage_provider: aws_storage_provider)
   end
-  let(:aws_hash_response) do
-    { 'type' => 'fixity_check_complete',
-      'data' => { 'checksum_hexdigest' => 'ABCDEF12345', 'object_size' => 1234 } }
-  end
-  let(:aws_error_hash_response) do
-    { 'type' => 'fixity_check_error',
-      'data' => { 'error_message' => 'Ooops!',
-                  'job_identifier' => 1234,
-                  'bucket_name' => 'cul_bucket',
-                  'object_path' => 'I/Am/An/Object',
-                  'checksum_algorithm_name' => 'SHA31415' } }
-  end
 
   describe '#fixity_checksum_object_size' do
-    context 'with an AWS response without errors ' do
-      it 'returns the object checksum and object size, and nil for the aws error message' do
-        allow(aws_fixity_check).to receive(:aws_fixity_websocket_channel_stream) { aws_hash_response }
+    let(:remote_fixity_check) do
+      dbl = instance_double(Atc::Aws::RemoteFixityCheck)
+      allow(dbl).to receive(:perform).and_return(remote_fixity_check_perform_response)
+      dbl
+    end
+
+    before do
+      allow(Atc::Aws::RemoteFixityCheck).to receive(:new).and_return(remote_fixity_check)
+    end
+
+    context 'with a response without errors' do
+      let(:remote_fixity_check_perform_response) do
+        {
+          'checksum_hexdigest' => 'ABCDEF12345',
+          'object_size' => 1234
+        }
+      end
+
+      it 'returns the object checksum and object size, and nil for the error message' do
         result = aws_fixity_check.fixity_checksum_object_size
-        expect(result).to eq(['ABCDEF12345', 1234, nil])
+        expect(result).to eq(
+          [
+            remote_fixity_check_perform_response['checksum_hexdigest'],
+            remote_fixity_check_perform_response['object_size'],
+            nil
+          ]
+        )
       end
     end
 
-    context 'with an AWS response with errors ' do
+    context 'with a response with errors' do
+      let(:remote_fixity_check_perform_response) do
+        {
+          'error_message' => 'Ooops!'
+        }
+      end
+
       it 'returns nil for the object checksum and object size' do
-        allow(aws_fixity_check).to receive(:aws_fixity_websocket_channel_stream) { aws_error_hash_response }
         result = aws_fixity_check.fixity_checksum_object_size
         expect(result[0]).to eq nil
         expect(result[1]).to eq nil
       end
 
-      it 'returns the error message (including data)' do
-        allow(aws_fixity_check).to receive(:aws_fixity_websocket_channel_stream) { aws_error_hash_response }
+      it 'returns the error message' do
         result = aws_fixity_check.fixity_checksum_object_size
-        expect(result[2]).to include('Ooops!')
-        expect(result[2]).to include('cul_bucket')
+        expect(result[2]).to eq(remote_fixity_check_perform_response['error_message'])
       end
     end
   end
-
-  describe '#response_data_as_string' do
-    it 'returns the aws response data info as a string' do
-      result = aws_fixity_check.response_data_as_string aws_error_hash_response
-      expect(result).to include('Ooops!')
-      expect(result).to include('cul_bucket')
-    end
-  end
 end