diff --git a/app/jobs/samples/batch_sample_import_job.rb b/app/jobs/samples/batch_sample_import_job.rb new file mode 100644 index 0000000000..979aa5c3ac --- /dev/null +++ b/app/jobs/samples/batch_sample_import_job.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Samples + # Job used to import metadata for samples + class BatchSampleImportJob < ApplicationJob + queue_as :default + + def perform(namespace, current_user, broadcast_target, blob_id, params) # rubocop:disable Lint/UnusedMethodArgument + ::Samples::BatchFileImportService.new(namespace, current_user, blob_id, params).execute + + # TODO: change all these from metadata to batch sample + # if namespace.errors.empty? + # Turbo::StreamsChannel.broadcast_replace_to( + # broadcast_target, + # target: 'import_metadata_dialog_content', + # partial: 'shared/samples/metadata/file_imports/success', + # locals: { + # type: :success, + # message: I18n.t('shared.samples.metadata.file_imports.success.description') + # } + # ) + + # elsif namespace.errors.include?(:sample) + # errors = namespace.errors.messages_for(:sample) + + # Turbo::StreamsChannel.broadcast_replace_to( + # broadcast_target, + # target: 'import_metadata_dialog_content', + # partial: 'shared/samples/metadata/file_imports/errors', + # locals: { + # type: :alert, + # message: I18n.t('shared.samples.metadata.file_imports.errors.description'), + # errors: errors + # } + # ) + # else + # errors = namespace.errors.full_messages_for(:base) + + # Turbo::StreamsChannel.broadcast_replace_to( + # broadcast_target, + # target: 'import_metadata_dialog_content', + # partial: 'shared/samples/metadata/file_imports/errors', + # locals: { + # type: :alert, + # errors: errors + # } + # ) + # end + end + end +end diff --git a/app/services/base_spreadsheet_import_service.rb b/app/services/base_spreadsheet_import_service.rb new file mode 100644 index 0000000000..7686e007f6 --- /dev/null +++ b/app/services/base_spreadsheet_import_service.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require 'roo' + +# Service base class for handling spreadsheet file imports +class BaseSpreadsheetImportService < BaseService + FileImportError = Class.new(StandardError) + + def initialize(namespace, user = nil, blob_id = nil, required_headers = [], minimum_additional_data_columns = 0, params = {}) # rubocop:disable Metrics/ParameterLists,Layout/LineLength + super(user, params) + @namespace = namespace + @file = ActiveStorage::Blob.find(blob_id) + @required_headers = required_headers + @minimum_additional_data_columns = minimum_additional_data_columns + @spreadsheet = nil + @headers = nil + @temp_import_file = Tempfile.new + end + + def execute + raise NotImplementedError + end + + protected + + def validate_file + extension = validate_file_extension + download_batch_import_file(extension) + + @headers = @spreadsheet.row(1).compact + validate_file_headers + + validate_file_rows + end + + def validate_file_extension + file_extension = File.extname(@file.filename.to_s).downcase + + return file_extension if %w[.csv .tsv .xls .xlsx].include?(file_extension) + + raise FileImportError, I18n.t('services.spreadsheet_import.invalid_file_extension') + end + + def download_batch_import_file(extension) + begin + @temp_import_file.binmode + @file.download do |chunk| + @temp_import_file.write(chunk) + end + ensure + @temp_import_file.close + end + @spreadsheet = if extension.eql? '.tsv' + Roo::Spreadsheet.open(@temp_import_file.path, + { extension: '.csv', csv_options: { col_sep: "\t" } }) + else + Roo::Spreadsheet.open(@temp_import_file.path, extension:) + end + end + + def validate_file_headers + duplicate_headers = @headers.find_all { |header| @headers.count(header) > 1 }.uniq + unless duplicate_headers.empty? + raise FileImportError, + I18n.t('services.spreadsheet_import.duplicate_column_names') + end + + missing_headers = @required_headers - @headers + unless missing_headers.empty? + raise FileImportError, I18n.t('services.spreadsheet_import.missing_header', header_title: missing_headers) + end + + return unless @headers.count < (@required_headers.count + @minimum_additional_data_columns) + + raise FileImportError, I18n.t('services.spreadsheet_import.missing_data_columns') + end + + def validate_file_rows + # Should have at least 2 rows + first_row = @spreadsheet.row(2) + return unless first_row.compact.empty? + + raise FileImportError, I18n.t('services.spreadsheet_import.missing_data_row') + end + + def perform_file_import + raise NotImplementedError + end + + def cleanup_files + # delete the blob and temporary file as we no longer require them + @file.purge + @temp_import_file.unlink + end +end diff --git a/app/services/samples/batch_file_import_service.rb b/app/services/samples/batch_file_import_service.rb new file mode 100644 index 0000000000..41b9e0d0c1 --- /dev/null +++ b/app/services/samples/batch_file_import_service.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require 'roo' + +module Samples + # Service used to batch create samples via a file + class BatchFileImportService < BaseSpreadsheetImportService + def initialize(namespace, user = nil, blob_id = nil, params = {}) + @sample_name_column = params[:sample_name_column] + @project_puid_column = params[:project_puid_column] + @sample_description_column = params[:sample_description_column] + required_headers = [@sample_name_column, @project_puid_column] + super(namespace, user, blob_id, required_headers, 0, params) + end + + def execute + authorize! @namespace, to: :update_sample_metadata? + validate_file + perform_file_import + rescue FileImportError => e + @namespace.errors.add(:base, e.message) + {} + end + + protected + + def perform_file_import # rubocop:disable Metrics/AbcSize,Metrics/MethodLength + response = {} + parse_settings = @headers.zip(@headers).to_h + + @spreadsheet.each_with_index(parse_settings) do |data, index| + next unless index.positive? + + # TODO: handle metadata headers + + sample_name = data[@sample_name_column] + project_puid = data[@project_puid_column] + description = data[@sample_description_column] + + error = errors_on_sample_row(sample_name, project_puid, response, index) + unless error.nil? + response["index #{index}"] = error + next + end + + project = Namespaces::ProjectNamespace.find_by(puid: project_puid)&.project + error = errors_with_project(project_puid, project) + unless error.nil? + response[sample_name] = error + next + end + + response[sample_name] = process_sample_row(sample_name, project, description) + end + cleanup_files + response + end + + private + + def accessible_from_namespace?(project) + if @namespace.project_namespace? + @namespace.id == project.namespace.id + elsif @namespace.group_namespace? + authorized_scope(Project, type: :relation, as: :group_projects, scope_options: { + group: @namespace, + minimum_access_level: Member::AccessLevel::UPLOADER + }).where(id: project.id).count.positive? + else + false + end + end + + def errors_on_sample_row(sample_name, project_puid, response, index) + if sample_name.nil? || project_puid.nil? + { + path: ['sample'], + message: I18n.t('services.spreadsheet_import.missing_field', index:) + } + elsif response.key?(sample_name) + { + path: ['sample'], + message: I18n.t('services.samples.batch_import.duplicate_sample_name', index:) + } + end + end + + def errors_with_project(project_puid, project) + if project.nil? + { + path: ['project'], + message: I18n.t('services.samples.batch_import.project_puid_not_found', project_puid: project_puid) + } + elsif !accessible_from_namespace?(project) + { + path: ['project'], + message: I18n.t('services.samples.batch_import.project_puid_not_in_namespace', + project_puid: project_puid, + namespace: @namespace.full_path) + } + end + end + + def process_sample_row(name, project, description) + # TODO: process metadata too + sample_params = { name:, description: } + sample = Samples::CreateService.new(current_user, project, sample_params).execute + + if sample.persisted? + sample + else + sample.errors.map do |error| + { + path: ['sample', error.attribute.to_s.camelize(:lower)], + message: error.message + } + end + end + end + end +end diff --git a/app/services/samples/metadata/file_import_service.rb b/app/services/samples/metadata/file_import_service.rb index a588720fb2..7e8e321950 100644 --- a/app/services/samples/metadata/file_import_service.rb +++ b/app/services/samples/metadata/file_import_service.rb @@ -5,110 +5,24 @@ module Samples module Metadata # Service used to import sample metadata via a file - class FileImportService < BaseService # rubocop:disable Metrics/ClassLength - SampleMetadataFileImportError = Class.new(StandardError) - + class FileImportService < BaseSpreadsheetImportService def initialize(namespace, user = nil, blob_id = nil, params = {}) - super(user, params) - @namespace = namespace - @file = ActiveStorage::Blob.find(blob_id) @sample_id_column = params[:sample_id_column] + required_headers = [@sample_id_column] @ignore_empty_values = params[:ignore_empty_values] - @spreadsheet = nil - @headers = nil - @temp_import_file = Tempfile.new + super(namespace, user, blob_id, required_headers, 1, params) end def execute authorize! @namespace, to: :update_sample_metadata? - - validate_sample_id_column - validate_file - perform_file_import - rescue Samples::Metadata::FileImportService::SampleMetadataFileImportError => e + rescue FileImportError => e @namespace.errors.add(:base, e.message) {} end - private - - def validate_sample_id_column - return unless @sample_id_column.nil? - - raise SampleMetadataFileImportError, - I18n.t('services.samples.metadata.import_file.empty_sample_id_column') - end - - def validate_file_extension - file_extension = File.extname(@file.filename.to_s).downcase - - return file_extension if %w[.csv .tsv .xls .xlsx].include?(file_extension) - - raise SampleMetadataFileImportError, - I18n.t('services.samples.metadata.import_file.invalid_file_extension') - end - - def validate_file_headers - duplicate_headers = @headers.find_all { |header| @headers.count(header) > 1 }.uniq - unless duplicate_headers.empty? - raise SampleMetadataFileImportError, - I18n.t('services.samples.metadata.import_file.duplicate_column_names') - end - - unless @headers.include?(@sample_id_column) - raise SampleMetadataFileImportError, - I18n.t('services.samples.metadata.import_file.missing_sample_id_column') - end - - return if @headers.count { |header| header != @sample_id_column }.positive? - - raise SampleMetadataFileImportError, - I18n.t('services.samples.metadata.import_file.missing_metadata_column') - end - - def validate_file_rows - first_row = @spreadsheet.row(2) - return unless first_row.compact.empty? - - raise SampleMetadataFileImportError, - I18n.t('services.samples.metadata.import_file.missing_metadata_row') - end - - def validate_file - if @file.nil? - raise SampleMetadataFileImportError, - I18n.t('services.samples.metadata.import_file.empty_file') - end - - extension = validate_file_extension - - download_metadata_import_file(extension) - - @headers = @spreadsheet.row(1).compact - - validate_file_headers - - validate_file_rows - end - - def download_metadata_import_file(extension) - begin - @temp_import_file.binmode - @file.download do |chunk| - @temp_import_file.write(chunk) - end - ensure - @temp_import_file.close - end - @spreadsheet = if extension.eql? '.tsv' - Roo::CSV.new(@temp_import_file.path, extension:, - csv_options: { col_sep: "\t" }) - else - Roo::Spreadsheet.open(@temp_import_file.path, extension:) - end - end + protected def perform_file_import response = {} @@ -123,19 +37,15 @@ def perform_file_import metadata_changes = process_sample_metadata_row(sample_id, metadata) response[sample_id] = metadata_changes if metadata_changes - cleanup_files - response rescue ActiveRecord::RecordNotFound @namespace.errors.add(:sample, error_message(sample_id)) end + + cleanup_files response end - def cleanup_files - # delete the blob and temporary file as we no longer require them - @file.purge - @temp_import_file.unlink - end + private def error_message(sample_id) if @namespace.type == 'Group' diff --git a/config/locales/en.yml b/config/locales/en.yml index d2656ac032..c33adbf633 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1730,6 +1730,10 @@ en: namespace_project_exists: Project with the same name or path already exists in the target namespace. project_in_namespace: Project in namespace samples: + batch_import: + duplicate_sample_name: Duplicate sample name on index '%{index}' + project_puid_not_found: Project PUID '%{project_puid}' could not be found + project_puid_not_in_namespace: Project PUID '%{project_puid}' is not accessible from namespace '%{namespace}, or you do not have permission to create samples on the shared project within the namespace' clone: empty_new_project_id: Destination project was not selected. empty_sample_ids: The sample ids are empty. @@ -1743,13 +1747,6 @@ en: sample_does_not_belong_to_project: Sample '%{sample_name}'' does not belong to project '%{project_name}' single_all_keys_exist: Metadata key '%{key}' already exist. import_file: - duplicate_column_names: The file has duplicate column header names. Please remove the columns with duplicate header names and retry uploading. - empty_file: The file cannot be empty. - empty_sample_id_column: The sample id column cannot be empty. - invalid_file_extension: The file can only be a csv or excel. - missing_metadata_column: The file does not have any metadata columns. Please add some columns of metadata to the file and retry uploading. - missing_metadata_row: The file does not have any metadata rows. Please add some rows of metadata to the file and retry uploading. - missing_sample_id_column: The file is missing the sample id column. Please make sure the sample id column exists and retry uploading. sample_metadata_fields_not_updated: Sample '%{sample_name}' with field(s) '%{metadata_fields}' cannot be updated. sample_not_found_within_group: Could not find sample with puid '%{sample_puid}' in this group. sample_not_found_within_project: Could not find sample with puid or name '%{sample_puid}' in this project. @@ -1767,6 +1764,13 @@ en: same_project: The samples already exist in the project. Please select a different project. sample_exists: 'Sample %{sample_puid}: Conflicts with sample named ''%{sample_name}'' in the target project' samples_not_found: 'Samples with the following sample ids could not be transferred as they were not found in the source project: %{sample_ids}' + spreadsheet_import: + duplicate_column_names: The file has duplicate column header names. Please remove the columns with duplicate header names and retry uploading. + invalid_file_extension: The file can only be a csv, tsv or excel. + missing_data_columns: File is missing data columns. Please add additional data columns and headers and retry uploading. + missing_data_row: The file does not have any data rows. Please add some rows of data to the file and retry uploading. + missing_field: Row with index '%{index}' is missing required fields + missing_header: File is missing the header '%{header_title}' shared: alert: list: diff --git a/config/locales/fr.yml b/config/locales/fr.yml index 387ebd3133..2a6ff22bd1 100644 --- a/config/locales/fr.yml +++ b/config/locales/fr.yml @@ -1730,6 +1730,10 @@ fr: namespace_project_exists: Project with the same name or path already exists in the target namespace. project_in_namespace: Project in namespace samples: + batch_import: + duplicate_sample_name: Duplicate sample name on index '%{index}' + project_puid_not_found: Project PUID '%{project_puid}' could not be found + project_puid_not_in_namespace: Project PUID '%{project_puid}' is not accessible from namespace '%{namespace}, or you do not have permission to create samples on the shared project within the namespace' clone: empty_new_project_id: Destination project was not selected. empty_sample_ids: The sample ids are empty. @@ -1743,13 +1747,6 @@ fr: sample_does_not_belong_to_project: Sample '%{sample_name}'' does not belong to project '%{project_name}' single_all_keys_exist: Metadata key '%{key}' already exist. import_file: - duplicate_column_names: The file has duplicate column header names. Please remove the columns with duplicate header names and retry uploading. - empty_file: The file cannot be empty. - empty_sample_id_column: The sample id column cannot be empty. - invalid_file_extension: The file can only be a csv or excel. - missing_metadata_column: The file does not have any metadata columns. Please add some columns of metadata to the file and retry uploading. - missing_metadata_row: The file does not have any metadata rows. Please add some rows of metadata to the file and retry uploading. - missing_sample_id_column: The file is missing the sample id column. Please make sure the sample id column exists and retry uploading. sample_metadata_fields_not_updated: Sample '%{sample_name}' with field(s) '%{metadata_fields}' cannot be updated. sample_not_found_within_group: Could not find sample with puid '%{sample_puid}' in this group. sample_not_found_within_project: Could not find sample with puid or name '%{sample_puid}' in this project. @@ -1767,6 +1764,13 @@ fr: same_project: The samples already exist in the project. Please select a different project. sample_exists: 'Sample %{sample_puid}: Conflicts with sample named ''%{sample_name}'' in the target project' samples_not_found: 'Samples with the following sample ids could not be transferred as they were not found in the source project: %{sample_ids}' + spreadsheet_import: + duplicate_column_names: The file has duplicate column header names. Please remove the columns with duplicate header names and retry uploading. + invalid_file_extension: The file can only be a csv, tsv or excel. + missing_data_columns: File is missing data columns. Please add additional data columns and headers and retry uploading. + missing_data_row: The file does not have any data rows. Please add some rows of data to the file and retry uploading. + missing_field: Row with index '%{index}' is missing required fields + missing_header: File is missing the header '%{header_title}' shared: alert: list: diff --git a/test/fixtures/files/batch_sample_import_invalid_blank_line.csv b/test/fixtures/files/batch_sample_import_invalid_blank_line.csv new file mode 100644 index 0000000000..0cf2499b7a --- /dev/null +++ b/test/fixtures/files/batch_sample_import_invalid_blank_line.csv @@ -0,0 +1,4 @@ +sample_name,project_puid,description +my new sample,INXT_PRJ_AAAAAAAAAA,my description +,, +my new sample 2,INXT_PRJ_AAAAAAAAAA,my description 2 diff --git a/test/fixtures/files/batch_sample_import_invalid_missing_puid.csv b/test/fixtures/files/batch_sample_import_invalid_missing_puid.csv new file mode 100644 index 0000000000..e2c87d5554 --- /dev/null +++ b/test/fixtures/files/batch_sample_import_invalid_missing_puid.csv @@ -0,0 +1,3 @@ +sample_name,project_puid,description +my new sample,INXT_PRJ_AAAAAAAAAA,my description +my new sample 2,,my description 2 diff --git a/test/fixtures/files/batch_sample_import_invalid_project.csv b/test/fixtures/files/batch_sample_import_invalid_project.csv new file mode 100644 index 0000000000..d3273c6443 --- /dev/null +++ b/test/fixtures/files/batch_sample_import_invalid_project.csv @@ -0,0 +1,3 @@ +sample_name,project_puid,description +my new sample,INXT_PRJ_AAAAAAAAAA,my description +my new sample 2,invalid_puid,my description 2 diff --git a/test/fixtures/files/batch_sample_import_invalid_sample_dup_in_file.csv b/test/fixtures/files/batch_sample_import_invalid_sample_dup_in_file.csv new file mode 100644 index 0000000000..30b7ed0d6f --- /dev/null +++ b/test/fixtures/files/batch_sample_import_invalid_sample_dup_in_file.csv @@ -0,0 +1,3 @@ +sample_name,project_puid,description +my new sample,INXT_PRJ_AAAAAAAAAA,my description +my new sample,INXT_PRJ_AAAAAAAAAA,my description 2 diff --git a/test/fixtures/files/batch_sample_import_invalid_sample_exists.csv b/test/fixtures/files/batch_sample_import_invalid_sample_exists.csv new file mode 100644 index 0000000000..82f33b83ad --- /dev/null +++ b/test/fixtures/files/batch_sample_import_invalid_sample_exists.csv @@ -0,0 +1,3 @@ +sample_name,project_puid,description +my new sample,INXT_PRJ_AAAAAAAAAA,my description +Project 1 Sample 1,INXT_PRJ_AAAAAAAAAA,my description 2 diff --git a/test/fixtures/files/batch_sample_import_invalid_short_sample_name.csv b/test/fixtures/files/batch_sample_import_invalid_short_sample_name.csv new file mode 100644 index 0000000000..cdbf8ebd6b --- /dev/null +++ b/test/fixtures/files/batch_sample_import_invalid_short_sample_name.csv @@ -0,0 +1,3 @@ +sample_name,project_puid,description +my new sample,INXT_PRJ_AAAAAAAAAA,my description +m,INXT_PRJ_AAAAAAAAAA,my description 2 diff --git a/test/fixtures/files/batch_sample_import_valid.csv b/test/fixtures/files/batch_sample_import_valid.csv new file mode 100644 index 0000000000..97ad42c6bc --- /dev/null +++ b/test/fixtures/files/batch_sample_import_valid.csv @@ -0,0 +1,3 @@ +sample_name,project_puid,description +my new sample,INXT_PRJ_AAAAAAAAAA,my description +my new sample 2,INXT_PRJ_AAAAAAAAAA,my description 2 diff --git a/test/fixtures/files/metadata/empty.csv b/test/fixtures/files/metadata/empty.csv new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/services/samples/batch_file_import_service_test.rb b/test/services/samples/batch_file_import_service_test.rb new file mode 100644 index 0000000000..61bdc62822 --- /dev/null +++ b/test/services/samples/batch_file_import_service_test.rb @@ -0,0 +1,268 @@ +# frozen_string_literal: true + +require 'test_helper' + +module Samples + class BatchFileImportServiceTest < ActiveSupport::TestCase + def setup # rubocop:disable Metrics/MethodLength + @john_doe = users(:john_doe) + @jane_doe = users(:jane_doe) + @group = groups(:group_one) + @group2 = groups(:group_two) + @project = projects(:project1) + @project2 = projects(:project2) + + file = Rack::Test::UploadedFile.new(Rails.root.join('test/fixtures/files/batch_sample_import_valid.csv')) + @blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + @default_params = { + sample_name_column: 'sample_name', + project_puid_column: 'project_puid', + sample_description_column: 'description' + } + end + + test 'import samples with permission for project namespace' do + assert_equal 3, @project.samples.count + + assert_authorized_to(:update_sample_metadata?, @project.namespace, + with: Namespaces::ProjectNamespacePolicy, + context: { user: @john_doe }) do + Samples::BatchFileImportService.new(@project.namespace, @john_doe, @blob.id, @default_params).execute + end + + assert_equal 5, @project.samples.count + assert_equal 1, @project.samples.where(name: 'my new sample').count + assert_equal 1, @project.samples.where(name: 'my new sample 2').count + end + + test 'import samples with permission for group' do + assert_equal 3, @project.samples.count + + assert_authorized_to(:update_sample_metadata?, @group, + with: GroupPolicy, + context: { user: @john_doe }) do + Samples::BatchFileImportService.new(@group, @john_doe, @blob.id, @default_params).execute + end + + assert_equal 5, @project.samples.count + assert_equal 1, @project.samples.where(name: 'my new sample').count + assert_equal 1, @project.samples.where(name: 'my new sample 2').count + end + + test 'import samples without permission for project namespace' do + assert_equal 3, @project.samples.count + + exception = assert_raises(ActionPolicy::Unauthorized) do + Samples::BatchFileImportService.new(@project.namespace, @jane_doe, @blob.id, @default_params).execute + end + assert_equal Namespaces::ProjectNamespacePolicy, exception.policy + assert_equal :update_sample_metadata?, exception.rule + assert exception.result.reasons.is_a?(::ActionPolicy::Policy::FailureReasons) + assert_equal I18n.t(:'action_policy.policy.namespaces/project_namespace.update_sample_metadata?', + name: @project.name), exception.result.message + + assert_equal 3, @project.samples.count + end + + test 'import samples without permission for group' do + assert_equal 3, @project.samples.count + + exception = assert_raises(ActionPolicy::Unauthorized) do + Samples::BatchFileImportService.new(@group, @jane_doe, @blob.id, @default_params).execute + end + assert_equal GroupPolicy, exception.policy + assert_equal :update_sample_metadata?, exception.rule + assert exception.result.reasons.is_a?(::ActionPolicy::Policy::FailureReasons) + assert_equal I18n.t(:'action_policy.policy.group.update_sample_metadata?', + name: @group.name), exception.result.message + assert_equal 3, @project.samples.count + end + + test 'import samples with empty file' do + file = Rack::Test::UploadedFile.new(Rails.root.join('test/fixtures/files/metadata/empty.csv')) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + Samples::BatchFileImportService.new(@project.namespace, @john_doe, blob.id, @default_params).execute + assert_equal(@project.namespace.errors.full_messages_for(:base).first, + I18n.t('services.spreadsheet_import.missing_header', + header_title: %w[sample_name project_puid])) + end + + test 'import samples into a project that does not belong to project namespace' do + assert_equal 3, @project.samples.count + assert_equal 20, @project2.samples.count + + response = Samples::BatchFileImportService.new(@project2.namespace, @john_doe, @blob.id, + @default_params).execute + + assert_equal 3, @project.samples.count + assert_equal 20, @project2.samples.count + + assert_equal I18n.t('services.samples.batch_import.project_puid_not_in_namespace', + project_puid: @project.puid, + namespace: @project2.namespace.full_path), + response['my new sample'][:message] + assert_equal I18n.t('services.samples.batch_import.project_puid_not_in_namespace', + project_puid: @project.puid, + namespace: @project2.namespace.full_path), + response['my new sample 2'][:message] + end + + test 'import samples into a project that does not belong to group namespace' do + assert_equal 3, @project.samples.count + assert_equal 20, @project2.samples.count + + response = Samples::BatchFileImportService.new(@group2, @john_doe, @blob.id, @default_params).execute + + assert_equal 3, @project.samples.count + assert_equal 20, @project2.samples.count + + assert_equal I18n.t('services.samples.batch_import.project_puid_not_in_namespace', + project_puid: @project.puid, + namespace: @group2.full_path), + response['my new sample'][:message] + assert_equal I18n.t('services.samples.batch_import.project_puid_not_in_namespace', + project_puid: @project.puid, + namespace: @group2.full_path), + response['my new sample 2'][:message] + end + + test 'import with bad data invalid project' do + assert_equal 3, @project.samples.count + + file = Rack::Test::UploadedFile.new( + Rails.root.join('test/fixtures/files/batch_sample_import_invalid_project.csv') + ) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + response = Samples::BatchFileImportService.new(@project.namespace, @john_doe, blob.id, + @default_params).execute + + assert_equal 4, @project.samples.count + + assert_equal I18n.t('services.samples.batch_import.project_puid_not_found', + project_puid: 'invalid_puid'), + response['my new sample 2'][:message] + end + + test 'import with bad data missing puid' do + assert_equal 3, @project.samples.count + + file = Rack::Test::UploadedFile.new( + Rails.root.join('test/fixtures/files/batch_sample_import_invalid_missing_puid.csv') + ) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + response = Samples::BatchFileImportService.new(@project.namespace, @john_doe, blob.id, + @default_params).execute + + assert_equal 4, @project.samples.count + + assert_equal I18n.t('services.spreadsheet_import.missing_field', + index: 2), + response['index 2'][:message] + end + + test 'import with bad data blank line' do + assert_equal 3, @project.samples.count + + file = Rack::Test::UploadedFile.new( + Rails.root.join('test/fixtures/files/batch_sample_import_invalid_blank_line.csv') + ) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + response = Samples::BatchFileImportService.new(@project.namespace, @john_doe, blob.id, + @default_params).execute + + assert_equal 5, @project.samples.count + + assert_equal I18n.t('services.spreadsheet_import.missing_field', + index: 2), + response['index 2'][:message] + end + + test 'import with bad data short sample name' do + assert_equal 3, @project.samples.count + + file = Rack::Test::UploadedFile.new( + Rails.root.join('test/fixtures/files/batch_sample_import_invalid_short_sample_name.csv') + ) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + response = Samples::BatchFileImportService.new(@project.namespace, @john_doe, blob.id, + @default_params).execute + + assert_equal 4, @project.samples.count + + assert_equal ['sample', 'name'], response['m'][0][:path] # rubocop:disable Style/WordArray + assert_equal 'is too short (minimum is 3 characters)', response['m'][0][:message] + end + + test 'import with bad data sample already exists' do + assert_equal 3, @project.samples.count + + file = Rack::Test::UploadedFile.new( + Rails.root.join('test/fixtures/files/batch_sample_import_invalid_sample_exists.csv') + ) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + response = Samples::BatchFileImportService.new(@project.namespace, @john_doe, blob.id, + @default_params).execute + + assert_equal 4, @project.samples.count + + assert_equal ['sample', 'name'], response['Project 1 Sample 1'][0][:path] # rubocop:disable Style/WordArray + assert_equal 'has already been taken', response['Project 1 Sample 1'][0][:message] + end + + test 'import with bad data invalid duplicate sample name in file' do + assert_equal 3, @project.samples.count + + file = Rack::Test::UploadedFile.new( + Rails.root.join('test/fixtures/files/batch_sample_import_invalid_sample_dup_in_file.csv') + ) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + response = Samples::BatchFileImportService.new(@project.namespace, @john_doe, blob.id, + @default_params).execute + + assert_equal 4, @project.samples.count + + assert_equal I18n.t('services.samples.batch_import.duplicate_sample_name', + index: 2), + response['index 2'][:message] + end + end +end diff --git a/test/services/samples/metadata/file_import_service_test.rb b/test/services/samples/metadata/file_import_service_test.rb index 0af8657464..288da0af7f 100644 --- a/test/services/samples/metadata/file_import_service_test.rb +++ b/test/services/samples/metadata/file_import_service_test.rb @@ -62,19 +62,20 @@ def setup name: @group.name), exception.result.message end - test 'import sample metadata with empty params' do - assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, @blob.id, {}).execute + test 'import sample metadata with empty file' do + file = Rack::Test::UploadedFile.new(Rails.root.join('test/fixtures/files/metadata/empty.csv')) + blob = ActiveStorage::Blob.create_and_upload!( + io: file, + filename: file.original_filename, + content_type: file.content_type + ) + + params = { sample_id_column: 'sample_name' } + assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, blob.id, params).execute assert_equal(@project.namespace.errors.full_messages_for(:base).first, - I18n.t('services.samples.metadata.import_file.empty_sample_id_column')) + I18n.t('services.spreadsheet_import.missing_header', header_title: ['sample_name'])) end - # test 'import sample metadata with no file' do - # params = { sample_id_column: 'sample_name' } - # assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, params).execute - # assert_equal(@project.namespace.errors.full_messages_for(:base).first, - # I18n.t('services.samples.metadata.import_file.empty_file')) - # end - test 'import sample metadata via csv file using sample names for project namespace' do assert_equal({}, @sample1.metadata) assert_equal({}, @sample2.metadata) @@ -240,7 +241,7 @@ def setup params = { sample_id_column: 'sample_name' } assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, blob.id, params).execute assert_equal(@project.namespace.errors.full_messages_for(:base).first, - I18n.t('services.samples.metadata.import_file.invalid_file_extension')) + I18n.t('services.spreadsheet_import.invalid_file_extension')) end test 'import sample metadata with no sample_id_column' do @@ -257,7 +258,7 @@ def setup params = { sample_id_column: 'sample_name' } assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, blob.id, params).execute assert_equal(@project.namespace.errors.full_messages_for(:base).first, - I18n.t('services.samples.metadata.import_file.missing_sample_id_column')) + I18n.t('services.spreadsheet_import.missing_header', header_title: ['sample_name'])) end test 'import sample metadata with duplicate column names' do @@ -272,7 +273,7 @@ def setup params = { sample_id_column: 'sample_name' } assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, blob.id, params).execute assert_equal(@project.namespace.errors.full_messages_for(:base).first, - I18n.t('services.samples.metadata.import_file.duplicate_column_names')) + I18n.t('services.spreadsheet_import.duplicate_column_names')) end test 'import sample metadata with no metadata columns' do @@ -289,7 +290,7 @@ def setup params = { sample_id_column: 'sample_name' } assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, blob.id, params).execute assert_equal(@project.namespace.errors.full_messages_for(:base).first, - I18n.t('services.samples.metadata.import_file.missing_metadata_column')) + I18n.t('services.spreadsheet_import.missing_data_columns')) end test 'import sample metadata with no metadata rows' do @@ -304,7 +305,7 @@ def setup params = { sample_id_column: 'sample_name' } assert_empty Samples::Metadata::FileImportService.new(@project.namespace, @john_doe, blob.id, params).execute assert_equal(@project.namespace.errors.full_messages_for(:base).first, - I18n.t('services.samples.metadata.import_file.missing_metadata_row')) + I18n.t('services.spreadsheet_import.missing_data_row')) end test 'import sample metadata with an empty header' do diff --git a/test/system/groups/samples_test.rb b/test/system/groups/samples_test.rb index ceb0e07749..0ae155e7a6 100644 --- a/test/system/groups/samples_test.rb +++ b/test/system/groups/samples_test.rb @@ -809,7 +809,7 @@ def retrieve_puids perform_enqueued_jobs only: [::Samples::MetadataImportJob] end within %(turbo-frame[id="samples_dialog"]) do - assert_text I18n.t('services.samples.metadata.import_file.invalid_file_extension') + assert_text I18n.t('services.spreadsheet_import.invalid_file_extension') end end @@ -887,7 +887,7 @@ def retrieve_puids perform_enqueued_jobs only: [::Samples::MetadataImportJob] end within %(turbo-frame[id="samples_dialog"]) do - assert_text I18n.t('services.samples.metadata.import_file.duplicate_column_names') + assert_text I18n.t('services.spreadsheet_import.duplicate_column_names') end end @@ -902,7 +902,7 @@ def retrieve_puids perform_enqueued_jobs only: [::Samples::MetadataImportJob] end within %(turbo-frame[id="samples_dialog"]) do - assert_text I18n.t('services.samples.metadata.import_file.missing_metadata_row') + assert_text I18n.t('services.spreadsheet_import.missing_data_row') end end @@ -917,7 +917,7 @@ def retrieve_puids perform_enqueued_jobs only: [::Samples::MetadataImportJob] end within %(turbo-frame[id="samples_dialog"]) do - assert_text I18n.t('services.samples.metadata.import_file.missing_metadata_column') + assert_text I18n.t('services.spreadsheet_import.missing_data_columns') end end diff --git a/test/system/projects/samples_test.rb b/test/system/projects/samples_test.rb index 1716ebf949..1c68ccef8b 100644 --- a/test/system/projects/samples_test.rb +++ b/test/system/projects/samples_test.rb @@ -1314,7 +1314,7 @@ class SamplesTest < ApplicationSystemTestCase ### VERIFY START ### within('#dialog') do # error msg - assert_text I18n.t('services.samples.metadata.import_file.invalid_file_extension') + assert_text I18n.t('services.spreadsheet_import.invalid_file_extension') end ### VERIFY END ### end @@ -1420,7 +1420,7 @@ class SamplesTest < ApplicationSystemTestCase ### VERIFY START ### # error msg - assert_text I18n.t('services.samples.metadata.import_file.duplicate_column_names') + assert_text I18n.t('services.spreadsheet_import.duplicate_column_names') ### VERIFY END ### end end @@ -1445,7 +1445,7 @@ class SamplesTest < ApplicationSystemTestCase ### VERIFY START ### # error msg - assert_text I18n.t('services.samples.metadata.import_file.missing_metadata_row') + assert_text I18n.t('services.spreadsheet_import.missing_data_row') ### VERIFY END ### end end @@ -1470,7 +1470,7 @@ class SamplesTest < ApplicationSystemTestCase ### VERIFY START ### # error msg - assert_text I18n.t('services.samples.metadata.import_file.missing_metadata_column') + assert_text I18n.t('services.spreadsheet_import.missing_data_columns') ### VERIFY END ### end end