From 28d31c46d2946e5dd909990a06471c1907d6af8a Mon Sep 17 00:00:00 2001 From: Laurent Martin Date: Mon, 12 Feb 2024 22:11:16 +0100 Subject: [PATCH] async remote ws certificates --- lib/aspera/fasp/parameters.rb | 74 ++++++++++++++++------------------- lib/aspera/fasp/sync.rb | 48 +++++++++++++++++++---- lib/aspera/rest.rb | 14 +++++++ 3 files changed, 89 insertions(+), 47 deletions(-) diff --git a/lib/aspera/fasp/parameters.rb b/lib/aspera/fasp/parameters.rb index 7844c9c0..34ca2955 100644 --- a/lib/aspera/fasp/parameters.rb +++ b/lib/aspera/fasp/parameters.rb @@ -121,45 +121,6 @@ def file_list_folder=(v) TempFileManager.instance.cleanup_expired(@file_list_folder) end - def remote_certificates(builder, transfer_spec, options) - certificates_to_use = [] - # use web socket secure for session ? - if builder.read_param('wss_enabled') && (options[:wss] || !transfer_spec.key?('fasp_port')) - # by default use web socket session if available, unless removed by user - builder.add_command_line_options(['--ws-connect']) - # TODO: option to give order ssh,ws (legacy http is implied by ssh) - # This will need to be cleaned up in aspera core - transfer_spec['ssh_port'] = builder.read_param('wss_port') - transfer_spec.delete('fasp_port') - transfer_spec.delete('EX_ssh_key_paths') - transfer_spec.delete('sshfp') - # ignore cert for wss ? - if options[:check_ignore]&.call(transfer_spec['remote_host'], transfer_spec['wss_port']) - wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert') - # initiate a session to retrieve remote certificate - http_session = Rest.start_http_session("https://#{transfer_spec['remote_host']}:#{transfer_spec['wss_port']}") - begin - # retrieve underlying openssl socket - File.write(wss_cert_file, Rest.io_http_session(http_session).io.peer_cert_chain.reverse.map(&:to_pem).join("\n")) - rescue - File.write(wss_cert_file, http_session.peer_cert.to_pem) - end - http_session.finish - certificates_to_use.push(wss_cert_file) - end - # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR - options[:trusted_certs]&.each {|file|certificates_to_use.push(file)} - else - # remove unused parameter (avoid warning) - transfer_spec.delete('wss_port') - # add SSH bypass keys when authentication is token and no auth is provided - if transfer_spec.key?('token') && !transfer_spec.key?('remote_password') - # transfer_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase - Installation.instance.aspera_token_ssh_key_paths.each { |path| certificates_to_use.push(path) } - end - end - return certificates_to_use - end # static methods attr_reader :file_list_folder end # self @@ -229,6 +190,39 @@ def process_file_list @builder.add_command_line_options(["#{option}=#{file_list_file}"]) unless option.nil? end + def remote_certificates + certificates_to_use = [] + # use web socket secure for session ? + if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port')) + # by default use web socket session if available, unless removed by user + @builder.add_command_line_options(['--ws-connect']) + # TODO: option to give order ssh,ws (legacy http is implied by ssh) + # This will need to be cleaned up in aspera core + @job_spec['ssh_port'] = @builder.read_param('wss_port') + @job_spec.delete('fasp_port') + @job_spec.delete('EX_ssh_key_paths') + @job_spec.delete('sshfp') + # ignore cert for wss ? + if @options[:check_ignore]&.call(@job_spec['remote_host'], @job_spec['wss_port']) + wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert') + wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}" + File.write(wss_cert_file, Rest.remote_certificates(wss_url)) + certificates_to_use.push(wss_cert_file) + end + # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR + certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs] + else + # remove unused parameter (avoid warning) + @job_spec.delete('wss_port') + # add SSH bypass keys when authentication is token and no auth is provided + if @job_spec.key?('token') && !@job_spec.key?('remote_password') + # @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase + certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths) + end + end + return certificates_to_use + end + # translate transfer spec to env vars and command line arguments for ascp # NOTE: parameters starting with "EX_" (extended) are not standard def ascp_args @@ -245,7 +239,7 @@ def ascp_args assert(!@builder.read_param('multi_session')) # add ssh or wss certificates - self.class.remote_certificates(@builder, @job_spec, @options).each do |cert| + remote_certificates.each do |cert| Log.log.trace1{"adding certificate: #{cert}"} env_args[:args].unshift('-i', cert) end diff --git a/lib/aspera/fasp/sync.rb b/lib/aspera/fasp/sync.rb index 01da69f1..d3ac7515 100644 --- a/lib/aspera/fasp/sync.rb +++ b/lib/aspera/fasp/sync.rb @@ -17,6 +17,7 @@ module Fasp module Sync # sync direction, default is push DIRECTIONS = %i[push pull bidi].freeze + # custom JSON for async instance command line options PARAMS_VX_INSTANCE = { 'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string}, @@ -104,6 +105,32 @@ def update_remote_dir(sync_params, remote_dir_key, transfer_spec) nil end + def remote_certificates(remote) + certificates_to_use = [] + # use web socket secure for session ? + if remote['connect_mode']&.eql?('ws') + remote.delete('port') + remote.delete('fingerprint') + # ignore cert for wss ? + if false # @options[:check_ignore]&.call(remote['host'], remote['ws_port']) + wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert') + wss_url = "https://#{remote['host']}:#{remote['ws_port']}" + File.write(wss_cert_file, Rest.remote_certificates(wss_url)) + certificates_to_use.push(wss_cert_file) + end + # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR + # certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs] + else + # remove unused parameter (avoid warning) + remote.delete('ws_port') + # add SSH bypass keys when authentication is token and no auth is provided + if remote.key?('token') && !remote.key?('pass') + certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths) + end + end + return certificates_to_use + end + # @param sync_params [Hash] sync parameters, old or new format # @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir def start(sync_params, &block) @@ -113,12 +140,15 @@ def start(sync_params, &block) env: {} } if sync_params.key?('local') + remote = sync_params['remote'] # async native JSON format (v2) - assert_type(sync_params['remote'], Hash) + assert_type(remote, Hash) + # get transfer spec if possible, and feed back to new structure if block - transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], sync_params['remote']['path']) + transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], remote['path']) # async native JSON format assert_type(sync_params['local'], Hash) + # translate transfer spec to async parameters TS_TO_PARAMS_V2.each do |ts_param, sy_path| next unless transfer_spec.key?(ts_param) sy_dig = sy_path.split('.') @@ -127,10 +157,15 @@ def start(sync_params, &block) hash = sync_params[sy_dig.first] = {} if hash.nil? hash[param] = transfer_spec[ts_param] end - sync_params['remote']['connect_mode'] ||= sync_params['remote'].key?('ws_port') ? 'ws' : 'ssh' - sync_params['remote']['private_key_paths'] ||= Fasp::Installation.instance.aspera_token_ssh_key_paths if transfer_spec.key?('token') - update_remote_dir(sync_params['remote'], 'path', transfer_spec) + update_remote_dir(remote, 'path', transfer_spec) end + remote['connect_mode'] ||= remote.key?('ws_port') ? 'ws' : 'ssh' + add_certificates = remote_certificates(remote) + if !add_certificates.empty? + remote['private_key_paths'] ||= [] + remote['private_key_paths'].concat(add_certificates) + end + assert_type(sync_params, Hash) env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"] elsif sync_params.key?('sessions') # ascli JSON format (v1) @@ -169,7 +204,6 @@ def start(sync_params, &block) raise 'At least one of `local` or `sessions` must be present in async parameters' end Log.log.debug{Log.dump(:sync_params, sync_params)} - Log.log.debug{"execute: #{env_args[:env].map{|k, v| "#{k}=\"#{v}\""}.join(' ')} \"#{ASYNC_EXECUTABLE}\" \"#{env_args[:args].join('" "')}\""} res = system(env_args[:env], [ASYNC_EXECUTABLE, ASYNC_EXECUTABLE], *env_args[:args]) Log.log.debug{"result=#{res}"} @@ -233,7 +267,7 @@ def admin_status(sync_params, session_name) raise "Sync failed: #{status.exitstatus} : #{stderr}" unless status.success? return parse_status(stdout) end - end + end # end self end # end Sync end # end Fasp end # end Aspera diff --git a/lib/aspera/rest.rb b/lib/aspera/rest.rb index 5ec521a1..146b0fcf 100644 --- a/lib/aspera/rest.rb +++ b/lib/aspera/rest.rb @@ -112,6 +112,20 @@ def io_http_session(http_session) return result end + # @return [String] PEM certificates of remote server + def remote_certificates(url) + # initiate a session to retrieve remote certificate + http_session = Rest.start_http_session(url) + begin + # retrieve underlying openssl socket + return Rest.io_http_session(http_session).io.peer_cert_chain.reverse.map(&:to_pem).join("\n") + rescue + return http_session.peer_cert.to_pem + ensure + http_session.finish + end + end + # set global parameters def set_parameters(**options) options.each do |key, value|