diff --git a/lib/aspera/api/node.rb b/lib/aspera/api/node.rb index f9280f5c..48c4ee06 100644 --- a/lib/aspera/api/node.rb +++ b/lib/aspera/api/node.rb @@ -47,15 +47,11 @@ class Node < Aspera::Rest @use_node_cache = true class << self + # set to false to read transfer parameters from download_setup attr_accessor :use_standard_ports + # set to false to bypass cache in redis attr_accessor :use_node_cache - def cache_control_headers - h = {'Accept' => 'application/json'} - h[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache - h - end - # For access keys: provide expression to match entry in folder # @param match_expression one of supported types # @return lambda function @@ -160,7 +156,13 @@ def initialize(app_info: nil, add_tspec: nil, **rest_args) # Call node API, possibly adding cache control header, as globally specified def read_with_cache(subpath, query=nil) - return call(operation: 'GET', subpath: subpath, headers: self.class.cache_control_headers, query: query)[:data] + headers = {'Accept' => 'application/json'} + headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache + return call( + operation: 'GET', + subpath: subpath, + headers: headers, + query: query)[:data] end # update transfer spec with special additional tags @@ -204,7 +206,7 @@ def entry_has_link_information(entry) # @param state [Object] state object sent to processing method # @param top_file_id [String] file id to start at (default = access key root file id) # @param top_file_path [String] path of top folder (default = /) - def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/') + def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil) Aspera.assert(!top_file_path.nil?){'top_file_path not set'} Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id}, path=#{top_file_path}"} # start at top folder @@ -217,7 +219,7 @@ def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/') # get folder content folder_contents = begin - read("files/#{current_item[:id]}/files") + read_with_cache("files/#{current_item[:id]}/files") rescue StandardError => e Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"} [] @@ -230,21 +232,21 @@ def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/') end next end - relative_path = File.join(current_item[:path], entry['name']) - Log.log.debug{"process_folder_tree: checking #{relative_path}"} + current_path = File.join(current_item[:path], entry['name']) + Log.log.debug{"process_folder_tree: checking #{current_path}"} # call block, continue only if method returns true - next unless send(method_sym, entry, relative_path, state) + next unless send(method_sym, entry, current_path, state) # entry type is file, folder or link case entry['type'] when 'folder' - folders_to_explore.push({id: entry['id'], path: relative_path}) + folders_to_explore.push({id: entry['id'], path: current_path}) when 'link' if entry_has_link_information(entry) node_id_to_node(entry['target_node_id'])&.process_folder_tree( method_sym: method_sym, state: state, top_file_id: entry['target_id'], - top_file_path: relative_path) + top_file_path: current_path) end end end @@ -277,6 +279,12 @@ def find_files(top_file_id, test_lambda) return find_state[:found] end + def list_files(top_file_id) + find_state = {found: []} + process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id) + return find_state[:found] + end + def refreshed_transfer_token return oauth.token(refresh: true) end @@ -359,8 +367,8 @@ def transfer_spec_gen4(file_id, direction, ts_merge=nil) private + # method called in loop for each entry for `resolve_api_fid` def process_api_fid(entry, path, state) - # this block is called recursively for each entry in folder # stop digging here if not in right path return false unless entry['name'].eql?(state[:path].first) # ok it matches, so we remove the match, and continue digging @@ -403,12 +411,18 @@ def process_api_fid(entry, path, state) return true end - # method called in loop for each entry for "find" + # method called in loop for each entry for `find_files` def process_find_files(entry, path, state) state[:found].push(entry.merge({'path' => path})) if state[:test_lambda].call(entry) # test all files deeply return true end + + # method called in loop for each entry for `list_files` + def process_list_files(entry, path, state) + state[:found].push(entry.merge({'path' => path})) + return false + end end end end diff --git a/lib/aspera/cli/plugins/node.rb b/lib/aspera/cli/plugins/node.rb index e47d8727..f59a387b 100644 --- a/lib/aspera/cli/plugins/node.rb +++ b/lib/aspera/cli/plugins/node.rb @@ -131,6 +131,8 @@ def gen3_entry_folder?(entry) COMMANDS_SHARES = (BASE_ACTIONS - %i[search]).freeze COMMANDS_FASPEX = COMMON_ACTIONS + GEN4_LS_FIELDS = %w[name type recursive_size size modified_time access_level].freeze + def initialize(api: nil, **env) super(**env, basic_options: api.nil?) Node.declare_options(options) if api.nil? @@ -484,22 +486,15 @@ def execute_command_gen4(command_repo, top_file_id) when :browse apifid = apifid_from_next_arg(top_file_id) file_info = apifid[:api].read_with_cache("files/#{apifid[:file_id]}") - if file_info['type'].eql?('folder') - result = apifid[:api].call( - operation: 'GET', - subpath: "files/#{apifid[:file_id]}/files", - headers: Api::Node.cache_control_headers, - query: query_read_delete) - items = result[:data] - formatter.display_item_count(result[:data].length, result[:http]['X-Total-Count']) - else - items = [file_info] + unless file_info['type'].eql?('folder') + # a single file + return {type: :object_list, data: [file_info], fields: GEN4_LS_FIELDS} end - return {type: :object_list, data: items, fields: %w[name type recursive_size size modified_time access_level]} + return {type: :object_list, data: apifid[:api].list_files(apifid[:file_id]), fields: GEN4_LS_FIELDS} when :find apifid = apifid_from_next_arg(top_file_id) - test_block = Api::Node.file_matcher_from_argument(options) - return {type: :object_list, data: @api_node.find_files(apifid[:file_id], test_block), fields: ['path']} + find_lambda = Api::Node.file_matcher_from_argument(options) + return {type: :object_list, data: @api_node.find_files(apifid[:file_id], find_lambda), fields: ['path']} when :mkdir containing_folder_path = options.get_next_argument('path').split(Api::Node::PATH_SEPARATOR) new_folder = containing_folder_path.pop diff --git a/tests/Makefile b/tests/Makefile index 91f1a6f4..aea82185 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -930,7 +930,10 @@ $(T)aocp2: $(T).exists @$(STOP_TEST) $@ $(T)aocp3: $(T).exists @$(START_TEST) $@ - $(EXE_NO_MAN) aoc packages list --format=csv --fields=id --display=data --query=@json:'{"max":1}' --output=$(DIR_TMP)package_id3 + :>$(DIR_TMP)package_id3;while test -z "$$(cat $(DIR_TMP)package_id3)";do \ + $(EXE_NO_MAN) aoc packages list --format=csv --fields=id --display=data --query=@json:'{"max":1}' --output=$(DIR_TMP)package_id3;\ + sleep 3;\ + done $(EXE_MAN) aoc packages receive $$(cat $(DIR_TMP)package_id3) --to-folder=$(DIR_TMP). $(EXE_MAN) aoc packages browse $$(cat $(DIR_TMP)package_id3) /contents @$(STOP_TEST) $@ @@ -1019,7 +1022,7 @@ $(T)aoc_adm_wsm_lst: $(T).exists @$(STOP_TEST) $@ $(T)aoc15: $(T).exists @$(START_TEST) $@ - $(EXE_MAN) --preset=aoc_admin aoc admin analytics transfers organization --query=@json:'{"status":"completed","direction":"receive"}' --notify-to=@preset:misc.email_external --notify-template=@ruby:'%Q{From: <%=from_name%> <<%=from_email%>>\nTo: <<%=to%>>\nSubject: <%=ev["files_completed"]%> files received\n\n<%=ev.to_yaml%>}' + $(EXE_MAN) --preset=aoc_admin aoc admin analytics transfers organization --query=@json:'{"status":"completed","direction":"receive","limit":2}' --notify-to=@preset:misc.email_external --notify-template=@ruby:'%Q{From: <%=from_name%> <<%=from_email%>>\nTo: <<%=to%>>\nSubject: <%=ev["files_completed"]%> files received\n\n<%=ev.to_yaml%>}' @$(STOP_TEST) $@ $(T)aoc16: $(T).exists @$(START_TEST) $@ @@ -1428,6 +1431,7 @@ $(T)conf_gem: $(T).exists $(EXE_MAN) config smtp_settings $(EXE_MAN) $(TMP_CONF) config initdemo $(EXE_NO_MAN) $(TMP_CONF) config initdemo + $(EXE_NO_MAN) config gem path --http-proxy=http://test1 --http-options=@json:'{"user_agent":"test1"}' @$(STOP_TEST) $@ $(T)conf_vault_macos: $(T).exists @$(START_TEST) $@