use browse from node api with/out cache (recursive)

IBM · Feb 10, 2025 · f5f6a60 · f5f6a60
1 parent 609d2fd
commit f5f6a60
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 31 deletions.
diff --git a/lib/aspera/api/node.rb b/lib/aspera/api/node.rb
@@ -47,15 +47,11 @@ class Node < Aspera::Rest
       @use_node_cache = true
 
       class << self
+        # set to false to read transfer parameters from download_setup
         attr_accessor :use_standard_ports
+        # set to false to bypass cache in redis
         attr_accessor :use_node_cache
 
-        def cache_control_headers
-          h = {'Accept' => 'application/json'}
-          h[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
-          h
-        end
-
         # For access keys: provide expression to match entry in folder
         # @param match_expression one of supported types
         # @return lambda function
@@ -160,7 +156,13 @@ def initialize(app_info: nil, add_tspec: nil, **rest_args)
 
       # Call node API, possibly adding cache control header, as globally specified
       def read_with_cache(subpath, query=nil)
-        return call(operation: 'GET', subpath: subpath, headers: self.class.cache_control_headers, query: query)[:data]
+        headers = {'Accept' => 'application/json'}
+        headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
+        return call(
+          operation: 'GET',
+          subpath:   subpath,
+          headers:   headers,
+          query:     query)[:data]
       end
 
       # update transfer spec with special additional tags
@@ -204,7 +206,7 @@ def entry_has_link_information(entry)
       # @param state [Object] state object sent to processing method
       # @param top_file_id [String] file id to start at (default = access key root file id)
       # @param top_file_path [String] path of top folder (default = /)
-      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
+      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
         Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
         Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
         # start at top folder
@@ -217,7 +219,7 @@ def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
           # get folder content
           folder_contents =
             begin
-              read("files/#{current_item[:id]}/files")
+              read_with_cache("files/#{current_item[:id]}/files")
             rescue StandardError => e
               Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
               []
@@ -230,21 +232,21 @@ def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
               end
               next
             end
-            relative_path = File.join(current_item[:path], entry['name'])
-            Log.log.debug{"process_folder_tree: checking #{relative_path}"}
+            current_path = File.join(current_item[:path], entry['name'])
+            Log.log.debug{"process_folder_tree: checking #{current_path}"}
             # call block, continue only if method returns true
-            next unless send(method_sym, entry, relative_path, state)
+            next unless send(method_sym, entry, current_path, state)
             # entry type is file, folder or link
             case entry['type']
             when 'folder'
-              folders_to_explore.push({id: entry['id'], path: relative_path})
+              folders_to_explore.push({id: entry['id'], path: current_path})
             when 'link'
               if entry_has_link_information(entry)
                 node_id_to_node(entry['target_node_id'])&.process_folder_tree(
                   method_sym:    method_sym,
                   state:         state,
                   top_file_id:   entry['target_id'],
-                  top_file_path: relative_path)
+                  top_file_path: current_path)
               end
             end
           end
@@ -277,6 +279,12 @@ def find_files(top_file_id, test_lambda)
         return find_state[:found]
       end
 
+      def list_files(top_file_id)
+        find_state = {found: []}
+        process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id)
+        return find_state[:found]
+      end
+
       def refreshed_transfer_token
         return oauth.token(refresh: true)
       end
@@ -359,8 +367,8 @@ def transfer_spec_gen4(file_id, direction, ts_merge=nil)
 
       private
 
+      # method called in loop for each entry for `resolve_api_fid`
       def process_api_fid(entry, path, state)
-        # this block is called recursively for each entry in folder
         # stop digging here if not in right path
         return false unless entry['name'].eql?(state[:path].first)
         # ok it matches, so we remove the match, and continue digging
@@ -403,12 +411,18 @@ def process_api_fid(entry, path, state)
         return true
       end
 
-      # method called in loop for each entry for "find"
+      # method called in loop for each entry for `find_files`
       def process_find_files(entry, path, state)
         state[:found].push(entry.merge({'path' => path})) if state[:test_lambda].call(entry)
         # test all files deeply
         return true
       end
+
+      # method called in loop for each entry for `list_files`
+      def process_list_files(entry, path, state)
+        state[:found].push(entry.merge({'path' => path}))
+        return false
+      end
     end
   end
 end
diff --git a/lib/aspera/cli/plugins/node.rb b/lib/aspera/cli/plugins/node.rb
@@ -131,6 +131,8 @@ def gen3_entry_folder?(entry)
         COMMANDS_SHARES = (BASE_ACTIONS - %i[search]).freeze
         COMMANDS_FASPEX = COMMON_ACTIONS
 
+        GEN4_LS_FIELDS = %w[name type recursive_size size modified_time access_level].freeze
+
         def initialize(api: nil, **env)
           super(**env, basic_options: api.nil?)
           Node.declare_options(options) if api.nil?
@@ -484,22 +486,15 @@ def execute_command_gen4(command_repo, top_file_id)
           when :browse
             apifid = apifid_from_next_arg(top_file_id)
             file_info = apifid[:api].read_with_cache("files/#{apifid[:file_id]}")
-            if file_info['type'].eql?('folder')
-              result = apifid[:api].call(
-                operation: 'GET',
-                subpath:   "files/#{apifid[:file_id]}/files",
-                headers:   Api::Node.cache_control_headers,
-                query:     query_read_delete)
-              items = result[:data]
-              formatter.display_item_count(result[:data].length, result[:http]['X-Total-Count'])
-            else
-              items = [file_info]
+            unless file_info['type'].eql?('folder')
+              # a single file
+              return {type: :object_list, data: [file_info], fields: GEN4_LS_FIELDS}
             end
-            return {type: :object_list, data: items, fields: %w[name type recursive_size size modified_time access_level]}
+            return {type: :object_list, data: apifid[:api].list_files(apifid[:file_id]), fields: GEN4_LS_FIELDS}
           when :find
             apifid = apifid_from_next_arg(top_file_id)
-            test_block = Api::Node.file_matcher_from_argument(options)
-            return {type: :object_list, data: @api_node.find_files(apifid[:file_id], test_block), fields: ['path']}
+            find_lambda = Api::Node.file_matcher_from_argument(options)
+            return {type: :object_list, data: @api_node.find_files(apifid[:file_id], find_lambda), fields: ['path']}
           when :mkdir
             containing_folder_path = options.get_next_argument('path').split(Api::Node::PATH_SEPARATOR)
             new_folder = containing_folder_path.pop

diff --git a/tests/Makefile b/tests/Makefile
@@ -930,7 +930,10 @@ $(T)aocp2: $(T).exists
 	@$(STOP_TEST) $@
 $(T)aocp3: $(T).exists
 	@$(START_TEST) $@
-	$(EXE_NO_MAN) aoc packages list --format=csv --fields=id --display=data --query=@json:'{"max":1}' --output=$(DIR_TMP)package_id3
+	:>$(DIR_TMP)package_id3;while test -z "$$(cat $(DIR_TMP)package_id3)";do \
+	$(EXE_NO_MAN) aoc packages list --format=csv --fields=id --display=data --query=@json:'{"max":1}' --output=$(DIR_TMP)package_id3;\
+	sleep 3;\
+	done
 	$(EXE_MAN) aoc packages receive $$(cat $(DIR_TMP)package_id3) --to-folder=$(DIR_TMP).
 	$(EXE_MAN) aoc packages browse $$(cat $(DIR_TMP)package_id3) /contents
 	@$(STOP_TEST) $@
@@ -1019,7 +1022,7 @@ $(T)aoc_adm_wsm_lst: $(T).exists
 	@$(STOP_TEST) $@
 $(T)aoc15: $(T).exists
 	@$(START_TEST) $@
-	$(EXE_MAN) --preset=aoc_admin aoc admin analytics transfers organization --query=@json:'{"status":"completed","direction":"receive"}' --notify-to=@preset:misc.email_external --notify-template=@ruby:'%Q{From: <%=from_name%> <<%=from_email%>>\nTo: <<%=to%>>\nSubject: <%=ev["files_completed"]%> files received\n\n<%=ev.to_yaml%>}'
+	$(EXE_MAN) --preset=aoc_admin aoc admin analytics transfers organization --query=@json:'{"status":"completed","direction":"receive","limit":2}' --notify-to=@preset:misc.email_external --notify-template=@ruby:'%Q{From: <%=from_name%> <<%=from_email%>>\nTo: <<%=to%>>\nSubject: <%=ev["files_completed"]%> files received\n\n<%=ev.to_yaml%>}'
 	@$(STOP_TEST) $@
 $(T)aoc16: $(T).exists
 	@$(START_TEST) $@
@@ -1428,6 +1431,7 @@ $(T)conf_gem: $(T).exists
 	$(EXE_MAN) config smtp_settings
 	$(EXE_MAN) $(TMP_CONF) config initdemo
 	$(EXE_NO_MAN) $(TMP_CONF) config initdemo
+	$(EXE_NO_MAN) config gem path --http-proxy=http://test1 --http-options=@json:'{"user_agent":"test1"}'
 	@$(STOP_TEST) $@
 $(T)conf_vault_macos: $(T).exists
 	@$(START_TEST) $@