Merge pull request #1206 from soutaro/load-files

Load files in main process

lib/steep.rb

@@ -15,6 +15,7 @@
 require 'uri'
 require "yaml"
 require "securerandom"
+require "base64"
 
 require "concurrent/utility/processor_counter"
 require "terminal-table"
@@ -160,7 +161,11 @@ def self.ui_logger
   end
 
   def self.new_logger(output, prev_level)
-    ActiveSupport::TaggedLogging.new(Logger.new(output)).tap do |logger|
+    logger = Logger.new(output)
+    logger.formatter = proc do |severity, datetime, progname, msg|
+      "#{datetime.strftime('%Y-%m-%d %H:%M:%S')}: #{severity}: #{msg}\n"
+    end
+    ActiveSupport::TaggedLogging.new(logger).tap do |logger|
       logger.push_tags "Steep #{VERSION}"
       logger.level = prev_level || Logger::ERROR
     end

lib/steep/server/change_buffer.rb

@@ -24,16 +24,14 @@ def pop_buffer
         end
       end
 
-      def load_files(project:, commandline_args:)
+      def load_files(input)
         Steep.logger.tagged "#load_files" do
           push_buffer do |changes|
-            loader = Services::FileLoader.new(base_dir: project.base_dir)
-
-            Steep.measure "load changes from disk" do
-              project.targets.each do |target|
-                loader.load_changes(target.source_pattern, commandline_args, changes: changes)
-                loader.load_changes(target.signature_pattern, changes: changes)
+            input.each do |filename, content|
+              if content.is_a?(Hash)
+                content = Base64.decode64(content[:text]).force_encoding(Encoding::UTF_8)
               end
+              changes[Pathname(filename.to_s)] = [Services::ContentChange.new(text: content)]
             end
           end
         end

lib/steep/server/interaction_worker.rb

@@ -82,14 +82,17 @@ def queue_job(job)
       def handle_request(request)
         case request[:method]
         when "initialize"
-          load_files(project: project, commandline_args: [])
-          queue_job ApplyChangeJob.new
           writer.write({ id: request[:id], result: nil })
 
         when "textDocument/didChange"
           collect_changes(request)
           queue_job ApplyChangeJob.new
 
+        when "$/file/load"
+          input = request[:params][:content]
+          load_files(input)
+          queue_job ApplyChangeJob.new
+
         when "$/file/reset"
           uri = request[:params][:uri]
           text = request[:params][:content]

lib/steep/server/master.rb

@@ -150,6 +150,8 @@ def initialize(project:)
         def load(command_line_args:)
           loader = Services::FileLoader.new(base_dir: project.base_dir)
 
+          files = {} #: Hash[String, String]
+
           target_paths.each do |paths|
             target = paths.target
 
@@ -158,13 +160,25 @@ def load(command_line_args:)
 
             loader.each_path_in_patterns(target.source_pattern, command_line_args) do |path|
               paths.code_paths << project.absolute_path(path)
+              files[path.to_s] = project.absolute_path(path).read
+              if files.size > 1000
+                yield files.dup
+                files.clear
+              end
             end
             loader.each_path_in_patterns(target.signature_pattern) do |path|
               paths.signature_paths << project.absolute_path(path)
+              files[path.to_s] = project.absolute_path(path).read
+              if files.size > 1000
+                yield files.dup
+                files.clear
+              end
             end
 
             changed_paths.merge(paths.all_paths)
           end
+
+          yield files.dup unless files.empty?
         end
 
         def push_changes(path)
@@ -380,7 +394,7 @@ def response?
         include MessageUtils
       end
 
-      SendMessageJob = _ = Struct.new(:dest, :message, keyword_init: true) do
+      class SendMessageJob < Struct.new(:dest, :message, keyword_init: true)
         # @implements SendMessageJob
 
         def self.to_worker(worker, message:)
@@ -556,9 +570,21 @@ def process_message_from_client(message)
               group << send_request(method: "initialize", params: message[:params], worker: worker)
             end
 
-            group.on_completion do
-              controller.load(command_line_args: commandline_args)
+            Steep.measure("Load files from disk...") do
+              controller.load(command_line_args: commandline_args) do |input|
+                input.transform_values! do |content|
+                  content.is_a?(String) or raise
+                  if content.valid_encoding?
+                    content
+                  else
+                    { text: Base64.encode64(content), binary: true }
+                  end
+                end
+                broadcast_notification({ method: "$/file/load", params: { content: input } })
+              end
+            end
 
+            group.on_completion do
               job_queue << SendMessageJob.to_client(
                 message: {
                   id: id,

lib/steep/server/type_check_worker.rb

@@ -11,9 +11,7 @@ class TypeCheckWorker < BaseWorker
       TypeCheckCodeJob = _ = Struct.new(:guid, :path, keyword_init: true)
       ValidateAppSignatureJob = _ = Struct.new(:guid, :path, keyword_init: true)
       ValidateLibrarySignatureJob = _ = Struct.new(:guid, :path, keyword_init: true)
-      GotoJob = _ = Struct.new(:id, :kind, :params, keyword_init: true) do
-        # @implements GotoJob
-
+      class GotoJob < Struct.new(:id, :kind, :params, keyword_init: true)
         def self.implementation(id:, params:)
           new(
             kind: :implementation,
@@ -68,12 +66,15 @@ def initialize(project:, reader:, writer:, assignment:, commandline_args:)
       def handle_request(request)
         case request[:method]
         when "initialize"
-          load_files(project: project, commandline_args: commandline_args)
           writer.write({ id: request[:id], result: nil})
 
         when "textDocument/didChange"
           collect_changes(request)
 
+        when "$/file/load"
+          input = request[:params][:content]
+          load_files(input)
+
         when "$/file/reset"
           uri = request[:params][:uri]
           text = request[:params][:content]

rbs_collection.steep.yaml

@@ -23,3 +23,4 @@ gems:
   - name: securerandom
   - name: ffi
     ignore: true
+  - name: base64

sig/steep/server/change_buffer.rbs

@@ -22,9 +22,11 @@ module Steep
       def pop_buffer: [A] () { (changes) -> A } -> A
                     | () -> changes
 
+      type content = String | { text: String, binary: true }
+
       # Load files from `project` to `buffered_changes`
       #
-      def load_files: (project: Project, commandline_args: Array[String]) -> void
+      def load_files: (Hash[String, content] input) -> void
 
       # Load changes from a request with `DidChangeTextDocumentParams` into `buffered_changes`
       #

sig/steep/server/master.rbs

@@ -110,7 +110,7 @@ module Steep
 
         def initialize: (project: Project) -> void
 
-        def load: (command_line_args: Array[String]) -> void
+        def load: (command_line_args: Array[String]) { (Hash[String, ChangeBuffer::content]) -> void } -> void
 
         def push_changes: (Pathname path) -> void
 
@@ -213,7 +213,7 @@ module Steep
 
         attr_reader message: untyped
 
-        def initialize: (dest: WorkerProcess | :client, message: untyped) -> void
+        def self.new: (dest: WorkerProcess | :client, message: untyped) -> instance
 
         def self.to_worker: (WorkerProcess, message: untyped) -> SendMessageJob
 

sig/steep/server/type_check_worker.rbs

@@ -82,7 +82,7 @@ module Steep
 
         attr_reader params: params
 
-        def initialize: (id: String, params: params, kind: kind) -> void
+        def self.new: (id: String, params: params, kind: kind) -> instance
 
         def self.implementation: (id: String, params: params) -> GotoJob
 

test/interaction_worker_test.rb

@@ -53,6 +53,7 @@ def test_handle_request_initialize
       worker = InteractionWorker.new(project: project, reader: worker_reader, writer: worker_writer)
 
       worker.handle_request({ method: "initialize", id: 1, params: nil })
+      worker.handle_request({ method: "$/file/load", params: { content: {} } })
 
       q = flush_queue(worker.queue)
       assert_equal 1, q.size

test/master_type_check_controller_test.rb

@@ -104,7 +104,7 @@ class Customer
       Project::DSL.parse(project, steepfile.read)
 
       controller = Server::Master::TypeCheckController.new(project: project)
-      controller.load(command_line_args: [])
+      controller.load(command_line_args: []) {}
 
       controller.target_paths[0].tap do |paths|
         assert_equal Set[current_dir + "lib/customer.rb"], paths.code_paths
@@ -137,7 +137,7 @@ class Customer
       Project::DSL.parse(project, steepfile.read)
 
       controller = Server::Master::TypeCheckController.new(project: project)
-      controller.load(command_line_args: [])
+      controller.load(command_line_args: []) {}
       controller.changed_paths.clear()
 
       controller.push_changes(current_dir + "lib/customer.rb")
@@ -177,7 +177,7 @@ class Customer
       Project::DSL.parse(project, steepfile.read)
 
       controller = Server::Master::TypeCheckController.new(project: project)
-      controller.load(command_line_args: [])
+      controller.load(command_line_args: []) {}
       controller.changed_paths.clear()
 
       controller.update_priority(open: current_dir + "lib/customer.rb")
@@ -213,7 +213,7 @@ class Customer
       Project::DSL.parse(project, steepfile.read)
 
       controller = Server::Master::TypeCheckController.new(project: project)
-      controller.load(command_line_args: [])
+      controller.load(command_line_args: []) {}
       controller.changed_paths.clear()
 
       assert_nil controller.make_request()
@@ -241,7 +241,7 @@ class Customer
       Project::DSL.parse(project, steepfile.read)
 
       controller = Server::Master::TypeCheckController.new(project: project)
-      controller.load(command_line_args: [])
+      controller.load(command_line_args: []) {}
       controller.changed_paths.clear()
 
       controller.update_priority(open: current_dir + "lib/customer.rb")
@@ -278,7 +278,7 @@ class Customer
       Project::DSL.parse(project, steepfile.read)
 
       controller = Server::Master::TypeCheckController.new(project: project)
-      controller.load(command_line_args: [])
+      controller.load(command_line_args: []) {}
       controller.changed_paths.clear()
 
       controller.update_priority(open: current_dir + "lib/customer.rb")
@@ -319,7 +319,7 @@ class Account
       Project::DSL.parse(project, steepfile.read)
 
       controller = Server::Master::TypeCheckController.new(project: project)
-      controller.load(command_line_args: [])
+      controller.load(command_line_args: []) {}
       controller.changed_paths.clear()
 
       controller.update_priority(open: current_dir + "lib/customer.rb")

test/type_check_worker_test.rb

@@ -186,7 +186,6 @@ def test_handle_request_typecheck_start
           reader: worker_reader,
           writer: worker_writer
         )
-        worker.load_files(project: worker.project, commandline_args: [])
 
         worker.handle_request(
           {
@@ -250,7 +249,6 @@ def test_handle_job_start_typecheck
           reader: worker_reader,
           writer: worker_writer
         )
-        worker.load_files(project: worker.project, commandline_args: [])
 
         changes = {}
         changes[Pathname("lib/hello.rb")] = [Services::ContentChange.string(<<~RUBY)]
@@ -298,7 +296,6 @@ def test_handle_job_validate_app_signature
           writer: worker_writer
         )
 
-        worker.load_files(project: worker.project, commandline_args: [])
         worker.instance_variable_set(:@current_type_check_guid, "guid")
 
         {}.tap do |changes|
@@ -349,7 +346,6 @@ def test_handle_job_validate_app_signature_skip
           writer: worker_writer
         )
 
-        worker.load_files(project: worker.project, commandline_args: [])
         worker.instance_variable_set(:@current_type_check_guid, nil)
 
         {}.tap do |changes|
@@ -396,7 +392,6 @@ def test_handle_job_validate_lib_signature
           writer: worker_writer
         )
 
-        worker.load_files(project: worker.project, commandline_args: [])
         worker.instance_variable_set(:@current_type_check_guid, "guid")
 
         {}.tap do |changes|
@@ -450,7 +445,6 @@ def test_handle_job_validate_lib_signature_skip
           writer: worker_writer
         )
 
-        worker.load_files(project: worker.project, commandline_args: [])
         worker.instance_variable_set(:@current_type_check_guid, nil)
 
         {}.tap do |changes|
@@ -498,7 +492,6 @@ def test_handle_job_typecheck_code
           writer: worker_writer
         )
 
-        worker.load_files(project: worker.project, commandline_args: [])
         worker.instance_variable_set(:@current_type_check_guid, "guid")
 
         {}.tap do |changes|
@@ -557,7 +550,6 @@ def test_handle_job_typecheck_code_diagnostics
           writer: worker_writer
         )
 
-        worker.load_files(project: worker.project, commandline_args: [])
         worker.instance_variable_set(:@current_type_check_guid, "guid")
 
         {}.tap do |changes|
@@ -619,7 +611,6 @@ def test_handle_job_typecheck_skip
           writer: worker_writer
         )
 
-        worker.load_files(project: worker.project, commandline_args: [])
         worker.instance_variable_set(:@current_type_check_guid, nil)
 
         {}.tap do |changes|
@@ -685,35 +676,6 @@ def new_class_method: () -> void
     end
   end
 
-  def test_loading_files_with_args
-    in_tmpdir do
-      project = Project.new(steepfile_path: current_dir + "Steepfile")
-      Project::DSL.parse(project, <<EOF)
-target :lib do
-  check "lib"
-  signature "sig"
-end
-EOF
-
-      (current_dir + "lib").mkdir
-      (current_dir + "lib/foo.rb").write("")
-      (current_dir + "lib/bar.rb").write("")
-
-      worker = Server::TypeCheckWorker.new(
-        project: project,
-        assignment: assignment,
-        commandline_args: [],
-        reader: worker_reader,
-        writer: worker_writer
-      )
-
-      worker.load_files(project: worker.project, commandline_args: ["lib/foo.rb"])
-      worker.service.update(changes: worker.pop_buffer) {}
-
-      assert_equal [Pathname("lib/foo.rb")], worker.service.source_files.keys
-    end
-  end
-
   def test_job_stats
     in_tmpdir do
       project = Project.new(steepfile_path: current_dir + "Steepfile")