draft of a multithreaded sync command with a configurable pool size (…

…issue #19)

Gemfile.lock

@@ -2,6 +2,7 @@ PATH
   remote: .
   specs:
     nightcrawler_swift (0.7.0)
+      concurrent-ruby (~> 0.8.0)
       multi_mime (>= 1.0.1)
       rest-client
 
@@ -15,6 +16,8 @@ GEM
     codeclimate-test-reporter (0.4.0)
       simplecov (>= 0.7.1, < 1.0.0)
     columnize (0.9.0)
+    concurrent-ruby (0.8.0)
+      ref (~> 1.0, >= 1.0.5)
     debugger-linecache (1.2.0)
     diff-lcs (1.2.5)
     docile (1.1.5)
@@ -23,6 +26,7 @@ GEM
     multi_mime (1.0.1)
     netrc (0.10.2)
     rake (10.3.2)
+    ref (1.0.5)
     rest-client (1.7.2)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)

lib/nightcrawler_swift.rb

@@ -21,6 +21,7 @@
 require "nightcrawler_swift/commands/list"
 require "nightcrawler_swift/commands/delete"
 require "nightcrawler_swift/commands/sync"
+require "nightcrawler_swift/commands/multithread_sync"
 require "nightcrawler_swift/railtie" if defined?(Rails)
 
 module NightcrawlerSwift

lib/nightcrawler_swift/commands/multithread_sync.rb

@@ -0,0 +1,44 @@
+module NightcrawlerSwift
+  class MultithreadSync < Command
+    DEFAULT_POOL_SIZE = 5
+
+    def initialize
+      require 'concurrent/utilities'
+      require 'concurrent/executors'
+      @logger = NightcrawlerSwift.logger
+    end
+
+    def execute args = {}
+      pool_size = args[:pool_size] || DEFAULT_POOL_SIZE
+      dir_path = args[:dir_path]
+
+      @logger.info "[NightcrawlerSwift] dir_path: #{dir_path}"
+      @logger.info "[NightcrawlerSwift] multithread sync, #{Concurrent.processor_count} processors"
+
+      assets = Dir["#{dir_path}/**/**"].
+        reject {|fullpath| File.directory?(fullpath)}.
+        map {|fullpath|
+          path = fullpath.gsub("#{dir_path}/", "")
+          OpenStruct.new(path: path, fullpath: fullpath)
+        }
+
+      pool = Concurrent::FixedThreadPool.new pool_size
+
+      assets.each do |asset|
+        pool.post do
+          @logger.info "[NightcrawlerSwift] #{asset.path}"
+
+          upload = Upload.new
+          upload.execute asset.path, File.open(asset.fullpath, "r")
+        end
+      end
+
+      sleep(1) while pool.queue_length > 0
+
+      @logger.info "[NightcrawlerSwift] shutting down"
+      pool.shutdown
+      pool.wait_for_termination
+    end
+
+  end
+end

nightcrawler_swift.gemspec

@@ -19,7 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "rest-client"
-  spec.add_dependency "multi_mime",  ">= 1.0.1"
+  spec.add_dependency "multi_mime",      ">= 1.0.1"
+  spec.add_dependency "concurrent-ruby", "~> 0.8.0"
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"