changeing stop_crawl behaviour to stop new links, but finish processi…

…ng all pages in queue

lib/anemone/core.rb

@@ -180,23 +180,22 @@ def run
       loop do
         page = page_queue.deq
         @pages.touch_key page.url
-        puts "#{page.url} Queue: #{link_queue.size}" if @opts[:verbose]
+        puts "#{page.url} Queue: #{link_queue.size} PageQueue #{page_queue.size}" if @opts[:verbose]
         do_page_blocks page
         page.discard_doc! if @opts[:discard_page_bodies]
 
-        links = links_to_follow page
-        if link_queue.num_waiting < @opts[:links_limit]
+
+        if link_queue.size < @opts[:links_limit] and !@stop_crawl
+          links = links_to_follow page
           links.each do |link|
             link_queue << [link, page.url.dup, page.depth + 1]
           end
           @pages.touch_keys links
         end
 
-
         @pages[page.url] = page
 
         if @stop_crawl
-          page_queue.clear
           link_queue.clear
         end
 
@@ -205,7 +204,7 @@ def run
           until link_queue.num_waiting == @tentacles.size
             Thread.pass
           end
-          if page_queue.empty? || @stop_crawl
+          if page_queue.empty? 
             @tentacles.size.times { link_queue << :END }
             break
           end

spec/core_spec.rb

@@ -230,13 +230,13 @@ module Anemone
 
         it "should stop crawl if requested" do
           num_pages = 0
-          Anemone.crawl(@pages[0].url) do |anemone|
+          Anemone.crawl(@pages[0].url, @opts.merge({:pages_queue_limit => 1})) do |anemone|
           anemone.on_every_page do
             num_pages += 1
-            anemone.stop_crawl
+            anemone.stop_crawl if num_pages == 2
           end
         end
-        num_pages.should == 1
+        num_pages.should == 2
         end
 
         it "should limit number of links per crawl" do