From 0fa087ab2d6008211555a21227c72552a9d2f7a0 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 20 Jan 2012 11:31:39 +0100 Subject: [PATCH 1/4] Fix default filename extension for the kyotocabinet adapter. --- lib/anemone/storage.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/anemone/storage.rb b/lib/anemone/storage.rb index d5a15d2d..9c4ab096 100644 --- a/lib/anemone/storage.rb +++ b/lib/anemone/storage.rb @@ -18,7 +18,7 @@ def self.TokyoCabinet(file = 'anemone.tch') self::TokyoCabinet.new(file) end - def self.KyotoCabinet(file = 'anemone.tch') + def self.KyotoCabinet(file = 'anemone.kch') require 'anemone/storage/kyoto_cabinet' self::KyotoCabinet.new(file) end From 7ef8836b4daa2efd36995f8baba3568d97fe1627 Mon Sep 17 00:00:00 2001 From: Chris Kite Date: Fri, 20 Jan 2012 11:24:55 -0600 Subject: [PATCH 2/4] add test for Storage.KyotoCabinet --- .gitignore | 3 +- Rakefile | 6 ++-- anemone.gemspec | 5 +-- spec/storage_spec.rb | 83 ++++++++++++++++++++++++++++---------------- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index c5bbd5c2..453303f4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ Gemfile.lock test.db test.tch -test.kch +*.kch +rdoc diff --git a/Rakefile b/Rakefile index ad3798d2..81d0b581 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,5 @@ -require 'rubygems' -require 'rake' require 'rspec/core/rake_task' +require 'rake/rdoctask' desc "Run all specs" RSpec::Core::RakeTask.new(:rspec) do |spec| @@ -14,8 +13,7 @@ end task :default => :rspec -require 'rdoc/task' -RDoc::Task.new do |rdoc| +Rake::RDocTask.new(:rdoc) do |rdoc| version = File.exist?('VERSION') ? File.read('VERSION') : "" rdoc.rdoc_dir = 'rdoc' diff --git a/anemone.gemspec b/anemone.gemspec index 67655432..5201cfc7 100644 --- a/anemone.gemspec +++ b/anemone.gemspec @@ -14,8 +14,9 @@ spec = Gem::Specification.new do |s| s.add_dependency("nokogiri", ">= 1.3.0") s.add_dependency("robots", ">= 0.7.2") - s.add_development_dependency "rake", ">=0.8.7" - s.add_development_dependency "rspec", ">=2.6.0" + s.add_development_dependency "rake", ">=0.9.2" + s.add_development_dependency "rdoc", ">=3.12" + s.add_development_dependency "rspec", ">=2.8.0" s.add_development_dependency "fakeweb", ">=1.3.0" s.add_development_dependency "redis", ">=2.2.0" s.add_development_dependency "mongo", ">=1.3.1" diff --git a/spec/storage_spec.rb b/spec/storage_spec.rb index 02ab63c4..eeaa51f7 100644 --- a/spec/storage_spec.rb +++ b/spec/storage_spec.rb @@ -6,45 +6,70 @@ module Anemone describe Storage do - it "should have a class method to produce a Hash" do - Anemone::Storage.should respond_to(:Hash) - Anemone::Storage.Hash.should be_an_instance_of(Hash) + describe ".Hash" do + it "returns a Hash adapter" do + Anemone::Storage.Hash.should be_an_instance_of(Hash) + end end - it "should have a class method to produce a PStore" do - test_file = 'test.pstore' - Anemone::Storage.should respond_to(:PStore) - Anemone::Storage.PStore(test_file).should be_an_instance_of(Anemone::Storage::PStore) + describe ".PStore" do + it "returns a PStore adapter" do + test_file = 'test.pstore' + Anemone::Storage.PStore(test_file).should be_an_instance_of(Anemone::Storage::PStore) + end end - it "should have a class method to produce a TokyoCabinet" do - test_file = 'test.tch' - Anemone::Storage.should respond_to(:TokyoCabinet) - store = Anemone::Storage.TokyoCabinet(test_file) - store.should be_an_instance_of(Anemone::Storage::TokyoCabinet) - store.close + describe ".TokyoCabinet" do + it "returns a TokyoCabinet adapter" do + test_file = 'test.tch' + store = Anemone::Storage.TokyoCabinet(test_file) + store.should be_an_instance_of(Anemone::Storage::TokyoCabinet) + store.close + end end - it "should have a class method to produce a SQLite3" do - test_file = 'test.db' - Anemone::Storage.should respond_to(:SQLite3) - store = Anemone::Storage.SQLite3(test_file) - store.should be_an_instance_of(Anemone::Storage::SQLite3) - store.close + describe ".KyotoCabinet" do + context "when the file is specified" do + it "returns a KyotoCabinet adapter using that file" do + test_file = 'test.kch' + store = Anemone::Storage.KyotoCabinet(test_file) + store.should be_an_instance_of(Anemone::Storage::KyotoCabinet) + store.close + end + end + + context "when no file is specified" do + it "returns a KyotoCabinet adapter using the default filename" do + store = Anemone::Storage.KyotoCabinet + store.should be_an_instance_of(Anemone::Storage::KyotoCabinet) + store.close + end + end + end + + describe ".SQLite3" do + it "returns a SQLite3 adapter" do + test_file = 'test.db' + store = Anemone::Storage.SQLite3(test_file) + store.should be_an_instance_of(Anemone::Storage::SQLite3) + store.close + end end - it "should have a class method to produce a MongoDB" do - Anemone::Storage.should respond_to(:MongoDB) - store = Anemone::Storage.MongoDB - store.should be_an_instance_of(Anemone::Storage::MongoDB) - store.close + describe ".MongoDB" do + it "returns a MongoDB adapter" do + store = Anemone::Storage.MongoDB + store.should be_an_instance_of(Anemone::Storage::MongoDB) + store.close + end end - it "should have a class method to produce a Redis" do - Anemone::Storage.should respond_to(:Redis) - store = Anemone::Storage.Redis - store.should be_an_instance_of(Anemone::Storage::Redis) - store.close + describe ".MongoDB" do + it "returns a Redis adapter" do + store = Anemone::Storage.Redis + store.should be_an_instance_of(Anemone::Storage::Redis) + store.close + end end module Storage From 55d3f1c592f0705f2f50cc84a0defacc61ca7a44 Mon Sep 17 00:00:00 2001 From: Chris Kite Date: Fri, 20 Jan 2012 16:15:29 -0600 Subject: [PATCH 3/4] switch from robots to robotex --- anemone.gemspec | 2 +- lib/anemone/core.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/anemone.gemspec b/anemone.gemspec index 5201cfc7..ad7cf723 100644 --- a/anemone.gemspec +++ b/anemone.gemspec @@ -12,7 +12,7 @@ spec = Gem::Specification.new do |s| s.rdoc_options << '-m' << 'README.rdoc' << '-t' << 'Anemone' s.extra_rdoc_files = ["README.rdoc"] s.add_dependency("nokogiri", ">= 1.3.0") - s.add_dependency("robots", ">= 0.7.2") + s.add_dependency("robotex", ">= 1.0.0") s.add_development_dependency "rake", ">=0.9.2" s.add_development_dependency "rdoc", ">=3.12" diff --git a/lib/anemone/core.rb b/lib/anemone/core.rb index 0e98b41a..bd8a65ef 100644 --- a/lib/anemone/core.rb +++ b/lib/anemone/core.rb @@ -1,5 +1,5 @@ require 'thread' -require 'robots' +require 'robotex' require 'anemone/tentacle' require 'anemone/page' require 'anemone/exceptions' @@ -199,7 +199,7 @@ def process_options @opts[:threads] = 1 if @opts[:delay] > 0 storage = Anemone::Storage::Base.new(@opts[:storage] || Anemone::Storage.Hash) @pages = PageStore.new(storage) - @robots = Robots.new(@opts[:user_agent]) if @opts[:obey_robots_txt] + @robots = Robotex.new(@opts[:user_agent]) if @opts[:obey_robots_txt] freeze_options end From 57a58bc91fd342bbce85635c75234edfd6e00edf Mon Sep 17 00:00:00 2001 From: Chris Kite Date: Fri, 20 Jan 2012 16:22:35 -0600 Subject: [PATCH 4/4] bump version to 0.7.1 --- CHANGELOG.rdoc | 10 ++++++++++ README.rdoc | 3 ++- VERSION | 2 +- anemone.gemspec | 2 +- lib/anemone/core.rb | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rdoc b/CHANGELOG.rdoc index 0e200a57..7f03a144 100644 --- a/CHANGELOG.rdoc +++ b/CHANGELOG.rdoc @@ -1,3 +1,13 @@ +== 0.7.1 / 2012-01-20 + +* Minor enhancements + + * Switch from robots gem (which people reported problems with) to new robotex gem + +* Bug fixes + + * Fix incorrect default file extension for KyotoCabinet + == 0.7.0 / 2012-01-19 * Major enhancements diff --git a/README.rdoc b/README.rdoc index 4654ac5e..a115b186 100644 --- a/README.rdoc +++ b/README.rdoc @@ -30,8 +30,9 @@ To test and develop this gem, additional requirements are: * rspec * fakeweb * tokyocabinet +* kyotocabinet-ruby * mongo * redis * sqlite3 -You will need to have {Tokyo Cabinet}[http://fallabs.com/tokyocabinet/], {MongoDB}[http://www.mongodb.org/], and {Redis}[http://code.google.com/p/redis/] installed on your system and running. +You will need to have KyotoCabinet, {Tokyo Cabinet}[http://fallabs.com/tokyocabinet/], {MongoDB}[http://www.mongodb.org/], and {Redis}[http://code.google.com/p/redis/] installed on your system and running. diff --git a/VERSION b/VERSION index faef31a4..39e898a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.0 +0.7.1 diff --git a/anemone.gemspec b/anemone.gemspec index ad7cf723..2cf69cb7 100644 --- a/anemone.gemspec +++ b/anemone.gemspec @@ -1,6 +1,6 @@ spec = Gem::Specification.new do |s| s.name = "anemone" - s.version = "0.7.0" + s.version = "0.7.1" s.author = "Chris Kite" s.homepage = "http://anemone.rubyforge.org" s.rubyforge_project = "anemone" diff --git a/lib/anemone/core.rb b/lib/anemone/core.rb index bd8a65ef..208c6d4c 100644 --- a/lib/anemone/core.rb +++ b/lib/anemone/core.rb @@ -9,7 +9,7 @@ module Anemone - VERSION = '0.7.0'; + VERSION = '0.7.1'; # # Convenience method to start a crawl