add SQLite3 storage

lib/anemone/storage.rb

@@ -29,6 +29,11 @@ def self.Redis(opts = {})
       require 'anemone/storage/redis'
       self::Redis.new(opts)
     end
+
+    def self.SQLite3(file = 'anemone.db')
+      require 'anemone/storage/sqlite3'
+      self::SQLite3.new(file)
+    end
 
   end
 end

lib/anemone/storage/sqlite3.rb

@@ -0,0 +1,90 @@
+begin
+  require 'sqlite3'
+rescue LoadError
+  puts "You need the sqlite3 gem to use Anemone::Storage::Sqlite"
+  exit
+end
+
+module Anemone
+  module Storage
+    class SQLite3 
+
+      def initialize(file)
+        @db = ::SQLite3::Database.new(file)
+        create_schema
+      end
+
+      def [](url)
+        value = @db.get_first_value('SELECT data FROM anemone_storage WHERE key = ?', url.to_s)
+        if value
+          Marshal.load(value)
+        end
+      end
+
+      def []=(url, value)
+        data = Marshal.dump(value)
+        if has_key?(url)
+          @db.execute('UPDATE anemone_storage SET data = ? WHERE key = ?', data, url.to_s)
+        else
+          @db.execute('INSERT INTO anemone_storage (data, key) VALUES(?, ?)', data, url.to_s)
+        end
+      end
+
+      def delete(url)
+        page = self[url]
+        @db.execute('DELETE FROM anemone_storage WHERE key = ?', url.to_s)
+        page
+      end
+
+      def each
+        @db.execute("SELECT key, data FROM anemone_storage ORDER BY id") do |row|
+          value = Marshal.load(row[1])
+          yield row[0], value
+        end
+      end
+
+      def merge!(hash)
+        hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      def size
+        @db.get_first_value('SELECT COUNT(id) FROM anemone_storage')
+      end
+
+      def keys
+        @db.execute("SELECT key FROM anemone_storage ORDER BY id").map{|t| t[0]}
+      end
+
+      def has_key?(url)
+        !!@db.get_first_value('SELECT id FROM anemone_storage WHERE key = ?', url.to_s)
+      end
+
+      def close
+        @db.close
+      end
+
+      private
+
+      def create_schema
+        @db.execute_batch <<SQL
+          create table if not exists anemone_storage (
+            id INTEGER PRIMARY KEY ASC,
+            key TEXT,
+            data BLOB
+          );
+          create index  if not exists anemone_key_idx on anemone_storage (key);
+SQL
+      end
+
+      def load_page(hash)
+        BINARY_FIELDS.each do |field|
+          hash[field] = hash[field].to_s
+        end
+        Page.from_hash(hash)
+      end
+
+    end
+  end
+end
+

spec/core_spec.rb

@@ -1,6 +1,6 @@
 $:.unshift(File.dirname(__FILE__))
 require 'spec_helper'
-%w[pstore tokyo_cabinet].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe Core do
@@ -277,6 +277,27 @@ module Anemone
       end
     end
 
+    describe Storage::SQLite3 do
+      it_should_behave_like "crawl"
+
+      before(:all) do
+        @test_file = 'test.db'
+      end
+
+      before(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+      end
+
+      after(:each) do
+        @store.close
+      end
+
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
     describe "options" do
       it "should accept options for the crawl" do
         core = Anemone.crawl(SPEC_DOMAIN, :verbose => false,

spec/page_store_spec.rb

@@ -1,6 +1,6 @@
 $:.unshift(File.dirname(__FILE__))
 require 'spec_helper'
-%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe PageStore do
@@ -125,6 +125,24 @@ module Anemone
       end
     end
 
+    describe Storage::SQLite3 do
+      it_should_behave_like "page storage"
+
+      before(:each) do
+        @test_file = 'test.db'
+        File.delete(@test_file) if File.exists?(@test_file)
+        @opts = {:storage => @store = Storage.SQLite3(@test_file)}
+      end
+
+      after(:each) do
+        @store.close
+      end
+
+      after(:each) do
+        File.delete(@test_file) if File.exists?(@test_file)
+      end
+    end
+
     describe Storage::MongoDB do
       it_should_behave_like "page storage"
 

spec/storage_spec.rb

@@ -1,7 +1,7 @@
 $:.unshift(File.dirname(__FILE__))
 require 'spec_helper'
 
-%w[pstore tokyo_cabinet mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
+%w[pstore tokyo_cabinet sqlite3 mongodb redis].each { |file| require "anemone/storage/#{file}.rb" }
 
 module Anemone
   describe Storage do
@@ -25,6 +25,14 @@ module Anemone
       store.close
     end
 
+    it "should have a class method to produce a SQLite3" do
+      test_file = 'test.db'
+      Anemone::Storage.should respond_to(:SQLite3)
+      store = Anemone::Storage.SQLite3(test_file)
+      store.should be_an_instance_of(Anemone::Storage::SQLite3)
+      store.close
+    end
+
     it "should have a class method to produce a MongoDB" do
       Anemone::Storage.should respond_to(:MongoDB)
       store = Anemone::Storage.MongoDB
@@ -143,6 +151,25 @@ module Storage
         end
       end
 
+      describe SQLite3 do
+        it_should_behave_like "storage engine"
+
+        before(:each) do
+          @test_file = 'test.db'
+          File.delete @test_file rescue nil
+          @store =  Anemone::Storage.SQLite3(@test_file)
+        end
+
+        after(:each) do
+          @store.close
+        end
+
+        after(:all) do
+          File.delete @test_file rescue nil
+        end
+
+      end
+
       describe Storage::MongoDB do
         it_should_behave_like "storage engine"