Merge branch 'release/v3.3.0'

.travis.yml

@@ -2,8 +2,9 @@ language: ruby
 rvm:
   - 2.0.0
   - 2.1
-  - 2.2
-  - jruby-19mode
+  - 2.2.2
+  - 2.3.1
+  - jruby-9.0.5.0
 before_install:
   - gem update --system
   - gem install bundler
@@ -24,3 +25,6 @@ addons:
   postgresql: "9.4"
 services:
   - postgresql
+matrix:
+  allow_failures:
+  - rvm: jruby-9.0.5.0

Appraisals

@@ -1,15 +1,28 @@
 appraise 'rails_3_2' do
   gem 'rails',  '~> 3.2.22.2'
+  gem 'rack',   '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
 
 appraise 'rails_4_0' do
   gem 'rails', '~> 4.0.13'
+  gem 'rack',   '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
 
 appraise 'rails_4_1' do
   gem 'rails', '~> 4.1.15'
+  gem 'rack',  '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
 
 appraise 'rails_4_2' do
   gem 'rails', '~> 4.2.6'
+  gem 'rack',  '~> 1.0', :platforms => [:ruby_20, :ruby_21]
 end
+
+appraise 'rails_5_0' do
+  gem 'rails', '~> 5.0.0'
+  # gem 'activerecord-jdbc-adapter',
+  #   :git      => 'git://github.com/jruby/activerecord-jdbc-adapter.git',
+  #   :branch   => 'rails-5',
+  #   :platform => :jruby,
+  #   :ref      => 'c3570ce730'
+end if RUBY_VERSION.to_f >= 2.2 && RUBY_PLATFORM != 'java'

Gemfile

@@ -3,9 +3,10 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'mysql2', '~> 0.3.12b4', :platform => :ruby
-gem 'pg',     '~> 0.16.0',   :platform => :ruby
+gem 'pg',     '~> 0.18.4',   :platform => :ruby
 
 gem 'jdbc-mysql',                          '5.1.35',   :platform => :jruby
 gem 'activerecord-jdbcmysql-adapter',      '~> 1.3.4', :platform => :jruby
 gem 'activerecord-jdbcpostgresql-adapter', '~> 1.3.4', :platform => :jruby
 
+gem 'rack', '~> 1.0' if RUBY_VERSION.to_f <= 2.1

HISTORY

@@ -1,3 +1,22 @@
+2016-12-13: 3.3.0
+* [FEATURE] Real-time callbacks can now be used with after_commit hooks if that's preferred over after_save.
+* [CHANGE] Only toggle the delta value if the record has changed or is new (rather than on every single save call).
+* [CHANGE] Delta indexing is now quiet by default (rather than verbose).
+* [FIX] Explicit source method in the SQLQuery Builder instead of relying on method missing, thus avoiding any global methods named 'source' (Asaf Bartov).
+* [CHANGE] Use Riddle's reworked command interface for interacting with Sphinx's command-line tools.
+* [CHANGE] Respect Rake's quiet and silent flags for the Thinking Sphinx rake tasks.
+* [CHANGE] ts:start and ts:stop tasks default to verbose.
+* [FIX] Load indices before deleting index files, to ensure the files are actually found and deleted.
+* [FIX] Avoid loading ActiveRecord earlier than necessary. This avoids loading Rails out of order, which caused problems with Rails 5.
+* [FEATURE] Allow for custom batch sizes when populating real-time indices.
+* [FIX] Handle queries that are too long for Sphinx.
+* [FIX] Improve Rails 5 / JRuby support.
+* [CHANGE] Sort engine paths for loading indices to ensure they're consistent.
+* [CHANGE] Custom exception class for invalid database adapters.
+* [FIX] Fixed handling of multiple field tokens in wildcarding logic.
+* [CHANGE] Memoize the default primary keys per context.
+* [FIX] Ensure custom primary key columns are handled consistently (Julio Monteiro).
+
 2016-05-13: 3.2.0
 * [FIX] Ensure SQL table aliases are reliable for SQL-backed index queries.
 * [FEATURE] Add JSON attribute support for real-time indices.

README.textile

@@ -1,11 +1,12 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v3.2.0.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v3.3.0.
 
 h2. Upgrading
 
 Please refer to the release notes for any changes you need to make when upgrading:
 
+* "v3.3.0":https://github.com/pat/thinking-sphinx/releases/tag/v3.3.0
 * "v3.2.0":https://github.com/pat/thinking-sphinx/releases/tag/v3.2.0
 * "v3.1.4":https://github.com/pat/thinking-sphinx/releases/tag/v3.1.4
 * "v3.1.3":https://github.com/pat/thinking-sphinx/releases/tag/v3.1.3
@@ -22,7 +23,7 @@ It's a gem, so install it like you would any other gem. You will also need to sp
 
 <pre><code>gem 'mysql2',          '~> 0.3.18', :platform => :ruby
 gem 'jdbc-mysql',      '= 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 3.2.0'</code></pre>
+gem 'thinking-sphinx', '~> 3.3.0'</code></pre>
 
 The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. If you're using JRuby, there is "currently an issue with Sphinx and jdbc-mysql 5.1.36 or newer":http://sphinxsearch.com/forum/view.html?id=13939, so you'll need to stick to nothing more recent than 5.1.35.
 

gemfiles/rails_3_2.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 3.2.22.2"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

gemfiles/rails_4_0.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 4.0.13"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

gemfiles/rails_4_1.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 4.1.15"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

gemfiles/rails_4_2.gemfile

@@ -3,10 +3,11 @@
 source "https://rubygems.org"
 
 gem "mysql2", "~> 0.3.12b4", :platform => :ruby
-gem "pg", "~> 0.16.0", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
 gem "jdbc-mysql", "5.1.35", :platform => :jruby
 gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
 gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
 gem "rails", "~> 4.2.6"
+gem "rack", "~> 1.0", :platforms => [:ruby_20, :ruby_21]
 
 gemspec :path => "../"

gemfiles/rails_5_0.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "mysql2", "~> 0.3.12b4", :platform => :ruby
+gem "pg", "~> 0.18.4", :platform => :ruby
+gem "jdbc-mysql", "5.1.35", :platform => :jruby
+gem "activerecord-jdbcmysql-adapter", "~> 1.3.4", :platform => :jruby
+gem "activerecord-jdbcpostgresql-adapter", "~> 1.3.4", :platform => :jruby
+gem "rails", "~> 5.0.0"
+
+gemspec :path => "../"

lib/thinking_sphinx.rb

@@ -15,6 +15,8 @@
 require 'active_support/core_ext/module/attribute_accessors'
 
 module ThinkingSphinx
+  MAXIMUM_STATEMENT_LENGTH = (2 ** 23) - 1
+
   def self.count(query = '', options = {})
     search_for_ids(query, options).total_entries
   end

lib/thinking_sphinx/active_record/attribute/type.rb

@@ -41,7 +41,7 @@ def associations
   end
 
   def big_integer?
-    database_column.type == :integer && database_column.sql_type[/bigint/i]
+    type_symbol == :integer && database_column.sql_type[/bigint/i]
   end
 
   def column_name
@@ -76,15 +76,29 @@ def type_from_database
 
     return :bigint if big_integer?
 
-    case database_column.type
+    case type_symbol
     when :datetime, :date
       :timestamp
     when :text
       :string
     when :decimal
       :float
+    when :integer, :boolean, :timestamp, :float, :string, :bigint, :json
+      type_symbol
     else
-      database_column.type
+      raise ThinkingSphinx::UnknownAttributeType,
+        <<-ERROR
+Unable to determine an equivalent Sphinx attribute type from #{database_column.type.class.name} for attribute #{attribute.name}. You may want to manually set the type.
+
+e.g.
+  has my_column, :type => :integer
+        ERROR
     end
   end
+
+  def type_symbol
+    return database_column.type if database_column.type.is_a?(Symbol)
+
+    database_column.type.class.name.demodulize.downcase.to_sym
+  end
 end

lib/thinking_sphinx/active_record/callbacks/delta_callbacks.rb

@@ -4,9 +4,7 @@ class ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks <
   callbacks :after_commit, :before_save
 
   def after_commit
-    return unless !suspended? && delta_indices? && processors.any? { |processor|
-      processor.toggled?(instance)
-    }
+    return unless !suspended? && delta_indices? && toggled?
 
     delta_indices.each do |index|
       index.delta_processor.index index
@@ -18,7 +16,9 @@ def after_commit
   end
 
   def before_save
-    return unless !ThinkingSphinx::Callbacks.suspended? && delta_indices?
+    return unless new_or_changed?           &&
+      !ThinkingSphinx::Callbacks.suspended? &&
+      delta_indices?
 
     processors.each { |processor| processor.toggle instance }
   end
@@ -45,11 +45,19 @@ def indices
     @indices ||= config.index_set_class.new :classes => [instance.class]
   end
 
+  def new_or_changed?
+    instance.new_record? || instance.changed?
+  end
+
   def processors
     delta_indices.collect &:delta_processor
   end
 
   def suspended?
     ThinkingSphinx::Callbacks.suspended? || ThinkingSphinx::Deltas.suspended?
   end
+
+  def toggled?
+    processors.any? { |processor| processor.toggled?(instance) }
+  end
 end

lib/thinking_sphinx/active_record/database_adapters.rb

@@ -12,7 +12,7 @@ def adapter_for(model)
       when :postgresql
         PostgreSQLAdapter
       else
-        raise "Invalid Database Adapter '#{adapter}': Thinking Sphinx only supports MySQL and PostgreSQL."
+        raise ThinkingSphinx::InvalidDatabaseAdapter, "Invalid adapter '#{adapter}': Thinking Sphinx only supports MySQL and PostgreSQL."
       end
 
       klass.new model

lib/thinking_sphinx/active_record/sql_builder/query.rb

@@ -45,6 +45,10 @@ def scope_by_utf8
         self.scope += utf8_query_pre if source.options[:utf8?]
       end
 
+      def source
+        report.source
+      end
+
       def method_missing(*args, &block)
         report.send *args, &block
       end

lib/thinking_sphinx/active_record/sql_source.rb

@@ -90,7 +90,7 @@ def type
         when DatabaseAdapters::PostgreSQLAdapter
           'pgsql'
         else
-          raise "Unknown Adapter Type: #{adapter.class.name}"
+          raise UnknownDatabaseAdapter, "Provided type: #{adapter.class.name}"
         end
       end
 

lib/thinking_sphinx/configuration.rb

@@ -1,7 +1,7 @@
 require 'pathname'
 
 class ThinkingSphinx::Configuration < Riddle::Configuration
-  attr_accessor :configuration_file, :indices_location, :version
+  attr_accessor :configuration_file, :indices_location, :version, :batch_size
   attr_reader :index_paths
   attr_writer :controller, :index_set_class, :indexing_strategy
 
@@ -48,7 +48,7 @@ def framework=(framework)
   def engine_index_paths
     return [] unless defined?(Rails)
 
-    engine_indice_paths.flatten.compact
+    engine_indice_paths.flatten.compact.sort
   end
 
   def engine_indice_paths
@@ -121,6 +121,7 @@ def setup
       'db', 'sphinx', environment
     ).to_s
     @version = settings['version'] || '2.1.4'
+    @batch_size = settings['batch_size'] || 1000
 
     if settings['common_sphinx_configuration']
       common.common_sphinx_configuration  = true

lib/thinking_sphinx/connection.rb

@@ -70,26 +70,39 @@ def close
     end
 
     def execute(statement)
-      query(statement).first
+      check_and_perform(statement).first
     end
 
     def query_all(*statements)
-      query *statements
+      check_and_perform statements.join('; ')
     end
 
     private
 
+    def check(statements)
+      if statements.length > ThinkingSphinx::MAXIMUM_STATEMENT_LENGTH
+        exception           = ThinkingSphinx::QueryLengthError.new
+        exception.statement = statements
+        raise exception
+      end
+    end
+
+    def check_and_perform(statements)
+      check statements
+      perform statements
+    end
+
     def close_and_clear
       client.close
       @client = nil
     end
 
-    def query(*statements)
-      results_for *statements
+    def perform(statements)
+      results_for statements
     rescue => error
-      message           = "#{error.message} - #{statements.join('; ')}"
+      message           = "#{error.message} - #{statements}"
       wrapper           = ThinkingSphinx::QueryExecutionError.new message
-      wrapper.statement = statements.join('; ')
+      wrapper.statement = statements
       raise wrapper
     ensure
       close_and_clear unless ThinkingSphinx::Connection.persistent?
@@ -117,8 +130,8 @@ def client
       raise ThinkingSphinx::SphinxError.new_from_mysql error
     end
 
-    def results_for(*statements)
-      results  = [client.query(statements.join('; '))]
+    def results_for(statements)
+      results  = [client.query(statements)]
       results << client.store_result while client.next_result
       results
     end
@@ -145,9 +158,9 @@ def client
       raise ThinkingSphinx::SphinxError.new_from_mysql error
     end
 
-    def results_for(*statements)
+    def results_for(statements)
       statement = client.createStatement
-      statement.execute statements.join('; ')
+      statement.execute statements
 
       results   = [set_to_array(statement.getResultSet)]
       results  << set_to_array(statement.getResultSet) while statement.getMoreResults