Merge branch 'release/v5.0.0'

.travis.yml

@@ -1,39 +1,30 @@
 language: ruby
 dist: xenial
 rvm:
-- 2.3.8
-- 2.4.6
-- 2.5.5
-- 2.6.3
+- 2.4.10
+- 2.5.8
+- 2.6.6
+- 2.7.1
 before_install:
-- pip install --upgrade --user awscli
 - gem update --system
-- gem uninstall -v '>= 2' -i $(rvm gemdir)@global -ax bundler || true
-- gem install bundler -v '< 2'
+- gem install bundler -v '1.17.3'
+install: bundle _1.17.3_ install --jobs=3 --retry=3
 before_script:
 - mysql -e 'create database thinking_sphinx;' > /dev/null
 - psql -c 'create database thinking_sphinx;' -U postgres >/dev/null
 - "./bin/loadsphinx $SPHINX_VERSION $SPHINX_ENGINE"
-- bundle exec appraisal install
-script: bundle exec appraisal rspec
+- bundle _1.17.3_ exec appraisal install
+script: bundle _1.17.3_ exec appraisal rspec
 env:
-  global:
-  - secure: cUPinkilBafqDSPsTkl/PXYc2aXNKUQKXGK8poBBMqKN9/wjfJx1DWgtowDKalekdZELxDhc85Ye3bL1xlW4nLjOu+U6Tkt8eNw2Nhs1flodHzA/RyENdBLr/tBHt43EjkrDehZx5sBHmWQY4miHs8AJz0oKO9Ae2inTOHx9Iuc=
   matrix:
-  - DATABASE=mysql2 SPHINX_VERSION=2.1.9 SPHINX_ENGINE=sphinx
-  - DATABASE=postgresql SPHINX_VERSION=2.1.9 SPHINX_ENGINE=sphinx
   - DATABASE=mysql2 SPHINX_VERSION=2.2.11 SPHINX_ENGINE=sphinx
   - DATABASE=postgresql SPHINX_VERSION=2.2.11 SPHINX_ENGINE=sphinx
-  - DATABASE=mysql2 SPHINX_VERSION=3.0.3 SPHINX_ENGINE=sphinx
-  - DATABASE=postgresql SPHINX_VERSION=3.0.3 SPHINX_ENGINE=sphinx
-  - DATABASE=mysql2 SPHINX_VERSION=3.1.1 SPHINX_ENGINE=sphinx
-  - DATABASE=mysql2 SPHINX_VERSION=2.6.4 SPHINX_ENGINE=manticore
-  - DATABASE=postgresql SPHINX_VERSION=2.6.4 SPHINX_ENGINE=manticore
-  - DATABASE=mysql2 SPHINX_VERSION=2.7.5 SPHINX_ENGINE=manticore
-  - DATABASE=postgresql SPHINX_VERSION=2.7.5 SPHINX_ENGINE=manticore
+  - DATABASE=mysql2 SPHINX_VERSION=3.2.1 SPHINX_ENGINE=sphinx
   - DATABASE=mysql2 SPHINX_VERSION=2.8.2 SPHINX_ENGINE=manticore
   - DATABASE=postgresql SPHINX_VERSION=2.8.2 SPHINX_ENGINE=manticore
-  # - DATABASE=postgresql SPHINX_VERSION=3.1.1 SPHINX_ENGINE=sphinx
+  - DATABASE=mysql2 SPHINX_VERSION=3.4.2 SPHINX_ENGINE=manticore
+  - DATABASE=postgresql SPHINX_VERSION=3.4.2 SPHINX_ENGINE=manticore
+  # - DATABASE=postgresql SPHINX_VERSION=3.2.1 SPHINX_ENGINE=sphinx
 sudo: false
 addons:
   postgresql: '9.4'

Appraisals

@@ -1,22 +1,7 @@
-appraise 'rails_3_2' do
-  gem 'rails',  '~> 3.2.22.2'
-  gem 'mysql2', '~> 0.3.10', :platform => :ruby
-end if RUBY_VERSION.to_f <= 2.3
-
-appraise 'rails_4_0' do
-  gem 'rails',  '~> 4.0.13'
-  gem 'mysql2', '~> 0.3.10', :platform => :ruby
-end if RUBY_VERSION.to_f <= 2.3
-
-appraise 'rails_4_1' do
-  gem 'rails',  '~> 4.1.15'
-  gem 'mysql2', '~> 0.3.13', :platform => :ruby
-end if RUBY_VERSION.to_f <= 2.3
-
 appraise 'rails_4_2' do
   gem 'rails',  '~> 4.2.6'
   gem 'mysql2', '~> 0.4.0', :platform => :ruby
-end if RUBY_VERSION.to_f <= 2.3
+end if RUBY_VERSION.to_f <= 2.4
 
 appraise 'rails_5_0' do
   if RUBY_PLATFORM == "java"

CHANGELOG.markdown

@@ -2,6 +2,26 @@
 
 All notable changes to this project (at least, from v3.0.0 onwards) are documented in this file.
 
+## 5.0.0 - 2020-07-20
+
+[Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.0.0)
+
+### Added
+
+* New interface for adding callbacks to indexed models (which is no longer done automatically). Discussed in [#1173](https://github.com/pat/thinking-sphinx/issues/1173) and committed via [#1175](https://github.com/pat/thinking-sphinx/pull/1175). **This is a breaking change - you will need to add these callbacks. See [the full release notes](https://github.com/pat/thinking-sphinx/releases/tag/v5.0.0) for examples.**
+* Fields and attributes can be overriden - whichever's defined last with a given name is the definition that's used. This is an edge case, but useful if you want to override any of the default fields/indices. (Requested by @kalsan in [#1172](https://github.com/pat/thinking-sphinx/issues/1172).)
+* Custom index_set_class implementations can now expect the `:instances` option to be set alongside `:classes`, which is useful in cases to limit the indices returned if you're splitting index data for given classes/models into shards. (Introduced in PR [#1171](https://github.com/pat/thinking-sphinx/pull/1171) after discussions with @lunaru in [#1166](https://github.com/pat/thinking-sphinx/issues/1166).)
+
+### Changed
+
+* Sphinx 2.2.11 or newer is required, or Manticore 2.8.2 or newer.
+* Ruby 2.4 or newer is required.
+* Rails 4.2 or newer is required.
+* Remove internal uses of `send`, replaced with `public_send` as that's available in all supported Ruby versions.
+* Deletion statements are simplified by avoiding the need to calculate document keys/offsets (@njakobsen via [#1134](https://github.com/pat/thinking-sphinx/issues/1134)).
+* Real-time data is deleted before replacing it, to avoid duplicate data when offsets change (@njakobsen via [#1134](https://github.com/pat/thinking-sphinx/issues/1134)).
+* Use `reference_name` as per custom `index_set_class` definitions. Previously, the class method was called on `ThinkingSphinx::IndexSet` even if a custom subclass was configured. (As per discussinos with @kalsan in [#1172](https://github.com/pat/thinking-sphinx/issues/1172).)
+
 ## 4.4.1 - 2019-08-23
 
 [Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v4.4.1)

README.textile

@@ -1,22 +1,22 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v4.4.1.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v5.0.0.
 
 h2. Upgrading
 
 Please refer to "the changelog":https://github.com/pat/thinking-sphinx/blob/develop/CHANGELOG.markdown and "release notes":https://github.com/pat/thinking-sphinx/releases for any changes you need to make when upgrading. The release notes in particular are quite good at covering breaking changes and more details for new features.
 
-The documentation also has more details on what's involved for upgrading from "v3 to v4":https://freelancing-gods.com/thinking-sphinx/v4/upgrading.html, and "v1/v2 to v3":https://freelancing-gods.com/thinking-sphinx/v3/upgrading.html.
+The documentation also has more details on what's involved for upgrading from "v4 to v5":https://freelancing-gods.com/thinking-sphinx/v5/upgrading.html, "v3 to v4":https://freelancing-gods.com/thinking-sphinx/v4/upgrading.html, and "v1/v2 to v3":https://freelancing-gods.com/thinking-sphinx/v3/upgrading.html.
 
 h2. Installation
 
 It's a gem, so install it like you would any other gem. You will also need to specify the mysql2 gem if you're using MRI, or jdbc-mysql if you're using JRuby:
 
-<pre><code>gem 'mysql2',          '~> 0.3',    :platform => :ruby
+<pre><code>gem 'mysql2',          '~> 0.4',    :platform => :ruby
 gem 'jdbc-mysql',      '~> 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 4.4'</code></pre>
+gem 'thinking-sphinx', '~> 5.0'</code></pre>
 
-The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. If you're using JRuby with a version of Sphinx prior to 2.2.11, there is "currently an issue with Sphinx and jdbc-mysql 5.1.36 or newer":http://sphinxsearch.com/forum/view.html?id=13939, so you'll need to stick to nothing more recent than 5.1.35, or upgrade Sphinx.
+The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database.
 
 You'll also need to install Sphinx - this is covered in "the extended documentation":https://freelancing-gods.com/thinking-sphinx/installing_sphinx.html.
 
@@ -29,30 +29,28 @@ h2. Requirements
 The current release of Thinking Sphinx works with the following versions of its dependencies:
 
 |_. Library |_. Minimum |_. Tested Against |
-| Ruby | v2.3 | v2.3.8, v2.4.5, v2.5.3, v2.6.1 |
-| Sphinx | v2.1.2 | v2.1.9, v2.2.11, v3.0.3, v3.1.1 |
-| Manticore | v2.6.3 | v2.6.4, v2.7.5, v2.8.1 |
-| ActiveRecord | v3.2 | v3.2..v6.0 |
+| Ruby | v2.4 | v2.4, v2.5, v2.6, v2.7 |
+| Sphinx | v2.2.11 | v2.2.11, v3.2.1 |
+| Manticore | v2.8 | v2.8, v3.4 |
+| ActiveRecord | v4.2 | v4.2..v6.0 |
 
 It _might_ work with older versions of Ruby, but it's highly recommended to update to a supported release.
 
 It should also work with JRuby, but the test environment on Travis CI has been timing out, hence that's not actively tested against at the moment.
 
 h3. Sphinx or Manticore
 
-Thinking Sphinx v3 is currently built for Sphinx 2.1.2 or newer, or Manticore v2.6+.
+Thinking Sphinx is currently built for Sphinx 2.2.11 or newer (though it'll likely work with 2.1.x releases), or Manticore v2.8+.
 
 h3. Rails and ActiveRecord
 
-Currently Thinking Sphinx 3 is built to support Rails/ActiveRecord 3.2 or newer. If you're using Sinatra and ActiveRecord instead of Rails, that's fine - just make sure you add the @:require => 'thinking_sphinx/sinatra'@ option when listing @thinking-sphinx@ in your Gemfile.
+Currently Thinking Sphinx is built to support Rails/ActiveRecord 4.2 or newer. If you're using Sinatra and ActiveRecord instead of Rails, that's fine - just make sure you add the @:require => 'thinking_sphinx/sinatra'@ option when listing @thinking-sphinx@ in your Gemfile.
 
-Please note that if you're referring to polymorphic associations in your index definitions, you'll want to be using Rails/ActiveRecord 4.0 or newer. Supporting polymorphic associations and Rails/ActiveRecord 3.2 is problematic, and likely will not be addressed in the future.
-
-If you want ActiveRecord 3.1 support, then refer to the 3.0.x releases of Thinking Sphinx. Anything older than that, then you're stuck with Thinking Sphinx v2.x (for Rails/ActiveRecord 3.0) or v1.x (Rails 2.3). Please note that these older versions are no longer actively supported.
+If you want ActiveRecord 3.2-4.1 support, then refer to the 4.x releases of Thinking Sphinx. Or, for ActiveRecord 3.1 support, then refer to the 3.0.x releases. Anything older than that, then you're stuck with Thinking Sphinx v2.x (for Rails/ActiveRecord 3.0) or v1.x (Rails 2.3). Please note that these older versions are no longer actively supported.
 
 h3. Ruby
 
-You'll need either the standard Ruby (v2.3 or newer) or JRuby (9.1 or newer).
+You'll need either the standard Ruby (v2.4 or newer) or JRuby (9.1 or newer).
 
 h3. Database Versions
 
@@ -81,4 +79,4 @@ You can then run the unit tests with @rake spec:unit@, the acceptance tests with
 
 h2. Licence
 
-Copyright (c) 2007-2019, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.
+Copyright (c) 2007-2020, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.

bin/loadsphinx

@@ -22,6 +22,9 @@ load_sphinx () {
     3.1.1)
       url="http://sphinxsearch.com/files/sphinx-3.1.1-612d99f-linux-amd64.tar.gz"
       format="gz";;
+    3.2.1)
+      url="http://www.sphinxsearch.com/files/sphinx-3.2.1-f152e0b-linux-amd64.tar.gz"
+      format="gz";;
     *)
       echo "No Sphinx version $version available"
       exit 1;;
@@ -52,6 +55,8 @@ load_manticore () {
       url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.7.5/manticore_2.7.5-181204-4a31c54-release-stemmer.xenial_amd64-bin.deb";;
     2.8.2)
       url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.8.2/manticore_2.8.2-190402-4e81114-release-stemmer.xenial_amd64-bin.deb";;
+    3.4.2)
+      url="https://github.com/manticoresoftware/manticoresearch/releases/download/3.4.2/manticore_3.4.2-200410-6903305-release.xenial_amd64-bin.deb";;
     *)
       echo "No Manticore version $version available"
       exit 1;;

lib/thinking_sphinx/active_record/association_proxy/attribute_finder.rb

@@ -31,7 +31,7 @@ def indices
     @indices ||= begin
       configuration.preload_indices
       configuration.indices_for_references(
-        *ThinkingSphinx::IndexSet.reference_name(@association.klass)
+        *configuration.index_set_class.reference_name(@association.klass)
       ).reject &:distributed?
     end
   end

lib/thinking_sphinx/active_record/base.rb

@@ -4,11 +4,6 @@ module ThinkingSphinx::ActiveRecord::Base
   extend ActiveSupport::Concern
 
   included do
-    after_destroy ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks
-    before_save   ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
-    after_update  ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks
-    after_commit  ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
-
     if ActiveRecord::VERSION::STRING.to_i >= 5
       [
         ::ActiveRecord::Reflection::HasManyReflection,
@@ -17,8 +12,9 @@ module ThinkingSphinx::ActiveRecord::Base
         reflection_class.include DefaultReflectionAssociations
       end
     else
-      ::ActiveRecord::Associations::CollectionProxy.send :include,
+      ::ActiveRecord::Associations::CollectionProxy.include(
         ThinkingSphinx::ActiveRecord::AssociationProxy
+      )
     end
   end
 

lib/thinking_sphinx/active_record/callbacks/delete_callbacks.rb

@@ -27,7 +27,7 @@ def delete_from_sphinx
 
   def indices
     ThinkingSphinx::Configuration.instance.index_set_class.new(
-      :classes => [instance.class]
+      :instances => [instance], :classes => [instance.class]
     ).to_a
   end
 end

lib/thinking_sphinx/active_record/callbacks/delta_callbacks.rb

@@ -43,8 +43,9 @@ def delta_indices?
   end
 
   def indices
-    @indices ||= config.index_set_class.new(:classes => [instance.class]).
-      select { |index| index.type == "plain" }
+    @indices ||= config.index_set_class.new(
+      :instances => [instance], :classes => [instance.class]
+    ).select { |index| index.type == "plain" }
   end
 
   def new_or_changed?

lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb

@@ -47,7 +47,7 @@ def indices
   end
 
   def reference
-    ThinkingSphinx::IndexSet.reference_name(instance.class)
+    configuration.index_set_class.reference_name(instance.class)
   end
 
   def update(index)

lib/thinking_sphinx/active_record/interpreter.rb

@@ -13,15 +13,15 @@ def group_by(*columns)
   end
 
   def has(*columns)
-    __source.attributes += build_properties(
+    build_properties(
       ::ThinkingSphinx::ActiveRecord::Attribute, columns
-    )
+    ).each { |attribute| __source.add_attribute attribute }
   end
 
   def indexes(*columns)
-    __source.fields += build_properties(
+    build_properties(
       ::ThinkingSphinx::ActiveRecord::Field, columns
-    )
+    ).each { |field| __source.add_field field }
   end
 
   def join(*columns)

lib/thinking_sphinx/active_record/sql_source.rb

@@ -39,6 +39,18 @@ def adapter
         @adapter ||= DatabaseAdapters.adapter_for(@model)
       end
 
+      def add_attribute(attribute)
+        attributes.delete_if { |existing| existing.name == attribute.name }
+
+        attributes << attribute
+      end
+
+      def add_field(field)
+        fields.delete_if { |existing| existing.name == field.name }
+
+        fields << field
+      end
+
       def delta_processor
         options[:delta_processor].try(:new, adapter, @options[:delta_options] || {})
       end

lib/thinking_sphinx/active_record/sql_source/template.rb

@@ -18,14 +18,14 @@ def apply
   private
 
   def add_attribute(column, name, type, options = {})
-    source.attributes << ThinkingSphinx::ActiveRecord::Attribute.new(
+    source.add_attribute ThinkingSphinx::ActiveRecord::Attribute.new(
       source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
       options.merge(:as => name, :type => type)
     )
   end
 
   def add_field(column, name, options = {})
-    source.fields << ThinkingSphinx::ActiveRecord::Field.new(
+    source.add_field ThinkingSphinx::ActiveRecord::Field.new(
       source.model, ThinkingSphinx::ActiveRecord::Column.new(column),
       options.merge(:as => name)
     )

lib/thinking_sphinx/callbacks.rb

@@ -3,6 +3,13 @@
 class ThinkingSphinx::Callbacks
   attr_reader :instance
 
+  def self.append(model, reference = nil, options, &block)
+    reference ||= ThinkingSphinx::Configuration.instance.index_set_class.
+      reference_name(model)
+
+    ThinkingSphinx::Callbacks::Appender.call(model, reference, options, &block)
+  end
+
   def self.callbacks(*methods)
     mod = Module.new
     methods.each do |method|
@@ -33,3 +40,5 @@ def initialize(instance)
     @instance = instance
   end
 end
+
+require "thinking_sphinx/callbacks/appender"

lib/thinking_sphinx/callbacks/appender.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class ThinkingSphinx::Callbacks::Appender
+  def self.call(model, reference, options, &block)
+    new(model, reference, options, &block).call
+  end
+
+  def initialize(model, reference, options, &block)
+    @model = model
+    @reference = reference
+    @options = options
+    @block = block
+  end
+
+  def call
+    model.after_destroy ThinkingSphinx::ActiveRecord::Callbacks::DeleteCallbacks
+
+    if behaviours.include?(:deltas)
+      model.before_save  ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
+      model.after_commit ThinkingSphinx::ActiveRecord::Callbacks::DeltaCallbacks
+    end
+
+    if behaviours.include?(:real_time)
+      model.after_save ThinkingSphinx::RealTime.callback_for(
+        reference, path, &block
+      )
+    end
+
+    if behaviours.include?(:updates)
+      model.after_update(
+        ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks
+      )
+    end
+  end
+
+  private
+
+  attr_reader :model, :reference, :options, :block
+
+  def behaviours
+    options[:behaviours] || []
+  end
+
+  def path
+    options[:path] || []
+  end
+end

lib/thinking_sphinx/core/index.rb

@@ -11,7 +11,6 @@ module ThinkingSphinx::Core::Index
 
   def initialize(reference, options = {})
     @reference    = reference.to_sym
-    @docinfo      = :extern unless config.settings["skip_docinfo"]
     @options      = options
     @offset       = config.next_offset(options[:offset_as] || reference)
     @type         = 'plain'
@@ -40,7 +39,7 @@ def document_id_for_key(key)
   def interpret_definition!
     table_exists = model.table_exists?
     unless table_exists
-      Rails.logger.info "No table exists for #{model}. Index can not be created" 
+      Rails.logger.info "No table exists for #{model}. Index can not be created"
       return
     end
     return if @interpreted_definition