Merge branch 'release/v3.0.6'

HISTORY

@@ -1,4 +1,25 @@
-Edge:
+2013-10-20: 3.0.6
+* [FEATURE] Raise an error if no indices match the search criteria (Bryan Ricker).
+* [FEATURE] skip_time_zone setting is now available per environment via config/thinking_sphinx.yml to avoid the sql_query_pre time zone command.
+* [CHANGE] Updating Riddle dependency to be >= 1.5.9.
+* [FEATURE] Added new search options in Sphinx 2.1.x.
+* [FEATURE] Added ability to disable UTF-8 forced encoding, now that Sphinx 2.1.2 returns UTF-8 strings by default. This will be disabled by default in Thinking Sphinx 3.1.0.
+* [FEATURE] Added ability to switch between Sphinx special variables and the equivalent functions. Sphinx 2.1.x requires the latter, and that behaviour will become the default in Sphinx 3.1.0.
+* [FIX] Cast every column to a timestamp for timestamp attributes with multiple columns.
+* [CHANGE] Separated directory preparation from data generation for real-time index (re)generation tasks.
+* [CHANGE] Have tests index UTF-8 characters where appropriate (Pedro Cunha).
+* [FIX] Don't use Sphinx ordering if SQL order option is supplied to a search.
+* [CHANGE] Always use DISTINCT in group concatenation.
+* [CHANGE] Sphinx connection failures now have their own class, ThinkingSphinx::ConnectionError, instead of the standard Mysql2::Error.
+* [FIX] Custom middleware and mask options now function correctly with model-scoped searches.
+* [FEATURE] Adding search_for_ids on scoped search calls.
+* [CHANGE] Don't clobber custom :select options for facet searches (Timo Virkkala).
+* [CHANGE] Automatically load Riddle's Sphinx 2.0.5 compatability changes.
+* [FIX] Suspended deltas now no longer update core indices as well.
+* [CHANGE] Realtime fields and attributes now accept symbols as well as column objects, and fields can be sortable (with a _sort prefix for the matching attribute).
+* [FEATURE] MySQL users can enable a minimal GROUP BY statement, to speed up queries: set_property :minimal_group_by? => true.
+* [CHANGE] Insist on the log directory existing, to ensure correct behaviour for symlinked paths. (Michael Pearson).
+* [FIX] Use alphabetical ordering for index paths consistently (@grin).
 * [FIX] Convert very small floats to fixed format for geo-searches.
 * [CHANGE] Rake's silent mode is respected for indexing (@endoscient).
 

README.textile

@@ -7,7 +7,7 @@ h2. Installation
 It's a gem, so install it like you would any other gem. You will also need to specify the Mysql2 gem as well (this is not an inbuilt dependency because JRuby, when supported, will need something different):
 
 <pre><code>gem 'mysql2',          '0.3.13'
-gem 'thinking-sphinx', '3.0.5'</code></pre>
+gem 'thinking-sphinx', '3.0.6'</code></pre>
 
 The mysql2 gem is required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database.
 

lib/thinking_sphinx.rb

@@ -11,6 +11,7 @@
 require 'active_record'
 require 'innertube'
 require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/module/attribute_accessors'
 
 module ThinkingSphinx
   def self.count(query = '', options = {})
@@ -60,8 +61,10 @@ module Subscribers; end
 require 'thinking_sphinx/rake_interface'
 require 'thinking_sphinx/scopes'
 require 'thinking_sphinx/search'
+require 'thinking_sphinx/sphinxql'
 require 'thinking_sphinx/subscribers/populator_subscriber'
 require 'thinking_sphinx/test'
+require 'thinking_sphinx/utf8'
 # Extended
 require 'thinking_sphinx/active_record'
 require 'thinking_sphinx/deltas'

lib/thinking_sphinx/active_record/sql_builder/query.rb

@@ -24,16 +24,20 @@ def filter_by_query_pre
       end
 
       def scope_by_delta_processor
-        self.scope << delta_processor.reset_query if delta_processor && !source.delta?
+        return unless delta_processor && !source.delta?
+
+        self.scope << delta_processor.reset_query
       end
 
       def scope_by_session
-        if max_len = source.options[:group_concat_max_len]
-          self.scope << "SET SESSION group_concat_max_len = #{max_len}"
-        end
+        return unless max_len = source.options[:group_concat_max_len]
+
+        self.scope << "SET SESSION group_concat_max_len = #{max_len}"
       end
 
       def scope_by_time_zone
+        return if config.settings['skip_time_zone']
+
         self.scope += time_zone_query_pre
       end
 

lib/thinking_sphinx/configuration.rb

@@ -100,7 +100,7 @@ def settings
   def configure_searchd
     configure_searchd_log_files
 
-    searchd.binlog_path = framework_root.realpath.join('tmp', 'binlog', environment).to_s
+    searchd.binlog_path = tmp_path.join('binlog', environment).to_s
     searchd.address = settings['address'].presence || Defaults::ADDRESS
     searchd.mysql41 = settings['mysql41'] || settings['port'] || Defaults::PORT
     searchd.workers = 'threads'
@@ -147,6 +147,11 @@ def setup
     @offsets = {}
   end
 
+  def tmp_path
+    path = framework_root.join('tmp')
+    File.exists?(path) ? path.realpath : path
+  end
+
   def apply_sphinx_settings!
     [indexer, searchd].each do |object|
       settings.each do |key, value|

lib/thinking_sphinx/errors.rb

@@ -39,3 +39,6 @@ class ThinkingSphinx::QueryExecutionError < StandardError
 
 class ThinkingSphinx::MixedScopesError < StandardError
 end
+
+class ThinkingSphinx::NoIndicesError < StandardError
+end

lib/thinking_sphinx/excerpter.rb

@@ -18,8 +18,8 @@ def excerpt!(text)
       connection.query(statement_for(text)).first['snippet']
     end
 
-    result.encode!("ISO-8859-1")
-    result.force_encoding("UTF-8")
+    ThinkingSphinx::Configuration.instance.settings['utf8'] ? result :
+      ThinkingSphinx::UTF8.encode(result)
   end
 
   private

lib/thinking_sphinx/facet.rb

@@ -11,15 +11,15 @@ def filter_type
 
   def results_from(raw)
     raw.inject({}) { |hash, row|
-      hash[row[group_column]] = row['@count']
+      hash[row[group_column]] = row[ThinkingSphinx::SphinxQL.count]
       hash
     }
   end
 
   private
 
   def group_column
-    @properties.any?(&:multi?) ? '@groupby' : name
+    @properties.any?(&:multi?) ? ThinkingSphinx::SphinxQL.group_by : name
   end
 
   def use_field?

lib/thinking_sphinx/facet_search.rb

@@ -101,7 +101,8 @@ def limit
 
   def options_for(facet)
     options.merge(
-      :select      => (options[:select] || '*') + ', @groupby, @count',
+      :select      => (options[:select] || '*') +
+        ", #{ThinkingSphinx::SphinxQL.group_by}, #{ThinkingSphinx::SphinxQL.count}",
       :group_by    => facet.name,
       :indices     => index_names_for(facet),
       :max_matches => limit,

lib/thinking_sphinx/index_set.rb

@@ -1,6 +1,8 @@
 class ThinkingSphinx::IndexSet
   include Enumerable
 
+  delegate :each, :empty?, :to => :indices
+
   def initialize(classes, index_names, configuration = nil)
     @classes       = classes || []
     @index_names   = index_names
@@ -11,10 +13,6 @@ def ancestors
     classes_and_ancestors - classes
   end
 
-  def each(&block)
-    indices.each { |index| yield index }
-  end
-
   def to_a
     indices
   end

lib/thinking_sphinx/masks/group_enumerators_mask.rb

@@ -9,19 +9,20 @@ def can_handle?(method)
 
   def each_with_count(&block)
     @search.raw.each_with_index do |row, index|
-      yield @search[index], row['@count']
+      yield @search[index], row[ThinkingSphinx::SphinxQL.count]
     end
   end
 
   def each_with_group(&block)
     @search.raw.each_with_index do |row, index|
-      yield @search[index], row['@groupby']
+      yield @search[index], row[ThinkingSphinx::SphinxQL.group_by]
     end
   end
 
   def each_with_group_and_count(&block)
     @search.raw.each_with_index do |row, index|
-      yield @search[index], row['@groupby'], row['@count']
+      yield @search[index], row[ThinkingSphinx::SphinxQL.group_by],
+        row[ThinkingSphinx::SphinxQL.count]
     end
   end
 end

lib/thinking_sphinx/masks/weight_enumerator_mask.rb

@@ -9,7 +9,7 @@ def can_handle?(method)
 
   def each_with_weight(&block)
     @search.raw.each_with_index do |row, index|
-      yield @search[index], row['@weight']
+      yield @search[index], row[ThinkingSphinx::SphinxQL.weight]
     end
   end
 end

lib/thinking_sphinx/middlewares/sphinxql.rb

@@ -3,7 +3,8 @@ class ThinkingSphinx::Middlewares::SphinxQL <
 
   SELECT_OPTIONS = [:ranker, :max_matches, :cutoff, :max_query_time,
     :retry_count, :retry_delay, :field_weights, :index_weights, :reverse_scan,
-    :comment]
+    :comment, :agent_query_timeout, :boolean_simplify, :global_idf, :idf,
+    :sort_method]
 
   def call(contexts)
     contexts.each do |context|
@@ -131,7 +132,11 @@ def index_options
     end
 
     def indices
-      @indices ||= ThinkingSphinx::IndexSet.new classes, options[:indices]
+      @indices ||= begin
+        set = ThinkingSphinx::IndexSet.new classes, options[:indices]
+        raise ThinkingSphinx::NoIndicesError if set.empty?
+        set
+      end
     end
 
     def order_clause
@@ -150,7 +155,7 @@ def select_options
     end
 
     def values
-      options[:select] ||= '*, @groupby, @count' if group_attribute.present?
+      options[:select] ||= "*, #{ThinkingSphinx::SphinxQL.group_by}, #{ThinkingSphinx::SphinxQL.count}" if group_attribute.present?
       options[:select]
     end
 

lib/thinking_sphinx/middlewares/utf8.rb

@@ -5,7 +5,7 @@ def call(contexts)
     contexts.each do |context|
       context[:results].each { |row| update_row row }
       update_row context[:meta]
-    end
+    end unless ThinkingSphinx::Configuration.instance.settings['utf8']
 
     app.call contexts
   end
@@ -16,8 +16,7 @@ def update_row(row)
     row.each do |key, value|
       next unless value.is_a?(String)
 
-      value.encode!("ISO-8859-1")
-      row[key] = value.force_encoding("UTF-8")
+      row[key] = ThinkingSphinx::UTF8.encode value
     end
   end
 end

lib/thinking_sphinx/panes/weight_pane.rb

@@ -4,6 +4,6 @@ def initialize(context, object, raw)
   end
 
   def weight
-    @raw['@weight']
+    @raw[ThinkingSphinx::SphinxQL.weight]
   end
 end

lib/thinking_sphinx/sphinxql.rb

@@ -0,0 +1,17 @@
+module ThinkingSphinx::SphinxQL
+  mattr_accessor :weight, :group_by, :count
+
+  def self.functions!
+    self.weight   = 'weight()'
+    self.group_by = 'groupby()'
+    self.count    = 'count(*)'
+  end
+
+  def self.variables!
+    self.weight   = '@weight'
+    self.group_by = '@groupby'
+    self.count    = '@count'
+  end
+
+  self.variables!
+end

lib/thinking_sphinx/utf8.rb

@@ -0,0 +1,16 @@
+class ThinkingSphinx::UTF8
+  attr_reader :string
+
+  def self.encode(string)
+    new(string).encode
+  end
+
+  def initialize(string)
+    @string = string
+  end
+
+  def encode
+    string.encode!('ISO-8859-1')
+    string.force_encoding('UTF-8')
+  end
+end

spec/acceptance/attribute_access_spec.rb

@@ -15,7 +15,8 @@
     Book.create! :title => 'American Gods', :year => 2001
     index
 
-    search = Book.search('gods', :select => '*, @weight')
+    search = Book.search 'gods',
+      :select => "*, #{ThinkingSphinx::SphinxQL.weight}"
     search.context[:panes] << ThinkingSphinx::Panes::WeightPane
 
     search.first.weight.should == 2500
@@ -25,7 +26,8 @@
     gods = Book.create! :title => 'American Gods', :year => 2001
     index
 
-    search = Book.search('gods', :select => '*, @weight')
+    search = Book.search 'gods',
+      :select => "*, #{ThinkingSphinx::SphinxQL.weight}"
     search.masks << ThinkingSphinx::Masks::WeightEnumeratorMask
 
     expectations = [[gods, 2500]]

spec/acceptance/searching_within_a_model_spec.rb

@@ -65,6 +65,12 @@
       User.recent.search
     }.should raise_error(ThinkingSphinx::MixedScopesError)
   end
+
+  it "raises an error if the model has no indices defined" do
+    lambda {
+      Category.search.to_a
+    }.should raise_error(ThinkingSphinx::NoIndicesError)
+  end
 end
 
 describe 'Searching within a model with a realtime index', :live => true do

spec/acceptance/support/sphinx_controller.rb

@@ -9,6 +9,11 @@ def setup
 
     ThinkingSphinx::Configuration.reset
 
+    if ENV['SPHINX_VERSION'].try :[], /2.1.\d/
+      ThinkingSphinx::SphinxQL.functions!
+      ThinkingSphinx::Configuration.instance.settings['utf8'] = true
+    end
+
     ActiveSupport::Dependencies.loaded.each do |path|
       $LOADED_FEATURES.delete "#{path}.rb"
     end

spec/thinking_sphinx/active_record/sql_builder_spec.rb

@@ -12,7 +12,7 @@
     :quoted_table_name => '`users`', :name => 'User') }
   let(:connection)   { double('connection') }
   let(:relation)     { double('relation') }
-  let(:config)       { double('config', :indices => indices) }
+  let(:config)       { double('config', :indices => indices, :settings => {}) }
   let(:indices)      { double('indices', :count => 5) }
   let(:presenter)    { double('presenter', :to_select => '`name` AS `name`',
     :to_group => '`name`') }
@@ -596,6 +596,16 @@
 
       builder.sql_query_pre.should_not include('SET UTF8')
     end
+
+    it "adds a time-zone query by default" do
+      expect(builder.sql_query_pre).to include('SET TIME ZONE')
+    end
+
+    it "does not add a time-zone query if requested" do
+      config.settings['skip_time_zone'] = true
+
+      expect(builder.sql_query_pre).to_not include('SET TIME ZONE')
+    end
   end
 
   describe 'sql_query_range' do

spec/thinking_sphinx/facet_search_spec.rb

@@ -29,12 +29,12 @@ module ThinkingSphinx; end
   DumbSearch = ::Struct.new(:query, :options) do
     def raw
       [{
-        'sphinx_internal_class' => 'Foo',
-        'price_bracket'         => 3,
-        'tag_ids'               => '1,2',
-        'category_id'           => 11,
-        '@count'                => 5,
-        '@groupby'              => 2
+        'sphinx_internal_class'           => 'Foo',
+        'price_bracket'                   => 3,
+        'tag_ids'                         => '1,2',
+        'category_id'                     => 11,
+        ThinkingSphinx::SphinxQL.count    => 5,
+        ThinkingSphinx::SphinxQL.group_by => 2
       }]
     end
   end