diff --git a/HISTORY b/HISTORY index 922f90956..3cb2a61a8 100644 --- a/HISTORY +++ b/HISTORY @@ -1,4 +1,25 @@ -Edge: +2013-10-20: 3.0.6 +* [FEATURE] Raise an error if no indices match the search criteria (Bryan Ricker). +* [FEATURE] skip_time_zone setting is now available per environment via config/thinking_sphinx.yml to avoid the sql_query_pre time zone command. +* [CHANGE] Updating Riddle dependency to be >= 1.5.9. +* [FEATURE] Added new search options in Sphinx 2.1.x. +* [FEATURE] Added ability to disable UTF-8 forced encoding, now that Sphinx 2.1.2 returns UTF-8 strings by default. This will be disabled by default in Thinking Sphinx 3.1.0. +* [FEATURE] Added ability to switch between Sphinx special variables and the equivalent functions. Sphinx 2.1.x requires the latter, and that behaviour will become the default in Sphinx 3.1.0. +* [FIX] Cast every column to a timestamp for timestamp attributes with multiple columns. +* [CHANGE] Separated directory preparation from data generation for real-time index (re)generation tasks. +* [CHANGE] Have tests index UTF-8 characters where appropriate (Pedro Cunha). +* [FIX] Don't use Sphinx ordering if SQL order option is supplied to a search. +* [CHANGE] Always use DISTINCT in group concatenation. +* [CHANGE] Sphinx connection failures now have their own class, ThinkingSphinx::ConnectionError, instead of the standard Mysql2::Error. +* [FIX] Custom middleware and mask options now function correctly with model-scoped searches. +* [FEATURE] Adding search_for_ids on scoped search calls. +* [CHANGE] Don't clobber custom :select options for facet searches (Timo Virkkala). +* [CHANGE] Automatically load Riddle's Sphinx 2.0.5 compatability changes. +* [FIX] Suspended deltas now no longer update core indices as well. +* [CHANGE] Realtime fields and attributes now accept symbols as well as column objects, and fields can be sortable (with a _sort prefix for the matching attribute). +* [FEATURE] MySQL users can enable a minimal GROUP BY statement, to speed up queries: set_property :minimal_group_by? => true. +* [CHANGE] Insist on the log directory existing, to ensure correct behaviour for symlinked paths. (Michael Pearson). +* [FIX] Use alphabetical ordering for index paths consistently (@grin). * [FIX] Convert very small floats to fixed format for geo-searches. * [CHANGE] Rake's silent mode is respected for indexing (@endoscient). diff --git a/README.textile b/README.textile index c25602167..f1e0d40b9 100644 --- a/README.textile +++ b/README.textile @@ -7,7 +7,7 @@ h2. Installation It's a gem, so install it like you would any other gem. You will also need to specify the Mysql2 gem as well (this is not an inbuilt dependency because JRuby, when supported, will need something different):
gem 'mysql2',          '0.3.13'
-gem 'thinking-sphinx', '3.0.5'
+gem 'thinking-sphinx', '3.0.6' The mysql2 gem is required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. diff --git a/lib/thinking_sphinx.rb b/lib/thinking_sphinx.rb index 4d251727b..797cddf7f 100644 --- a/lib/thinking_sphinx.rb +++ b/lib/thinking_sphinx.rb @@ -11,6 +11,7 @@ require 'active_record' require 'innertube' require 'active_support/core_ext/module/delegation' +require 'active_support/core_ext/module/attribute_accessors' module ThinkingSphinx def self.count(query = '', options = {}) @@ -60,8 +61,10 @@ module Subscribers; end require 'thinking_sphinx/rake_interface' require 'thinking_sphinx/scopes' require 'thinking_sphinx/search' +require 'thinking_sphinx/sphinxql' require 'thinking_sphinx/subscribers/populator_subscriber' require 'thinking_sphinx/test' +require 'thinking_sphinx/utf8' # Extended require 'thinking_sphinx/active_record' require 'thinking_sphinx/deltas' diff --git a/lib/thinking_sphinx/active_record/sql_builder/query.rb b/lib/thinking_sphinx/active_record/sql_builder/query.rb index d0177a86c..b93bf1212 100644 --- a/lib/thinking_sphinx/active_record/sql_builder/query.rb +++ b/lib/thinking_sphinx/active_record/sql_builder/query.rb @@ -24,16 +24,20 @@ def filter_by_query_pre end def scope_by_delta_processor - self.scope << delta_processor.reset_query if delta_processor && !source.delta? + return unless delta_processor && !source.delta? + + self.scope << delta_processor.reset_query end def scope_by_session - if max_len = source.options[:group_concat_max_len] - self.scope << "SET SESSION group_concat_max_len = #{max_len}" - end + return unless max_len = source.options[:group_concat_max_len] + + self.scope << "SET SESSION group_concat_max_len = #{max_len}" end def scope_by_time_zone + return if config.settings['skip_time_zone'] + self.scope += time_zone_query_pre end diff --git a/lib/thinking_sphinx/configuration.rb b/lib/thinking_sphinx/configuration.rb index 4ba2430bb..8db6fef7f 100644 --- a/lib/thinking_sphinx/configuration.rb +++ b/lib/thinking_sphinx/configuration.rb @@ -100,7 +100,7 @@ def settings def configure_searchd configure_searchd_log_files - searchd.binlog_path = framework_root.realpath.join('tmp', 'binlog', environment).to_s + searchd.binlog_path = tmp_path.join('binlog', environment).to_s searchd.address = settings['address'].presence || Defaults::ADDRESS searchd.mysql41 = settings['mysql41'] || settings['port'] || Defaults::PORT searchd.workers = 'threads' @@ -147,6 +147,11 @@ def setup @offsets = {} end + def tmp_path + path = framework_root.join('tmp') + File.exists?(path) ? path.realpath : path + end + def apply_sphinx_settings! [indexer, searchd].each do |object| settings.each do |key, value| diff --git a/lib/thinking_sphinx/errors.rb b/lib/thinking_sphinx/errors.rb index 0fffa801e..7547266f3 100644 --- a/lib/thinking_sphinx/errors.rb +++ b/lib/thinking_sphinx/errors.rb @@ -39,3 +39,6 @@ class ThinkingSphinx::QueryExecutionError < StandardError class ThinkingSphinx::MixedScopesError < StandardError end + +class ThinkingSphinx::NoIndicesError < StandardError +end diff --git a/lib/thinking_sphinx/excerpter.rb b/lib/thinking_sphinx/excerpter.rb index 414cb23bf..21b736905 100644 --- a/lib/thinking_sphinx/excerpter.rb +++ b/lib/thinking_sphinx/excerpter.rb @@ -18,8 +18,8 @@ def excerpt!(text) connection.query(statement_for(text)).first['snippet'] end - result.encode!("ISO-8859-1") - result.force_encoding("UTF-8") + ThinkingSphinx::Configuration.instance.settings['utf8'] ? result : + ThinkingSphinx::UTF8.encode(result) end private diff --git a/lib/thinking_sphinx/facet.rb b/lib/thinking_sphinx/facet.rb index ac5035b28..f5f9cdced 100644 --- a/lib/thinking_sphinx/facet.rb +++ b/lib/thinking_sphinx/facet.rb @@ -11,7 +11,7 @@ def filter_type def results_from(raw) raw.inject({}) { |hash, row| - hash[row[group_column]] = row['@count'] + hash[row[group_column]] = row[ThinkingSphinx::SphinxQL.count] hash } end @@ -19,7 +19,7 @@ def results_from(raw) private def group_column - @properties.any?(&:multi?) ? '@groupby' : name + @properties.any?(&:multi?) ? ThinkingSphinx::SphinxQL.group_by : name end def use_field? diff --git a/lib/thinking_sphinx/facet_search.rb b/lib/thinking_sphinx/facet_search.rb index fa0c2c825..7fd6eb8f8 100644 --- a/lib/thinking_sphinx/facet_search.rb +++ b/lib/thinking_sphinx/facet_search.rb @@ -101,7 +101,8 @@ def limit def options_for(facet) options.merge( - :select => (options[:select] || '*') + ', @groupby, @count', + :select => (options[:select] || '*') + + ", #{ThinkingSphinx::SphinxQL.group_by}, #{ThinkingSphinx::SphinxQL.count}", :group_by => facet.name, :indices => index_names_for(facet), :max_matches => limit, diff --git a/lib/thinking_sphinx/index_set.rb b/lib/thinking_sphinx/index_set.rb index e57cf8919..8cdb67b96 100644 --- a/lib/thinking_sphinx/index_set.rb +++ b/lib/thinking_sphinx/index_set.rb @@ -1,6 +1,8 @@ class ThinkingSphinx::IndexSet include Enumerable + delegate :each, :empty?, :to => :indices + def initialize(classes, index_names, configuration = nil) @classes = classes || [] @index_names = index_names @@ -11,10 +13,6 @@ def ancestors classes_and_ancestors - classes end - def each(&block) - indices.each { |index| yield index } - end - def to_a indices end diff --git a/lib/thinking_sphinx/masks/group_enumerators_mask.rb b/lib/thinking_sphinx/masks/group_enumerators_mask.rb index a21fb03b2..2eb12f8e2 100644 --- a/lib/thinking_sphinx/masks/group_enumerators_mask.rb +++ b/lib/thinking_sphinx/masks/group_enumerators_mask.rb @@ -9,19 +9,20 @@ def can_handle?(method) def each_with_count(&block) @search.raw.each_with_index do |row, index| - yield @search[index], row['@count'] + yield @search[index], row[ThinkingSphinx::SphinxQL.count] end end def each_with_group(&block) @search.raw.each_with_index do |row, index| - yield @search[index], row['@groupby'] + yield @search[index], row[ThinkingSphinx::SphinxQL.group_by] end end def each_with_group_and_count(&block) @search.raw.each_with_index do |row, index| - yield @search[index], row['@groupby'], row['@count'] + yield @search[index], row[ThinkingSphinx::SphinxQL.group_by], + row[ThinkingSphinx::SphinxQL.count] end end end diff --git a/lib/thinking_sphinx/masks/weight_enumerator_mask.rb b/lib/thinking_sphinx/masks/weight_enumerator_mask.rb index 65ec81456..b8fdc6a67 100644 --- a/lib/thinking_sphinx/masks/weight_enumerator_mask.rb +++ b/lib/thinking_sphinx/masks/weight_enumerator_mask.rb @@ -9,7 +9,7 @@ def can_handle?(method) def each_with_weight(&block) @search.raw.each_with_index do |row, index| - yield @search[index], row['@weight'] + yield @search[index], row[ThinkingSphinx::SphinxQL.weight] end end end diff --git a/lib/thinking_sphinx/middlewares/sphinxql.rb b/lib/thinking_sphinx/middlewares/sphinxql.rb index f373e3195..97aec226d 100644 --- a/lib/thinking_sphinx/middlewares/sphinxql.rb +++ b/lib/thinking_sphinx/middlewares/sphinxql.rb @@ -3,7 +3,8 @@ class ThinkingSphinx::Middlewares::SphinxQL < SELECT_OPTIONS = [:ranker, :max_matches, :cutoff, :max_query_time, :retry_count, :retry_delay, :field_weights, :index_weights, :reverse_scan, - :comment] + :comment, :agent_query_timeout, :boolean_simplify, :global_idf, :idf, + :sort_method] def call(contexts) contexts.each do |context| @@ -131,7 +132,11 @@ def index_options end def indices - @indices ||= ThinkingSphinx::IndexSet.new classes, options[:indices] + @indices ||= begin + set = ThinkingSphinx::IndexSet.new classes, options[:indices] + raise ThinkingSphinx::NoIndicesError if set.empty? + set + end end def order_clause @@ -150,7 +155,7 @@ def select_options end def values - options[:select] ||= '*, @groupby, @count' if group_attribute.present? + options[:select] ||= "*, #{ThinkingSphinx::SphinxQL.group_by}, #{ThinkingSphinx::SphinxQL.count}" if group_attribute.present? options[:select] end diff --git a/lib/thinking_sphinx/middlewares/utf8.rb b/lib/thinking_sphinx/middlewares/utf8.rb index 239c7baf0..c1253d75e 100644 --- a/lib/thinking_sphinx/middlewares/utf8.rb +++ b/lib/thinking_sphinx/middlewares/utf8.rb @@ -5,7 +5,7 @@ def call(contexts) contexts.each do |context| context[:results].each { |row| update_row row } update_row context[:meta] - end + end unless ThinkingSphinx::Configuration.instance.settings['utf8'] app.call contexts end @@ -16,8 +16,7 @@ def update_row(row) row.each do |key, value| next unless value.is_a?(String) - value.encode!("ISO-8859-1") - row[key] = value.force_encoding("UTF-8") + row[key] = ThinkingSphinx::UTF8.encode value end end end diff --git a/lib/thinking_sphinx/panes/weight_pane.rb b/lib/thinking_sphinx/panes/weight_pane.rb index 92e51b858..9d9828790 100644 --- a/lib/thinking_sphinx/panes/weight_pane.rb +++ b/lib/thinking_sphinx/panes/weight_pane.rb @@ -4,6 +4,6 @@ def initialize(context, object, raw) end def weight - @raw['@weight'] + @raw[ThinkingSphinx::SphinxQL.weight] end end diff --git a/lib/thinking_sphinx/sphinxql.rb b/lib/thinking_sphinx/sphinxql.rb new file mode 100644 index 000000000..0509183ab --- /dev/null +++ b/lib/thinking_sphinx/sphinxql.rb @@ -0,0 +1,17 @@ +module ThinkingSphinx::SphinxQL + mattr_accessor :weight, :group_by, :count + + def self.functions! + self.weight = 'weight()' + self.group_by = 'groupby()' + self.count = 'count(*)' + end + + def self.variables! + self.weight = '@weight' + self.group_by = '@groupby' + self.count = '@count' + end + + self.variables! +end diff --git a/lib/thinking_sphinx/utf8.rb b/lib/thinking_sphinx/utf8.rb new file mode 100644 index 000000000..08f157016 --- /dev/null +++ b/lib/thinking_sphinx/utf8.rb @@ -0,0 +1,16 @@ +class ThinkingSphinx::UTF8 + attr_reader :string + + def self.encode(string) + new(string).encode + end + + def initialize(string) + @string = string + end + + def encode + string.encode!('ISO-8859-1') + string.force_encoding('UTF-8') + end +end diff --git a/spec/acceptance/attribute_access_spec.rb b/spec/acceptance/attribute_access_spec.rb index 795c1e86d..dc25046ff 100644 --- a/spec/acceptance/attribute_access_spec.rb +++ b/spec/acceptance/attribute_access_spec.rb @@ -15,7 +15,8 @@ Book.create! :title => 'American Gods', :year => 2001 index - search = Book.search('gods', :select => '*, @weight') + search = Book.search 'gods', + :select => "*, #{ThinkingSphinx::SphinxQL.weight}" search.context[:panes] << ThinkingSphinx::Panes::WeightPane search.first.weight.should == 2500 @@ -25,7 +26,8 @@ gods = Book.create! :title => 'American Gods', :year => 2001 index - search = Book.search('gods', :select => '*, @weight') + search = Book.search 'gods', + :select => "*, #{ThinkingSphinx::SphinxQL.weight}" search.masks << ThinkingSphinx::Masks::WeightEnumeratorMask expectations = [[gods, 2500]] diff --git a/spec/acceptance/searching_within_a_model_spec.rb b/spec/acceptance/searching_within_a_model_spec.rb index 6cd8e1d9e..b7b6b4d83 100644 --- a/spec/acceptance/searching_within_a_model_spec.rb +++ b/spec/acceptance/searching_within_a_model_spec.rb @@ -65,6 +65,12 @@ User.recent.search }.should raise_error(ThinkingSphinx::MixedScopesError) end + + it "raises an error if the model has no indices defined" do + lambda { + Category.search.to_a + }.should raise_error(ThinkingSphinx::NoIndicesError) + end end describe 'Searching within a model with a realtime index', :live => true do diff --git a/spec/acceptance/support/sphinx_controller.rb b/spec/acceptance/support/sphinx_controller.rb index 4a797e8ca..601eb6302 100644 --- a/spec/acceptance/support/sphinx_controller.rb +++ b/spec/acceptance/support/sphinx_controller.rb @@ -9,6 +9,11 @@ def setup ThinkingSphinx::Configuration.reset + if ENV['SPHINX_VERSION'].try :[], /2.1.\d/ + ThinkingSphinx::SphinxQL.functions! + ThinkingSphinx::Configuration.instance.settings['utf8'] = true + end + ActiveSupport::Dependencies.loaded.each do |path| $LOADED_FEATURES.delete "#{path}.rb" end diff --git a/spec/thinking_sphinx/active_record/sql_builder_spec.rb b/spec/thinking_sphinx/active_record/sql_builder_spec.rb index ecffecbb2..8a6a4b83c 100644 --- a/spec/thinking_sphinx/active_record/sql_builder_spec.rb +++ b/spec/thinking_sphinx/active_record/sql_builder_spec.rb @@ -12,7 +12,7 @@ :quoted_table_name => '`users`', :name => 'User') } let(:connection) { double('connection') } let(:relation) { double('relation') } - let(:config) { double('config', :indices => indices) } + let(:config) { double('config', :indices => indices, :settings => {}) } let(:indices) { double('indices', :count => 5) } let(:presenter) { double('presenter', :to_select => '`name` AS `name`', :to_group => '`name`') } @@ -596,6 +596,16 @@ builder.sql_query_pre.should_not include('SET UTF8') end + + it "adds a time-zone query by default" do + expect(builder.sql_query_pre).to include('SET TIME ZONE') + end + + it "does not add a time-zone query if requested" do + config.settings['skip_time_zone'] = true + + expect(builder.sql_query_pre).to_not include('SET TIME ZONE') + end end describe 'sql_query_range' do diff --git a/spec/thinking_sphinx/facet_search_spec.rb b/spec/thinking_sphinx/facet_search_spec.rb index 64455f663..b63478855 100644 --- a/spec/thinking_sphinx/facet_search_spec.rb +++ b/spec/thinking_sphinx/facet_search_spec.rb @@ -29,12 +29,12 @@ module ThinkingSphinx; end DumbSearch = ::Struct.new(:query, :options) do def raw [{ - 'sphinx_internal_class' => 'Foo', - 'price_bracket' => 3, - 'tag_ids' => '1,2', - 'category_id' => 11, - '@count' => 5, - '@groupby' => 2 + 'sphinx_internal_class' => 'Foo', + 'price_bracket' => 3, + 'tag_ids' => '1,2', + 'category_id' => 11, + ThinkingSphinx::SphinxQL.count => 5, + ThinkingSphinx::SphinxQL.group_by => 2 }] end end diff --git a/spec/thinking_sphinx/middlewares/sphinxql_spec.rb b/spec/thinking_sphinx/middlewares/sphinxql_spec.rb index 636882a7e..d54351b36 100644 --- a/spec/thinking_sphinx/middlewares/sphinxql_spec.rb +++ b/spec/thinking_sphinx/middlewares/sphinxql_spec.rb @@ -6,11 +6,14 @@ module ActiveRecord class Base; end end +require 'active_support/core_ext/module/attribute_accessors' require 'active_support/core_ext/module/delegation' require 'active_support/core_ext/object/blank' require 'active_support/core_ext/string/inflections' require 'thinking_sphinx/middlewares/middleware' require 'thinking_sphinx/middlewares/sphinxql' +require 'thinking_sphinx/errors' +require 'thinking_sphinx/sphinxql' describe ThinkingSphinx::Middlewares::SphinxQL do let(:app) { double('app', :call => true) } @@ -58,6 +61,14 @@ class Base; end middleware.call [context] end + it "raises an exception if there's no matching indices" do + index_set.clear + + expect { + middleware.call [context] + }.to raise_error(ThinkingSphinx::NoIndicesError) + end + it "generates a Sphinx query from the provided keyword and conditions" do search.stub :query => 'tasty' search.options[:conditions] = {:title => 'pancakes'} diff --git a/spec/thinking_sphinx/panes/weight_pane_spec.rb b/spec/thinking_sphinx/panes/weight_pane_spec.rb index 14cb9dff7..d1ea77e11 100644 --- a/spec/thinking_sphinx/panes/weight_pane_spec.rb +++ b/spec/thinking_sphinx/panes/weight_pane_spec.rb @@ -12,7 +12,7 @@ module Panes; end describe '#weight' do it "returns the object's weight by default" do - raw['@weight'] = 101 + raw[ThinkingSphinx::SphinxQL.weight] = 101 pane.weight.should == 101 end diff --git a/thinking-sphinx.gemspec b/thinking-sphinx.gemspec index 134974010..458080e98 100644 --- a/thinking-sphinx.gemspec +++ b/thinking-sphinx.gemspec @@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__) Gem::Specification.new do |s| s.name = 'thinking-sphinx' - s.version = '3.0.5' + s.version = '3.0.6' s.platform = Gem::Platform::RUBY s.authors = ["Pat Allan"] s.email = ["pat@freelancing-gods.com"] @@ -25,7 +25,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'builder', '>= 2.1.2' s.add_runtime_dependency 'middleware', '>= 0.1.0' s.add_runtime_dependency 'innertube', '>= 1.0.2' - s.add_runtime_dependency 'riddle', '>= 1.5.8' + s.add_runtime_dependency 'riddle', '>= 1.5.9' s.add_development_dependency 'appraisal', '~> 0.4.0' s.add_development_dependency 'combustion', '~> 0.4.0'