Add RSpec parallelization

Added a Rake task for running tests in parallel, 1 thread per concurrency available on SL.

TestBroker handles the number of tests, the platform brokerage and the concurrency.

Relying on the "worker" facility of parallel_tests to dispatch tests to available Sauce VMs.

Saucerspec runner replaces the rspec runner for parallel_tests.  The runner creates a duplicate test group for each set of environment variables and also attaches variables from the broker for each group.

Config now only returns the first entry of browsers if not running in || tests.

Increased version to 3

Rakefile

@@ -9,14 +9,22 @@ Bundler::GemHelper.install_tasks
 namespace :spec do
   rspec_options = '--color --format d --fail-fast --order random'
   RSpec::Core::RakeTask.new(:unit) do |s|
-    s.pattern = 'spec/sauce/**_spec.rb'
+    s.pattern = 'spec/sauce/**/*_spec.rb'
     s.rspec_opts = rspec_options
   end
 
   RSpec::Core::RakeTask.new(:integration) do |s|
     s.pattern = 'spec/integration/**_spec.rb'
     s.rspec_opts = rspec_options
   end
+
+  namespace :rspec do
+    desc "Run an integration test with rspec and capybara"
+    task :capybara do |t|
+      ensure_rvm!
+      sh "(cd spec/integration/rspec && ./run-test.sh)"
+    end
+  end
 end
 
 def ensure_rvm!

gems/sauce-connect/sauce-connect.gemspec

@@ -14,5 +14,5 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency('sauce', "~> #{Sauce::MAJOR_VERSION}.0")
+  gem.add_dependency('sauce', "~> #{Sauce.version}")
 end

gems/sauce-cucumber/sauce-cucumber.gemspec

@@ -13,8 +13,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "sauce-cucumber"
   gem.require_paths = ["lib"]
-  gem.version       = "#{Sauce::MAJOR_VERSION}.1"
+  gem.version       = "#{Sauce::MAJOR_VERSION}.0"
 
-  gem.add_dependency('sauce', ">= #{Sauce::MAJOR_VERSION}.0")
+  gem.add_dependency('sauce', "~> #{Sauce.version}")
   gem.add_dependency('cucumber', '>= 1.2.0')
 end

lib/parallel_tests/saucerspec/runner.rb

@@ -0,0 +1,23 @@
+require "yaml"
+require "parallel_tests/rspec/runner"
+
+module ParallelTests
+  module Saucerspec
+    class Runner < ParallelTests::RSpec::Runner
+
+      def self.run_tests(test_files, process_number, num_processes, options)
+        exe = executable # expensive, so we cache
+        version = (exe =~ /\brspec\b/ ? 2 : 1)
+        cmd = [exe, options[:test_options], (rspec_2_color if version == 2), spec_opts, *test_files].compact.join(" ")
+        env = Sauce::TestBroker.next_environment(test_files)
+        env << " #{rspec_1_color}" if version == 1
+        options = options.merge(:env => env)
+        execute_command(cmd, process_number, num_processes, options)
+      end
+
+      def self.tests_in_groups(tests, num_groups, options={})
+        super * Sauce::TestBroker.test_platforms.length
+      end
+    end
+  end
+end

lib/sauce.rb

@@ -5,3 +5,5 @@
 require 'sauce/config'
 require 'sauce/selenium'
 require 'sauce/integrations'
+require 'tasks/parallel_testing'
+require 'parallel_tests/saucerspec/runner'

lib/sauce/config.rb

@@ -127,7 +127,7 @@ def browser
       if @undefaulted_opts[:browser]
         return @undefaulted_opts[:browser]
       end
-      if @opts[:browsers]
+      if !ENV["TEST_ENV_NUMBER"] && @opts[:browsers]
         @opts[:browsers][0][1]
       else
         @opts[:browser]
@@ -138,7 +138,7 @@ def os
       if @undefaulted_opts[:os]
         return @undefaulted_opts[:os]
       end
-      if @opts[:browsers]
+      if !ENV["TEST_ENV_NUMBER"] && @opts[:browsers]
         @opts[:browsers][0][0]
       else
         @opts[:os]
@@ -149,7 +149,7 @@ def browser_version
       if @undefaulted_opts[:browser_version]
         return @undefaulted_opts[:browser_version]
       end
-      if @opts[:browsers]
+      if !ENV["TEST_ENV_NUMBER"] && @opts[:browsers]
         @opts[:browsers][0][2]
       else
         @opts[:browser_version]

lib/sauce/parallel/test_broker.rb

@@ -0,0 +1,51 @@
+require "rest-client"
+require "json"
+require "yaml"
+require "sauce/parallel/test_group"
+
+module Sauce
+  class TestBroker
+
+    def self.next_environment(group)
+      unless test_groups.has_key? group
+        test_groups[group] = TestGroup.new(self.test_platforms)
+      end
+
+      test_groups[group].next_platform
+    end
+
+    def self.test_groups
+      @@groups ||= {}
+    end
+
+    def self.test_platforms
+      unless defined? @@platforms
+        load_sauce_config
+        brokers = Sauce.get_config
+        @@platforms ||= brokers[:browsers]
+      end
+      @@platforms
+    end
+
+    def self.concurrencies
+      response = RestClient.get "#{rest_jobs_url}/#{SAUCE_USERNAME}/limits"
+      res = JSON.parse(response)["concurrency"]
+    end
+
+    def self.rest_jobs_url
+      "https://#{AUTH_DETAILS}@saucelabs.com/rest/v1"
+    end
+
+    def self.load_sauce_config
+      if File.exists? "./spec/sauce_helper.rb"
+        require "./spec/sauce_helper"
+      else
+        require "./spec/spec_helper"
+      end
+    end
+
+    SAUCE_USERNAME = ENV["SAUCE_USERNAME"]
+    SAUCE_ACCESS_KEY = ENV["SAUCE_ACCESS_KEY"]
+    AUTH_DETAILS = "#{SAUCE_USERNAME}:#{SAUCE_ACCESS_KEY}"
+  end
+end

lib/sauce/parallel/test_group.rb

@@ -0,0 +1,22 @@
+module Sauce
+  class TestGroup
+    def initialize(platforms)
+      @platforms = platforms
+      @index = 0
+    end
+
+    def next_platform
+      platform = @platforms[@index]
+      begin
+        @index = @index + 1
+        {
+          :SAUCE_OS => "'#{platform[0]}'",
+          :SAUCE_BROWSER => "'#{platform[1]}'",
+          :SAUCE_BROWSER_VERSION => "'#{platform[2]}'"
+        }
+      rescue NoMethodError => e
+        puts "I don't have any config"
+      end
+    end
+  end
+end

lib/sauce/version.rb

@@ -1,6 +1,6 @@
 module Sauce
-  MAJOR_VERSION = '2.4'
-  PATCH_VERSION = '4'
+  MAJOR_VERSION = '3.0'
+  PATCH_VERSION = '0.beta.1'
 
   def version
     "#{MAJOR_VERSION}.#{PATCH_VERSION}"

lib/tasks/parallel_testing.rb

@@ -0,0 +1,9 @@
+require "sauce/parallel/test_broker"
+require "parallel_tests"
+require "parallel_tests/tasks"
+
+namespace :sauce do
+  task :spec do
+    ParallelTests::CLI.new.run(["--type", "saucerspec"] + ["-n #{[20]}", "spec"])
+  end
+end

release_notes.MARKDOWN

@@ -0,0 +1,14 @@
+RELEASE NOTES
+-------------
+
+3.0.0.beta.1
+============
+
+* Removed multiple browsers for RSpec in lieu of magic parallelization via Rake Task
+* Added said Rake Task
+* Added test brokerage and a runner for Parallel Tests
+
+The plan at the moment is to beta ||ism for RSpec 2, including placing it in the
+tutorial and providing beta docs to accompany it.  Once accepted for release,
+multiple browsers sequentially will be removed from the integrations.rb file and
+we'll recommend parallel testing for everyone.

sauce.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.summary = "A Ruby helper for running tests in Sauce Labs"
   s.description = "A Ruby helper for running tests in Sauce Labs' browser testing cloud service"
   # Include pretty much everything in Git except the examples/ directory
-  s.files = Dir['lib/**/*.rb']
+  s.files = Dir['lib/**/*.rb'] + Dir['lib/**/**/*.rb']
   s.executables = ['sauce']
   s.default_executable = 'sauce'
   s.require_paths = ["lib"]
@@ -28,4 +28,5 @@ Gem::Specification.new do |s|
   s.add_dependency('json', [">= 1.2.0"])
   s.add_dependency('cmdparse', [">= 2.0.2"])
   s.add_dependency('highline', [">= 1.5.0"])
+  s.add_dependency('parallel_tests', ["= 0.12.4"])
 end

spec/integration/rspec/run-test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -xe
+#!/bin/bash
 
 RAKE="bundle exec rake"
 
@@ -17,4 +17,4 @@ source .rvmrc
 
 bundle install
 
-${RAKE} spec
+${RAKE} spec

spec/parallel_tests/sauce_rspec_runner_spec.rb

@@ -0,0 +1,23 @@
+require "rspec"
+require "sauce"
+require "parallel_tests"
+
+describe "Sauce Rspec Runner" do
+  describe "#self.tests_in_groups" do
+    it "should return a group for every environment" do
+      Sauce.config do |c|
+        c.browsers = [
+            ["Windows 7", "Opera", "10"],
+            ["Linux", "Firefox", "19"],
+            ["Windows 8", "Chrome", ""]
+        ]
+      end
+
+      ParallelTests::Test::Runner.stub(:tests_in_groups).with(anything, anything, anything) {
+        ["spec/one_spec", "spec/two_spec"]
+      }
+      test_groups = ParallelTests::Saucerspec::Runner.tests_in_groups ["spec/one_spec.rb"], "3"
+      test_groups.length.should eq 6
+    end
+  end
+end

spec/sauce/parallel/test_broker_spec.rb

@@ -0,0 +1,43 @@
+require "rspec"
+require "sauce/parallel/test_broker"
+
+describe Sauce::TestBroker do
+
+  describe "#next_environment" do
+
+    before :all do
+      Sauce.config do |c|
+        c.browsers = [
+            ["Windows 7", "Opera", "10"],
+            ["Linux", "Firefox", "19"]
+        ]
+      end
+    end
+
+    it "returns the first environment for new entries" do
+      first_environment = Sauce::TestBroker.next_environment "spec/a_spec"
+      first_environment.should eq({
+        :SAUCE_OS => "'Windows 7'",
+        :SAUCE_BROWSER => "'Opera'",
+        :SAUCE_BROWSER_VERSION => "'10'"
+      })
+    end
+
+    it "should only return an environment once" do
+      Sauce::TestBroker.next_environment "spec/b_spec"
+      second_environment = Sauce::TestBroker.next_environment "spec/b_spec"
+
+      second_environment.should eq({
+          :SAUCE_OS => "'Linux'",
+          :SAUCE_BROWSER => "'Firefox'",
+          :SAUCE_BROWSER_VERSION => "'19'"
+      })
+    end
+  end
+
+  describe "#test_platforms" do
+    it "should report the same groups as configured in Sauce.config" do
+      Sauce::TestBroker.test_platforms.should eq Sauce.get_config.browsers
+    end
+  end
+end

spec/sauce_helper.rb

@@ -0,0 +1,8 @@
+require "sauce"
+
+Sauce.config do |c|
+  c[:browsers] = [
+      ["Windows 7", "Opera", 10],
+      ["Linux", "Firefox", 19]
+  ]
+end