squashme: add spec, bump robotex

Gemfile.lock

@@ -48,9 +48,10 @@ GIT
 
 GIT
   remote: https://github.com/MothOnMars/robotex
-  revision: e51e6085d21ba532aaa3d54e979c92090c842af4
+  revision: 7e91000a3592e96ead47591e6c5c15233c19ab43
   specs:
     robotex (1.0.0)
+      open_uri_redirections (~> 0.2.1)
 
 GIT
   remote: https://github.com/MothOnMars/sitemaps
@@ -433,6 +434,7 @@ GEM
     nokogumbo (1.5.0)
       nokogiri
     oj (3.3.10)
+    open_uri_redirections (0.2.1)
     os (0.9.6)
     paperclip (5.2.1)
       activemodel (>= 4.2.0)

spec/models/searchgov_domain_spec.rb

@@ -78,6 +78,17 @@
       let(:robots) { "User-agent: *\nCrawl-delay: 10" }
 
       it { is_expected.to eq 10 }
+
+      context 'when the domain is redirected' do
+        before do
+          stub_request(:get, "http://#{domain}/robots.txt").
+            to_return(status: 301, headers: { location: "https://#{domain}/robots.txt" }, body: "")
+          stub_request(:get, "https://#{domain}/robots.txt").
+            to_return(status: [200, "OK"], headers: { content_type: 'text/plain' }, body: robots)
+        end
+
+        it { is_expected.to eq 10 }
+      end
     end
 
     context 'when no delay is specified' do