From 2d245b07920716ddfcf42e0fd75f365ce12f6490 Mon Sep 17 00:00:00 2001
From: MothOnMars <mothonmars@gmail.com>
Date: Mon, 7 May 2018 12:24:58 -0700
Subject: [PATCH] squashme: add spec, bump robotex

---
 Gemfile.lock                         |  4 +++-
 spec/models/searchgov_domain_spec.rb | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index bb0fd32ad2..e5c6a5d092 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -48,9 +48,10 @@ GIT
 
 GIT
   remote: https://github.com/MothOnMars/robotex
-  revision: e51e6085d21ba532aaa3d54e979c92090c842af4
+  revision: 7e91000a3592e96ead47591e6c5c15233c19ab43
   specs:
     robotex (1.0.0)
+      open_uri_redirections (~> 0.2.1)
 
 GIT
   remote: https://github.com/MothOnMars/sitemaps
@@ -433,6 +434,7 @@ GEM
     nokogumbo (1.5.0)
       nokogiri
     oj (3.3.10)
+    open_uri_redirections (0.2.1)
     os (0.9.6)
     paperclip (5.2.1)
       activemodel (>= 4.2.0)
diff --git a/spec/models/searchgov_domain_spec.rb b/spec/models/searchgov_domain_spec.rb
index c93ca13088..0c6688d0bc 100644
--- a/spec/models/searchgov_domain_spec.rb
+++ b/spec/models/searchgov_domain_spec.rb
@@ -78,6 +78,17 @@
       let(:robots) { "User-agent: *\nCrawl-delay: 10" }
 
       it { is_expected.to eq 10 }
+
+      context 'when the domain is redirected' do
+        before do
+          stub_request(:get, "http://#{domain}/robots.txt").
+            to_return(status: 301, headers: { location: "https://#{domain}/robots.txt" }, body: "")
+          stub_request(:get, "https://#{domain}/robots.txt").
+            to_return(status: [200, "OK"], headers: { content_type: 'text/plain' }, body: robots)
+        end
+
+        it { is_expected.to eq 10 }
+      end
     end
 
     context 'when no delay is specified' do