Handle rate limited UAA (#4171)

app/controllers/v3/roles_controller.rb

@@ -58,6 +58,9 @@ def create
     render status: :created, json: Presenters::V3::RolePresenter.new(role)
   rescue RoleCreate::Error => e
     unprocessable!(e)
+  rescue UaaRateLimited
+    headers['Retry-After'] = rand(5..20).to_s
+    raise CloudController::Errors::V3::ApiError.new_from_details('UaaRateLimited')
   rescue UaaUnavailable
     raise CloudController::Errors::ApiError.new_from_details('UaaUnavailable')
   end

app/controllers/v3/users_controller.rb

@@ -54,6 +54,9 @@ def create
     else
       render status: :created, json: Presenters::V3::UserPresenter.new(user, uaa_users: User.uaa_users_info([user.guid]))
     end
+  rescue UaaRateLimited
+    headers['Retry-After'] = rand(5..20).to_s
+    raise CloudController::Errors::V3::ApiError.new_from_details('UaaRateLimited')
   rescue VCAP::CloudController::UaaUnavailable
     raise CloudController::Errors::ApiError.new_from_details('UaaUnavailable')
   rescue UserCreate::Error => e

errors/v3.yml

@@ -12,3 +12,8 @@
   name: ServiceBrokerGone
   http_code: 422
   message: "The service broker was removed before the synchronization completed"
+
+20008:
+  name: UaaRateLimited
+  http_code: 429
+  message: "The UAA is currently rate limited. Please try again later"

lib/cloud_controller/uaa/errors.rb

@@ -30,4 +30,10 @@ def message
       'The UAA returned an unexpected error'
     end
   end
+
+  class UaaRateLimited < UaaError
+    def message
+      'The UAA is currently rate limited. Please try again later'
+    end
+  end
 end

lib/cloud_controller/uaa/uaa_client.rb

@@ -102,6 +102,14 @@ def create_shadow_user(username, origin)
       raise e unless e.info['error'] == 'scim_resource_already_exists'
 
       { 'id' => e.info['user_id'] }
+    rescue CF::UAA::BadResponse => e
+      unless e.message == 'invalid status response: 429'
+        logger.error("UAA request for creating a user failed: #{e.inspect}")
+        raise UaaUnavailable
+      end
+
+      logger.warn("UAA request for creating a user ran into rate limits: #{e.inspect}")
+      raise UaaRateLimited
     rescue CF::UAA::UAAError => e
       logger.error("UAA request for creating a user failed: #{e.inspect}")
       raise UaaUnavailable

spec/request/roles_spec.rb

@@ -1044,6 +1044,22 @@
                 expect(parsed_response).to match_json_response(expected_response)
                 expect(uaa_client).to have_received(:create_shadow_user)
               end
+
+              context 'when UAA is rate limited' do
+                before do
+                  allow(uaa_client).to receive(:create_shadow_user).and_raise(VCAP::CloudController::UaaRateLimited)
+                end
+
+                it 'raises a 429 with a helpful message and Retry-After header' do
+                  post '/v3/roles', params.to_json, admin_header
+
+                  expect(last_response).to have_status_code(429)
+                  expect(last_response).to have_error_message(
+                    'The UAA is currently rate limited. Please try again later'
+                  )
+                  expect(last_response.headers).to include('Retry-After')
+                end
+              end
             end
           end
         end

spec/request/users_spec.rb

@@ -968,6 +968,22 @@
           expect(uaa_client).not_to have_received(:users_for_ids)
         end
 
+        context 'when UAA is rate limited' do
+          before do
+            allow(uaa_client).to receive(:create_shadow_user).and_raise(VCAP::CloudController::UaaRateLimited)
+          end
+
+          it 'raises a 429 with a helpful message and Retry-After header' do
+            post '/v3/users', params.to_json, admin_header
+
+            expect(last_response).to have_status_code(429)
+            expect(last_response).to have_error_message(
+              'The UAA is currently rate limited. Please try again later'
+            )
+            expect(last_response.headers).to include('Retry-After')
+          end
+        end
+
         context 'when user already exists in UAA' do
           before do
             allow(uaa_client).to receive(:ids_for_usernames_and_origins).and_return([user_guid])

spec/unit/lib/uaa/uaa_client_spec.rb

@@ -522,6 +522,38 @@ module VCAP::CloudController
           expect(mock_logger).to have_received(:error).with("UAA request for creating a user failed: #{uaa_error.inspect}")
         end
       end
+
+      context 'when uaa is rate limited' do
+        let(:uaa_error) { CF::UAA::BadResponse.new('invalid status response: 429') }
+        let(:mock_logger) { double(:steno_logger, warn: nil) }
+
+        before do
+          scim = instance_double(CF::UAA::Scim)
+          allow(scim).to receive(:add).and_raise(uaa_error)
+          allow(uaa_client).to receive_messages(scim: scim, logger: mock_logger)
+        end
+
+        it 'logs the error and raises UaaRateLimited' do
+          expect { uaa_client.create_shadow_user('[email protected]', 'idp.local') }.to raise_error(UaaRateLimited)
+          expect(mock_logger).to have_received(:warn).with("UAA request for creating a user ran into rate limits: #{uaa_error.inspect}")
+        end
+      end
+
+      context 'when a BadResponse is raised' do
+        let(:uaa_error) { CF::UAA::BadResponse.new('invalid status response: 433') }
+        let(:mock_logger) { double(:steno_logger, error: nil) }
+
+        before do
+          scim = instance_double(CF::UAA::Scim)
+          allow(scim).to receive(:add).and_raise(uaa_error)
+          allow(uaa_client).to receive_messages(scim: scim, logger: mock_logger)
+        end
+
+        it 'logs and raises the error' do
+          expect { uaa_client.create_shadow_user('[email protected]', 'idp.local') }.to raise_error(UaaUnavailable)
+          expect(mock_logger).to have_received(:error).with("UAA request for creating a user failed: #{uaa_error.inspect}")
+        end
+      end
     end
 
     describe '#id_for_username' do