Add Canary deployments (#3892)

* Introduce Canary deployments as an experimental feature. Closes #3837 Co-authored-by: Sam Gunaratne <[email protected]> Co-authored-by: Seth Boyles <[email protected]>
cloudfoundry · Jul 29, 2024 · d66f46a · d66f46a
1 parent 3d09a70
commit d66f46a
Show file tree

Hide file tree

Showing 31 changed files with 1,044 additions and 41 deletions.
diff --git a/app/actions/deployment_continue.rb b/app/actions/deployment_continue.rb
@@ -0,0 +1,42 @@
+module VCAP::CloudController
+  class DeploymentContinue
+    class Error < StandardError
+    end
+    class InvalidStatus < Error
+    end
+
+    class << self
+      def continue(deployment:, user_audit_info:)
+        deployment.db.transaction do
+          deployment.lock!
+          reject_invalid_state!(deployment) unless deployment.continuable?
+
+          record_audit_event(deployment, user_audit_info)
+          deployment.update(
+            state: DeploymentModel::DEPLOYING_STATE,
+            status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
+            status_reason: DeploymentModel::DEPLOYING_STATUS_REASON
+          )
+        end
+      end
+
+      private
+
+      def reject_invalid_state!(deployment)
+        raise InvalidStatus.new("Cannot continue a deployment with status: #{deployment.status_value} and reason: #{deployment.status_reason}")
+      end
+
+      def record_audit_event(deployment, user_audit_info)
+        app = deployment.app
+        Repositories::DeploymentEventRepository.record_continue(
+          deployment,
+          deployment.droplet,
+          user_audit_info,
+          app.name,
+          app.space_guid,
+          app.space.organization_guid
+        )
+      end
+    end
+  end
+end
diff --git a/app/actions/deployment_create.rb b/app/actions/deployment_create.rb
@@ -13,14 +13,16 @@ def create(app:, user_audit_info:, message:)
         DeploymentModel.db.transaction do
           app.lock!
 
+          message.strategy ||= DeploymentModel::ROLLING_STRATEGY
+
           target_state = DeploymentTargetState.new(app, message)
 
           previous_droplet = app.droplet
           target_state.apply_to_app(app, user_audit_info)
 
           if target_state.rollback_target_revision
             revision = RevisionResolver.rollback_app_revision(app, target_state.rollback_target_revision, user_audit_info)
-            log_rollback_event(app.guid, user_audit_info.user_guid, target_state.rollback_target_revision.guid)
+            log_rollback_event(app.guid, user_audit_info.user_guid, target_state.rollback_target_revision.guid, message.strategy)
           else
             revision = RevisionResolver.update_app_revision(app, user_audit_info)
           end
@@ -41,15 +43,15 @@ def create(app:, user_audit_info:, message:)
 
           deployment = DeploymentModel.create(
             app: app,
-            state: DeploymentModel::DEPLOYING_STATE,
+            state: starting_state(message),
             status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
             status_reason: DeploymentModel::DEPLOYING_STATUS_REASON,
             droplet: target_state.droplet,
             previous_droplet: previous_droplet,
             original_web_process_instance_count: desired_instances(app.oldest_web_process, previous_deployment),
             revision_guid: revision&.guid,
             revision_version: revision&.version,
-            strategy: DeploymentModel::ROLLING_STRATEGY
+            strategy: message.strategy
           )
           MetadataUpdate.update(deployment, message)
 
@@ -201,15 +203,23 @@ def supersede_deployment(previous_deployment)
         )
       end
 
-      def log_rollback_event(app_guid, user_id, revision_id)
+      def starting_state(message)
+        if message.strategy == DeploymentModel::CANARY_STRATEGY
+          DeploymentModel::PREPAUSED_STATE
+        else
+          DeploymentModel::DEPLOYING_STATE
+        end
+      end
+
+      def log_rollback_event(app_guid, user_id, revision_id, strategy)
         TelemetryLogger.v3_emit(
           'rolled-back-app',
           {
             'app-id' => app_guid,
             'user-id' => user_id,
             'revision-id' => revision_id
           },
-          { 'strategy' => 'rolling' }
+          { 'strategy' => strategy }
         )
       end
     end

diff --git a/app/controllers/v3/deployments_controller.rb b/app/controllers/v3/deployments_controller.rb
@@ -6,6 +6,7 @@
 require 'actions/deployment_create'
 require 'actions/deployment_update'
 require 'actions/deployment_cancel'
+require 'actions/deployment_continue'
 require 'cloud_controller/telemetry_logger'
 
 class DeploymentsController < ApplicationController
@@ -57,7 +58,7 @@ def create
           'app-id' => app.guid,
           'user-id' => current_user.guid
         },
-        { 'strategy' => 'rolling' }
+        { 'strategy' => deployment.strategy }
       )
     rescue DeploymentCreate::Error => e
       unprocessable!(e.message)
@@ -97,6 +98,22 @@ def cancel
     head :ok
   end
 
+  def continue
+    deployment = DeploymentModel.find(guid: hashed_params[:guid])
+
+    resource_not_found!(:deployment) unless deployment && permission_queryer.can_manage_apps_in_active_space?(deployment.app.space.id) &&
+                                            permission_queryer.is_space_active?(deployment.app.space.id)
+
+    begin
+      DeploymentContinue.continue(deployment:, user_audit_info:)
+      logger.info("Continued deployment #{deployment.guid} for app #{deployment.app_guid}")
+    rescue DeploymentContinue::Error => e
+      unprocessable!(e.message)
+    end
+
+    head :ok
+  end
+
   private
 
   def deployments_not_enabled!

diff --git a/app/jobs/runtime/prune_completed_deployments.rb b/app/jobs/runtime/prune_completed_deployments.rb
@@ -25,7 +25,7 @@ def perform
                                   select(:id)
 
             deployments_to_delete = deployments_dataset.
-                                    exclude(state: [DeploymentModel::DEPLOYING_STATE, DeploymentModel::CANCELING_STATE]).
+                                    exclude(state: DeploymentModel::ACTIVE_STATES).
                                     exclude(id: deployments_to_keep)
 
             delete_count = DeploymentDelete.delete(deployments_to_delete)

diff --git a/app/messages/deployment_create_message.rb b/app/messages/deployment_create_message.rb
@@ -11,7 +11,7 @@ class DeploymentCreateMessage < MetadataBaseMessage
 
     validates_with NoAdditionalKeysValidator
     validates :strategy,
-              inclusion: { in: %w[rolling], message: "'%<value>s' is not a supported deployment strategy" },
+              inclusion: { in: %w[rolling canary], message: "'%<value>s' is not a supported deployment strategy" },
               allow_nil: true
     validate :mutually_exclusive_droplet_sources
 

diff --git a/app/models/runtime/app_model.rb b/app/models/runtime/app_model.rb
@@ -132,7 +132,7 @@ def stopped?
     end
 
     def deploying?
-      deployments_dataset.where(state: DeploymentModel::DEPLOYING_STATE).any?
+      deployments_dataset.where(state: DeploymentModel::PROGRESSING_STATES).any?
     end
 
     def self.user_visibility_filter(user)

diff --git a/app/models/runtime/deployment_model.rb b/app/models/runtime/deployment_model.rb
@@ -2,6 +2,8 @@ module VCAP::CloudController
   class DeploymentModel < Sequel::Model(:deployments)
     DEPLOYMENT_STATES = [
       DEPLOYING_STATE = 'DEPLOYING'.freeze,
+      PREPAUSED_STATE = 'PREPAUSED'.freeze,
+      PAUSED_STATE = 'PAUSED'.freeze,
       DEPLOYED_STATE = 'DEPLOYED'.freeze,
       CANCELING_STATE = 'CANCELING'.freeze,
       CANCELED_STATE = 'CANCELED'.freeze
@@ -13,15 +15,28 @@ class DeploymentModel < Sequel::Model(:deployments)
     ].freeze
 
     STATUS_REASONS = [
-      DEPLOYED_STATUS_REASON = 'DEPLOYED'.freeze,
       DEPLOYING_STATUS_REASON = 'DEPLOYING'.freeze,
+      PAUSED_STATUS_REASON = 'PAUSED'.freeze,
+      DEPLOYED_STATUS_REASON = 'DEPLOYED'.freeze,
       CANCELED_STATUS_REASON = 'CANCELED'.freeze,
       CANCELING_STATUS_REASON = 'CANCELING'.freeze,
       SUPERSEDED_STATUS_REASON = 'SUPERSEDED'.freeze
     ].freeze
 
     DEPLOYMENT_STRATEGIES = [
-      ROLLING_STRATEGY = 'rolling'.freeze
+      ROLLING_STRATEGY = 'rolling'.freeze,
+      CANARY_STRATEGY = 'canary'.freeze
+    ].freeze
+
+    PROGRESSING_STATES = [
+      DEPLOYING_STATE,
+      PREPAUSED_STATE,
+      PAUSED_STATE
+    ].freeze
+
+    ACTIVE_STATES = [
+      *PROGRESSING_STATES,
+      CANCELING_STATE
     ].freeze
 
     many_to_one :app,
@@ -63,7 +78,7 @@ class DeploymentModel < Sequel::Model(:deployments)
 
     dataset_module do
       def deploying_count
-        where(state: DeploymentModel::DEPLOYING_STATE).count
+        where(state: DeploymentModel::PROGRESSING_STATES).count
       end
     end
 
@@ -73,13 +88,15 @@ def before_update
     end
 
     def deploying?
-      state == DEPLOYING_STATE
+      DeploymentModel::PROGRESSING_STATES.include?(state)
     end
 
     def cancelable?
-      valid_states_for_cancel = [DeploymentModel::DEPLOYING_STATE,
-                                 DeploymentModel::CANCELING_STATE]
-      valid_states_for_cancel.include?(state)
+      DeploymentModel::ACTIVE_STATES.include?(state)
+    end
+
+    def continuable?
+      state == DeploymentModel::PAUSED_STATE
     end
 
     private

diff --git a/app/presenters/v3/deployment_presenter.rb b/app/presenters/v3/deployment_presenter.rb
@@ -76,6 +76,13 @@ def build_links
             method: 'POST'
           }
         end
+      end.tap do |links|
+        if deployment.continuable?
+          links[:continue] = {
+            href: url_builder.build_url(path: "/v3/deployments/#{deployment.guid}/actions/continue"),
+            method: 'POST'
+          }
+        end
       end
     end
   end

diff --git a/app/repositories/deployment_event_repository.rb b/app/repositories/deployment_event_repository.rb
@@ -11,7 +11,8 @@ def self.record_create(deployment, droplet, user_audit_info, v3_app_name, space_
           droplet_guid: droplet&.guid,
           type: type,
           revision_guid: deployment.revision_guid,
-          request: params
+          request: params,
+          strategy: deployment.strategy
         }
 
         Event.create(
@@ -53,6 +54,30 @@ def self.record_cancel(deployment, droplet, user_audit_info, v3_app_name, space_
           organization_guid: org_guid
         )
       end
+
+      def self.record_continue(deployment, droplet, user_audit_info, v3_app_name, space_guid, org_guid)
+        VCAP::AppLogEmitter.emit(deployment.app_guid, "Continuing deployment for app with guid #{deployment.app_guid}")
+
+        metadata = {
+          deployment_guid: deployment.guid,
+          droplet_guid: droplet&.guid
+        }
+
+        Event.create(
+          type: EventTypes::APP_DEPLOYMENT_CONTINUE,
+          actor: user_audit_info.user_guid,
+          actor_type: 'user',
+          actor_name: user_audit_info.user_email,
+          actor_username: user_audit_info.user_name,
+          actee: deployment.app_guid,
+          actee_type: 'app',
+          actee_name: v3_app_name,
+          timestamp: Sequel::CURRENT_TIMESTAMP,
+          metadata: metadata,
+          space_guid: space_guid,
+          organization_guid: org_guid
+        )
+      end
     end
   end
 end
diff --git a/app/repositories/event_types.rb b/app/repositories/event_types.rb
@@ -47,6 +47,7 @@ class EventTypesError < StandardError
         APP_REVISION_ENV_VARS_SHOW = 'audit.app.revision.environment_variables.show'.freeze,
         APP_DEPLOYMENT_CANCEL = 'audit.app.deployment.cancel'.freeze,
         APP_DEPLOYMENT_CREATE = 'audit.app.deployment.create'.freeze,
+        APP_DEPLOYMENT_CONTINUE = 'audit.app.deployment.continue'.freeze,
         APP_COPY_BITS = 'audit.app.copy-bits'.freeze,
         APP_UPLOAD_BITS = 'audit.app.upload-bits'.freeze,
         APP_APPLY_MANIFEST = 'audit.app.apply_manifest'.freeze,

diff --git a/config/routes.rb b/config/routes.rb
@@ -87,6 +87,7 @@
   get '/deployments/', to: 'deployments#index'
   get '/deployments/:guid', to: 'deployments#show'
   post '/deployments/:guid/actions/cancel', to: 'deployments#cancel'
+  post '/deployments/:guid/actions/continue', to: 'deployments#continue'
 
   # domains
   post '/domains', to: 'domains#create'

diff --git a/docs/v3/source/includes/api_resources/_deployments.erb b/docs/v3/source/includes/api_resources/_deployments.erb
@@ -9,7 +9,7 @@
       "last_successful_healthcheck": "2018-04-25T22:42:10Z"
     }
   },
-  "strategy": "rolling",
+  "strategy": "canary",
   "droplet": {
     "guid": "44ccfa61-dbcf-4a0d-82fe-f668e9d2a962"
   },
@@ -50,6 +50,10 @@
       "href": "https://api.example.org/v3/deployments/59c3d133-2b83-46f3-960e-7765a129aea4/actions/cancel",
       "method": "POST"
     }
+    "continue": {
+      "href": "https://api.example.org/v3/deployments/59c3d133-2b83-46f3-960e-7765a129aea4/actions/continue",
+      "method": "POST"
+    }
   }
 }
 <% end %>

diff --git a/docs/v3/source/includes/resources/audit_events/_header.md.erb b/docs/v3/source/includes/resources/audit_events/_header.md.erb
@@ -14,6 +14,7 @@ For more information, see the [Cloud Foundry docs](https://docs.cloudfoundry.org
 - `audit.app.delete-request`
 - `audit.app.deployment.cancel`
 - `audit.app.deployment.create`
+- `audit.app.deployment.continue`
 - `audit.app.droplet.create`
 - `audit.app.droplet.delete`
 - `audit.app.droplet.download`

diff --git a/docs/v3/source/includes/resources/deployments/_continue.md.erb b/docs/v3/source/includes/resources/deployments/_continue.md.erb
@@ -0,0 +1,30 @@
+### Continue a deployment
+
+```
+Example Request
+```
+
+```shell
+curl "https://api.example.org/v3/deployments/[guid]/actions/continue" \
+  -X POST \
+  -H "Authorization: bearer [token]"
+```
+
+```
+Example Response
+```
+
+```http
+HTTP/1.1 200 OK
+
+```
+
+#### Definition
+`POST /v3/deployments/:guid/actions/continue`
+
+#### Permitted roles
+ |
+--- |
+Admin |
+Space Developer |
+Space Supporter |
diff --git a/docs/v3/source/includes/resources/deployments/_header.md b/docs/v3/source/includes/resources/deployments/_header.md
@@ -8,7 +8,11 @@ They can either:
 
 * Roll an app back to a specific [revision](#revisions) along with its associated droplet
 
+Deployments are different than the traditional method of pushing app updates which performs start/stop deployments.
 
-It is possible to use [rolling deployments](https://docs.cloudfoundry.org/devguide/deploy-apps/rolling-deploy.html) for
-applications without incurring downtime. This is different from the traditional method of pushing app updates which performs start/stop deployments.
+Deployment strategies supported:
 
+* [Rolling deployments](https://docs.cloudfoundry.org/devguide/deploy-apps/rolling-deploy.html) allows for
+applications to be deployed without incurring downtime by gradually rolling out instances.
+
+* Canary deployments deploy a single instance and pause for user evaluation. If the canary instance is deemed successful, the deployment can be resumed via the [continue action](#continue-a-deployment). The deployment then continues like a rolling deployment. This feature is experimental and is subject to change.
diff --git a/docs/v3/source/includes/resources/deployments/_list.md.erb b/docs/v3/source/includes/resources/deployments/_list.md.erb
@@ -32,7 +32,7 @@ Name | Type | Description
 ---- | ---- | ------------
 **app_guids** | _list of strings_ | Comma-delimited list of app guids to filter by
 **states** | _list of strings_ | Comma-delimited list of states to filter by
-**status_reasons** | _list of strings_ | Comma-delimited list of status reasons to filter by;<br>valid values include `DEPLOYING`, `CANCELING`, `DEPLOYED`, `CANCELED`, `SUPERSEDED`
+**status_reasons** | _list of strings_ | Comma-delimited list of status reasons to filter by;<br>valid values include `DEPLOYING`, `PAUSED`, `CANCELING`, `DEPLOYED`, `CANCELED`, `SUPERSEDED`
 **status_values** | _list of strings_ | Comma-delimited list of status values to filter by;<br>valid values include `ACTIVE` and `FINALIZED`
 **page** | _integer_ | Page to display; valid values are integers >= 1
 **per_page** | _integer_ | Number of results per page; <br>valid values are 1 through 5000