diff --git a/concourse/pipelines/create-cloudfoundry.yml b/concourse/pipelines/create-cloudfoundry.yml index e09d7bf796..e802691586 100644 --- a/concourse/pipelines/create-cloudfoundry.yml +++ b/concourse/pipelines/create-cloudfoundry.yml @@ -3545,7 +3545,7 @@ jobs: echo 'Getting variables' APP_AUTOSCALER_NS="/${DEPLOY_ENV}/app-autoscaler" - APP_AUTOSCALER_SERVICE_BROKER_PASSWORD="$(credhub get -q -n "${APP_AUTOSCALER_NS}/autoscaler_service_broker_password")" + APP_AUTOSCALER_SERVICE_BROKER_PASSWORD="$(credhub get -q -n "${APP_AUTOSCALER_NS}/service_broker_password")" echo 'Setting secrets' TEAM_NS="/concourse/main" @@ -3567,9 +3567,9 @@ jobs: CF_ADMIN: ((cf_admin)) CF_PASS: ((cf_pass)) DEPLOY_ENV: ((deploy_env)) - APP_AUTOSCALER_SERVICE_BROKER_USERNAME: autoscaler_service_broker_user + APP_AUTOSCALER_SERVICE_BROKER_USERNAME: autoscaler-broker-user APP_AUTOSCALER_SERVICE_BROKER_PASSWORD: ((autoscaler_service_broker_password)) - APP_AUTOSCALER_SERVICE_BROKER_URL: https://autoscalerservicebroker.((system_dns_zone_name)) + APP_AUTOSCALER_SERVICE_BROKER_URL: https://app-autoscalerservicebroker.((system_dns_zone_name)) run: path: bash args: diff --git a/concourse/tasks/app-autoscaler-acceptance-tests-run.yml b/concourse/tasks/app-autoscaler-acceptance-tests-run.yml index 2fe6c4a106..0e9db7b894 100644 --- a/concourse/tasks/app-autoscaler-acceptance-tests-run.yml +++ b/concourse/tasks/app-autoscaler-acceptance-tests-run.yml @@ -47,6 +47,12 @@ run: "admin_user": "$(cat admin-creds/username)", "admin_password": "$(cat admin-creds/password)", + "eventgenerator_health_endpoint": "app-autoscaler-eventgenerator.$SYSTEM_DOMAIN/health", + "scalingengine_health_endpoint": "app-autoscaler-scalingengine.$SYSTEM_DOMAIN/health", + "operator_health_endpoint": "app-autoscaler-operator.$SYSTEM_DOMAIN/health", + "metricsforwarder_health_endpoint": "app-autoscaler-metricsforwarder.$SYSTEM_DOMAIN/health", + "scheduler_health_endpoint": "app-autoscaler-scheduler.$SYSTEM_DOMAIN/health", + "enable_service_access": false } EOF @@ -58,7 +64,11 @@ run: echo "Running tests" cd paas-cf/manifests/app-autoscaler/upstream + git apply ../diffs/patch_dynamic_policy_test.patch PATH=$(go env GOPATH)/bin:${PATH} export PATH - cd src/acceptance + cd src/acceptance/assets/app/go_app + echo "Building tests" + make build + cd ../../.. ./bin/test_default -procs 4 --compilers 4 --poll-progress-after=120s --poll-progress-interval=30s diff --git a/manifests/app-autoscaler/diffs/patch_dynamic_policy_test.patch b/manifests/app-autoscaler/diffs/patch_dynamic_policy_test.patch new file mode 100644 index 0000000000..1f16b696ef --- /dev/null +++ b/manifests/app-autoscaler/diffs/patch_dynamic_policy_test.patch @@ -0,0 +1,13 @@ +diff --git a/src/acceptance/app/dynamic_policy_test.go b/src/acceptance/app/dynamic_policy_test.go +index 1339ff514..ff0a2fe47 100644 +--- a/src/acceptance/app/dynamic_policy_test.go ++++ b/src/acceptance/app/dynamic_policy_test.go +@@ -50,7 +50,7 @@ var _ = Describe("AutoScaler dynamic policy", func() { + + It("should scale out and then back in.", Label(acceptance.LabelSmokeTests), func() { + By(fmt.Sprintf("Use heap %d mb of heap on app", heapToUse)) +- CurlAppInstance(cfg, appName, 0, fmt.Sprintf("/memory/%d/5", heapToUse)) ++ CurlAppInstance(cfg, appName, 0, fmt.Sprintf("/memory/%d/5", heapToUse+20)) + + By("wait for scale to 2") + WaitForNInstancesRunning(appGUID, 2, 5*time.Minute) diff --git a/manifests/app-autoscaler/operations.d/001-release.yml b/manifests/app-autoscaler/operations.d/001-release.yml index 2f3a1fe090..836b509f68 100644 --- a/manifests/app-autoscaler/operations.d/001-release.yml +++ b/manifests/app-autoscaler/operations.d/001-release.yml @@ -1,9 +1,17 @@ --- +# UPGRADE WARNING! +# The 11.4.2 release of app-autscaler-release has been patched to fix the acceptance tests. +# See this issue: https://github.com/cloudfoundry/app-autoscaler-release/issues/2434 +# The patch file can be found here: +# paas-cf/manifests/app-autoscaler/diffs/patch_dynamic_policy_test.patch +# This patch is applied as part of the concourse task. See: +# concourse/tasks/app-autoscaler-acceptance-tests-run.yml + - type: replace path: /releases/name=app-autoscaler? value: - name: app-autoscaler - version: "10.0.0" - url: "https://bosh.io/d/github.com/cloudfoundry-incubator/app-autoscaler-release?v=10.0.0" - sha1: "f691986b6f14e5bc1974e8c0b02996e9e2497a61" + name: "app-autoscaler" + version: "11.4.2" + url: "https://bosh.io/d/github.com/cloudfoundry-incubator/app-autoscaler-release?v=11.4.2" + sha1: "e3e3c292122acd8f14b825f24051ad4b7330ffd5" \ No newline at end of file diff --git a/manifests/app-autoscaler/operations.d/002-links.yml b/manifests/app-autoscaler/operations.d/002-links.yml index 3c43674a84..1360733d96 100644 --- a/manifests/app-autoscaler/operations.d/002-links.yml +++ b/manifests/app-autoscaler/operations.d/002-links.yml @@ -4,6 +4,14 @@ path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=reverse-log-proxy.service.cf.internal/targets/instance_group=log-api/deployment value: ((deploy_env)) +- type: replace + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=log-cache.service.cf.internal/targets/instance_group=log-cache/deployment + value: ((deploy_env)) + +- type: replace + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=logcache/targets/instance_group=log-cache/deployment + value: ((deploy_env)) + - type: replace path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=nats.service.cf.internal/targets/instance_group=nats/deployment value: ((deploy_env)) @@ -13,22 +21,34 @@ value: ((deploy_env)) - type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/consumes/nats/deployment + path: /instance_groups/name=apiserver/jobs/name=route_registrar/consumes/nats/deployment + value: ((deploy_env)) + +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=loggregator_agent/consumes/doppler/deployment + value: ((deploy_env)) + +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=route_registrar/consumes/nats/deployment + value: ((deploy_env)) + +- type: replace + path: /instance_groups/name=operator/jobs/name=route_registrar/consumes/nats/deployment value: ((deploy_env)) - type: replace - path: /instance_groups/name=asapi/jobs/name=loggregator_agent/consumes/doppler/deployment + path: /instance_groups/name=scheduler/jobs/name=route_registrar/consumes/nats/deployment value: ((deploy_env)) - type: replace - path: /instance_groups/name=asactors/jobs/name=route_registrar/consumes/nats/deployment + path: /instance_groups/name=metricsserver/jobs/name=route_registrar/consumes/nats/deployment value: ((deploy_env)) - type: replace - path: /instance_groups/name=asmetrics/jobs/name=route_registrar/consumes/nats/deployment + path: /instance_groups/name=eventgenerator/jobs/name=route_registrar/consumes/nats/deployment value: ((deploy_env)) - type: replace - path: /instance_groups/name=asnozzle/jobs/name=route_registrar/consumes/nats/deployment + path: /instance_groups/name=metricsgateway/jobs/name=route_registrar/consumes/nats/deployment value: ((deploy_env)) - \ No newline at end of file + diff --git a/manifests/app-autoscaler/operations.d/025-add-postgres-variables-UPSTREAM.yml b/manifests/app-autoscaler/operations.d/025-add-postgres-variables-UPSTREAM.yml new file mode 120000 index 0000000000..9c6aa4259f --- /dev/null +++ b/manifests/app-autoscaler/operations.d/025-add-postgres-variables-UPSTREAM.yml @@ -0,0 +1 @@ +../upstream/operations/add-postgres-variables.yml \ No newline at end of file diff --git a/manifests/app-autoscaler/operations.d/031-bosh-dns-aliases.yml b/manifests/app-autoscaler/operations.d/031-bosh-dns-aliases.yml index 221997278d..7a68ac58cc 100644 --- a/manifests/app-autoscaler/operations.d/031-bosh-dns-aliases.yml +++ b/manifests/app-autoscaler/operations.d/031-bosh-dns-aliases.yml @@ -1,25 +1,25 @@ --- - type: remove - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=autoscalerpostgres.service.cf.internal + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).autoscalerpostgres.service.cf.internal - type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=apiserver.service.cf.internal/targets/0/network + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).apiserver.service.cf.internal/targets/0/network value: cf - type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=autoscalerscheduler.service.cf.internal/targets/0/network + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).autoscalerscheduler.service.cf.internal/targets/0/network value: cf - type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=servicebroker.service.cf.internal/targets/0/network + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).servicebroker.service.cf.internal/targets/0/network value: cf - type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=eventgenerator.service.cf.internal/targets/0/network + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).eventgenerator.service.cf.internal/targets/0/network value: cf - type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=scalingengine.service.cf.internal/targets/0/network + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).scalingengine.service.cf.internal/targets/0/network value: cf - type: replace @@ -27,11 +27,19 @@ value: cf - type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=metricsgateway.service.cf.internal/targets/0/network + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=log-cache.service.cf.internal/targets/0/network value: cf - type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=metricsserver.service.cf.internal/targets/0/network + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=logcache/targets/0/network + value: cf + +- type: replace + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).metricsgateway.service.cf.internal/targets/0/network + value: cf + +- type: replace + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).metricsserver.service.cf.internal/targets/0/network value: cf - type: replace @@ -43,7 +51,13 @@ value: cf - type: replace - path: /variables/name=metricsserver_server/options/alternative_names + path: /variables/name=eventgenerator_server_cert/options/alternative_names + value: + - "((deployment_name)).eventgenerator.service.cf.internal" + - "*.eventgenerator.cf.((deployment_name)).bosh" + +- type: replace + path: /variables/name=metricsserver_server_cert/options/alternative_names value: - - "metricsserver.service.cf.internal" - - "*.asmetrics.cf.app-autoscaler.bosh" + - "((deployment_name)).metricsserver.service.cf.internal" + - "*.metricsserver.cf.((deployment_name)).bosh" diff --git a/manifests/app-autoscaler/operations.d/033-apiserver-public-endpoint.yml b/manifests/app-autoscaler/operations.d/033-apiserver-public-endpoint.yml new file mode 100644 index 0000000000..fdf853b087 --- /dev/null +++ b/manifests/app-autoscaler/operations.d/033-apiserver-public-endpoint.yml @@ -0,0 +1,8 @@ +--- + +# autoscaler.((system_domain)) remains the cf cli autoscaler plugin's +# default presumed endpoint. +- type: replace + path: /instance_groups/name=apiserver/jobs/name=route_registrar/properties/route_registrar/routes/name=api_server/uris + value: + - autoscaler.((system_domain)) diff --git a/manifests/app-autoscaler/operations.d/035-internal-cert-update-mode-converge.yml b/manifests/app-autoscaler/operations.d/035-internal-cert-update-mode-converge.yml index 5bdad00984..013b877635 100644 --- a/manifests/app-autoscaler/operations.d/035-internal-cert-update-mode-converge.yml +++ b/manifests/app-autoscaler/operations.d/035-internal-cert-update-mode-converge.yml @@ -1,33 +1,24 @@ --- - type: replace - path: /variables/name=scalingengine_server/update_mode? + path: /variables/name=scalingengine_server_cert/update_mode? value: converge - type: replace - path: /variables/name=eventgenerator_server/update_mode? + path: /variables/name=eventgenerator_server_cert/update_mode? value: converge - type: replace - path: /variables/name=apiserver_server/update_mode? + path: /variables/name=apiserver_server_cert/update_mode? value: converge - type: replace - path: /variables/name=apiserver_public_server/update_mode? + path: /variables/name=servicebroker_server_cert/update_mode? value: converge - type: replace - path: /variables/name=servicebroker_server/update_mode? + path: /variables/name=scheduler_server_cert/update_mode? value: converge - type: replace - path: /variables/name=servicebroker_public_server/update_mode? + path: /variables/name=metricsserver_server_cert/update_mode? value: converge - -- type: replace - path: /variables/name=scheduler_server/update_mode? - value: converge - -- type: replace - path: /variables/name=postgres_server/update_mode? - value: converge - diff --git a/manifests/app-autoscaler/operations.d/036-set-password-length-72.yml b/manifests/app-autoscaler/operations.d/036-set-password-length-72.yml new file mode 100644 index 0000000000..995d88d66b --- /dev/null +++ b/manifests/app-autoscaler/operations.d/036-set-password-length-72.yml @@ -0,0 +1,18 @@ +--- +# https://github.com/cloudfoundry/app-autoscaler-release/pull/1295 +- type: replace + path: /variables/name=service_broker_password/options/length? + value: 72 + +- type: replace + path: /variables/name=service_broker_password/update_mode? + value: converge + +- type: replace + path: /variables/name=service_broker_password_blue/options/length? + value: 72 + +- type: replace + path: /variables/name=service_broker_password_blue/update_mode? + value: converge + diff --git a/manifests/app-autoscaler/operations.d/050-enable-nats-tls-UPSTREAM.yml b/manifests/app-autoscaler/operations.d/050-enable-nats-tls-UPSTREAM.yml index 460118698b..c14b4588aa 120000 --- a/manifests/app-autoscaler/operations.d/050-enable-nats-tls-UPSTREAM.yml +++ b/manifests/app-autoscaler/operations.d/050-enable-nats-tls-UPSTREAM.yml @@ -1 +1 @@ -../upstream/example/operation/enable-nats-tls.yml \ No newline at end of file +../upstream/operations/enable-nats-tls.yml \ No newline at end of file diff --git a/manifests/app-autoscaler/operations.d/051-enable-nats-tls-fix-deployment-name.yml b/manifests/app-autoscaler/operations.d/051-enable-nats-tls-fix-deployment-name.yml index e678924bff..b1dd329a03 100644 --- a/manifests/app-autoscaler/operations.d/051-enable-nats-tls-fix-deployment-name.yml +++ b/manifests/app-autoscaler/operations.d/051-enable-nats-tls-fix-deployment-name.yml @@ -1,16 +1,32 @@ --- - type: replace - path: /instance_groups/name=asactors/jobs/name=route_registrar/consumes/nats-tls/deployment? + path: /instance_groups/name=scalingengine/jobs/name=route_registrar/consumes/nats-tls/deployment? value: ((deploy_env)) - type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/consumes/nats-tls/deployment? + path: /instance_groups/name=scheduler/jobs/name=route_registrar/consumes/nats-tls/deployment? value: ((deploy_env)) - type: replace - path: /instance_groups/name=asnozzle/jobs/name=route_registrar/consumes/nats-tls/deployment? + path: /instance_groups/name=operator/jobs/name=route_registrar/consumes/nats-tls/deployment? value: ((deploy_env)) - type: replace - path: /instance_groups/name=asmetrics/jobs/name=route_registrar/consumes/nats-tls/deployment? + path: /instance_groups/name=apiserver/jobs/name=route_registrar/consumes/nats-tls/deployment? + value: ((deploy_env)) + +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/consumes/nats-tls/deployment? + value: ((deploy_env)) + +- type: replace + path: /instance_groups/name=metricsgateway/jobs/name=route_registrar/consumes/nats-tls/deployment? + value: ((deploy_env)) + +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=route_registrar/consumes/nats-tls/deployment? + value: ((deploy_env)) + +- type: replace + path: /instance_groups/name=metricsserver/jobs/name=route_registrar/consumes/nats-tls/deployment? value: ((deploy_env)) diff --git a/manifests/app-autoscaler/operations.d/100-update-networks.yml b/manifests/app-autoscaler/operations.d/100-update-networks.yml index b0c4ad3134..192c47c14f 100644 --- a/manifests/app-autoscaler/operations.d/100-update-networks.yml +++ b/manifests/app-autoscaler/operations.d/100-update-networks.yml @@ -1,21 +1,41 @@ --- - type: replace - path: /instance_groups/name=asactors/networks + path: /instance_groups/name=scalingengine/networks value: - name: cf - type: replace - path: /instance_groups/name=asmetrics/networks + path: /instance_groups/name=scheduler/networks value: - name: cf - type: replace - path: /instance_groups/name=asapi/networks + path: /instance_groups/name=operator/networks value: - name: cf - type: replace - path: /instance_groups/name=asnozzle/networks + path: /instance_groups/name=metricsforwarder/networks + value: + - name: cf + +- type: replace + path: /instance_groups/name=metricsserver/networks + value: + - name: cf + +- type: replace + path: /instance_groups/name=eventgenerator/networks + value: + - name: cf + +- type: replace + path: /instance_groups/name=apiserver/networks + value: + - name: cf + +- type: replace + path: /instance_groups/name=metricsgateway/networks value: - name: cf diff --git a/manifests/app-autoscaler/operations.d/100-update-vm-types.yml b/manifests/app-autoscaler/operations.d/100-update-vm-types.yml index 3c043f43ba..71668e1555 100644 --- a/manifests/app-autoscaler/operations.d/100-update-vm-types.yml +++ b/manifests/app-autoscaler/operations.d/100-update-vm-types.yml @@ -1,33 +1,65 @@ --- - type: replace - path: /instance_groups/name=asactors/vm_type + path: /instance_groups/name=scheduler/vm_type value: small - type: replace - path: /instance_groups/name=asactors/instances + path: /instance_groups/name=scheduler/instances value: 2 - type: replace - path: /instance_groups/name=asmetrics/vm_type + path: /instance_groups/name=operator/vm_type value: small - type: replace - path: /instance_groups/name=asmetrics/instances + path: /instance_groups/name=operator/instances value: 2 - type: replace - path: /instance_groups/name=asapi/vm_type + path: /instance_groups/name=scalingengine/vm_type value: small - type: replace - path: /instance_groups/name=asapi/instances + path: /instance_groups/name=scalingengine/instances value: 2 - type: replace - path: /instance_groups/name=asnozzle/vm_type + path: /instance_groups/name=eventgenerator/vm_type value: small - type: replace - path: /instance_groups/name=asnozzle/instances + path: /instance_groups/name=eventgenerator/instances + value: 2 + +- type: replace + path: /instance_groups/name=metricsserver/vm_type + value: small + +- type: replace + path: /instance_groups/name=metricsserver/instances + value: 2 + +- type: replace + path: /instance_groups/name=apiserver/vm_type + value: small + +- type: replace + path: /instance_groups/name=apiserver/instances + value: 2 + +- type: replace + path: /instance_groups/name=metricsforwarder/vm_type + value: small + +- type: replace + path: /instance_groups/name=metricsforwarder/instances + value: 2 + +- type: replace + path: /instance_groups/name=metricsgateway/vm_type + value: small + +- type: replace + path: /instance_groups/name=metricsgateway/instances value: 2 diff --git a/manifests/app-autoscaler/operations.d/200-external-db-UPSTREAM.yml b/manifests/app-autoscaler/operations.d/200-external-db-UPSTREAM.yml deleted file mode 120000 index fa6bcb3dce..0000000000 --- a/manifests/app-autoscaler/operations.d/200-external-db-UPSTREAM.yml +++ /dev/null @@ -1 +0,0 @@ -../upstream/example/operation/external-db.yml \ No newline at end of file diff --git a/manifests/app-autoscaler/operations.d/200-external-db.yml b/manifests/app-autoscaler/operations.d/200-external-db.yml new file mode 100644 index 0000000000..18ed979b8a --- /dev/null +++ b/manifests/app-autoscaler/operations.d/200-external-db.yml @@ -0,0 +1,162 @@ +# based on upstream external-db.yml + +- type: remove + path: /instance_groups/name=postgres + +# scalingengine/scalingengine +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scalingengine_db + value: &external_database + sslmode: &sslmode ((database.sslmode)) + tls: &database_tls + ca: ((database.tls.ca)) + databases: + - name: ((database.name)) + tag: default + address: ((database.host)) + db_scheme: ((database.scheme)) + port: ((database.port)) + roles: + - name: ((database.username)) + password: ((database.password)) + tag: default + +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scalingengine_db_connection_config? + value: &databaseConnectionConfig + max_open_connections: 100 + max_idle_connections: 10 + connection_max_lifetime: 60s + +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scheduler_db + value: *external_database +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scheduler_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/policy_db + value: *external_database +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/policy_db_connection_config? + value: *databaseConnectionConfig + +# scheduler/scheduler +- type: replace + path: /instance_groups/name=scheduler/jobs/name=scheduler/properties/autoscaler/policy_db + value: *external_database + +- type: replace + path: /instance_groups/name=scheduler/jobs/name=scheduler/properties/autoscaler/scheduler_db + value: *external_database + +# operator/operator +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/policy_db + value: *external_database +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/policy_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/appmetrics_db + value: *external_database +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/appmetrics_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db + value: *external_database +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/scalingengine_db + value: *external_database +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/scalingengine_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/lock_db + value: *external_database +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/lock_db_connection_config? + value: *databaseConnectionConfig + +# metricsserver/metricsserver +- type: replace + path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db + value: *external_database +- type: replace + path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/policy_db + value: *external_database +- type: replace + path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/policy_db_connection_config? + value: *databaseConnectionConfig + +# eventgenerator/eventgenerator +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db + value: *external_database +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/policy_db + value: *external_database +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/policy_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/lock_db + value: *external_database +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/lock_db_connection_config? + value: *databaseConnectionConfig + +# metricsgateway/metricsgateway +- type: replace + path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/policy_db + value: *external_database +- type: replace + path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/policy_db_connection_config? + value: *databaseConnectionConfig + +# apiserver/golangapiserver +- type: replace + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/policy_db + value: *external_database +- type: replace + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/policy_db_connection_config? + value: *databaseConnectionConfig +- type: replace + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/binding_db + value: *external_database +- type: replace + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/binding_db_connection_config? + value: *databaseConnectionConfig + +# metricsforwarder/metricsforwarder +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/policy_db + value: *external_database +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/policy_db_connection_config? + value: *databaseConnectionConfig + +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/storedprocedure_db + value: *external_database +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/storedprocedure_db_connection_config? + value: *databaseConnectionConfig diff --git a/manifests/app-autoscaler/operations.d/201-external-db-access.yml b/manifests/app-autoscaler/operations.d/201-external-db-access.yml index b4bd48d28a..5b2f670a5e 100644 --- a/manifests/app-autoscaler/operations.d/201-external-db-access.yml +++ b/manifests/app-autoscaler/operations.d/201-external-db-access.yml @@ -1,17 +1,33 @@ --- - type: replace - path: /instance_groups/name=asactors/vm_extensions?/- + path: /instance_groups/name=scalingengine/vm_extensions?/- value: cf_rds_client_sg - type: replace - path: /instance_groups/name=asapi/vm_extensions?/- + path: /instance_groups/name=operator/vm_extensions?/- value: cf_rds_client_sg - type: replace - path: /instance_groups/name=asmetrics/vm_extensions?/- + path: /instance_groups/name=scheduler/vm_extensions?/- value: cf_rds_client_sg - type: replace - path: /instance_groups/name=asnozzle/vm_extensions?/- + path: /instance_groups/name=apiserver/vm_extensions?/- + value: cf_rds_client_sg + +- type: replace + path: /instance_groups/name=metricsforwarder/vm_extensions?/- + value: cf_rds_client_sg + +- type: replace + path: /instance_groups/name=eventgenerator/vm_extensions?/- + value: cf_rds_client_sg + +- type: replace + path: /instance_groups/name=metricsserver/vm_extensions?/- + value: cf_rds_client_sg + +- type: replace + path: /instance_groups/name=metricsgateway/vm_extensions?/- value: cf_rds_client_sg diff --git a/manifests/app-autoscaler/operations.d/300-set-plans.yml b/manifests/app-autoscaler/operations.d/300-set-plans.yml index 9dce30befe..5e1d24d0cd 100644 --- a/manifests/app-autoscaler/operations.d/300-set-plans.yml +++ b/manifests/app-autoscaler/operations.d/300-set-plans.yml @@ -1,12 +1,14 @@ --- - type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/broker/server/catalog/services/name=autoscaler + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/apiserver/broker/server/catalog/services/name=((deployment_name)) value: id: 656a9276-a1f6-4646-aade-08f0a51b29ed name: autoscaler description: App instances autoscaler bindable: true + instances_retrievable: true + bindings_retrievable: true AdditionalMetadata: usecase: - Scaling at peak times @@ -17,6 +19,7 @@ - instance - schedule - rules + - app-autoscaler metadata: displayName: Autoscaler longDescription: Apps can be configured to dynamically increase or decrease the number of running instances using autoscaling. diff --git a/manifests/app-autoscaler/operations.d/800-set-vcap-password.yml b/manifests/app-autoscaler/operations.d/800-set-vcap-password.yml index 422919b21c..4a6bcdfa07 100644 --- a/manifests/app-autoscaler/operations.d/800-set-vcap-password.yml +++ b/manifests/app-autoscaler/operations.d/800-set-vcap-password.yml @@ -1,14 +1,26 @@ --- - type: replace - path: /instance_groups/name=asactors/env?/bosh/password + path: /instance_groups/name=scheduler/env?/bosh/password value: ((vcap_password)) - type: replace - path: /instance_groups/name=asmetrics/env?/bosh/password + path: /instance_groups/name=operator/env?/bosh/password value: ((vcap_password)) - type: replace - path: /instance_groups/name=asapi/env?/bosh/password + path: /instance_groups/name=scalingengine/env?/bosh/password value: ((vcap_password)) - type: replace - path: /instance_groups/name=asnozzle/env?/bosh/password + path: /instance_groups/name=eventgenerator/env?/bosh/password + value: ((vcap_password)) +- type: replace + path: /instance_groups/name=metricsserver/env?/bosh/password + value: ((vcap_password)) +- type: replace + path: /instance_groups/name=metricsforwarder/env?/bosh/password + value: ((vcap_password)) +- type: replace + path: /instance_groups/name=apiserver/env?/bosh/password + value: ((vcap_password)) +- type: replace + path: /instance_groups/name=metricsgateway/env?/bosh/password value: ((vcap_password)) diff --git a/manifests/app-autoscaler/operations.d/900-temp-fix-for-v10.0.0.yml b/manifests/app-autoscaler/operations.d/900-temp-fix-for-v10.0.0.yml deleted file mode 100644 index 3a36eda4e1..0000000000 --- a/manifests/app-autoscaler/operations.d/900-temp-fix-for-v10.0.0.yml +++ /dev/null @@ -1,11 +0,0 @@ -# We have experienced an issue with the v10.0.0 upgrade -# See https://github.com/cloudfoundry/app-autoscaler-release/issues/1461 . -# This file is a temp workaround and can be removed for the next release. - -- type: replace - path: /releases/name=postgres? - value: - name: "postgres" - version: "44" - url: "https://bosh.io/d/github.com/cloudfoundry/postgres-release?v=44" - sha1: "582b1de9522077102dfa44ff7164cd8f499dbfc8" \ No newline at end of file diff --git a/manifests/app-autoscaler/operations.d/910-add-plan-options.yml b/manifests/app-autoscaler/operations.d/910-add-plan-options.yml deleted file mode 100644 index 512f682021..0000000000 --- a/manifests/app-autoscaler/operations.d/910-add-plan-options.yml +++ /dev/null @@ -1,7 +0,0 @@ -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/broker/server/catalog/services/name=autoscaler/instances_retrievable? - value: true - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/broker/server/catalog/services/name=autoscaler/bindings_retrievable? - value: true diff --git a/manifests/app-autoscaler/operations/scale-down-dev.yml b/manifests/app-autoscaler/operations/scale-down-dev.yml index 85374d6131..f2200f05ab 100644 --- a/manifests/app-autoscaler/operations/scale-down-dev.yml +++ b/manifests/app-autoscaler/operations/scale-down-dev.yml @@ -1,33 +1,65 @@ --- - type: replace - path: /instance_groups/name=asactors/vm_type + path: /instance_groups/name=scalingengine/vm_type value: nano - type: replace - path: /instance_groups/name=asactors/instances + path: /instance_groups/name=scalingengine/instances value: 1 - type: replace - path: /instance_groups/name=asmetrics/vm_type + path: /instance_groups/name=operator/vm_type value: nano - type: replace - path: /instance_groups/name=asmetrics/instances + path: /instance_groups/name=operator/instances value: 1 - type: replace - path: /instance_groups/name=asapi/vm_type + path: /instance_groups/name=scheduler/vm_type value: nano - type: replace - path: /instance_groups/name=asapi/instances + path: /instance_groups/name=scheduler/instances value: 1 - type: replace - path: /instance_groups/name=asnozzle/vm_type + path: /instance_groups/name=eventgenerator/vm_type value: nano - type: replace - path: /instance_groups/name=asnozzle/instances + path: /instance_groups/name=eventgenerator/instances + value: 1 + +- type: replace + path: /instance_groups/name=metricsserver/vm_type + value: nano + +- type: replace + path: /instance_groups/name=metricsserver/instances + value: 1 + +- type: replace + path: /instance_groups/name=metricsforwarder/vm_type + value: nano + +- type: replace + path: /instance_groups/name=metricsforwarder/instances + value: 1 + +- type: replace + path: /instance_groups/name=apiserver/vm_type + value: nano + +- type: replace + path: /instance_groups/name=apiserver/instances + value: 1 + +- type: replace + path: /instance_groups/name=metricsgateway/vm_type + value: nano + +- type: replace + path: /instance_groups/name=metricsgateway/instances value: 1 diff --git a/manifests/app-autoscaler/scripts/generate-manifest.sh b/manifests/app-autoscaler/scripts/generate-manifest.sh index 24a7e0d1c3..d0b24210d0 100755 --- a/manifests/app-autoscaler/scripts/generate-manifest.sh +++ b/manifests/app-autoscaler/scripts/generate-manifest.sh @@ -28,6 +28,7 @@ app_domain: $APPS_DNS_ZONE_NAME aws_account: $AWS_ACCOUNT bosh_ca_cert: $BOSH_CA_CERT vcap_password: $VCAP_PASSWORD +deployment_name: app-autoscaler cf_client_id: app_autoscaler database: @@ -53,7 +54,7 @@ bosh interpolate \ ${opsfile_args} \ --vars-file="${variables_file}" \ --vars-file="${WORKDIR}/terraform-outputs/cf.yml" \ - "${APP_AUTOSCALER_BOSHRELEASE_DIR}/templates/app-autoscaler-deployment.yml" \ + "${APP_AUTOSCALER_BOSHRELEASE_DIR}/templates/app-autoscaler.yml" \ | sed "s@dns_api_client_tls[.]@/$DEPLOY_ENV/$DEPLOY_ENV/dns_api_client_tls.@g" \ | sed "s@dns_api_server_tls[.]@/$DEPLOY_ENV/$DEPLOY_ENV/dns_api_server_tls.@g" \ | sed "s@/bosh-autoscaler/cf/nats_client_cert[.]@/$DEPLOY_ENV/$DEPLOY_ENV/nats_client_cert.@g" \ diff --git a/manifests/app-autoscaler/spec/autoscaler_spec.rb b/manifests/app-autoscaler/spec/autoscaler_spec.rb index 470a731b39..b5856a24bf 100644 --- a/manifests/app-autoscaler/spec/autoscaler_spec.rb +++ b/manifests/app-autoscaler/spec/autoscaler_spec.rb @@ -5,12 +5,12 @@ let(:manifest) { manifest_with_defaults } - describe "actors" do - subject(:actors) { manifest["instance_groups.asactors"] } + describe "scalingengine" do + subject(:scalingengine) { manifest["instance_groups.scalingengine"] } let(:jobs) { subject["jobs"] } - it_behaves_like "a cf rds client" + it_behaves_like "an autoscaler rds client" describe "scalingengine" do let(:scalingengine) { jobs.find { |j| j["name"] == "scalingengine" } } @@ -22,6 +22,20 @@ expect(cf["secret"]).to eq("((/test/test/uaa_clients_app_autoscaler_secret))") end end + end + + describe "scheduler" do + subject(:scheduler) { manifest["instance_groups.scheduler"] } + + it_behaves_like "an autoscaler rds client" + end + + describe "operator" do + subject(:operator) { manifest["instance_groups.operator"] } + + let(:jobs) { subject["jobs"] } + + it_behaves_like "an autoscaler rds client" describe "operator" do let(:operator) { jobs.find { |j| j["name"] == "operator" } } @@ -35,12 +49,12 @@ end end - describe "api" do - subject(:api) { manifest["instance_groups.asapi"] } + describe "apiserver" do + subject(:apiserver) { manifest["instance_groups.apiserver"] } let(:jobs) { subject["jobs"] } - it_behaves_like "a cf rds client" + it_behaves_like "an autoscaler rds client" describe "golangapiserver" do let(:apiserver) { jobs.find { |j| j["name"] == "golangapiserver" } } @@ -81,15 +95,27 @@ end end - describe "metrics" do - subject(:metrics) { manifest["instance_groups.asmetrics"] } + describe "eventgenerator" do + subject(:eventgenerator) { manifest["instance_groups.eventgenerator"] } + + it_behaves_like "an autoscaler rds client" + end + + describe "metricsforwarder" do + subject(:metricsforwarder) { manifest["instance_groups.metricsforwarder"] } + + it_behaves_like "an autoscaler rds client" + end + + describe "metricsserver" do + subject(:metricsserver) { manifest["instance_groups.metricsserver"] } - it_behaves_like "a cf rds client" + it_behaves_like "an autoscaler rds client" end - describe "nozzle" do - subject(:nozzle) { manifest["instance_groups.asnozzle"] } + describe "metricsgateway" do + subject(:metricsgateway) { manifest["instance_groups.metricsgateway"] } - it_behaves_like "a cf rds client" + it_behaves_like "an autoscaler rds client" end end diff --git a/manifests/app-autoscaler/spec/support/shared_examples.rb b/manifests/app-autoscaler/spec/support/shared_examples.rb new file mode 100644 index 0000000000..d1ef23234e --- /dev/null +++ b/manifests/app-autoscaler/spec/support/shared_examples.rb @@ -0,0 +1,30 @@ +def each_kv_recursive(obj, &block) + case obj + when Hash + obj.each do |k, v| + block.call(k, v) + each_kv_recursive(v, &block) + end + when Array + obj.each do |v| + each_kv_recursive(v, &block) + end + end +end + +RSpec.shared_examples "an autoscaler rds client" do + let(:jobs) { subject["jobs"] } + + it_behaves_like "a cf rds client" + + it "uses the rds database for all jobs" do + jobs.each do |job| + each_kv_recursive job do |k, v| + next unless k.end_with?("_db") && v.is_a?(Hash) + + expect(v).to have_key("address") + expect(v["address"]).to eq("abcd.postgres.aws"), "#{k} doesn't appear to be using RDS: db address is #{v['address']}" + end + end + end +end diff --git a/manifests/app-autoscaler/upstream b/manifests/app-autoscaler/upstream index ed6637290b..4d3796dd9d 160000 --- a/manifests/app-autoscaler/upstream +++ b/manifests/app-autoscaler/upstream @@ -1 +1 @@ -Subproject commit ed6637290b34f15c21c69a268a2d803634c51f0f +Subproject commit 4d3796dd9d491aed0e102925dc3279175ccd279b