diff --git a/bin/bucc b/bin/bucc index e63dcdd6..44b68856 100755 --- a/bin/bucc +++ b/bin/bucc @@ -5,7 +5,6 @@ repo_root=$(dirname $(realpath "$0") | rev | cut -c5- | rev) state=${repo_root}/state mkdir -p $state/manifests manifest="${repo_root}/src/bosh-deployment/bosh.yml" -ssh_private_key="${state}/ssh.key" state_arg="--state ${state}/state.json" vars_store=${state}/creds.yml vars_store_arg="--vars-store ${vars_store}" @@ -45,7 +44,7 @@ lite() { } cpis() { - find ${repo_root}/ops/cpis -mindepth 1 -maxdepth 1 ! -type l -exec basename {} +; + find ${repo_root}/ops/cpis -mindepth 1 -maxdepth 1 ! -type l -exec basename -a {} +; } validate_vars_file() { @@ -236,6 +235,9 @@ env() { echo "export BOSH_CA_CERT='$(cat $(ca_cert))'" echo "export BOSH_CLIENT=$(get_var bosh_client)" echo "export BOSH_CLIENT_SECRET=$(get_var bosh_client_secret)" + echo "export BOSH_GW_USER=$(int --path /instance_groups/name=bosh/jobs/name=user_add/properties/users/0/name)" + echo "export BOSH_GW_HOST=$(ip)" + echo "export BOSH_GW_PRIVATE_KEY=$(ssh_private_key)" } routes() { @@ -324,11 +326,16 @@ ip() { int --path /networks/name=default/subnets/0/static/0 } -_ssh() { - local user=$(int --path /instance_groups/name=bosh/jobs/name=user_add/properties/users/0/name) +ssh_private_key() { + local ssh_private_key="${state}/ssh.key" store_int --path /jumpbox_ssh/private_key > $ssh_private_key chmod 600 $ssh_private_key - ssh -i $ssh_private_key $user@$(ip) -t 'command; sudo su -' + echo $ssh_private_key +} + +_ssh() { + local user=$(int --path /instance_groups/name=bosh/jobs/name=user_add/properties/users/0/name) + ssh -i $(ssh_private_key) $user@$(ip) -t 'command; sudo su -' } header() { diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 6bc573ad..43cf69a2 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -12,6 +12,7 @@ meta: owner: (( param "Please specify the name of the user / organization that owns the Github repository" )) repo: (( param "Please specify the name of the Github repository" )) branch: develop + master-branch: master private_key: (( param "Please generate an SSH Deployment Key for this repo and specify it here" )) access_token: (( param "Please generate a Personal Access Token and specify it here" )) @@ -63,14 +64,13 @@ jobs: - { get: bucc-ci } - task: bucc-manifests file: bucc-ci/ci/tasks/bucc-manifests/task.yml - - # on_failure: - # put: notify - # params: - # channel: (( grab meta.slack.channel )) - # username: (( grab meta.slack.username )) - # icon_url: (( grab meta.slack.icon )) - # text: '(( concat meta.slack.fail_url " " meta.pipeline ": test job failed" ))' + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": sanity-check job failed" ))' - name: test public: true @@ -104,6 +104,14 @@ jobs: ensure: task: bucc-down-vsphere file: bucc-ci/ci/tasks/bucc-down/task.yml + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": test job failed" ))' + - name: rc public: true @@ -114,13 +122,13 @@ jobs: - { get: version, trigger: true, params: {pre: rc} } - put: version params: {file: version/number} - # on_failure: - # put: notify - # params: - # channel: (( grab meta.slack.channel )) - # username: (( grab meta.slack.username )) - # icon_url: (( grab meta.slack.icon )) - # text: '(( concat meta.slack.fail_url " " meta.pipeline ": rc job failed" ))' + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": rc job failed" ))' - name: bump-patch public: true @@ -174,6 +182,9 @@ jobs: - { get: version, passed: [rc], params: {bump: final} } - { get: bucc, passed: [rc] } - { get: bucc-ci } + - { get: bucc-stable } + - task: generate-release-notes + file: bucc-ci/ci/tasks/generate-release-notes/task.yml - task: release file: bucc-ci/ci/tasks/shipit/task.yml - put: version @@ -182,20 +193,20 @@ jobs: params: rebase: true repository: bucc-out - tag: github/tag + tag: release/tag - name: github put: github params: - name: github/name - tag: github/tag - body: github/notes.md - # on_failure: - # put: notify - # params: - # channel: (( grab meta.slack.channel )) - # username: (( grab meta.slack.username )) - # icon_url: (( grab meta.slack.icon )) - # text: '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))' + name: release/name + tag: release/tag + body: release-notes/notes.md + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))' - name: compile-releases public: true @@ -218,6 +229,13 @@ jobs: - try: put: compiled-releases-credhub params: { file: "compiled-releases/credhub*.tgz", acl: public-read } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": compiled-releases job failed" ))' - name: concourse-compiled-release-ops-file public: true @@ -238,6 +256,13 @@ jobs: params: rebase: true repository: bucc-out + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": concourse-compiled-release-ops-file job failed" ))' - name: os-conf-compiled-release-ops-file public: true @@ -258,6 +283,13 @@ jobs: params: rebase: true repository: bucc-out + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": os-conf-compiled-release-ops-file job failed" ))' - name: credhub-compiled-release-ops-file public: true @@ -278,6 +310,13 @@ jobs: params: rebase: true repository: bucc-out + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": concourse-compiled-release-ops-file job failed" ))' - name: bosh-deployment-repo public: true @@ -297,6 +336,13 @@ jobs: params: rebase: true repository: bucc-out + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": bosh-deployment-repo job failed" ))' - name: concourse-release public: true @@ -310,7 +356,7 @@ jobs: trigger: true params: tarball: false - - task: bosh-deployment + - task: update-concourse-release file: bucc-ci/ci/tasks/update-concourse-release/task.yml privileged: true - task: bucc-manifests @@ -319,6 +365,13 @@ jobs: params: rebase: true repository: bucc-out + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": concourse-release job failed" ))' resource_types: - name: bosh-deployment @@ -344,11 +397,18 @@ resources: type: git source: uri: (( grab meta.github.uri )) - branch: master + branch: (( grab meta.github.master-branch )) private_key: (( grab meta.github.private_key )) - ignore_paths: [ci/*] submodules: all + - name: bucc-stable + type: git + source: + uri: (( grab meta.github.uri )) + branch: (( grab meta.github.master-branch )) + private_key: (( grab meta.github.private_key )) + tag_filter: "v*" + - name: bucc-ci type: git source: @@ -356,12 +416,13 @@ resources: branch: (( grab meta.github.branch )) private_key: (( grab meta.github.private_key )) paths: [ci/*] + disable_ci_skip: true - name: bosh-deployment type: git source: uri: https://github.com/cloudfoundry/bosh-deployment.git - branch: master + branch: (( grab meta.github.master-branch )) ignore_paths: [ci/*] - name: version diff --git a/ci/settings.yml b/ci/settings.yml index b06091f5..0126c3a7 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -18,5 +18,5 @@ meta: access_key_id: (( vault "secret/pipelines/bucc/aws:access_key_id" )) secret_access_key: (( vault "secret/pipelines/bucc/aws:secret_access_key_id" )) slack: - webhook: (( vault "secret/pipelines/habitat-boshrelease/slack:webhook" )) - channel: "#cf-community-pipeline" + webhook: (( vault "secret/pipelines/bucc/slack:webhook" )) + channel: "#bucc" diff --git a/ci/tasks/compile-releases/task b/ci/tasks/compile-releases/task index f103f056..e25abc51 100755 --- a/ci/tasks/compile-releases/task +++ b/ci/tasks/compile-releases/task @@ -36,7 +36,10 @@ generate_compile_manifest() { continue fi if [[ ${compiled_release_url} != *"${version}"*"${stemcell_version}"* ]]; then + already_exist=$(curl -s "https://s3-eu-west-1.amazonaws.com/bucc-compiled-releases/?prefix${name}" | grep "${version}-ubuntu-trusty-${stemcell_version}") + if [ -z $alread_exist]; then releases_to_compile+="- $( echo ${release} | jq -c '.version = "latest"' )\n" + fi fi done diff --git a/ci/tasks/generate-release-notes/task b/ci/tasks/generate-release-notes/task new file mode 100755 index 00000000..954bf820 --- /dev/null +++ b/ci/tasks/generate-release-notes/task @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +custom_notes=$(pwd)/bucc/ci/release_notes.md +notes=$(pwd)/release-notes/notes.md + +touch ${custom_notes} +cp ${custom_notes} ${notes} + +curl -L >$(which bosh2) https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-2.0.10-linux-amd64 && chmod 755 $(which bosh2) + +get_releases() { + mkdir -p state + echo "virtualbox" > state/cpi + echo "foo: bar" > vars.yml + ops=$(find ops/cpis/*/*.yml | grep -v 'virtualbox\|google' | sed 's/^/-o /' | xargs) + spruce json <(echo "releases:"; ./bin/bucc int ${ops} --path /releases) | jq -c -r -M '.releases[]' +} + +get_stemcell_version() { + mkdir -p state + echo "virtualbox" > state/cpi + echo "foo: bar" > vars.yml + spruce json <(echo "releases:"; ./bin/bucc int --path /releases) | jq '.releases[0].url' \ + | tr '-' '\n' | grep trusty -A 1 | tail -1 +} + +get_release_url() { + # sort results to prefere cloudfoundry over cloudfoundry-{incubator,community} + local path=$(curl -s bosh.io/releases/ | grep ' /dev/null +stable_releases=$(get_releases) +stable_stemcell=$(get_stemcell_version) +popd > /dev/null + +pushd bucc > /dev/null +new_releases=$(get_releases) +new_stemcell=$(get_stemcell_version) +popd > /dev/null + +notes_releases="" +for release in $(echo ${new_releases} | jq -c '.'); do + name=$(echo ${release} | jq -r '.name') + version=$(echo ${release} | jq -r '.version') + stable_version=$(echo ${stable_releases} | jq -s -r --arg name "${name}" '.[] | select(.name == $name).version') + + release_url=$(get_release_url ${name} ${version}) + if [ ! -z "${release_url}" ]; then + notes_releases+="| ${name} | [${version}](${release_url}) |" + else + notes_releases+="| ${name} | ${version} |" + fi + + if [[ "${version}" != "${stable_version}" ]]; then + notes_releases+=" yes |\n" + else + notes_releases+=" no |\n" + fi +done + +if [ ! -z "${notes_releases}" ]; then + echo "## Release updates:" >> ${notes} + echo "| Name | Version | Changed |" >> ${notes} + echo "| ---- | ------- | ------- |" >> ${notes} + echo -e "${notes_releases}" >> ${notes} +fi + +if [[ ${stemcell} != ${new_stemcell} ]]; then + echo "## Stemcell update:" >> ${notes} + stemcell_url="https://github.com/cloudfoundry/bosh-linux-stemcell-builder/releases/tag/v${new_stemcell}" + echo -e "Bumped to [${new_stemcell}](${stemcell_url})" >> ${notes} +fi + +cat ${notes} + diff --git a/ci/tasks/generate-release-notes/task.yml b/ci/tasks/generate-release-notes/task.yml new file mode 100644 index 00000000..33bac6ca --- /dev/null +++ b/ci/tasks/generate-release-notes/task.yml @@ -0,0 +1,20 @@ +--- +platform: linux + +image_resource: + type: docker-image + source: + repository: starkandwayne/concourse + +inputs: + - name: bucc-ci + - name: bucc + - name: bucc-stable + +outputs: + - name: release-notes + +params: [] + +run: + path: bucc-ci/ci/tasks/generate-release-notes/task diff --git a/ci/tasks/shipit/task b/ci/tasks/shipit/task index 34e6d786..e413179f 100755 --- a/ci/tasks/shipit/task +++ b/ci/tasks/shipit/task @@ -3,16 +3,11 @@ set -eu VERSION=$(cat version/number) -if [[ ! -f bucc-ci/ci/release_notes.md ]]; then - echo >&2 "ci/release_notes.md not found. Did you forget to write them?" - exit 1 -fi -echo "v${VERSION}" > github/tag -echo "BUCC v${VERSION}" > github/name -mv bucc-ci/ci/release_notes.md github/notes.md +echo "v${VERSION}" > release/tag +echo "BUCC v${VERSION}" > release/name -cat > github/notification < release/notification < New BUCC v${VERSION} released! EOF diff --git a/ci/tasks/shipit/task.yml b/ci/tasks/shipit/task.yml index 9f2c4ff0..77a1bb3f 100644 --- a/ci/tasks/shipit/task.yml +++ b/ci/tasks/shipit/task.yml @@ -12,7 +12,7 @@ inputs: - name: bucc-ci outputs: -- name: github +- name: release - name: bucc-out run: diff --git a/ci/tasks/update-compiled-release-ops-file/task b/ci/tasks/update-compiled-release-ops-file/task index b3356b03..a4728d11 100755 --- a/ci/tasks/update-compiled-release-ops-file/task +++ b/ci/tasks/update-compiled-release-ops-file/task @@ -32,7 +32,7 @@ if [[ -z $(git config --global user.name) ]]; then git config --global user.name "CI Bot" fi -git merge --no-edit master +git merge --no-edit develop if [[ -n $(git status -s) ]]; then git add ${OPS_FILE_PATH} git commit -m "Compiled releases updated: ${release_name}/${release_version}" diff --git a/ci/tasks/update-concourse-release/task b/ci/tasks/update-concourse-release/task index b07dbf92..3188487e 100755 --- a/ci/tasks/update-concourse-release/task +++ b/ci/tasks/update-concourse-release/task @@ -32,7 +32,7 @@ if [[ -z $(git config --global user.name) ]]; then git config --global user.name "CI Bot" fi -git merge --no-edit master +git merge --no-edit develop if [[ -n $(git status -s) ]]; then git add ${OPS_FILE_PATH} git commit -m "Releases updated: ${release_name}/${release_version}" diff --git a/dev/local-bosh-release-tarball.yml b/dev/local-bosh-release-tarball.yml new file mode 100644 index 00000000..f2b5abb7 --- /dev/null +++ b/dev/local-bosh-release-tarball.yml @@ -0,0 +1,7 @@ +# Use local bosh release tarball -- useful for testing + +- type: replace + path: /releases/name=bosh? + value: + name: bosh + url: file://((local_bosh_release_tarball)) diff --git a/ops/3-concourse-release.yml b/ops/3-concourse-release.yml index 50d4f91f..3987e35e 100644 --- a/ops/3-concourse-release.yml +++ b/ops/3-concourse-release.yml @@ -2,6 +2,6 @@ path: /releases/name=concourse? value: name: concourse - version: 3.2.1 - url: https://bosh.io/d/github.com/concourse/concourse?v=3.2.1 - sha1: 5d4fa5907d3733445ea0e8f9852e4fd3296b1ec7 + version: 3.3.1 + url: https://bosh.io/d/github.com/concourse/concourse?v=3.3.1 + sha1: 60e6c5e1806eee48931091566a56bd0897090e28 diff --git a/ops/9-concourse-compiled-release.yml b/ops/9-concourse-compiled-release.yml index 832cb7f6..c000fc84 100644 --- a/ops/9-concourse-compiled-release.yml +++ b/ops/9-concourse-compiled-release.yml @@ -2,6 +2,6 @@ path: /releases/name=concourse? value: name: concourse - version: 3.2.1 - url: https://s3-eu-west-1.amazonaws.com/bucc-compiled-releases/concourse-3.2.1-ubuntu-trusty-3421.9-20170621-192038-462654899-20170621192058.tgz - sha1: f7bc84ef32292754229e5366364abae3059ed4c2 + version: 3.3.1 + url: https://s3-eu-west-1.amazonaws.com/bucc-compiled-releases/concourse-3.3.1-ubuntu-trusty-3421.9-20170704-085722-763532943-20170704085751.tgz + sha1: 0119635c50c600fa367e2bffe9f630823f0d606b diff --git a/src/bosh-deployment/bosh.yml b/src/bosh-deployment/bosh.yml index d1234769..49d3c6ad 100644 --- a/src/bosh-deployment/bosh.yml +++ b/src/bosh-deployment/bosh.yml @@ -3,9 +3,9 @@ name: bosh releases: - name: bosh - version: "262.1" - url: https://s3.amazonaws.com/bosh-compiled-release-tarballs/bosh-262.1-ubuntu-trusty-3421.9-20170621-055124-244370454-20170621055129.tgz?versionId=lxNGZVeHOlvxh4LyMgNxnHC8wczKDP70 - sha1: 1ad2a2b9913c20eac5f27a5e9ad71739c633d37d + version: "262.2" + url: https://s3.amazonaws.com/bosh-compiled-release-tarballs/bosh-262.2-ubuntu-trusty-3421.9-20170628-223206-395348439-20170628223211.tgz?versionId=LSWubg7BOnAZxXFDaCHEpIXWcxyqkb1o + sha1: 07c07d8520b3508fbc25c4a747c9c038a4b8b672 resource_pools: - name: vms diff --git a/src/bosh-deployment/dev/nats-tls.yml b/src/bosh-deployment/dev/nats-tls.yml index 97d8adf8..75fe7206 100644 --- a/src/bosh-deployment/dev/nats-tls.yml +++ b/src/bosh-deployment/dev/nats-tls.yml @@ -4,20 +4,25 @@ - type: replace path: /instance_groups/name=bosh/jobs/- + value: {name: gonats, release: bosh} + +- type: replace + path: /instance_groups/name=bosh/properties/nats? value: - name: gonats - release: bosh - properties: - gonats: - listen_address: ((internal_ip)) - user: nats - password: ((nats_password)) - certificate: ((nats_ssl.certificate)) - private_key: ((nats_ssl.private_key)) + address: ((internal_ip)) + user: nats + password: ((nats_password)) + cert: + ca: ((nats_ssl.ca)) - type: replace - path: /instance_groups/name=bosh/properties/nats?/cert/ca - value: ((nats_ssl.ca)) + path: /instance_groups/name=bosh/properties/gonats? + value: + address: ((internal_ip)) + user: nats + password: ((nats_password)) + certificate: ((nats_ssl.certificate)) + private_key: ((nats_ssl.private_key)) - type: replace path: /variables/- @@ -27,5 +32,4 @@ options: ca: default_ca common_name: ((internal_ip)) - alternative_names: [((internal_ip))] - + alternative_names: [((internal_ip))] \ No newline at end of file