Merge ../bucc into HEAD

egen · Nov 6, 2017 · e4003d8 · e4003d8
2 parents d46a45a + 4dd4241
commit e4003d8
Show file tree

Hide file tree

Showing 54 changed files with 548 additions and 1,109 deletions.
diff --git a/.envrc b/.envrc
@@ -1,10 +1,9 @@
-export PATH=$PWD/bin:$PATH
 case $SHELL in
-     /bin/zsh)
-         eval $(bucc env)
+     *zsh)
+         eval "$(./bin/bucc env)"
      ;;
-     /bin/bash)
-        source <(bucc env)
+     *bash)
+        source <(./bin/bucc env)
 esac
 
 

diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,4 @@ local.yml
 /bin/fly
 /bin/credhub
 /vars.yml
+/vars.yml.bck
diff --git a/.versions b/.versions
@@ -1,2 +1,2 @@
-credhub=1.4.1
-bucc=0.1.3
+credhub=1.5.0
+bucc=0.1.4
diff --git a/bin/bucc b/bin/bucc
@@ -1,7 +1,5 @@
 #!/bin/bash
-version=$(grep bucc .versions | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
-
-repo_root=$(dirname $(realpath "$0") | rev | cut -c5- | rev)
+repo_root=$(dirname $(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd))
 state=${repo_root}/state
 mkdir -p $state/manifests
 manifest="${repo_root}/src/bosh-deployment/bosh.yml"
@@ -253,6 +251,7 @@ env() {
     if [[ ! -f ${vars_store} ]]; then
         exit 0
     fi
+    echo "export PATH=${PATH}"
     echo "export BOSH_ENVIRONMENT=$(get_var bosh_target)"
     echo "export BOSH_CA_CERT='$(cat $(ca_cert))'"
     echo "export BOSH_CLIENT=$(get_var bosh_client)"

diff --git a/ci/assets/acceptance-pipeline.yml b/ci/assets/acceptance-pipeline.yml
@@ -0,0 +1,69 @@
+jobs:
+- name: test
+  plan:
+  - aggregate:
+    - get: stemcell
+    - get: bosh-deployment
+    - get: docker-release
+  - put: cloud-config
+    params:
+      manifest: bosh-deployment/warden/cloud-config.yml
+      releases: []
+  - put: deploy
+    params:
+      manifest: docker-release/manifests/swarm/docker-swarm.yml
+      stemcells: [ stemcell/stemcell.tgz ]
+
+resources:
+- name: bosh-deployment
+  type: git
+  source:
+    uri: https://github.com/cloudfoundry/bosh-deployment.git
+    branch: master
+
+- name: docker-release
+  type: git
+  source:
+    uri: https://github.com/cloudfoundry-community/docker-boshrelease.git
+    branch: master
+
+- name: stemcell
+  type: bosh-io-stemcell
+  source:
+    name: bosh-warden-boshlite-ubuntu-trusty-go_agent
+
+- name: deploy
+  type: bosh-deployment
+  source: 
+    <<: *deployment
+    <<: *bosh_env
+
+- name: cloud-config
+  type: bosh-config
+  source:
+    type: cloud-config
+    <<: *bosh_env
+
+params:
+  bosh_env: &bosh_env
+    target: ((bosh_environment))
+    client: ((bosh_client))
+    client_id: ((bosh_client))
+    client_secret: ((bosh_client_secret))
+    ca_cert: ((bosh_ca_cert))
+  deployment: &deployment
+    deployment: docker-swarm
+
+resource_types:
+- name: bosh-config
+  type: docker-image
+  source:
+    repository: rkoster/bosh-config-resource
+    tag: latest
+
+- name: bosh-deployment
+  type: docker-image
+  source:
+    repository: cloudfoundry/bosh-deployment-resource
+    tag: latest
+
diff --git a/ci/pipeline.yml b/ci/pipeline.yml
@@ -48,7 +48,8 @@ groups:
       - compile-releases
       - concourse-compiled-release-ops-file
       - os-conf-compiled-release-ops-file
-      - credhub-compiled-release-ops-file
+      # - credhub-compiled-release-ops-file
+      - credhub-importer-compiled-release-ops-file
   - name: upstream
     jobs:
       - bosh-deployment-repo
@@ -108,9 +109,11 @@ jobs:
               vcenter_cluster: cluster1
         - task: bucc-test-vsphere
           file: bucc-ci/ci/tasks/bucc-test/task.yml
-          ensure:
-            task: bucc-down-vsphere
-            file: bucc-ci/ci/tasks/bucc-down/task.yml
+        - task: bucc-acceptance-test
+          file: bucc-ci/ci/tasks/bucc-acceptance-test/task.yml
+          attempts: 5
+        - task: bucc-down-vsphere
+          file: bucc-ci/ci/tasks/bucc-down/task.yml
         on_failure:
           put: notify
           params:
@@ -119,7 +122,6 @@ jobs:
             icon_url: (( grab meta.slack.icon ))
             text:    '(( concat meta.slack.fail_url " " meta.pipeline ": test job failed" ))'
 
-
   - name: rc
     public: true
     plan:
@@ -187,7 +189,7 @@ jobs:
         - name: inputs
           aggregate:
             - { get: version, passed: [rc], params: {bump: final} }
-            - { get: bucc,    passed: [rc] }
+            - { get: bucc,    passed: [test] }
             - { get: bucc-ci }
             - { get: bucc-stable }
         - task: generate-release-notes
@@ -204,7 +206,7 @@ jobs:
         - put: bucc
           params:
             rebase: true
-            repository: bucc
+            repository: bucc-out
         - name: github
           put: github
           params:
@@ -237,9 +239,12 @@ jobs:
           - try:
               put: compiled-releases-os-conf
               params: { file: "compiled-releases/os-conf*.tgz", acl: public-read }
+          # - try:
+          #     put: compiled-releases-credhub
+          #     params: { file: "compiled-releases/credhub-[0-9]*.tgz", acl: public-read }
           - try:
-              put: compiled-releases-credhub
-              params: { file: "compiled-releases/credhub*.tgz", acl: public-read }
+              put: compiled-releases-credhub-importer
+              params: { file: "compiled-releases/credhub-importer*.tgz", acl: public-read }
         on_failure:
           put: notify
           params:
@@ -302,21 +307,48 @@ jobs:
             icon_url: (( grab meta.slack.icon ))
             text:    '(( concat meta.slack.fail_url " " meta.pipeline ": os-conf-compiled-release-ops-file job failed" ))'
 
-  - name: credhub-compiled-release-ops-file
+  # - name: credhub-compiled-release-ops-file
+  #   public: true
+  #   serial: true
+  #   plan:
+  #     - do:
+  #       - aggregate:
+  #         - get: bucc
+  #         - get: bucc-ci
+  #         - { get: compiled-releases-credhub, trigger: true }
+  #       - task: update-credhub-compiled-release-ops-file
+  #         file: bucc-ci/ci/tasks/update-compiled-release-ops-file/task.yml
+  #         input_mapping:
+  #           compiled-release: compiled-releases-credhub
+  #         params:
+  #           OPS_FILE_PATH: ops/9-credhub-compiled-release.yml
+  #       - put: bucc
+  #         params:
+  #           merge: true
+  #           repository: bucc-out
+  #       on_failure:
+  #         put: notify
+  #         params:
+  #           channel:  (( grab meta.slack.channel ))
+  #           username: (( grab meta.slack.username ))
+  #           icon_url: (( grab meta.slack.icon ))
+  #           text:    '(( concat meta.slack.fail_url " " meta.pipeline ": credhub-compiled-release-ops-file job failed" ))'
+
+  - name: credhub-importer-compiled-release-ops-file
     public: true
     serial: true
     plan:
       - do:
         - aggregate:
           - get: bucc
           - get: bucc-ci
-          - { get: compiled-releases-credhub, trigger: true }
-        - task: update-credhub-compiled-release-ops-file
+          - { get: compiled-releases-credhub-importer, trigger: true }
+        - task: update-credhub-importer-compiled-release-ops-file
           file: bucc-ci/ci/tasks/update-compiled-release-ops-file/task.yml
           input_mapping:
-            compiled-release: compiled-releases-credhub
+            compiled-release: compiled-releases-credhub-importer
           params:
-            OPS_FILE_PATH: ops/9-credhub-compiled-release.yml
+            OPS_FILE_PATH: ops/9-credhub-importer-compiled-release.yml
         - put: bucc
           params:
             merge: true
@@ -327,7 +359,7 @@ jobs:
             channel:  (( grab meta.slack.channel ))
             username: (( grab meta.slack.username ))
             icon_url: (( grab meta.slack.icon ))
-            text:    '(( concat meta.slack.fail_url " " meta.pipeline ": concourse-compiled-release-ops-file job failed" ))'
+            text:    '(( concat meta.slack.fail_url " " meta.pipeline ": credhub-importer-compiled-release-ops-file job failed" ))'
 
   - name: bosh-deployment-repo
     public: true
@@ -340,7 +372,6 @@ jobs:
           - { get: bucc-ci, trigger: false}
         - task: bosh-deployment
           file: bucc-ci/ci/tasks/get-latest-bosh-deployment/task.yml
-          privileged: true
         - task: bucc-manifests
           file: bucc-ci/ci/tasks/bucc-manifests/task.yml
         - put: bucc
@@ -369,7 +400,6 @@ jobs:
               tarball: false
         - task: update-concourse-release
           file: bucc-ci/ci/tasks/update-concourse-release/task.yml
-          privileged: true
         - task: bucc-manifests
           file: bucc-ci/ci/tasks/bucc-manifests/task.yml
         - put: bucc
@@ -399,13 +429,12 @@ jobs:
                 - credhub-linux-*.tgz
         - task: update-credhub-cli
           file: bucc-ci/ci/tasks/update-credhub-cli/task.yml
-          privileged: true
         - task: bucc-manifests
           file: bucc-ci/ci/tasks/bucc-manifests/task.yml
         - put: bucc
           params:
             merge: true
-            repository: bucc-out          
+            repository: bucc-out
         on_failure:
           put: notify
           params:
@@ -494,14 +523,23 @@ resources:
       region_name: eu-west-1
       regexp: "os-conf.*-(\\d+).tgz"
 
-  - name: compiled-releases-credhub
+  # - name: compiled-releases-credhub
+  #   type: s3
+  #   source:
+  #     bucket: "bucc-compiled-releases"
+  #     access_key_id: (( grab meta.aws.access_key_id ))
+  #     secret_access_key: (( grab meta.aws.secret_access_key ))
+  #     region_name: eu-west-1
+  #     regexp: "credhub-\\d+.*-(\\d+).tgz"
+
+  - name: compiled-releases-credhub-importer
     type: s3
     source:
       bucket: "bucc-compiled-releases"
       access_key_id: (( grab meta.aws.access_key_id ))
       secret_access_key: (( grab meta.aws.secret_access_key ))
       region_name: eu-west-1
-      regexp: "credhub.*-(\\d+).tgz"
+      regexp: "credhub-importer.*-(\\d+).tgz"
 
   - name: notify
     type: slack-notification

diff --git a/ci/tasks/bucc-acceptance-test/task b/ci/tasks/bucc-acceptance-test/task
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+mv bucc-state/{vars.yml,state} bucc/
+source <(./bucc/bin/bucc env)
+
+bucc fly
+
+fly -t bucc set-pipeline -n -c bucc-ci/ci/assets/acceptance-pipeline.yml -p test
+fly -t bucc unpause-pipeline -p test
+fly -t bucc trigger-job -j test/test -w
+
+if ! fly -t bucc builds | grep succeed; then
+    >&2 echo "build did not succeed"; exit 1
+fi
diff --git a/ci/tasks/bucc-acceptance-test/task.yml b/ci/tasks/bucc-acceptance-test/task.yml
@@ -0,0 +1,15 @@
+---
+platform: linux
+
+image_resource:
+  type: docker-image
+  source:
+    repository: starkandwayne/concourse
+
+inputs:
+  - name: bucc-ci
+  - name: bucc
+  - name: bucc-state    
+
+run:
+  path: bucc-ci/ci/tasks/bucc-acceptance-test/task
diff --git a/ci/tasks/bucc-manifests/task b/ci/tasks/bucc-manifests/task
@@ -33,7 +33,7 @@ gen_manifest() {
 
     manifest="../bucc-generated-manifest/${manifest_name}.yml"
 
-    ./bin/bucc int > ${manifest}
+    ./bin/bucc int --var-errs > ${manifest}
 
     if ! grep -q ${cpi} ${manifest}; then
         echo "Invalid manifest missing cpi: ${cpi}"

diff --git a/ci/tasks/compile-releases/task b/ci/tasks/compile-releases/task
@@ -37,8 +37,8 @@ generate_compile_manifest() {
             continue
         fi
         if [[ ${compiled_release_url} != *"${version}"*"${stemcell_version}"* ]]; then
-          releases_in_s3=$(curl -s "https://s3-eu-west-1.amazonaws.com/bucc-compiled-releases/?prefix${name}")
-          if [[ ${releases_in_s3} != "*${version}-ubuntu-trusty-${stemcell_version}*" ]]; then
+          releases_in_s3=$(curl -s "https://s3-eu-west-1.amazonaws.com/bucc-compiled-releases/?prefix=${name}")
+          if [[ ${releases_in_s3} != *"${version}-ubuntu-trusty-${stemcell_version}"* ]]; then
             releases_to_compile+="- $( echo ${release} | jq -c '.version = "latest"' )\n"
           fi
         fi

diff --git a/ci/tasks/get-latest-bosh-deployment/task b/ci/tasks/get-latest-bosh-deployment/task
@@ -3,6 +3,8 @@
 set -e
 set -o pipefail
 
+ref=$(cat bosh-deployment/.git/ref | cut -c1-8)
+
 pushd bucc > /dev/null
 
 if [[ -z $(git config --global user.email) ]]; then
@@ -12,12 +14,15 @@ if [[ -z $(git config --global user.name) ]]; then
     git config --global user.name "CI Bot"
 fi
 
+echo "CHECKING git status"
 git status
 
-echo "MERGE bosh-deployment repo with git subtree"
-git subtree pull --prefix src/bosh-deployment \
-    ../bosh-deployment master --squash \
-    -m "merging the latest bosh-deployment repo"
+echo "MERGE bosh-deployment repo"
+rm -r src/bosh-deployment
+cp -r ../bosh-deployment src/
+rm -rf src/bosh-deployment/{ci,tests}
+git add src/bosh-deployment
+git commit -m "Update to cloudfoundry/bosh-deployment@$ref"
 
 popd > /dev/null
 

diff --git a/ci/tasks/shipit/task b/ci/tasks/shipit/task
@@ -16,11 +16,9 @@ git config --global user.name "CI Bot"
 
 pushd bucc > /dev/null
 
-git config core.fileMode false
-
 rm -f ci/release_notes.md
 
-sed "s/.*bucc.*/bucc=$version/" .versions
+sed -i "s/.*bucc.*/bucc=$VERSION/" .versions
 
 git add -A
 git commit -m "release: v${VERSION}"