Merge pull request #10 from cloudfoundry-incubator/PR-backup-and-rest…

…ore-#160017872

Refactor bbr scripts to work for multi-master

jobs/bbr-etcd/spec

@@ -4,6 +4,9 @@ name: bbr-etcd
 templates:
   backup.erb: bin/bbr/backup
   restore.erb: bin/bbr/restore
+  metadata.sh.erb: bin/bbr/metadata
+  pre-restore-lock.sh.erb: bin/bbr/pre-restore-lock
+  post-restore-unlock.sh.erb: bin/bbr/post-restore-unlock
 
 packages: []
 

jobs/bbr-etcd/templates/backup.erb

@@ -1,16 +1,20 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -euo pipefail
 
 source /var/vcap/jobs/etcd/bin/utils.sh
 
 BBR_ARTIFACT_FILE_PATH="${BBR_ARTIFACT_DIRECTORY}/snapshotdb"
-
 export ETCDCTL_API=3
 
+
+<% if spec.bootstrap %>
+echo "Taking etcd snapshot"
+
 /var/vcap/packages/etcd/bin/etcdctl \
   --cacert /var/vcap/jobs/etcd/config/etcdctl-ca.crt \
   --cert /var/vcap/jobs/etcd/config/etcdctl.crt \
   --key /var/vcap/jobs/etcd/config/etcdctl.key \
   --endpoints "${etcd_endpoint_address}" \
   snapshot save "${BBR_ARTIFACT_FILE_PATH}"
+<% end %>

jobs/bbr-etcd/templates/metadata.sh.erb

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+echo "---
+restore_name: cfcr-etcd-snapshot
+<% if spec.bootstrap %>
+backup_name: cfcr-etcd-snapshot
+<% end %>
+"

jobs/bbr-etcd/templates/post-restore-unlock.sh.erb

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "Starting etcd"
+
+TIMEOUT=60
+if timeout "$TIMEOUT" /bin/bash <<EOF
+  #!/bin/bash
+
+  until /var/vcap/bosh/bin/monit start etcd; do
+    echo "starting etcd"
+    sleep 5
+  done
+EOF
+then
+  echo "monit has started etcd"
+else
+  echo "monit was unable to start etcd after $TIMEOUT seconds"
+  exit 1
+fi
+
+if timeout "$TIMEOUT" /bin/bash <<EOF
+  #!/bin/bash
+
+  until /var/vcap/bosh/bin/monit summary | grep etcd | grep "running"; do
+    echo "waiting for etcd daemon to start"
+    sleep 5
+  done
+EOF
+then
+  echo "etcd daemon has started"
+else
+  echo "etcd daemon was unable to start after $TIMEOUT seconds"
+  exit 1
+fi

jobs/bbr-etcd/templates/pre-restore-lock.sh.erb

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "Stopping etcd"
+/var/vcap/bosh/bin/monit stop etcd
+
+TIMEOUT=60
+
+if timeout "$TIMEOUT" /bin/bash <<EOF
+  #!/bin/bash
+
+  until /var/vcap/bosh/bin/monit summary | grep etcd | grep "not monitored" ; do
+    echo "waiting for etcd to stop..."
+    sleep 5
+  done
+EOF
+then
+  echo "Etcd stopped"
+else
+  echo "Timed out stopping etcd after $TIMEOUT seconds"
+  exit 1
+fi
+
+echo "Deleting old etcd data"
+rm -rf /var/vcap/store/etcd

jobs/bbr-etcd/templates/restore.erb

@@ -1,36 +1,19 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -euo pipefail
 
 source /var/vcap/jobs/etcd/bin/utils.sh
 
-exec 1>&2
-
 export BBR_ARTIFACT_FILE_PATH="${BBR_ARTIFACT_DIRECTORY}/snapshotdb"
 export ETCDCTL_API=3
 
-/var/vcap/bosh/bin/monit unmonitor etcd
-/var/vcap/jobs/bpm/bin/bpm stop etcd
-
-while /var/vcap/jobs/bpm/bin/bpm pid etcd; do
-    echo "Waiting for etcd to exit"
-    sleep 1
-done
-
-echo "Deleting old etcd data"
-rm -rf /var/vcap/store/etcd
-
 echo "Restoring etcd data"
+
 /var/vcap/packages/etcd/bin/etcdctl \
     snapshot restore ${BBR_ARTIFACT_FILE_PATH} \
     --name="<%= spec.id %>" \
     --initial-cluster="${etcd_peers}" \
     --initial-advertise-peer-urls="${etcd_peer_address}" \
     --data-dir="/var/vcap/store/etcd"
 
-chown -R vcap:vcap /var/vcap/store/etcd
-
-echo "Starting etcd"
-/var/vcap/jobs/bpm/bin/bpm start etcd
-/var/vcap/bosh/bin/monit summary
-/var/vcap/bosh/bin/monit monitor etcd
+chown -R vcap:vcap /var/vcap/store/etcd