Merge pull request #2 from stakater-docker/add-manifest

[add-manifest]

Dockerfile

@@ -0,0 +1,28 @@
+
+FROM phusion/baseimage:0.9.19
+MAINTAINER Rasheed Amir <[email protected]>
+
+RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe" > /etc/apt/sources.list && \
+    apt-get -y update && \
+    apt-get install -y --no-install-recommends mysql-client && \
+    apt-get install -y python-pip && \
+    apt-get -y install sudo nano git sudo zip bzip2 fontconfig wget && \
+    pip install awscli && \
+    mkdir /backup
+
+ENV CRON_TIME="00 00 * * *" \
+    MYSQL_DB="--all-databases" \ 
+    S3_BUCKET_NAME="docker-backups.example.com" \
+    AWS_ACCESS_KEY_ID="**DefineMe**" \ 
+    AWS_SECRET_ACCESS_KEY="**DefineMe**" \
+    AWS_DEFAULT_REGION="us-east-1" \
+    PATHS_TO_BACKUP="/paths/to/backup" \
+    BACKUP_NAME="backup" \
+    LAST_BACKUP="" \
+    RESTORE="true"
+
+ADD run.sh /run.sh
+
+VOLUME ["/backup"]
+
+CMD ["/run.sh"]

Jenkinsfile

@@ -0,0 +1,7 @@
+#!/usr/bin/env groovy
+@Library('github.com/stakater/stakater-pipeline-library@update-helm-template-command') _
+
+pushDockerImage {
+    dockerRepositoryURL = "docker.io"
+    imagePrefix = "5.7"
+}

README.md

@@ -1,2 +1,22 @@
-# mysql-restore-backup
-A repo containing image to backup & restore Mysql snapshots
+# MySQL Data Backup and Restore Sidecar
+
+## Overview
+
+It is a sidecar that will run inside mysql pod. Its main purpose is to backup and restore mysql data.
+
+## Description
+
+mysql backup and restore [image](https://hub.docker.com/r/stakater/mysql-backup-restore-s3/) [MBR] container can be used to backup elasticsearch data on S3 bucket. It runs inside mysql pod as a sidecar container. Sidecar container dump the mysql data and store it in S3 as an object.
+
+
+| Environment Variable | Description | Default value |
+|---|---|---|
+| CRON_TIME | Data backup interval | Default backup interval is "00 */1 * * *", which means take backup after each hour. |
+| MYSQL_DB | Name of the database for backup | --all-databases is its default value which means that it will backup all databases. If a database name is provided then it will only take that database backup |
+| S3_BUCKET_NAME | AWS S3 bucket name | "" |
+| AWS_ACCESS_KEY_ID | AWS account access id | "" |
+| AWS_SECRET_ACCESS_KEY | AWS account access id secret | "" |
+| AWS_DEFAULT_REGION | AWS default region | "" |
+| BACKUP_NAME | Name of the backup. | Its default value is yyyy.mm.dd-HH-MM-SS.sql |
+| LAST_BACKUP | Name of the last backup. Last backup name in extracted from S3 bucket name using the script written in `run.sh` file. | None |
+| RESTORE | Variable for check to restore data from S3 bucket. If `true` data will be restored from S3 bucket and store in location given in `RESTORE_FOLDER` env variable otherwise only data backup script will execute in the container. | true |

mysql-restore-backup.yaml

@@ -0,0 +1,104 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: mysql
+  name: mysql-svc
+  namespace: <namespace>
+spec:
+  ports:
+  - name: "mysql-port"
+    port: 3306
+    targetPort: 3306
+  selector:
+    app: mysql
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mysql
+  namespace: <namespace>
+spec:
+  serviceName: "mysql"
+  selector:
+    matchLabels:
+      app: mysql
+  replicas: 1 
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:   
+      tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "app"
+        effect: "NoSchedule"   
+      containers:
+      - image: stakater/mysql-backup-restore-s3:0.0.1
+        name: mysql-backup-restore
+        volumeMounts:
+        - mountPath: /backup
+          name: mysql-pvc    
+        env:
+        - name: MYSQL_DB
+          value: "restore-db-name"
+        - name: CRON_TIME
+          value: "0 */1 * * *"
+        - name: MYSQL_USER
+          value: "root"
+        - name: MYSQL_PASS
+          valueFrom:
+            secretKeyRef:
+              name: mysql
+              key: mysql-root-password
+        - name: S3_BUCKET_NAME
+          valueFrom:
+            secretKeyRef:
+              name: aws-secrets
+              key: mysql_bucket
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: aws-secrets
+              key: aws_access_key_id
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: aws-secrets
+              key: aws_secret_access_key
+        - name: AWS_DEFAULT_REGION
+          valueFrom:
+            secretKeyRef:
+              name: aws-secrets
+              key: aws_default_region
+        - name: MYSQL_HOST
+          value: "127.0.0.1"
+        - name: MYSQL_PORT
+          value: "3306"
+        - name: RESTORE
+          value: "false"
+      - image: mysql:5.7
+        name: mysql
+        env:
+        - name: MYSQL_ROOT_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mysql
+              key: mysql-root-password
+        ports:
+        - containerPort: 3306
+          name: tcp
+        volumeMounts:
+        - mountPath: /var/lib/mysql
+          name: mysql-pvc
+        resources: {}          
+  volumeClaimTemplates:
+  - metadata:
+      name: mysql-pvc
+    spec:
+      accessModes: [ "ReadWriteMany" ]
+      storageClassName: efs
+      resources:
+        requests:
+          storage: 2Gi

run.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+export PATH=$PATH:/usr/bin:/usr/local/bin:/bin
+
+if [ "${MYSQL_ENV_MYSQL_PASS}" == "**Random**" ]; then
+        unset MYSQL_ENV_MYSQL_PASS
+fi
+
+MYSQL_HOST=${MYSQL_PORT_3306_TCP_ADDR:-${MYSQL_HOST}}
+MYSQL_HOST=${MYSQL_PORT_1_3306_TCP_ADDR:-${MYSQL_HOST}}
+MYSQL_PORT=${MYSQL_PORT_3306_TCP_PORT:-${MYSQL_PORT}}
+MYSQL_PORT=${MYSQL_PORT_1_3306_TCP_PORT:-${MYSQL_PORT}}
+MYSQL_USER=${MYSQL_USER:-${MYSQL_ENV_MYSQL_USER}}
+MYSQL_PASS=${MYSQL_PASS:-${MYSQL_ENV_MYSQL_PASS}}
+
+[ -z "${MYSQL_HOST}" ] && { echo "=> MYSQL_HOST cannot be empty" && exit 1; }
+echo "=> MYSQL_HOST ${MYSQL_HOST}" 
+
+[ -z "${MYSQL_PORT}" ] && { echo "=> MYSQL_PORT cannot be empty" && exit 1; }
+echo "=> MYSQL_PORT ${MYSQL_PORT}" 
+
+[ -z "${MYSQL_USER}" ] && { echo "=> MYSQL_USER cannot be empty" && exit 1; }
+[ -z "${MYSQL_PASS}" ] && { echo "=> MYSQL_PASS cannot be empty" && exit 1; }
+
+BACKUP_CMD="mysqldump -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} ${EXTRA_OPTS} ${MYSQL_DB} > /backup/"'${BACKUP_NAME}'
+
+echo "=> Creating backup script"
+rm -f /backup.sh
+cat <<EOF >> /backup.sh
+#!/bin/bash
+export PATH=$PATH:/usr/bin:/usr/local/bin:/bin
+MAX_BACKUPS=${MAX_BACKUPS}
+BACKUP_NAME=\$(date +\%Y.\%m.\%d.\%H\%M\%S).sql
+echo "=> Backup started: \${BACKUP_NAME}"
+if ${BACKUP_CMD} ;then
+    echo "   Backup succeeded"
+else
+    echo "   Backup failed"
+    rm -rf /backup/\${BACKUP_NAME}
+fi
+if [ -n "\${MAX_BACKUPS}" ]; then
+    while [ \$(ls /backup -N1 | wc -l) -gt \${MAX_BACKUPS} ];
+    do
+        BACKUP_TO_BE_DELETED=\$(ls /backup -N1 | sort | head -n 1)
+        echo "   Backup \${BACKUP_TO_BE_DELETED} is deleted"
+        rm -rf /backup/\${BACKUP_TO_BE_DELETED}
+    done
+fi
+echo "=> Backup done"
+if ${BACKUP_CMD} ;then
+    echo "=> Upload to s3 started: \${BACKUP_NAME}"
+    # Create bucket, if it doesn't already exist
+    BUCKET_EXIST=\$(aws s3 --region \${AWS_DEFAULT_REGION} ls | grep \${S3_BUCKET_NAME} | wc -l)
+    if [ \${BUCKET_EXIST} -eq 0 ];
+    then
+      aws s3 --region \${AWS_DEFAULT_REGION} mb s3://\${S3_BUCKET_NAME}
+    fi
+    # Upload the backup to S3 with timestamp
+    aws s3 --region \${AWS_DEFAULT_REGION} cp /backup/\${BACKUP_NAME} s3://\${S3_BUCKET_NAME}/\${BACKUP_NAME}
+    echo "=> Upload to s3 done"
+    echo "=> Cleanup started: \${BACKUP_NAME}"
+        rm -rf /backup/\${BACKUP_NAME}
+    echo "=> Cleanup done"
+fi
+EOF
+chmod +x /backup.sh
+
+echo "=> Creating restore script"
+rm -f /restore.sh
+cat <<EOF >> /restore.sh
+#!/bin/bash
+    BUCKET_EXIST=\$(aws s3 --region \${AWS_DEFAULT_REGION} ls | grep \${S3_BUCKET_NAME} | wc -l)
+    if [ \${BUCKET_EXIST} -eq 0 ];
+    then
+      echo "Bucket Doesnt Exist"
+      exit 1
+    fi
+    if [ -z "\${LAST_BACKUP}" ]; then
+    # Find last backup file
+    : ${LAST_BACKUP:=$(aws s3 ls s3://$S3_BUCKET_NAME | awk -F " " '{print $4}' | sort -r | head -n1)}
+    fi
+    
+    # Download backup from S3
+    echo "=> Restore from S3 => $LAST_BACKUP"
+    aws s3 cp s3://$S3_BUCKET_NAME/$LAST_BACKUP $LAST_BACKUP
+    echo "=> Restore database from \$1"
+    if mysql -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} --database=${MYSQL_DB} < \/$LAST_BACKUP ;then
+    echo "   Restore succeeded"
+    else
+    echo "   Restore failed"
+    fi
+    echo "=> Done"
+EOF
+chmod +x /restore.sh
+
+touch /mysql_backup.log
+tail -F /mysql_backup.log &
+
+if [ -n "${INIT_BACKUP}" ]; then
+    echo "=> Create a backup on the startup"
+    /backup.sh
+fi
+
+if [[ "$RESTORE" == "true" ]]; then
+  ./restore.sh
+fi
+
+echo "${CRON_TIME} export S3_BUCKET_NAME=${S3_BUCKET_NAME}; export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}; export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}; export AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}; /backup.sh >> /mysql_backup.log 2>&1" > /crontab.conf
+crontab  /crontab.conf
+echo "=> Running cron job"
+exec cron -f