Merge branch 'master' of git.assembla.com:lp-cdw.4

etc/cron/README.md

@@ -0,0 +1,18 @@
+# Overview
+
+This folder contains scripts that run on the NAS (ReadyNAS), under cron:
+
+    cron_table: crontab file for the context under which croned scripts will run
+    sync_*:     the script to run
+
+The cron_table file is loaded as follows:
+
+    crontab cron_table
+
+Push all the files to the NAS as follows:
+
+    rsync -avz -e ssh . [email protected]:/c/home/cdwmedia/cron/
+    # Then load the cron file updates
+    ssh [email protected] crontab /c/home/cdwmedia/cron/cron_table
+
+

etc/cron/cdw.s3cfg.sample

@@ -0,0 +1,33 @@
+[default]
+access_key = 
+secret_key = 
+host_base = s3.amazonaws.com
+host_bucket = civildebatewall.s3.amazonaws.com
+
+acl_public = True
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+cloudfront_resource = /2008-06-30/distribution
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+encoding = UTF-8
+encrypt = False
+force = False
+get_continue = False
+guess_mime_type = True
+human_readable_sizes = False
+list_md5 = False
+preserve_attrs = True
+progress_meter = True
+proxy_host =
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+urlencoding_mode = normal
+use_https = False
+verbosity = WARNING
+

etc/cron/cron_table

@@ -0,0 +1,25 @@
+# 
+# INSTRUCTIONS:
+# Push this file to the remote host along with the scripts defined (and into their respective paths)
+# Load this file with 
+#   crontab this-file-name
+#
+# m h  dom mon dow   command
+
+# Notify administrators on status of tasks
+[email protected]
+
+# @yearly   0 0 1 1 *
+# @daily    0 0 * * *
+# @hourly   0 * * * *
+# @reboot   Run at startup.
+
+# Run media sync every 7 minutes
+*/7 *  *   *   *     /c/home/cdwmedia/cron/sync_media
+
+# Sync logs every 30 mins
+30 *  *   *   *     /c/home/cdwmedia/cron/sync_logs
+
+# Sync binaries irregularly
+45 */4 *  *   *    /c/home/cdwmedia/cron/sync_binaries
+

etc/cron/sync_binaries

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+##
+# Synchronize binaries
+##
+
+#------------------------------------------------
+# Configuration
+#------------------------------------------------
+# Application name
+APPNAME="civildebatewall"
+
+# Email recipients. Blank list to disable email sending
+# Comma-delimited list to be sent to mailx
+RECIPIENTS=
+# Logfile path MUST exist 
+RUN_PATH="/var/run/$APPNAME"
+pidfile="/$RUN_PATH/sync_binaries.pid"
+
+LOG_PATH="/var/log/$APPNAME"
+LOGFILE="$LOG_PATH/sync_binaries.log"
+PROCESS_SUBJECT="CDW Binaries Sync Status"
+
+S3_BUCKET="$APPNAME"
+MEDIA_STORE_PATH=/c/home/cdwmedia
+S3CFG_FILE=/etc/cdw.s3cfg
+#------------------------------------------------
+
+usage() {
+  echo "$0 "
+}
+
+notify() {
+    TIMESTAMP=`date -u +%Y%m%d%H%M%S`
+    echo -e "($TIMESTAMP) $1" >> $LOGFILE
+    # Tell someone that there's something going on
+    if [ ! -z "$RECIPIENTS" ];then
+        echo -e "$1" | mailx -s "$PROCESS_SUBJECT" $RECIPIENTS
+    fi
+}
+
+initialize() { 
+    # Create a process-file so that we don't run the process multiple times
+    if [ -e $pidfile ]; then
+        pid=`cat $pidfile`
+        if kill -0 &>1 > /dev/null $pid; then
+            echo "Already running"
+            exit 1
+        else
+            rm $pidfile
+        fi
+    fi
+    echo $$ > $pidfile
+}
+
+run_sync() { 
+    notify "STARTED: Synchronizing media from $src to $dst"
+    s3cmd --config=$S3CFG_FILE --acl-private --no-delete-removed --recursive --rexclude="Thumbs.db" --rexclude=".AppleDouble" --rexclude=".DS_Store" sync $src $dst >> $LOGFILE
+    notify "FINISHED: Synchronizing files from $src to $dst"    
+
+    # Force set everything to private
+    s3cmd --config=$S3CFG_FILE --acl-private --recursive setacl $src >> $LOGFILE
+    notify "FINISHED: Setting permissions to private on $src"    
+}
+
+cleanup() {
+    rm $pidfile
+}
+
+sanity_check() {
+    # Check that the log folder exists
+    if [ ! -d "$LOG_PATH" ];then
+        mkdir -p $LOG_PATH
+    fi
+    if [ ! -d "$RUN_PATH" ];then
+        mkdir -p $RUN_PATH
+    fi
+}
+
+sanity_check
+initialize
+
+# Then pull content from S3
+src="s3://$S3_BUCKET/bin/"
+dst="$MEDIA_STORE_PATH/"
+run_sync
+
+cleanup

etc/cron/sync_logs

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+##
+# Synchronize log files up to S3
+##
+
+#------------------------------------------------
+# Configuration
+#------------------------------------------------
+# Application name
+APPNAME="civildebatewall"
+
+# Email recipients. Blank list to disable email sending
+# Comma-delimited list to be sent to mailx
+RECIPIENTS=
+# Logfile path MUST exist 
+RUN_PATH="/var/run/$APPNAME"
+pidfile="/$RUN_PATH/sync_logs.pid"
+
+LOG_PATH="/var/log/$APPNAME"
+LOGFILE="$LOG_PATH/sync_logs.log"
+PROCESS_SUBJECT="CDW Logs Sync Status"
+
+S3_BUCKET="$APPNAME"
+MEDIA_STORE_PATH=/c/home/cdwmedia
+S3CFG_FILE=/etc/cdw.s3cfg
+#------------------------------------------------
+
+usage() {
+  echo "$0 "
+}
+
+notify() {
+    TIMESTAMP=`date -u +%Y%m%d%H%M%S`
+    echo -e "($TIMESTAMP) $1" >> $LOGFILE
+    # Tell someone that there's something going on
+    if [ ! -z "$RECIPIENTS" ];then
+        echo -e "$1" | mailx -s "$PROCESS_SUBJECT" $RECIPIENTS
+    fi
+}
+
+initialize() { 
+    # Create a process-file so that we don't run the process multiple times
+    if [ -e $pidfile ]; then
+        pid=`cat $pidfile`
+        if kill -0 &>1 > /dev/null $pid; then
+            echo "Already running"
+            exit 1
+        else
+            rm $pidfile
+        fi
+    fi
+    echo $$ > $pidfile
+}
+
+run_sync() { 
+    notify "STARTED: Synchronizing media from $src to $dst"
+    s3cmd --config=$S3CFG_FILE --acl-private --no-delete-removed --recursive --rexclude="Thumbs.db" --rexclude=".AppleDouble" --rexclude=".DS_Store" sync $src $dst >> $LOGFILE
+    notify "FINISHED: Synchronizing files from $src to $dst"    
+
+    # Force set everything to private
+    s3cmd --config=$S3CFG_FILE --acl-private --recursive setacl $dst >> $LOGFILE
+    notify "FINISHED: Setting permissions to private on $src"    
+}
+
+cleanup() {
+    rm $pidfile
+}
+
+sanity_check() {
+    # Check that the log folder exists
+    if [ ! -d "$LOG_PATH" ];then
+        mkdir -p $LOG_PATH
+    fi
+    if [ ! -d "$RUN_PATH" ];then
+        mkdir -p $RUN_PATH
+    fi
+}
+
+sanity_check
+initialize
+
+# Then pull content from S3
+src="$MEDIA_STORE_PATH/var/"
+dst="s3://$S3_BUCKET/var/"
+run_sync
+
+cleanup

etc/cron/sync_media

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+##
+# Synchronize the media files
+##
+
+#------------------------------------------------
+# Configuration
+#------------------------------------------------
+# Application name
+APPNAME="civildebatewall"
+
+# Email recipients. Blank list to disable email sending
+# Comma-delimited list to be sent to mailx
+RECIPIENTS=
+# Logfile path MUST exist 
+RUN_PATH="/var/run/$APPNAME"
+pidfile="/$RUN_PATH/sync_media.pid"
+
+LOG_PATH="/var/log/$APPNAME"
+LOGFILE="$LOG_PATH/sync_media.log"
+PROCESS_SUBJECT="CDW Media Sync Status"
+
+S3_BUCKET="$APPNAME"
+MEDIA_STORE_PATH=/c/home/cdwmedia
+S3CFG_FILE=/etc/cdw.s3cfg
+#------------------------------------------------
+
+usage() {
+  echo "$0 "
+}
+
+notify() {
+    TIMESTAMP=`date -u +%Y%m%d%H%M%S`
+    echo -e "($TIMESTAMP) $1" >> $LOGFILE
+    # Tell someone that there's something going on
+    if [ ! -z "$RECIPIENTS" ];then
+        echo -e "$1" | mailx -s "$PROCESS_SUBJECT" $RECIPIENTS
+    fi
+}
+
+initialize() { 
+    # Create a process-file so that we don't run the process multiple times
+    if [ -e $pidfile ]; then
+        pid=`cat $pidfile`
+        if kill -0 &>1 > /dev/null $pid; then
+            echo "Already running"
+            exit 1
+        else
+            rm $pidfile
+        fi
+    fi
+    echo $$ > $pidfile
+}
+
+run_sync() { 
+    notify "STARTED: Synchronizing media from $src to $dst"
+    s3cmd --config=$S3CFG_FILE --acl-public --no-delete-removed --rexclude="Thumbs.db" --rexclude=".AppleDouble" --rexclude=".DS_Store" --recursive sync $src $dst >> $LOGFILE
+    notify "FINISHED: Synchronizing media from $src to $dst"    
+}
+
+cleanup() {
+    rm $pidfile
+}
+
+sanity_check() {
+    # Check to see if the pre-requisites pass
+    if [ ! -e "$MEDIA_STORE_PATH/media/" ];then
+        notify "Target folder $MEDIA_STORE_PATH/media does not exist! Cannot proceed!"
+        exit 1
+    fi
+
+    # Check that the log folder exists
+    if [ ! -d "$LOG_PATH" ];then
+        mkdir -p $LOG_PATH
+    fi
+    if [ ! -d "$RUN_PATH" ];then
+        mkdir -p $RUN_PATH
+    fi
+}
+
+sanity_check
+initialize
+
+# First push content to S3
+src="$MEDIA_STORE_PATH/media/"
+dst="s3://$S3_BUCKET/media/"
+run_sync
+
+# Then pull content from S3
+src="s3://$S3_BUCKET/media/"
+dst="$MEDIA_STORE_PATH/"
+run_sync
+
+cleanup