Clean up start and stop scripts

api/puzzle-massive-api.service.sh

@@ -10,12 +10,14 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=API puzzle-massive instance
-After=network.target
+After=multi-user.target
 
 [Service]
+Type=exec
 User=dev
 Group=dev
 WorkingDirectory=$SRCDIR
@@ -26,6 +28,7 @@ else
 echo "ExecStart=${SRCDIR}bin/puzzle-massive-api serve"
 fi
 cat <<HERE
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

api/puzzle-massive-artist.service.sh

@@ -9,17 +9,19 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=artist puzzle-massive instance
-After=network.target
-After=puzzle-massive-api.service
+After=multi-user.target
 
 [Service]
+Type=exec
 User=dev
 Group=dev
 WorkingDirectory=$SRCDIR
 ExecStart=${SRCDIR}bin/puzzle-massive-artist
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

api/puzzle-massive-backup-db.service.sh

@@ -10,13 +10,14 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=Puzzle Massive backup database
-After=network.target
-After=puzzle-massive-api.service
+After=multi-user.target
 
 [Service]
+Type=exec
 User=dev
 Group=dev
 WorkingDirectory=$SRCDIR

api/puzzle-massive-backup-db.timer.sh

@@ -9,13 +9,15 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=Run puzzle-massive-backup-db every 24 hours
 Requires=puzzle-massive-backup-db.service
 
 [Timer]
 Unit=puzzle-massive-backup-db.service
+# TODO: OnActiveSec?
 OnActiveSec=1min
 HERE
 if test "${ENVIRONMENT}" == 'development'; then

api/puzzle-massive-cache-purge.service.sh

@@ -17,16 +17,17 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=Purges the cached URLs listed in urls-to-purge.txt
-After=network.target
+After=multi-user.target
 
 [Service]
+Type=exec
 User=www-data
 Group=www-data
 WorkingDirectory=$SRCDIR
-Type=simple
 ExecStart=${SRCDIR}bin/purge_nginx_cache_file.sh ${PURGEURLLIST} ${origin_server} ${CACHEDIR}
 
 [Install]

api/puzzle-massive-janitor.service.sh

@@ -9,17 +9,19 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=janitor puzzle-massive instance
-After=network.target
-After=puzzle-massive-api.service
+After=multi-user.target
 
 [Service]
+Type=exec
 User=dev
 Group=dev
 WorkingDirectory=$SRCDIR
 ExecStart=${SRCDIR}bin/puzzle-massive-janitor
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

api/puzzle-massive-publish.service.sh

@@ -10,12 +10,14 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=Publish puzzle-massive instance
-After=network.target
+After=multi-user.target
 
 [Service]
+Type=exec
 User=dev
 Group=dev
 WorkingDirectory=$SRCDIR
@@ -26,6 +28,7 @@ else
 echo "ExecStart=${SRCDIR}bin/puzzle-massive-publish serve"
 fi
 cat <<HERE
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

api/puzzle-massive-scheduler.service.sh

@@ -9,17 +9,19 @@ cat <<HERE
 
 # File generated from $0
 # on ${DATE}
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html
 
 [Unit]
 Description=scheduler puzzle-massive instance
-After=network.target
-After=puzzle-massive-api.service
+After=multi-user.target
 
 [Service]
+Type=exec
 User=dev
 Group=dev
 WorkingDirectory=$SRCDIR
 ExecStart=${SRCDIR}bin/puzzle-massive-scheduler
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

bin/appctl.sh

@@ -1,45 +1,63 @@
 #!/usr/bin/env bash
 # not setting the -e or the option -o pipefail here. That way if a service fails
 # to stop the others will continue to stop.
-set -u
 
 COMMAND=$1
 
+SKIP_BACKUP=0
+if [ -n "${2-}" ]; then
+SKIP_BACKUP=$2
+fi
+
 # Simple convenience script to control the apps.
 
 # Switch out nginx puzzle-massive.conf to puzzle-massive--down.conf to
 # show the down page.
 if test "${COMMAND}" == 'start'; then
     rm -f /etc/nginx/sites-enabled/puzzle-massive--down.conf;
     ln -sf /etc/nginx/sites-available/puzzle-massive.conf /etc/nginx/sites-enabled/puzzle-massive.conf;
+    # Start the puzzle-massive-api first since other services depend on it.
+    systemctl start puzzle-massive-api;
+    systemctl start puzzle-massive-stream;
+    systemctl start puzzle-massive-publish;
+    systemctl start puzzle-massive-scheduler;
+    systemctl start puzzle-massive-chill;
+    systemctl start puzzle-massive-janitor;
+    systemctl start puzzle-massive-artist;
+    systemctl start puzzle-massive-backup-db.timer;
+    systemctl reload nginx;
+
 elif test "${COMMAND}" == 'stop'; then
     rm -f /etc/nginx/sites-enabled/puzzle-massive.conf;
     ln -sf /etc/nginx/sites-available/puzzle-massive--down.conf /etc/nginx/sites-enabled/puzzle-massive--down.conf;
-    # Stop the stream first before other apps.
+    systemctl stop puzzle-massive-artist;
+    systemctl stop puzzle-massive-chill;
+    systemctl stop puzzle-massive-publish;
+    systemctl stop puzzle-massive-scheduler;
+    systemctl stop puzzle-massive-backup-db.timer;
+    systemctl stop puzzle-massive-janitor;
     echo "Waiting for the puzzle-massive-stream connections...";
     systemctl stop puzzle-massive-stream;
-    systemctl stop puzzle-massive-chill;
-fi
-if test "${COMMAND}" == 'stop'; then
-    # Now that the app isn't accepting any outside requests; transfer piece data
-    # to SQL database from redis database.
-    su dev -c "./bin/python api/api/jobs/convertPiecesToDB.py";
-fi
+    if [ "$SKIP_BACKUP" != "-f" ]; then
+      su dev -c "bin/backup.sh -d /home/dev -c"
+    fi
+    # Stop the puzzle-massive-api last since other services depend on it.
+    systemctl stop puzzle-massive-api;
+    systemctl reload nginx;
 
-# Skipping the puzzle-massive-cache-purge.service since it is activated by path
-# Skipping divulger since it is not needed at the moment.
-# puzzle-massive-divulger \
-for app in puzzle-massive-chill \
-  puzzle-massive-api \
-  puzzle-massive-publish \
-  puzzle-massive-stream \
-  puzzle-massive-artist \
-  puzzle-massive-scheduler \
-  puzzle-massive-backup-db.timer \
-  puzzle-massive-janitor;
-do
-  echo "";
-  echo "systemctl $COMMAND $app;";
-  echo "----------------------------------------";
-  systemctl "$COMMAND" "$app" | cat;
-done;
+else
+  for app in puzzle-massive-chill \
+    puzzle-massive-api \
+    puzzle-massive-publish \
+    puzzle-massive-stream \
+    puzzle-massive-artist \
+    puzzle-massive-scheduler \
+    puzzle-massive-backup-db.timer \
+    puzzle-massive-janitor;
+  do
+    echo "";
+    echo "systemctl $COMMAND $app;";
+    echo "----------------------------------------";
+    systemctl "$COMMAND" "$app" | cat;
+  done;
+fi

bin/backup.sh

@@ -64,11 +64,15 @@ echo "Running one-off scheduler tasks to clean up any batched data";
 if [ -n "${1-}" ]; then
 DBDUMPFILE="$1";
 else
-    if [ -n "${WEEKDAY_BACKUP-}" ]; then
-        DBDUMPFILE="db-$(date --utc '+%a').dump.gz";
-    else
-        DBDUMPFILE="db-$(date --iso-8601 --utc).dump.gz";
-    fi;
+  if [ -n "${WEEKDAY_BACKUP-}" ]; then
+    DBDUMPFILE="db-$(date --utc '+%a').dump.gz";
+  else
+    DBDUMPFILE="db-$(date --iso-8601 --utc).dump.gz";
+    if [ -e "${BACKUP_DIRECTORY}/${DBDUMPFILE}" ]; then
+      DBDUMPFILE_bak="db-$(date --iso-8601 --utc).bak.dump.gz";
+      mv --backup=numbered "${BACKUP_DIRECTORY}/${DBDUMPFILE}" "${BACKUP_DIRECTORY}/${DBDUMPFILE_bak}"
+    fi
+  fi;
 fi;
 
 echo "";

bin/install.sh

@@ -178,39 +178,39 @@ fi
 cp resources/imagemagick-policy.xml ${IMAGEMAGICK_POLICY}
 
 mkdir -p "${SYSTEMDDIR}"
-cp chill/puzzle-massive-chill.service "${SYSTEMDDIR}"
-systemctl start puzzle-massive-chill || echo "can't start service"
-systemctl enable puzzle-massive-chill || echo "can't enable service"
-
 cp api/puzzle-massive-api.service "${SYSTEMDDIR}"
 systemctl start puzzle-massive-api || echo "can't start service"
 systemctl enable puzzle-massive-api || echo "can't enable service"
 
+cp stream/puzzle-massive-stream.service "${SYSTEMDDIR}"
+systemctl start puzzle-massive-stream || echo "can't start service"
+systemctl enable puzzle-massive-stream || echo "can't enable service"
+
 cp api/puzzle-massive-publish.service "${SYSTEMDDIR}"
 systemctl start puzzle-massive-publish || echo "can't start service"
 systemctl enable puzzle-massive-publish || echo "can't enable service"
 
-cp api/puzzle-massive-artist.service "${SYSTEMDDIR}"
-systemctl start puzzle-massive-artist || echo "can't start service"
-systemctl enable puzzle-massive-artist || echo "can't enable service"
+cp api/puzzle-massive-scheduler.service "${SYSTEMDDIR}"
+systemctl start puzzle-massive-scheduler || echo "can't start service"
+systemctl enable puzzle-massive-scheduler || echo "can't enable service"
+
+cp chill/puzzle-massive-chill.service "${SYSTEMDDIR}"
+systemctl start puzzle-massive-chill || echo "can't start service"
+systemctl enable puzzle-massive-chill || echo "can't enable service"
 
 cp api/puzzle-massive-janitor.service "${SYSTEMDDIR}"
 systemctl start puzzle-massive-janitor || echo "can't start service"
 systemctl enable puzzle-massive-janitor || echo "can't enable service"
 
+cp api/puzzle-massive-artist.service "${SYSTEMDDIR}"
+systemctl start puzzle-massive-artist || echo "can't start service"
+systemctl enable puzzle-massive-artist || echo "can't enable service"
+
 # Skipping divulger since it is not needed at the moment.
 #cp divulger/puzzle-massive-divulger.service "${SYSTEMDDIR}"
 #systemctl start puzzle-massive-divulger || echo "can't start service"
 #systemctl enable puzzle-massive-divulger || echo "can't enable service"
 
-cp stream/puzzle-massive-stream.service "${SYSTEMDDIR}"
-systemctl start puzzle-massive-stream || echo "can't start service"
-systemctl enable puzzle-massive-stream || echo "can't enable service"
-
-cp api/puzzle-massive-scheduler.service "${SYSTEMDDIR}"
-systemctl start puzzle-massive-scheduler || echo "can't start service"
-systemctl enable puzzle-massive-scheduler || echo "can't enable service"
-
 cp api/puzzle-massive-cache-purge.path "${SYSTEMDDIR}"
 cp api/puzzle-massive-cache-purge.service "${SYSTEMDDIR}"
 systemctl start puzzle-massive-cache-purge.path || echo "can't start service"
@@ -224,3 +224,5 @@ systemctl start puzzle-massive-backup-db.timer || echo "can't start service"
 systemctl enable puzzle-massive-backup-db.timer || echo "can't enable service"
 systemctl start puzzle-massive-backup-db.service || echo "can't start service"
 systemctl enable puzzle-massive-backup-db.service || echo "can't enable service"
+
+systemctl reload nginx;