Web container fixes and backups templates (#301)

* Add osmcha container

* Add chartpress config for osmcha

* Update start point in osmcha

* Add osmcha database helm template

* Update osmcha db config

* Add helm template for osmcha-web

* Rename files

* Update osmcha db helm template

* Update osmcha-web deployment template

* Add osm server env var for osmcha

* Update OSMCha Dockerfile and start script

* Remove pip uninstall command

* Update env vars for osmcha web

* Update env var

* Open port 81 for cgimap

* Fix service - web config

* Add cgimap endpoint

* Update to web-cgimap endpoint

* Update cgimap endpoint

* Build osmcha to ohm config + add env vars

* Fix git clone url

* Use npm install instead of yarn

* Install yarnpkg and use it instead of npm

* Try to fix error on yarn command

* Fix yarn instalation for osmcha

* Add extra osmcha env vars  in deployent

* Fix the path for static files - osmcha

* Build osmcha app

* Remove cgimap from web config

* Add multiple backup database jobs

* Add supervisor + nginx

* Update docker container for backup db

* Update Script to backup and restore db

* Update path for bucket and env vars for backups db

* Change port for osmcha web

* Add env vars for backups

* Update bakup db template

* Add function to clean up backup older than x days

* Update target port for osmcha

* Update env vars for backup db

* Update log and config for backups

* Add node selector for backups config

* Raising shared memory limit for web api

* Install wget for db-bakups container

* Store results of restoring db

* Fix name for backup

* Set file name

* osm-seed/values.yaml

* Add sharedMemorySize for db

* Remove comment

* Add liveness using bash

chartpress.yaml

@@ -1,44 +1,48 @@
 charts:
-  - name: osm-seed
-    imagePrefix: developmentseed/osmseed-
-    repo:
-       git: developmentseed/osm-seed-chart
-       published: https://devseed.com/osm-seed-chart
+- name: osm-seed
+  imagePrefix: developmentseed/osmseed-
+  repo:
+    git: developmentseed/osm-seed-chart
+    published: https://devseed.com/osm-seed-chart
 
-    images:
-      web:
-        valuesPath: web.image
-      db:
-        valuesPath: db.image
-      backup-restore:
-        valuesPath: dbBackupRestore.image
-      planet-dump:
-        valuesPath: planetDump.image
-      full-history:
-        valuesPath: fullHistory.image
-      replication-job:
-        valuesPath: replicationJob.image
-      populate-apidb:
-        valuesPath: populateApidb.image
-      osm-processor:
-        valuesPath: osmProcessor.image
-      tiler-db:
-        valuesPath: tilerDb.image
-      tiler-imposm:
-        valuesPath: tilerImposm.image
-      tiler-server:
-        valuesPath: tilerServer.image
-      tasking-manager-api:
-        valuesPath: tmApi.image
-      # tiler-visor:
-      #   valuesPath: tilerVisor.image
-      nominatim:
-        valuesPath: nominatimApi.image
-      overpass-api:
-        valuesPath: overpassApi.image
-      taginfo:
-        valuesPath: taginfo.image
-      osm-simple-metrics:
-        valuesPath: osmSimpleMetrics.image
-      changeset-replication-job:
-        valuesPath: changesetReplicationJob.image
+  images:
+    web:
+      valuesPath: web.image
+    db:
+      valuesPath: db.image
+    backup-restore:
+      valuesPath: dbBackupRestore.image
+    planet-dump:
+      valuesPath: planetDump.image
+    full-history:
+      valuesPath: fullHistory.image
+    replication-job:
+      valuesPath: replicationJob.image
+    populate-apidb:
+      valuesPath: populateApidb.image
+    osm-processor:
+      valuesPath: osmProcessor.image
+    tiler-db:
+      valuesPath: tilerDb.image
+    tiler-imposm:
+      valuesPath: tilerImposm.image
+    tiler-server:
+      valuesPath: tilerServer.image
+    tasking-manager-api:
+      valuesPath: tmApi.image
+    # tiler-visor:
+    #   valuesPath: tilerVisor.image
+    nominatim:
+      valuesPath: nominatimApi.image
+    overpass-api:
+      valuesPath: overpassApi.image
+    taginfo:
+      valuesPath: taginfo.image
+    osm-simple-metrics:
+      valuesPath: osmSimpleMetrics.image
+    changeset-replication-job:
+      valuesPath: changesetReplicationJob.image
+    osmcha-web:
+      valuesPath: osmchaWeb.image
+    osmcha-db:
+      valuesPath: osmchaDb.image

compose/db-backup-restore.yml

@@ -9,13 +9,14 @@ services:
       context: ../images/backup-restore
       dockerfile: Dockerfile
     volumes:
-      - ../data/db-backup-restore-data:/mnt/data
+    - ../data/db-backup-restore-data:/mnt/data
     command: >
       /bin/bash -c "
         echo 'Set cron job for backing up DB, every 4 minutes';
         while :; do echo 'Creating DB backup...'; /start.sh; sleep 4m; done;
       "
     env_file:
-      - ../envs/.env.db
-      - ../envs/.env.db-utils
-      - ../envs/.env.cloudprovider
+    - ../envs/.env.db
+    - ../envs/.env.osmcha
+    - ../envs/.env.db-utils
+    - ../envs/.env.cloudprovider

compose/osmcha.yml

@@ -0,0 +1,24 @@
+version: '3'
+services:
+  osmcha-db:
+    platform: linux/amd64
+    image: osmseed-osmcha-db:v1
+    build:
+      context: ../images/osmcha-db
+      dockerfile: Dockerfile
+    ports:
+    - '5432:5432'
+    volumes:
+    - ../data/osmcha-db-data:/var/lib/postgresql/data
+    env_file:
+    - ../envs/.env.osmcha
+  osmcha-web:
+    platform: linux/amd64
+    image: osmseed-osmcha-web:v1
+    build:
+      context: ../images/osmcha-web
+      dockerfile: Dockerfile
+    ports:
+    - '8000:8000'
+    env_file:
+    - ../envs/.env.osmcha

envs/.env.osmcha.example

@@ -0,0 +1,24 @@
+POSTGRES_DB=osmcha
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=1234
+PGHOST=osmcha-db
+DJANGO_DEBUG=true
+DJANGO_SECRET_KEY=abc
+OAUTH_OSM_KEY=xya
+OAUTH_OSM_SECRET=xya
+DJANGO_ANON_USER_THROTTLE_RATE=30/min
+DJANGO_COMMON_USER_THROTTLE_RATE=180/min
+DJANGO_NON_STAFF_USER_THROTTLE_RATE=3/min
+OSMCHA_URL=public-url
+OAUTH_REDIRECT_URI=$OSMCHA_URL/oauth-landing.html
+DJANGO_ENABLE_CHANGESET_COMMENTS=False
+DJANGO_OSM_COMMENTS_API_KEY=""
+DJANGO_ROOT=/app
+DJANGO_SETTINGS_MODULE=config.settings.local
+C_FORCE_ROOT=True
+REACT_APP_OSM_URL=https://www.openhistoricalmap.org
+REACT_APP_OSM_API=https://www.openhistoricalmap.org/api/0.6
+REACT_APP_OVERPASS_BASE=https://overpass-api.openhistoricalmap.org/api/interpreter
+REACT_APP_NOMINATIM_URL=https://www.openhistoricalmap.org/geocoder/search_osm_nominatim
+REACT_APP_PRODUCTION_API_URL=$OSMCHA_URL/api/v1
+REACT_APP_MAPBOX_ACCESS_TOKEN=''

images/backup-restore/Dockerfile

@@ -1,19 +1,23 @@
-FROM python:3.9.9
-RUN apt-get update
-RUN apt-get install -y \
-    curl \
-    postgresql-client
+FROM python:3.9.9-slim
 
-# Install AWS CLI
-RUN pip install awscli
+# Install Postgres client, GCP CLI, and Azure CLI
+RUN apt-get update \
+    && apt-get install -y curl apt-transport-https lsb-release gnupg \
+    && echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+    && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+    && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \
+    && apt-get update \
+    && apt-get install -y postgresql-client-14 wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install GCP CLI
-RUN curl -sSL https://sdk.cloud.google.com | bash
-RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
+RUN curl -sSL https://sdk.cloud.google.com | bash \
+    && ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
+RUN rm -rf /root/google-cloud-sdk/.install/.backup
 
-# Install Azure CLI
-RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+# Install AWS CLI
+RUN pip install awscli
 
-VOLUME /mnt/data
 COPY ./start.sh /
-CMD /start.sh
+CMD ["/start.sh"]

images/backup-restore/start.sh

@@ -1,65 +1,111 @@
 #!/usr/bin/env bash
+set -e
 export PGPASSWORD=$POSTGRES_PASSWORD
-export VOLUME_DIR=/mnt/data
+# Upload files
+cloudStorageOps() {
+	local LOCAL_STATE_FILE=state.txt
+	local filepath=$1
+	local cloudpath=$2
 
-date=$(date '+%y%m%d_%H%M')
-local_backupFile=$VOLUME_DIR/osmseed-db-${date}.sql.gz
-cloud_backupFile=database/osmseed-db-${date}.sql.gz
-stateFile=$VOLUME_DIR/state.txt
-restoreFile=$VOLUME_DIR/backup.sql.gz
-
-echo "Start...$DB_ACTION action"
-# Backing up DataBase
-if [ "$DB_ACTION" == "backup" ]; then
-	# Backup database and make maximum compression at the slowest speed
-	pg_dump -h $POSTGRES_HOST -U $POSTGRES_USER $POSTGRES_DB | gzip -9 >$local_backupFile
-
-	# AWS
-	if [ "$CLOUDPROVIDER" == "aws" ]; then
-		echo "$AWS_S3_BUCKET/$cloud_backupFile" > $stateFile
-		# Upload db backup file
-		aws s3 cp $local_backupFile $AWS_S3_BUCKET/$cloud_backupFile
-		# Upload state.txt file
-		aws s3 cp $stateFile $AWS_S3_BUCKET/database/state.txt
-	fi
-
-	# GCP
-	if [ "$CLOUDPROVIDER" == "gcp" ]; then
-		echo "$GCP_STORAGE_BUCKET/$cloud_backupFile" > $stateFile
-		# Upload db backup file
-		gsutil cp $local_backupFile $GCP_STORAGE_BUCKET/$cloud_backupFile
-		# Upload state.txt file
-		gsutil cp $stateFile $GCP_STORAGE_BUCKET/database/state.txt
-	fi
-
-	# Azure
-	if [ "$CLOUDPROVIDER" == "azure" ]; then
-		# Save the path file
-		echo "blob://$AZURE_STORAGE_ACCOUNT/$AZURE_CONTAINER_NAME/$cloud_backupFile" > $stateFile
-		# Upload db backup file
+	case "${CLOUDPROVIDER}" in
+	aws)
+		aws s3 cp ${filepath} s3://${AWS_S3_BUCKET}/${cloudpath}
+		echo s3://${AWS_S3_BUCKET}/${cloudpath} >${LOCAL_STATE_FILE}
+		aws s3 cp ${LOCAL_STATE_FILE} s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/state.txt
+		;;
+	gcp)
+		gsutil cp ${filepath} gs://${GCP_STORAGE_BUCKET}/${cloudpath}
+		echo "gs://${GCP_STORAGE_BUCKET}/${CLOUD_BACKUP_FILE}" >${LOCAL_STATE_FILE}
+		gsutil cp ${LOCAL_STATE_FILE} gs://${GCP_STORAGE_BUCKET}/${BACKUP_CLOUD_FOLDER}/state.txt
+		;;
+	azure)
 		az storage blob upload \
-			--container-name $AZURE_CONTAINER_NAME \
-			--file $local_backupFile \
-			--name $cloud_backupFile \
+			--container-name ${AZURE_CONTAINER_NAME} \
+			--file ${filepath} \
+			--name ${cloudpath} \
 			--output table
-		# Upload state.txt file
+		echo "blob://${AZURE_STORAGE_ACCOUNT}/${AZURE_CONTAINER_NAME}/${CLOUD_BACKUP_FILE}" >${LOCAL_STATE_FILE}
 		az storage blob upload \
-			--container-name $AZURE_CONTAINER_NAME \
-			--file $stateFile \
-			--name database/state.txt \
+			--container-name ${AZURE_CONTAINER_NAME} \
+			--file ${LOCAL_STATE_FILE} \
+			--name ${BACKUP_CLOUD_FOLDER}/state.txt \
 			--output table
+		;;
+	esac
+}
+
+backupDB() {
+	local LOCAL_BACKUP_FILE=${BACKUP_CLOUD_FILE}.sql.gz
+	local CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}.sql.gz"
+	if [ "$SET_DATE_AT_NAME" == "true" ]; then
+		local CURRENT_DATE=$(date '+%Y%m%d-%H%M')
+		LOCAL_BACKUP_FILE="${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.sql.gz"
+		CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.sql.gz"
 	fi
-fi
 
-# Restoring DataBase
-if [ "$DB_ACTION" == "restore" ]; then
-	# AWS
-	flag=true
+	# Backup database with max compression
+	echo "Backing up DB ${POSTGRES_DB} into ${LOCAL_BACKUP_FILE}"
+	pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} ${POSTGRES_DB} | gzip -9 >${LOCAL_BACKUP_FILE}
+
+	# Handle cloud storage based on the provider
+	cloudStorageOps "${LOCAL_BACKUP_FILE}" "${CLOUD_BACKUP_FILE}"
+}
+
+restoreDB() {
+	local CURRENT_DATE=$(date '+%Y%m%d-%H%M')
+	local RESTORE_FILE="backup.sql.gz"
+	local LOG_RESULT_FILE="restore_results-${CURRENT_DATE}.log"
+	local flag=true
+
 	while "$flag" = true; do
-	pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
-	flag=false
-	wget -O $restoreFile $RESTORE_URL_FILE
-	gunzip <$restoreFile | psql -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB
-	echo " Import data to  $POSTGRES_DB has finished ..."
+		pg_isready -h ${POSTGRES_HOST} -p 5432 >/dev/null 2>&2 || continue
+		flag=false
+		wget -O ${RESTORE_FILE} ${RESTORE_URL_FILE}
+		echo "Restoring ${RESTORE_URL_FILE} in ${POSTGRES_DB}"
+		gunzip -c <${RESTORE_FILE} | psql -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -d ${POSTGRES_DB} | tee ${LOG_RESULT_FILE}
+		aws s3 cp ${LOG_RESULT_FILE} s3://${AWS_S3_BUCKET}/${LOG_RESULT_FILE}
+		echo "Import data to ${POSTGRES_DB} has finished ..."
+	done
+}
+
+delete_old_s3_files() {
+	# Use RETENTION_DAYS from environment variable or default to 30 days
+	if [ -z "${RETENTION_DAYS}" ]; then
+		DAYS_AGO=30
+	else
+		DAYS_AGO="${RETENTION_DAYS}"
+	fi
+
+	echo "Files older than $DAYS_AGO days will be deleted."
+	echo "Processing s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/"
+	TARGET_DATE=$(date -d "${DAYS_AGO} days ago" +%Y-%m-%d)
+	aws s3 ls "s3://${AWS_S3_BUCKET}/${BACKUP_CLOUD_FOLDER}/" --recursive | while read -r line; do
+		FILE_DATE=$(echo "$line" | awk '{print $1}')
+		FILE_PATH=$(echo "$line" | awk '{print $4}')
+		if [[ "$FILE_DATE" < "$TARGET_DATE" && ! -z "$FILE_PATH" ]]; then
+			echo "Deleting ${FILE_PATH} which was modified on ${FILE_DATE}"
+			aws s3 rm "s3://${AWS_S3_BUCKET}/${FILE_PATH}"
+		fi
 	done
+}
+
+# Main logic
+case "${DB_ACTION}" in
+backup)
+	backupDB
+	;;
+restore)
+	restoreDB
+	;;
+*)
+	echo "Unknown action: ${DB_ACTION}"
+	exit 1
+	;;
+esac
+
+# Check for the CLEAN_BACKUPS var
+if [ "$CLEANUP_BACKUPS" == "true" ]; then
+	delete_old_s3_files
+else
+	echo "CLEANUP_BACKUPS is not set to true. Skipping deletion."
 fi

images/osmcha-db/Dockerfile

@@ -0,0 +1,9 @@
+FROM postgres:14
+# Install PostGIS
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    postgis \
+    postgresql-14-postgis-3 && \
+    rm -rf /var/lib/apt/lists/*
+# Initialize PostGIS
+COPY init-postgis.sql /docker-entrypoint-initdb.d/

images/osmcha-db/README.md

@@ -0,0 +1,12 @@
+# Postgres  14 for OpenStreetMap Changeset Analyzer (OSMCha)
+
+## Build
+```sh
+docker compose -f <PathTo>/compose/osmcha.yml build
+```
+
+## Run
+
+```sh
+docker compose -f <PathTo>/compose/osmcha.yml up
+```

images/osmcha-db/init-postgis.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS postgis;