Merge pull request #242 from OpenHistoricalMap/staging

Update Apache  cgimap config and  config for db backups

.github/workflows/chartpress.yaml

@@ -15,14 +15,14 @@ jobs:
     - name: Setup python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.6'
+        python-version: '3.7'
 
     - name: Setup git
       run: git config --global user.email "[email protected]" && git config --global user.name "Github Action"
 
     - name: Install Chartpress
       run: |
-        pip install -v chartpress && pip install six
+        pip install chartpress six ruamel.yaml
     - name: Run Chartpress
       run: chartpress --push
       env:

images/db/Dockerfile

@@ -1,5 +1,8 @@
 FROM postgres:11
 RUN rm /etc/apt/sources.list.d/pgdg.list
+RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list
+RUN sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list 
+RUN sed -i '/stretch-updates/d' /etc/apt/sources.list 
 RUN apt-get update && apt-get -y install apt-transport-https
 RUN echo "deb [ signed-by=/usr/local/share/keyrings/postgres.gpg.asc ] https://apt-archive.postgresql.org/pub/repos/apt/ stretch-pgdg main 11" >/etc/apt/sources.list.d/pgdg.list
 RUN apt-get update && apt-get install -y \

images/tasking-manager-api/Dockerfile

@@ -1,7 +1,7 @@
-FROM python:3.7-alpine
+FROM python:3.7-slim
 
-RUN apk update && \
-    apk add git
+RUN apt update && \
+    apt install -y git
 
 ENV workdir /usr/src/app
 
@@ -11,21 +11,21 @@ RUN cd $workdir && git checkout -f f0df07174f4014365220af09187b5f941f9770b0
 WORKDIR $workdir
 
 # Setup backend dependencies
-RUN apk update && \
-    apk add \
-        gcc \
-        g++ \
-        make \
-        musl-dev \
-        libffi-dev \
-        python3-dev \
-        postgresql-dev \
-        geos-dev \
-        proj-util \
-        proj-dev
+RUN apt update && apt install -y \
+    gcc \
+    g++ \
+    make \
+    libffi-dev \
+    python3-dev \
+    libpq-dev \
+    proj-bin && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
 
+RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 RUN pip install apscheduler==3.7.0
+RUN pip install --upgrade markdown
 
 ## INITIALIZATION
 EXPOSE 5000

images/web/config/production.conf

@@ -19,7 +19,10 @@
     RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301]
 
     <Location />
-        CGIPassAuth Off
+        # For TM, do not use cgimap auth.
+        <If "%{HTTP_REFERER} !~ m#https://tasks(-\w+)?\.openhistoricalmap\.org/#">
+            CGIPassAuth On
+        </If>
     </Location>
 
     # ======Proxying traffic to CGImap====
@@ -32,7 +35,11 @@
     RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P]
     RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P]
     RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P]
-    # RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P]
+
+    # For changeset requests originating from TM, do not use cgimap.
+    RewriteCond %{REQUEST_METHOD} ^POST$
+    RewriteCond %{HTTP_REFERER} !^https://tasks(-\w+)?\.openhistoricalmap\.org/ [NC]
+    RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ fcgi://127.0.0.1:8000$0 [P]
 
     # Relax Apache security settings
     <Directory /var/www/public>

ohm/requirements.yaml

@@ -1,4 +1,4 @@
 dependencies:
 - name: osm-seed
-  version: '0.1.0-n794.h5b6149a'
+  version: '0.1.0-n806.h6f59175'
   repository: https://devseed.com/osm-seed-chart/

values.production.template.yaml

@@ -262,25 +262,47 @@ osm-seed:
   # ====================================================================================================
 
   dbBackupRestore:
-    enabled: true
-    schedule: '0 0 * * *'
-    nodeSelector:
+    cronjobs:
+    - name: web-db
       enabled: true
-      label_key: nodegroup_type
-      label_value: job
-    env:
-      DB_ACTION: backup
-      RESTORE_URL_FILE: https://osmseed-production-db.s3.us-east-1.amazonaws.com/database/osmseed-db-220803_1606.sql.gz?response-content-disposition=inline&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEDcaCXNhLWVhc3QtMSJIMEYCIQC6oYShK01LFCHNNOP0%2Bch5uKeAcpSmfRLxnaaygUYd5QIhAKt0OtxclFyJrX1qqdOfOgcylfVGBi%2FizMIIISenxD%2FbKoQDCJD%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEQAhoMNjE4MzgwMjQyMjQ3IgzdtXrtIBspySZIJ0Yq2AJ0RovZahua04PQ%2FBWVGLhmQPxRiN6Pe5evt2FUuVFdrSMGaqfxo3YAGTezOmviYoqtrHf7NZHRcHQx9RTB4Nk2rmeZfQ9xepB7semK0G3VbzDqsVTzglFDxk52xqpESYYf6vVbORyBDJvjvlCjUAXLH1PqOlj%2FFDd5A3UK3%2FCSJc9JfYh1LVkz4md6ug3mh0KYpUL0%2FipBVpQTcN0PCkVJTTPWOBXmn9qnYMUJg2LNRVMkgIr7AA2MmmSVzPhj%2FfnaHIXCCbHlJ5RA9IgpftC%2FEvTrP2cvgcV03YOWSq9sSMg%2B%2B0xDWn7stI9PsKoakHX%2B%2BtsxHimJ4uGTY8I7DsV915Kzgx1aRlJ5MYjOtbSuMOfhZP6FSwLGRTPw7qvS0kLFKHAwCYLsd7GF6%2FHadTbLSJ0Zx0EqPYVjyIKfwZ5fu204xUkGhCqBlHuQBQhoG%2FMxcnYf53NoSTChm6qXBjqyAkr7doHwSC9HTJYlIq7gixhkZ29BfaHp8jq0gIBW7aAEC3idFdN2ZQd9pHQAgN4fKYrOmC%2Bbf9njZH91%2BEvzwpEHXyho8E8lPSQJGJZnAquRzJkzJ23%2F4zF51Rf2WMSpmYMHyWXodLLP4yYj%2Bgk%2Bg%2FbUsDYhkq4%2B%2F5%2Bf7YLCmvi%2BdO%2B5GEZEa%2B1iPghxd%2BH8eQk9jUlebsI3fh01f%2Bv6TbzgGieM7eqXYeH0JTZDZGEJlPdbUzMHKcj6G6kuTX4nvD9YDPQzUYAoVwu2gwjvrjaHRaK8NKMlEC7JCIlsWrVzjVR%2FtXlBybrZ5rSdZNeTIDDof5lS8O%2F9qqyhTel4pObDPNosj6jCOVPe3geM5ZNZVbL4ATUdJ5oU3BcTm%2FMcerATv%2FysoVrskKNuDMjHoT3fkA%3D%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20220803T172639Z&X-Amz-SignedHeaders=host&X-Amz-Expires=2400&X-Amz-Credential=ASIAY76SVVVDSEAYVNUG%2F20220803%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=6ed260d9d775bbc6b0d4f5e08575552270c970a81afa2254f87b62c3b97730c2
-      CLEAN_BACKUPS: false
-      AWS_S3_BUCKET: {{PRODUCTION_DB_BACKUP_S3_BUCKET}}
-    resources:
+      schedule: '0 0 * * *'
+      env:
+        # backup/restore
+        DB_ACTION: backup
+        # Naming backup files
+        SET_DATE_AT_NAME: true
+        BACKUP_CLOUD_FOLDER: database/web-api-db
+        BACKUP_CLOUD_FILE: ohm-api-web-db
+        AWS_S3_BUCKET: {{PRODUCTION_DB_BACKUP_S3_BUCKET}}
+        # Clean up backups options
+        CLEANUP_BACKUPS: true
+        RETENTION_DAYS: '30'
+      resources:
+        enabled: false
+      nodeSelector:
+        enabled: true
+        label_key: nodegroup_type
+        label_value: job
+    - name: osmcha-db
       enabled: false
-      requests:
-        memory: "10Gi"
-        cpu: "3"
-      limits:
-        memory: "10Gi"
-        cpu: "3"
+      schedule: '0 0 * * *'
+      env:
+        # backup/restore
+        DB_ACTION: backup
+        # Naming backup files
+        SET_DATE_AT_NAME: 'true'
+        BACKUP_CLOUD_FOLDER: database/osmcha-db
+        BACKUP_CLOUD_FILE: osmseed-osmcha-db
+        AWS_S3_BUCKET: {{PRODUCTION_DB_BACKUP_S3_BUCKET}}
+        # Clean up backups options
+        CLEANUP_BACKUPS: true
+        RETENTION_DAYS: '30'
+      resources:
+        enabled: false
+      nodeSelector:
+        enabled: true
+        label_key: nodegroup_type
+        label_value: job
 
   # ====================================================================================================
   # Variables for tiler-db

values.staging.template.yaml

@@ -278,28 +278,61 @@ osm-seed:
     # ====================================================================================================
     # Variables for restoring the DB
     # ====================================================================================================
-
     dbBackupRestore:
-      enabled: true
-      schedule: '0 0 * * *'
-      nodeSelector:
+      cronjobs:
+      - name: web-db
         enabled: true
-        label_key: nodegroup_type
-        label_value: job
-      env:
-        RESTORE_URL_FILE: https://osmseed-production-db.s3.us-east-1.amazonaws.com/database/osmseed-db-220802_0000.sql.gz?response-content-disposition=inline&X-Amz-Security-Token=IQoJb3JpZ2luX2VjECUaCXNhLWVhc3QtMSJIMEYCIQDaL%2BYJE0LIZ8dHisSghlpKYtIbqPM3r7OnlXYySl34FAIhAOYcyVdY7WxoWN9n3z%2Fkh6JkJkkHFV91PJ6qS4ZEloizKvsCCH4QAhoMNjE4MzgwMjQyMjQ3Igy6Fd%2BxFomzH%2F38Evwq2ALJG0LvPhoMjgxOeGv0uZQPjzMewvy0IzjExaMcMruXQkSxgCjLZzb2xJTmfPHqcT1iAN%2FYPcm87CuLMU6TZ7QdKQTbhPUIgIXsdQ1oscp8095EgaQr0pY%2F7gDfDlxrUjGnEN3FOUCYucFNn9ORlL%2BlNXCkOsaIIg0ByUyLHFfus7%2B8plA0GfAHwkXdJRyx4hmSXFzAhnDT0mqH0YjuiS5DevP0ykCLmF3hT3Xbwd81t%2FZ07mm8hwzAxLibIc5YqjrxxnLAuZR9AXHCH%2F4Q0HosKMVWpry%2BsjRuebUbNgKsb8M4e73xjN18b40feIFazI6Ypo2yTfsMhNM2t1swa%2FpMq%2B2qg5NlDM%2FP1KzwT2yvZ3cZnZ2o3wFTxCYRcVG0nz%2FYA6iirYwezO2JK37aaeqUaMzTSAwM46S9S2qtcyuz5qrP2%2FULcrILtxw8FHVk5uMbqcXpBOQxMDCkpKaXBjqyAsdeRIzom8r6pAwrr7yzldM3bEpDdpfah38i%2BVGoq4Vg1EfrTTrRJ1zM481yAK9VZ%2Bm8pH52NRXsf3KBvawHxfKWRS5RiY4lhU%2BcX0WxtgCicxSILzoD9xmU6PoiuCy4DUojES6CznkY1LXe10ccQn4URFYgSpx04JSvO6RW82BWXardXPtZXuaxw4bPD%2FNnp3UxXMpv8fTS9shlA98xaS4m7XYTbpNl5uere0HkGy4vi5IZYRY%2B%2B909WU9ALsvSHBE%2BZLiIND8r0V1y9zBL8jD9pgtZghVpc%2FyeQ4XPxxtxEDSvJi0m3Vr7IQC7kOxB%2BuHSafbTKH4oGCDD5con4b7GCImy7N0T5EbVEAkmmKvW9hAMDYhF0A3NOR8fEvNtPKZve0bEl0gmMJ07quqmOsd56A%3D%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20220802T223259Z&X-Amz-SignedHeaders=host&X-Amz-Expires=1200&X-Amz-Credential=ASIAY76SVVVDZNYKO7KN%2F20220802%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=2a60ec3375c1abffc83a9067732a76ee02be3611bbe0e3e297987dd37fc58365
-        DB_ACTION: backup
-        CLEAN_BACKUPS: false
-        AWS_S3_BUCKET: s3://osmseed-staging
-      resources:
-        enabled: false
-        requests:
-          memory: "10Gi"
-          cpu: "3"
-        limits:
-          memory: "10Gi"
-          cpu: "3"
-
+        schedule: '0 0 * * *'
+        env:
+          # backup/restore
+          DB_ACTION: backup
+          # Naming backup files
+          SET_DATE_AT_NAME: true
+          BACKUP_CLOUD_FOLDER: database/web-api-db
+          BACKUP_CLOUD_FILE: ohm-api-web-db
+          AWS_S3_BUCKET: osmseed-staging
+          # Clean up backups options
+          CLEANUP_BACKUPS: true
+          RETENTION_DAYS: '30'
+        resources:
+          enabled: false
+          requests:
+            memory: '300Mi'
+            cpu: '0.5'
+          limits:
+            memory: '400Mi'
+            cpu: '0.6'
+        nodeSelector:
+          enabled: true
+          label_key: nodegroup_type
+          label_value: job
+      - name: osmcha-db
+        enabled: true
+        schedule: '0 0 * * *'
+        env:
+          # backup/restore
+          DB_ACTION: backup
+          # Naming backup files
+          SET_DATE_AT_NAME: 'true'
+          BACKUP_CLOUD_FOLDER: database/osmcha-db
+          BACKUP_CLOUD_FILE: osmseed-osmcha-db
+          AWS_S3_BUCKET: osmseed-staging
+          # Clean up backups options
+          CLEANUP_BACKUPS: true
+          RETENTION_DAYS: '30'
+        resources:
+          enabled: false
+          requests:
+            memory: '300Mi'
+            cpu: '0.5'
+          limits:
+            memory: '400Mi'
+            cpu: '0.6'
+        nodeSelector:
+          enabled: true
+          label_key: nodegroup_type
+          label_value: job
+
     # ====================================================================================================
     # Variables for tiler-db
     # ====================================================================================================