diff --git a/.gitignore b/.gitignore index c2eaf90e..4c32d0b3 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,5 @@ tiler-server/imposm/ overpass-api-db/ data/*/ !data/README.md -images/changeset-replication-job/config.yaml \ No newline at end of file +images/changeset-replication-job/config.yaml +values_copy.yaml \ No newline at end of file diff --git a/chartpress.yaml b/chartpress.yaml index df170850..f5af3ddc 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -46,3 +46,4 @@ charts: valuesPath: osmchaWeb.image osmcha-db: valuesPath: osmchaDb.image + \ No newline at end of file diff --git a/compose/osmcha.yml b/compose/osmcha.yml index 5a1ae4e6..0761809d 100644 --- a/compose/osmcha.yml +++ b/compose/osmcha.yml @@ -1,24 +1,60 @@ -version: '3' +version: "3.8" services: - osmcha-db: + web: + platform: linux/amd64 + image: developmentseed/osmseed-osmcha-web:v16 + build: + context: ../images/osmcha-web + dockerfile: Dockerfile + env_file: + - ../envs/.env.osmcha + volumes: + - ../data/osmcha/staticfiles:/staticfiles + db: platform: linux/amd64 image: osmseed-osmcha-db:v1 build: context: ../images/osmcha-db dockerfile: Dockerfile ports: - - '5432:5432' + - "5432:5432" volumes: - ../data/osmcha-db-data:/var/lib/postgresql/data env_file: - ../envs/.env.osmcha - osmcha-web: + init: platform: linux/amd64 - image: osmseed-osmcha-web:v1 + image: ghcr.io/willemarcel/osmcha-django:b1f4e6afc90e08707cadc4d74580632ca3b93dd2 + command: > + /bin/bash -c " + set -x + python manage.py collectstatic + python manage.py migrate + mkdir -p /staticfiles/static + cp -r /app/staticfiles/* /staticfiles/static/ + " + env_file: + - ../envs/.env.osmcha + volumes: + - ../data/osmcha/staticfiles:/staticfiles + api: + platform: linux/amd64 + image: ghcr.io/willemarcel/osmcha-django:b1f4e6afc90e08707cadc4d74580632ca3b93dd2 build: - context: ../images/osmcha-web + context: ../images/osmcha-api dockerfile: Dockerfile + command: > + /bin/bash -c " + set -x + python manage.py collectstatic + python manage.py migrate + mkdir -p /staticfiles/static + cp -r /app/staticfiles/* /staticfiles/static/ + gunicorn --workers 4 --bind 0.0.0.0:5000 --log-file - --access-logfile - config.wsgi + " ports: - - '8000:8000' + - "5000:5000" env_file: - ../envs/.env.osmcha + volumes: + - ../data/osmcha/staticfiles:/staticfiles diff --git a/images/osmcha-web/Dockerfile b/images/osmcha-web/Dockerfile index ae3c6898..9b5f4181 100644 --- a/images/osmcha-web/Dockerfile +++ b/images/osmcha-web/Dockerfile @@ -1,66 +1,11 @@ -FROM ubuntu:20.04 -ARG DEBIAN_FRONTEND=noninteractive - -RUN apt-get update -y -qq -RUN apt-get install -y curl python3 python3-dev \ - python3-pip git libgeos-dev libyaml-dev \ - libcurl4-gnutls-dev librtmp-dev python3-gdal \ - locales nginx supervisor yarnpkg nodejs npm -RUN locale-gen en_US.UTF-8 -RUN apt-get clean -RUN rm -rf /var/lib/apt/lists/* - -# Install yarn. -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -RUN apt-get update -y -RUN apt-get remove -y yarnpkg cmdtest && apt-get install -y yarn && apt-get clean && rm -rf /var/lib/apt/lists/* - -# Setup nginx -RUN rm /etc/nginx/sites-enabled/default -COPY django.conf /etc/nginx/sites-available/ -RUN ln -s /etc/nginx/sites-available/django.conf /etc/nginx/sites-enabled/django.conf -RUN echo "daemon off;" >> /etc/nginx/nginx.conf - - -# Setting locale is required by unicode-slugify (osmcha-django python requirement) -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en - +FROM node:16-slim as builder +ENV DEBIAN_FRONTEND noninteractive +ENV BUILD_ENV=prod +ENV REACT_APP_PRODUCTION_API_URL=/api/v1 +RUN apt-get update && apt-get install -y git +RUN mkdir /app WORKDIR /app -ENV BACKEND_VERSION v4.19.0 -RUN git clone https://github.com/willemarcel/osmcha-django . -RUN git checkout $BACKEND_VERSION -RUN git pull origin $BACKEND_VERSION -RUN pip3 install -r requirements/production.txt -RUN pip3 install -r requirements/local.txt - -ADD . /app -ENV DJANGO_ROOT /app -ENV DJANGO_SETTINGS_MODULE config.settings.aws_production - -# Replace social-auth-core by a modified version that provides OHM authentication -RUN pip3 install git+https://github.com/OpenHistoricalMap/social-core.git@master - -# Upgrading requests to the latest version. -RUN pip3 install -U requests -RUN pip3 install certifi -RUN pip3 install django-extensions - -# Clone and build the frontend -RUN git clone https://github.com/mapbox/osmcha-frontend.git /osmcha-frontend -RUN cd /osmcha-frontend && yarn install - -RUN echo "alias python=python3" >> ~/.bashrc -RUN echo "alias pip=pip3" >> ~/.bashrc - -EXPOSE 80 - -# Supervisor config -RUN mkdir -p /var/log/supervisor -COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf -COPY gunicorn.conf /etc/supervisor/conf.d/gunicorn.conf - -COPY ./start.sh . -CMD ./start.sh +RUN git clone https://github.com/osmus/osmcha-frontend.git /app +RUN yarn install +COPY start.sh . +CMD [ "/app/start.sh" ] diff --git a/images/osmcha-web/README.md b/images/osmcha-web/README.md deleted file mode 100644 index c653ed65..00000000 --- a/images/osmcha-web/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# OpenStreetMap Changeset Analyzer (OSMCha) - -OSMCha in a Docker containe that integrated with the OSM-Seed Ecosystem - -## Build -```sh -docker compose -f /compose/osmcha.yml build -``` - -## Run - -```sh -docker compose -f /compose/osmcha.yml up -``` \ No newline at end of file diff --git a/images/osmcha-web/django.conf b/images/osmcha-web/django.conf deleted file mode 100644 index ff788d99..00000000 --- a/images/osmcha-web/django.conf +++ /dev/null @@ -1,31 +0,0 @@ -server { - listen 80; - charset utf-8; - client_max_body_size 20M; - - location /static/ { - gzip_static on; - alias /app/staticfiles/; - } - - location /index.html { - gzip_static on; - alias /app/osmchadjango/frontend/templates/frontend/index.html; - } - - location / { - # checks for static file, if not found proxy to app - try_files $uri @proxy_to_app; - } - - # cookiecutter-django app - location @proxy_to_app { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_pass http://127.0.0.1:5000; - fastcgi_connect_timeout 130; - fastcgi_send_timeout 130; - fastcgi_read_timeout 130; - } -} diff --git a/images/osmcha-web/gunicorn.conf b/images/osmcha-web/gunicorn.conf deleted file mode 100644 index 21d99bb7..00000000 --- a/images/osmcha-web/gunicorn.conf +++ /dev/null @@ -1,6 +0,0 @@ -[program:gunicorn] -command=/usr/local/bin/gunicorn config.wsgi -b localhost:5000 -w 4 --log-file /var/log/gunicorn.error --access-logfile /var/log/gunicorn.access --capture-output --timeout 120 -directory=/app -autostart=true -autorestart=true -redirect_stderr=true diff --git a/images/osmcha-web/start.sh b/images/osmcha-web/start.sh index b37767e0..661b6d28 100755 --- a/images/osmcha-web/start.sh +++ b/images/osmcha-web/start.sh @@ -1,15 +1,8 @@ #!/usr/bin/env bash -set -ex - -# Build frontend -cd /osmcha-frontend -REACT_APP_VERSION=ohm REACT_APP_STACK=PRODUCTION PUBLIC_URL=$OSMCHA_URL npx react-scripts build -cp -R build/*.html /app/osmchadjango/frontend/templates/frontend/ -cp -R build/* /app/osmchadjango/static/ -cp -R build/static/* /app/osmchadjango/static/ - -# Start service -cd /app -python3 manage.py collectstatic --noinput -python3 manage.py migrate -supervisord -c /etc/supervisor/supervisord.conf +set -x +export BUILD_ENV=prod +export REACT_APP_PRODUCTION_API_URL=/api/v1 +sed -i "s|https://osmcha.org|$OSMCHA_URL|g" package.json +yarn build:${BUILD_ENV} +find /app/build -type f -exec sed -i "s/www.openstreetmap.org/$OSMCHA_API_URL/g" {} + +cp -r /app/build/* /assets/ diff --git a/images/osmcha-web/supervisord.conf b/images/osmcha-web/supervisord.conf deleted file mode 100644 index 8d520a8c..00000000 --- a/images/osmcha-web/supervisord.conf +++ /dev/null @@ -1,5 +0,0 @@ -[supervisord] -nodaemon=true - -[program:nginx] -command=/usr/sbin/nginx diff --git a/images/overpass-api/Dockerfile b/images/overpass-api/Dockerfile index 9528668b..de488a70 100644 --- a/images/overpass-api/Dockerfile +++ b/images/overpass-api/Dockerfile @@ -1,4 +1,4 @@ -FROM wiktorn/overpass-api:0.7.56.8 +FROM wiktorn/overpass-api:latest COPY docker-entrypoint.sh /app/ RUN chmod a+rx /app/docker-entrypoint.sh CMD ["/app/docker-entrypoint.sh"] diff --git a/images/overpass-api/docker-entrypoint.sh b/images/overpass-api/docker-entrypoint.sh index 243ddac6..60b94687 100644 --- a/images/overpass-api/docker-entrypoint.sh +++ b/images/overpass-api/docker-entrypoint.sh @@ -1,119 +1,124 @@ #!/bin/bash -set -eo pipefail +set -eox pipefail shopt -s nullglob OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} OVERPASS_FLUSH_SIZE=${OVERPASS_FLUSH_SIZE:-16} -OVERPASS_CLONE_SOURCE=${OVERPASS_CLONE_SOURCE:-http://dev.overpass-api.de/api_drolbr/} +OVERPASS_CLONE_SOURCE=${OVERPASS_CLONE_SOURCE:-https://dev.overpass-api.de/api_drolbr/} # this is used by other processes, so needs to be exported export OVERPASS_MAX_TIMEOUT=${OVERPASS_MAX_TIMEOUT:-1000s} -if [[ "$OVERPASS_META" == "attic" ]] ; then - META="--keep-attic" -elif [[ "${OVERPASS_META}" == "yes" ]] ; then - META="--meta" +if [[ "$OVERPASS_META" == "attic" ]]; then + META="--keep-attic" +elif [[ "${OVERPASS_META}" == "yes" ]]; then + META="--meta" else - META="" + META="" fi for f in /docker-entrypoint-initdb.d/*; do - case "$f" in - *.sh) - if [[ -x "$f" ]]; then - echo "$0: running $f" - "$f" - else - echo "$0: sourcing $f" - . "$f" - fi - ;; - *) echo "$0: ignoring $f" ;; - esac - echo + case "$f" in + *.sh) + if [[ -x "$f" ]]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + # shellcheck disable=SC1090 # ignore SC1090 (unable to follow file) because they are dynamically provided + . "$f" + fi + ;; + *) echo "$0: ignoring $f" ;; + esac + echo done -function startAPIServer(){ - envsubst '${OVERPASS_MAX_TIMEOUT}' < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf - echo "Starting supervisord process" - exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf -} +if [[ ! -f /db/init_done ]]; then + echo "No database directory. Initializing" + if [[ "${USE_OAUTH_COOKIE_CLIENT}" = "yes" ]]; then + /app/venv/bin/python /app/bin/oauth_cookie_client.py -o /db/cookie.jar -s /secrets/oauth-settings.json --format netscape + # necessary to add newline at the end as oauth_cookie_client doesn't do that + echo >>/db/cookie.jar + else + echo "# Netscape HTTP Cookie File" >/db/cookie.jar + echo "${OVERPASS_COOKIE_JAR_CONTENTS}" >>/db/cookie.jar + fi + chown overpass /db/cookie.jar -if [[ ! -f /db/init_done ]] ; then - echo "No database directory. Initializing" - if [[ "${USE_OAUTH_COOKIE_CLIENT}" = "yes" ]]; then - /app/venv/bin/python /app/bin/oauth_cookie_client.py -o /db/cookie.jar -s /secrets/oauth-settings.json --format netscape - # necessary to add newline at the end as oauth_cookie_client doesn't do that - echo >> /db/cookie.jar - else - echo "# Netscape HTTP Cookie File" > /db/cookie.jar - echo "${OVERPASS_COOKIE_JAR_CONTENTS}" >> /db/cookie.jar - fi - chown overpass /db/cookie.jar + if [[ "$OVERPASS_MODE" = "clone" ]]; then + ( + mkdir -p /db/db && + /app/bin/download_clone.sh --db-dir=/db/db --source="${OVERPASS_CLONE_SOURCE}" --meta="${OVERPASS_META}" && + cp /db/db/replicate_id /db/replicate_id && + cp -r /app/etc/rules /db/db && + chown -R overpass:overpass /db/* && + touch /db/init_done + ) || ( + echo "Failed to clone overpass repository" + exit 1 + ) + if [[ "${OVERPASS_STOP_AFTER_INIT}" == "false" ]]; then + echo "Overpass container ready to receive requests" + else + echo "Overpass container initialization complete. Exiting." + exit 0 + fi + fi - if [[ "$OVERPASS_MODE" = "clone" ]]; then - mkdir -p /db/db \ - && /app/bin/download_clone.sh --db-dir=/db/db --source="${OVERPASS_CLONE_SOURCE}" --meta="${OVERPASS_META}" \ - && cp /db/db/replicate_id /db/replicate_id \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && touch /db/init_done \ - && echo "Overpass ready, you can start your container with docker start" - exit - fi - - if [[ "$OVERPASS_MODE" = "init" ]]; then - while true ; do - CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") - case "${CURL_STATUS_CODE}" in - 429) - echo "Server responded with 429 Too many requests. Trying again in 5 minutes..." - sleep 300 - continue - ;; - # for `file:///` scheme curl returns `000` HTTP status code - 200 | 000) - ( - if [[ ! -z "${OVERPASS_PLANET_PREPROCESS+x}" ]]; then - echo "Running preprocessing command: ${OVERPASS_PLANET_PREPROCESS}" - eval "${OVERPASS_PLANET_PREPROCESS}" - fi \ - && /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "${META}" "--version=$(osmium fileinfo -e -g data.timestamp.last /db/planet.osm.bz2) --compression-method=${OVERPASS_COMPRESSION} --map-compression-method=${OVERPASS_COMPRESSION} --flush-size=${OVERPASS_FLUSH_SIZE}" \ - && echo "Database created. Now updating it." \ - && cp -r /app/etc/rules /db/db \ - && chown -R overpass:overpass /db \ - && echo "Updating" \ - && /app/bin/update_overpass.sh "-O /db/planet.osm.bz2" \ - && /app/bin/osm3s_query --progress --rules --db-dir=/db/db < /db/db/rules/areas.osm3s \ - && touch /db/init_done \ - && rm /db/planet.osm.bz2 \ - && chown -R overpass:overpass /db \ - && echo $OVERPASS_REPLICATION_SEQUENCE_NUMBER > /db/replicate_id \ - && chmod 777 /db/replicate_id \ - && echo "Overpass ready, you can start your container with docker start" \ - && startAPIServer - ) || ( - echo "Failed to process planet file" - exit - ) - ;; - 403) - echo "Access denied when downloading planet file. Check your OVERPASS_PLANET_URL and OVERPASS_COOKIE_JAR_CONTENTS or USE_OAUTH_COOKIE_CLIENT" - cat /db/cookie.jar - exit - ;; - *) - echo "Failed to download planet file. HTTP status code: ${CURL_STATUS_CODE}" - cat /db/planet.osm.bz2 - exit - ;; - esac - exit - done - fi + if [[ "$OVERPASS_MODE" = "init" ]]; then + CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") + # try again until it's allowed + while [ "$CURL_STATUS_CODE" = "429" ]; do + echo "Server responded with 429 Too many requests. Trying again in 5 minutes..." + sleep 300 + CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") + done + # for `file:///` scheme curl returns `000` HTTP status code + if [[ $CURL_STATUS_CODE = "200" || $CURL_STATUS_CODE = "000" ]]; then + ( + if [[ -n "${OVERPASS_PLANET_PREPROCESS+x}" ]]; then + echo "Running preprocessing command: ${OVERPASS_PLANET_PREPROCESS}" + eval "${OVERPASS_PLANET_PREPROCESS}" + fi && + /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "${META}" "--version=$(osmium fileinfo -e -g data.timestamp.last /db/planet.osm.bz2) --compression-method=${OVERPASS_COMPRESSION} --map-compression-method=${OVERPASS_COMPRESSION} --flush-size=${OVERPASS_FLUSH_SIZE}" && + echo "Database created. Now updating it." && + cp -r /app/etc/rules /db/db && + chown -R overpass:overpass /db/* && + echo "Updating" && + /app/bin/update_overpass.sh -O /db/planet.osm.bz2 && + if [[ "${OVERPASS_USE_AREAS}" = "true" ]]; then + echo "Generating areas..." && /app/bin/osm3s_query --progress --rules --db-dir=/db/db /etc/nginx/nginx.conf + +echo "Starting supervisord process" +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf \ No newline at end of file diff --git a/images/taginfo/Dockerfile b/images/taginfo/Dockerfile index 1721e262..16745806 100644 --- a/images/taginfo/Dockerfile +++ b/images/taginfo/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:2.7 +FROM ruby:3.0 ENV workdir /apps # Install Taginfo site diff --git a/images/web/Dockerfile b/images/web/Dockerfile index bf8b4fed..0a1e5fb8 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -1,16 +1,52 @@ FROM ubuntu:20.04 + ENV DEBIAN_FRONTEND=noninteractive -ENV workdir /var/www +ENV RUBY_MAJOR 3.0 +ARG RUBY_VERSION=3.0.6 +ENV RUBY_VERSION $RUBY_VERSION +ENV PATH /opt/ruby/bin:$PATH:/opt/rbenv/plugins/ruby-build/bin + +# ruby-build +RUN set -ex \ + && mkdir -p /etc/network/interfaces.d \ + && BaseDeps=' \ + git \ + gcc \ + autoconf \ + bison \ + build-essential \ + libssl-dev \ + libyaml-dev \ + libreadline6-dev \ + zlib1g-dev \ + libncurses5-dev \ + libffi-dev \ + libgdbm6 \ + libgdbm-dev \ + make \ + wget \ + curl \ + iproute2 \ + net-tools \ + tzdata \ + locales \ + ca-certificates' \ + && apt-get update \ + && DEBCONF_NOWARNINGS=yes apt-get -y upgrade \ + && DEBCONF_NOWARNINGS=yes DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $BaseDeps \ + && rm -rf /var/lib/apt/lists/* \ + && git clone https://github.com/sstephenson/ruby-build.git /opt/rbenv/plugins/ruby-build \ + && ruby-build ${RUBY_VERSION} /opt/ruby -# Production OSM setup +ENV workdir /var/www ENV RAILS_ENV=production # Install the openstreetmap-website dependencies RUN apt-get update \ && apt-get install -y \ - ruby2.7 libruby2.7 ruby2.7-dev libmagickwand-dev libxml2-dev libxslt1-dev \ - nodejs npm apache2 apache2-dev build-essential git-core firefox-geckodriver postgresql-client \ - libpq-dev libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev yarnpkg curl \ + libmagickwand-dev libxml2-dev libxslt1-dev \ + nodejs npm libv8-dev apache2 apache2-dev build-essential git-core postgresql-client \ + libpq-dev libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev yarnpkg curl unzip \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -23,7 +59,8 @@ RUN apt-get update \ libcrypto++-dev libargon2-dev libyajl-dev automake autoconf libtool \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - + + # Install cgimap ENV cgimap /tmp/openstreetmap-cgimap RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap @@ -40,7 +77,7 @@ RUN npm install -g svgo # Install openstreetmap-website RUN rm -rf $workdir/html -# GITSHA value at 15-02-2022 + ENV OPENSTREETMAP_WEBSITE_GITSHA=c24b5481812aba9e83da1fd855ccb37f92c5d75e RUN curl -L https://github.com/openstreetmap/openstreetmap-website/archive/$OPENSTREETMAP_WEBSITE_GITSHA.zip --output website.zip && unzip website.zip RUN mv openstreetmap-website-$OPENSTREETMAP_WEBSITE_GITSHA/* $workdir/ @@ -50,6 +87,7 @@ RUN echo "gem 'image_optim_pack', :git => 'https://github.com/toy/image_optim_pa # Install Ruby packages RUN gem install bundler && bundle install + # Configure database.yml and secrets.yml RUN cp $workdir/config/example.database.yml $workdir/config/database.yml RUN touch $workdir/config/settings.local.yml @@ -78,9 +116,9 @@ RUN apt-get update && apt-get install -y libapache2-mod-passenger RUN echo "ServerName $(cat /etc/hostname)" >> /etc/apache2/apache2.conf RUN a2enmod passenger -# Check installation -RUN /usr/bin/passenger-config validate-install -RUN /usr/sbin/passenger-memory-stats +# # Check installation +# RUN /usr/bin/passenger-config validate-install +# RUN /usr/sbin/passenger-memory-stats # Enable required apache modules for the cgimap Apache service RUN a2enmod proxy proxy_http rewrite @@ -104,3 +142,5 @@ COPY start.sh $workdir/ COPY liveness.sh $workdir/ CMD $workdir/start.sh + + diff --git a/osm-seed/templates/osmcha-app/configMap.yaml b/osm-seed/templates/osmcha-app/configMap.yaml new file mode 100644 index 00000000..a3aa6ba3 --- /dev/null +++ b/osm-seed/templates/osmcha-app/configMap.yaml @@ -0,0 +1,126 @@ +{{- if .Values.osmchaApi.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-osmcha-common-env +data: + # DB + POSTGRES_HOST: {{ .Release.Name }}-osmcha-db + PGHOST: {{ .Release.Name }}-osmcha-db + POSTGRES_PORT: "5432" + POSTGRES_USER: {{ .Values.osmchaDb.env.POSTGRES_USER | quote }} + POSTGRES_PASSWORD: {{ .Values.osmchaDb.env.POSTGRES_PASSWORD | quote }} + POSTGRES_DATABASE: {{ .Values.osmchaDb.env.POSTGRES_DB | quote }} + POSTGRES_DB: {{ .Values.osmchaDb.env.POSTGRES_DB | quote }} + # API + DJANGO_SECRET_KEY: {{ .Values.osmchaApi.env.DJANGO_SECRET_KEY | quote }} + DJANGO_SETTINGS_MODULE: {{ .Values.osmchaApi.env.DJANGO_SETTINGS_MODULE | quote }} + OSMCHA_FRONTEND_VERSION: {{ .Values.osmchaApi.env.OSMCHA_FRONTEND_VERSION | quote }} + OAUTH_OSM_KEY: {{ .Values.osmchaApi.env.OAUTH_OSM_KEY | quote }} + OAUTH_OSM_SECRET: {{ .Values.osmchaApi.env.OAUTH_OSM_SECRET | quote }} + OAUTH_REDIRECT_URI: {{ .Values.osmchaApi.env.OAUTH_REDIRECT_URI | quote }} + DJANGO_SECURE_SSL_REDIRECT: "False" + DJANGO_DEBUG: "False" + DJANGO_ENV: "production" + OSM_SERVER_URL: {{ .Values.osmchaApi.env.OSM_SERVER_URL | quote }} + DJANGO_DEBUG: "False" + DJANGO_ENV: "production" + REDIS_URL: "redis://localhost:6379" + OSM_PLANET_BASE_URL: {{ .Values.osmchaApi.env.OSM_PLANET_BASE_URL | quote }} + # frontend + OSMCHA_URL: {{ .Values.osmchaApi.env.OSMCHA_URL | quote }} + OSMCHA_API_URL: {{ .Values.osmchaApi.env.OSMCHA_API_URL | quote }} + REACT_APP_OSM_URL: {{ .Values.osmchaApi.env.REACT_APP_OSM_URL | quote }} + REACT_APP_OSM_API: {{ .Values.osmchaApi.env.REACT_APP_OSM_API | quote }} + REACT_APP_OVERPASS_BASE: {{ .Values.osmchaApi.env.REACT_APP_OVERPASS_BASE | quote }} + REACT_APP_DISABLE_REAL_CHANGESETS: {{ .Values.osmchaApi.env.REACT_APP_DISABLE_REAL_CHANGESETS | quote }} + REACT_APP_MAPBOX_ACCESS_TOKEN: {{ .Values.osmchaApi.env.REACT_APP_MAPBOX_ACCESS_TOKEN | quote }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-osmcha-nginx-config +data: + nginx.conf: | + worker_processes 4; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + upstream app { + server localhost:5000; + } + + server { + listen 80; + charset utf-8; + + location ~ ^/static.* { + root /staticfiles; + try_files $uri @proxy_to_frontend; + } + + location @proxy_to_frontend { + root /assets; + try_files $uri $uri/ =404; + } + + location /api { + try_files $uri $uri/ @django; + } + + location /admin { + try_files $uri $uri/ @django; + } + + location @django { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://app; + } + + location / { + root /assets; + # checks for static file, if not found proxy to app + try_files $uri $uri/ /index.html; + } + } + } + mime.types: | + types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/x-icon ico; + image/x-jng jng; + application/font-woff woff; + application/json json; + application/zip zip; + } + +{{- end }} diff --git a/osm-seed/templates/osmcha-app/cronJob.yaml b/osm-seed/templates/osmcha-app/cronJob.yaml new file mode 100644 index 00000000..036d8df5 --- /dev/null +++ b/osm-seed/templates/osmcha-app/cronJob.yaml @@ -0,0 +1,36 @@ +{{- if .Values.osmchaApi.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-osmcha-fetch-changesets-cronjob + labels: + app: {{ template "osm-seed.name" . }} + component: osmcha-app-deployment + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + schedule: "*/2 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 2 + jobTemplate: + spec: + template: + spec: + containers: + - name: {{ .Release.Name }}-osmcha-fetch-changesets + image: "{{ .Values.osmchaApi.image.name }}:{{ .Values.osmchaApi.image.tag }}" + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - python manage.py fetchchangesets + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + restartPolicy: Never + {{- if .Values.osmchaApi.nodeSelector.enabled }} + nodeSelector: + {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }} + {{- end }} +{{- end }} diff --git a/osm-seed/templates/osmcha-app/deployment.yaml b/osm-seed/templates/osmcha-app/deployment.yaml new file mode 100644 index 00000000..43e2b981 --- /dev/null +++ b/osm-seed/templates/osmcha-app/deployment.yaml @@ -0,0 +1,124 @@ +{{- if .Values.osmchaApi.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-osmcha-app + labels: + app: {{ template "osm-seed.name" . }} + component: osmcha-app-deployment + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.osmchaApi.replicaCount }} + selector: + matchLabels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-app + template: + metadata: + labels: + app: {{ template "osm-seed.name" . }} + release: {{ .Release.Name }} + run: {{ .Release.Name }}-osmcha-app + spec: + initContainers: + - name: web-builder + image: "{{ .Values.osmchaWeb.image.name }}:{{ .Values.osmchaWeb.image.tag }}" + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + volumeMounts: + - name: nginx-assets + mountPath: /assets + - name: init + image: "{{ .Values.osmchaApi.image.name }}:{{ .Values.osmchaApi.image.tag }}" + command: + - /bin/sh + - -c + - | + set -x + flag=true + while "$flag" = true; do + pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue + flag=false + python manage.py migrate + python manage.py collectstatic + mkdir -p /staticfiles/static + cp -r /app/staticfiles/* /staticfiles/static/ + done + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + volumeMounts: + - name: staticfiles + mountPath: /staticfiles + # securityContext: + # runAsUser: 0 + containers: + - name: api + # securityContext: + # runAsUser: 0 + image: "{{ .Values.osmchaApi.image.name }}:{{ .Values.osmchaApi.image.tag }}" + command: + - /bin/sh + - -c + - | + set -x + echo "start app...." + gunicorn --workers 4 --bind 0.0.0.0:5000 --log-file - --access-logfile - config.wsgi + ports: + - containerPort: 5000 + livenessProbe: + httpGet: + path: / + port: 5000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + envFrom: + - configMapRef: + name: {{ .Release.Name }}-osmcha-common-env + volumeMounts: + - name: staticfiles + mountPath: /staticfiles + {{- if .Values.osmchaApi.resources.enabled }} + resources: + requests: + memory: {{ .Values.osmchaApi.resources.requests.memory }} + cpu: {{ .Values.osmchaApi.resources.requests.cpu }} + limits: + memory: {{ .Values.osmchaApi.resources.limits.memory }} + cpu: {{ .Values.osmchaApi.resources.limits.cpu }} + {{- end }} + + - name: nginx + image: "nginx:latest" + ports: + - containerPort: 80 + volumeMounts: + - name: osmcha-frontend-nginx-config + mountPath: /etc/nginx + readOnly: true + - name: staticfiles + mountPath: /staticfiles + - name: nginx-assets + mountPath: /assets + - name: redis + image: "redis:latest" + ports: + - containerPort: 6379 + volumes: + - name: staticfiles + emptyDir: {} + - name: nginx-assets + emptyDir: {} + - name: osmcha-frontend-nginx-config + configMap: + name: {{ .Release.Name }}-osmcha-nginx-config + defaultMode: 0777 + {{- if .Values.osmchaApi.nodeSelector.enabled }} + nodeSelector: + {{ .Values.osmchaApi.nodeSelector.label_key }} : {{ .Values.osmchaApi.nodeSelector.label_value }} + {{- end }} +{{- end }} + diff --git a/osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml b/osm-seed/templates/osmcha-app/ingress.yaml similarity index 71% rename from osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml rename to osm-seed/templates/osmcha-app/ingress.yaml index cf2df4bc..7360e2d1 100644 --- a/osm-seed/templates/osmcha-web/osmcha-web-ingress.yaml +++ b/osm-seed/templates/osmcha-app/ingress.yaml @@ -1,8 +1,8 @@ -{{- if and .Values.osmchaWeb.enabled (eq .Values.serviceType "ClusterIP") }} +{{- if and .Values.osmchaApi.enabled (eq .Values.serviceType "ClusterIP") }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ template "osm-seed.fullname" . }}-ingress-osmcha-web + name: {{ template "osm-seed.fullname" . }}-ingress-osmcha-app annotations: kubernetes.io/ingress.class: nginx cert-manager.io/cluster-issuer: letsencrypt-prod-issuer @@ -20,7 +20,7 @@ spec: pathType: Prefix backend: service: - name: {{ template "osm-seed.fullname" . }}-osmcha-web + name: {{ template "osm-seed.fullname" . }}-osmcha-app port: number: 80 -{{- end }} \ No newline at end of file +{{- end }} diff --git a/osm-seed/templates/osmcha-web/osmcha-web-service.yaml b/osm-seed/templates/osmcha-app/service.yaml similarity index 76% rename from osm-seed/templates/osmcha-web/osmcha-web-service.yaml rename to osm-seed/templates/osmcha-app/service.yaml index 612a35ab..985cf171 100644 --- a/osm-seed/templates/osmcha-web/osmcha-web-service.yaml +++ b/osm-seed/templates/osmcha-app/service.yaml @@ -1,18 +1,18 @@ -{{- if .Values.osmchaWeb.enabled -}} +{{- if .Values.osmchaApi.enabled -}} apiVersion: v1 kind: Service metadata: - name: {{ template "osm-seed.fullname" . }}-osmcha-web + name: {{ template "osm-seed.fullname" . }}-osmcha-app labels: app: {{ template "osm-seed.name" . }} - component: osmcha-web-service + component: osmcha-app-service environment: {{ .Values.environment }} release: {{ .Release.Name }} annotations: {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }} service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https {{- end }} {{- if eq .Values.serviceType "ClusterIP" }} kubernetes.io/ingress.class: nginx @@ -20,24 +20,24 @@ metadata: {{- else }} fake.annotation: fake {{- end }} - {{- with .Values.osmchaWeb.serviceAnnotations }} + {{- with .Values.osmchaApi.serviceAnnotations }} {{- toYaml . | nindent 4 }} {{- end }} spec: type: {{ .Values.serviceType }} ports: - - port: 80 - targetPort: http - protocol: TCP + - protocol: TCP + port: 80 + targetPort: 80 name: http {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} - port: 443 - targetPort: http + targetPort: 80 protocol: TCP name: https {{- end }} selector: app: {{ template "osm-seed.name" . }} release: {{ .Release.Name }} - run: {{ .Release.Name }}-osmcha-web -{{- end }} \ No newline at end of file + run: {{ .Release.Name }}-osmcha-app +{{- end }} diff --git a/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml b/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml deleted file mode 100644 index 81a86f9c..00000000 --- a/osm-seed/templates/osmcha-web/osmcha-web-deployment.yaml +++ /dev/null @@ -1,127 +0,0 @@ -{{- if .Values.osmchaWeb.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Release.Name }}-osmcha-web - labels: - app: {{ template "osm-seed.name" . }} - component: osmcha-web-deployment - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: - replicas: {{ .Values.osmchaWeb.replicaCount }} - selector: - matchLabels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-osmcha-web - template: - metadata: - labels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-osmcha-web - spec: - containers: - - name: {{ .Chart.Name }}-osmcha-web - image: "{{ .Values.osmchaWeb.image.name }}:{{ .Values.osmchaWeb.image.tag }}" - ports: - - name: http - containerPort: 5000 - protocol: TCP - livenessProbe: - httpGet: - path: / - port: 5000 - initialDelaySeconds: 600 # 10 min, because the compile process takes time. - timeoutSeconds: 30 - {{- if .Values.osmchaWeb.resources.enabled }} - resources: - requests: - memory: {{ .Values.osmchaWeb.resources.requests.memory }} - cpu: {{ .Values.osmchaWeb.resources.requests.cpu }} - limits: - memory: {{ .Values.osmchaWeb.resources.limits.memory }} - cpu: {{ .Values.osmchaWeb.resources.limits.cpu }} - {{- end }} - env: - - name: OSM_SERVER_URL - value: {{ .Values.osmchaDb.env.OSM_SERVER_URL | quote }} - - name: POSTGRES_USER - value: {{ .Values.osmchaDb.env.POSTGRES_USER | quote }} - - name: POSTGRES_PASSWORD - value: {{ .Values.osmchaDb.env.POSTGRES_PASSWORD | quote }} - - name: POSTGRES_DB - value: {{ .Values.osmchaDb.env.POSTGRES_DB | quote }} - - name: PGHOST - value: {{ .Release.Name }}-osmcha-db - - name: POSTGRES_HOST - value: {{ .Release.Name }}-osmcha-db - - name: DJANGO_CACHES - value: {{ .Values.osmchaWeb.env.DJANGO_CACHES | quote }} - - name: DJANGO_DEBUG - value: {{ .Values.osmchaWeb.env.DJANGO_DEBUG | quote }} - - name: DJANGO_SECRET_KEY - value: {{ .Values.osmchaWeb.env.DJANGO_SECRET_KEY | quote }} - - name: DJANGO_SECURE_BROWSER_XSS_FILTER - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_BROWSER_XSS_FILTER | quote }} - - name: DJANGO_SECURE_SSL_REDIRECT - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_SSL_REDIRECT | quote }} - - name: DJANGO_SECURE_CONTENT_TYPE_NOSNIFF - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_CONTENT_TYPE_NOSNIFF | quote }} - - name: DJANGO_SECURE_FRAME_DENY - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_FRAME_DENY | quote }} - - name: DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS - value: {{ .Values.osmchaWeb.env.DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS | quote }} - - name: DJANGO_SESSION_COOKIE_HTTPONLY - value: {{ .Values.osmchaWeb.env.DJANGO_SESSION_COOKIE_HTTPONLY | quote }} - - name: DJANGO_SESSION_COOKIE_SECURE - value: {{ .Values.osmchaWeb.env.DJANGO_SESSION_COOKIE_SECURE | quote }} - - name: DJANGO_DEFAULT_FROM_EMAIL - value: {{ .Values.osmchaWeb.env.DJANGO_DEFAULT_FROM_EMAIL | quote }} - - name: DJANGO_SERVER_EMAIL - value: {{ .Values.osmchaWeb.env.DJANGO_SERVER_EMAIL | quote }} - - name: DJANGO_EMAIL_SUBJECT_PREFIX - value: {{ .Values.osmchaWeb.env.DJANGO_EMAIL_SUBJECT_PREFIX | quote }} - - name: DJANGO_CHANGESETS_FILTER - value: {{ .Values.osmchaWeb.env.DJANGO_CHANGESETS_FILTER | quote }} - - name: OAUTH_OSM_KEY - value: {{ .Values.osmchaWeb.env.OAUTH_OSM_KEY | quote }} - - name: OAUTH_OSM_SECRET - value: {{ .Values.osmchaWeb.env.OAUTH_OSM_SECRET | quote }} - - name: DJANGO_ANON_USER_THROTTLE_RATE - value: {{ .Values.osmchaWeb.env.DJANGO_ANON_USER_THROTTLE_RATE | quote }} - - name: DJANGO_COMMON_USER_THROTTLE_RATE - value: {{ .Values.osmchaWeb.env.DJANGO_COMMON_USER_THROTTLE_RATE | quote }} - - name: DJANGO_NON_STAFF_USER_THROTTLE_RATE - value: {{ .Values.osmchaWeb.env.DJANGO_NON_STAFF_USER_THROTTLE_RATE | quote }} - - name: OAUTH_REDIRECT_URI - value: {{ .Values.osmchaWeb.env.OAUTH_REDIRECT_URI | quote }} - - name: OSMCHA_FRONTEND_VERSION - value: {{ .Values.osmchaWeb.env.OSMCHA_FRONTEND_VERSION | quote }} - - name: DJANGO_ENABLE_CHANGESET_COMMENTS - value: {{ .Values.osmchaWeb.env.DJANGO_ENABLE_CHANGESET_COMMENTS | quote }} - - name: DJANGO_OSM_COMMENTS_API_KEY - value: {{ .Values.osmchaWeb.env.DJANGO_OSM_COMMENTS_API_KEY | quote }} - - name: DJANGO_ROOT - value: {{ .Values.osmchaWeb.env.DJANGO_ROOT | quote }} - - name: DJANGO_SETTINGS_MODULE - value: {{ .Values.osmchaWeb.env.DJANGO_SETTINGS_MODULE | quote }} - - name: C_FORCE_ROOT - value: {{ .Values.osmchaWeb.env.C_FORCE_ROOT | quote }} - - name: REACT_APP_OSM_URL - value: {{ .Values.osmchaWeb.env.REACT_APP_OSM_URL | quote }} - - name: REACT_APP_OSM_API - value: {{ .Values.osmchaWeb.env.REACT_APP_OSM_API | quote }} - - name: REACT_APP_OVERPASS_BASE - value: {{ .Values.osmchaWeb.env.REACT_APP_OVERPASS_BASE | quote }} - - name: REACT_APP_NOMINATIM_URL - value: {{ .Values.osmchaWeb.env.REACT_APP_NOMINATIM_URL | quote }} - - name: OSMCHA_URL - value: {{ .Values.osmchaWeb.env.OSMCHA_URL | quote }} - {{- if .Values.osmchaWeb.nodeSelector.enabled }} - nodeSelector: - {{ .Values.osmchaWeb.nodeSelector.label_key }} : {{ .Values.osmchaWeb.nodeSelector.label_value }} - {{- end }} -{{- end }} - diff --git a/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml b/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml index 0f5902e1..3b0e6af0 100644 --- a/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml +++ b/osm-seed/templates/overpass-api/overpass-api-statefulset.yaml @@ -30,12 +30,12 @@ spec: - name: http containerPort: 80 protocol: TCP - # livenessProbe: - # httpGet: - # path: / - # port: 80 - # initialDelaySeconds: 600 # 10 min, because the compile process takes time. - # timeoutSeconds: 30 + livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 14400 # Normal update takes 2 hours aprrox, afther that start checking liveness + timeoutSeconds: 30 {{- if .Values.overpassApi.resources.enabled }} resources: requests: @@ -46,7 +46,6 @@ spec: cpu: {{ .Values.overpassApi.resources.limits.cpu }} {{- end }} env: - # overpass-api env variables - name: OVERPASS_META value: {{ .Values.overpassApi.env.OVERPASS_META | quote}} - name: OVERPASS_MODE diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index a1a534f0..4f4c5e79 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -15,7 +15,7 @@ cloudProvider: minikube AWS_S3_BUCKET: s3://osm-seed # ==================================================================================================== -# GCP: In case you are using the cloudProvider=gcp set the below variables, We are assuming the nodes has a policies to upload files to cloud storage +# GCP: In case you are using the cloudProvider=gcp set the below variables, We are assuming the nodes has a policies to upload files to cloud storage # ==================================================================================================== GCP_STORAGE_BUCKET: gs://osm-seed-test @@ -44,9 +44,9 @@ AWS_SSL_ARN: false # If you specify `LoadBalancer` as the service type, if you also specify # an `AWS_SSL_ARN` that is a wildcart certificate, that will be configured # as the SSL certificate for your services. Else, you will need to configure -# SSL separately. -serviceType: NodePort - +# SSL separately. +# serviceType: NodePort +serviceType: LoadBalancer # Domain that is pointed to the clusterIP # You will need to create an A record like *.osmseed.example.com pointed to the ClusterIP @@ -77,12 +77,12 @@ adminEmail: admin@example.com db: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: POSTGRES_DB: openstreetmap POSTGRES_USER: postgres - POSTGRES_PASSWORD: '1234' + POSTGRES_PASSWORD: "1234" persistenceDisk: enabled: false mountPath: /var/lib/postgresql/data @@ -103,11 +103,11 @@ db: resources: enabled: false requests: - memory: '1Gi' - cpu: '1' + memory: "1Gi" + cpu: "1" limits: - memory: '2Gi' - cpu: '1' + memory: "2Gi" + cpu: "1" nodeSelector: enabled: false sharedMemorySize: 64Mi @@ -117,29 +117,29 @@ db: web: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" replicaCount: 1 serviceAnnotations: {} env: MAILER_ADDRESS: smtp.gmail.com MAILER_DOMAIN: gmail.com MAILER_USERNAME: example@gmail.com - MAILER_PASSWORD: '1234' + MAILER_PASSWORD: "1234" MAILER_FROM: no-reply@example.com MAILER_PORT: 25 - NOMINATIM_URL: 'nominatim' - OVERPASS_URL: 'overpassApi' - OAUTH_CLIENT_ID: 'abcd' # Create client ID and key as OAuth2 Application - OAUTH_KEY: 'xyz' + NOMINATIM_URL: "nominatim" + OVERPASS_URL: "overpassApi" + OAUTH_CLIENT_ID: "abcd" # Create client ID and key as OAuth2 Application + OAUTH_KEY: "xyz" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false autoscaling: @@ -161,11 +161,11 @@ memcached: resources: enabled: false requests: - memory: '8Gi' - cpu: '2' + memory: "8Gi" + cpu: "2" limits: - memory: '8Gi' - cpu: '2' + memory: "8Gi" + cpu: "2" # ==================================================================================================== # Variables for id-editor @@ -179,13 +179,13 @@ idEditor: fullHistory: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" nodeSelector: enabled: false label_key: nodegroup-type label_value: ohm-non-tiler - schedule: '* * * * *' + schedule: "* * * * *" env: OVERWRITE_FHISTORY_FILE: false resources: @@ -202,12 +202,12 @@ fullHistory: # ==================================================================================================== dbBackupRestore: image: - name: 'developmentseed/osmseed-backup-restore' - tag: '0.1.0-0.dev.git.942.h994af82' + name: "developmentseed/osmseed-backup-restore" + tag: "0.1.0-0.dev.git.942.h994af82" cronjobs: - name: web-db - enabled: true - schedule: '* * * * *' + enabled: false + schedule: "* * * * *" env: # DB action: backup/restore DB_ACTION: backup @@ -222,16 +222,16 @@ dbBackupRestore: resources: enabled: false requests: - memory: '300Mi' - cpu: '0.5' + memory: "300Mi" + cpu: "0.5" limits: - memory: '400Mi' - cpu: '0.6' + memory: "400Mi" + cpu: "0.6" nodeSelector: enabled: false - name: osmcha-db - enabled: true - schedule: '* * * * *' + enabled: false + schedule: "* * * * *" env: # DB action: backup/restore DB_ACTION: backup @@ -246,11 +246,11 @@ dbBackupRestore: resources: enabled: false requests: - memory: '300Mi' - cpu: '0.5' + memory: "300Mi" + cpu: "0.5" limits: - memory: '400Mi' - cpu: '0.6' + memory: "400Mi" + cpu: "0.6" nodeSelector: enabled: false @@ -260,19 +260,19 @@ dbBackupRestore: planetDump: enabled: false image: - name: '' - tag: '' - schedule: '* * * * *' + name: "" + tag: "" + schedule: "* * * * *" env: OVERWRITE_PLANET_FILE: false resources: enabled: false requests: - memory: '14Gi' - cpu: '4' + memory: "14Gi" + cpu: "4" limits: - memory: '16Gi' - cpu: '4' + memory: "16Gi" + cpu: "4" nodeSelector: enabled: false @@ -282,16 +282,16 @@ planetDump: replicationJob: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" nodeSelector: enabled: false @@ -301,18 +301,18 @@ replicationJob: populateApidb: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: - URL_FILE_TO_IMPORT: 'http://download.geofabrik.de/europe/monaco-latest.osm.pbf' + URL_FILE_TO_IMPORT: "http://download.geofabrik.de/europe/monaco-latest.osm.pbf" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2.5' + memory: "2Gi" + cpu: "2.5" # ==================================================================================================== # Variables to start a pod to process osm files @@ -320,19 +320,19 @@ populateApidb: osmProcessor: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: - URL_FILE_TO_PROCESS: 'http://download.geofabrik.de/europe/monaco-latest.osm.pbf' + URL_FILE_TO_PROCESS: "http://download.geofabrik.de/europe/monaco-latest.osm.pbf" OSM_FILE_ACTION: simple_pbf resources: enabled: false requests: - memory: '14Gi' - cpu: '4' + memory: "14Gi" + cpu: "4" limits: - memory: '16Gi' - cpu: '4' + memory: "16Gi" + cpu: "4" # ==================================================================================================== # Variables for tiler-db @@ -340,13 +340,13 @@ osmProcessor: tilerDb: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: POSTGRES_HOST: tiler-db POSTGRES_DB: tiler-osm POSTGRES_USER: postgres - POSTGRES_PASSWORD: '1234' + POSTGRES_PASSWORD: "1234" POSTGRES_PORT: 5432 POSTGRES_DB_MAX_CONNECTIONS: 100 POSTGRES_DB_SHARED_BUFFERS: 128MB @@ -370,11 +370,11 @@ tilerDb: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2.5' + memory: "2Gi" + cpu: "2.5" nodeSelector: enabled: false label_key: nodegroup_type @@ -386,8 +386,8 @@ tilerDb: tilerImposm: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" env: TILER_IMPORT_FROM: osm TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf @@ -408,11 +408,11 @@ tilerImposm: resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" nodeSelector: enabled: false label_key: nodegroup_type @@ -423,8 +423,8 @@ tilerImposm: tilerServer: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" replicaCount: 1 serviceAnnotations: {} env: @@ -437,7 +437,7 @@ tilerServer: TILER_CACHE_REGION: us-east-1 TILER_CACHE_AWS_ACCESS_KEY_ID: xyz TILER_CACHE_AWS_SECRET_ACCESS_KEY: xyz/q - command: './start.sh' # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh + command: "./start.sh" # ./start.sh & ./tile_cache_downloader.sh & ./expire-watcher.sh # In case you use TILER_CACHE_TYPE: file with persistenceDisk persistenceDisk: enabled: false @@ -455,30 +455,30 @@ tilerServer: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false label_key: nodegroup_type label_value: tiler # ==================================================================================================== -# Variables for tiler-server cache cleaner, only avaliable in case the TILER_CACHE_TYPE = s3 +# Variables for tiler-server cache cleaner, only avaliable in case the TILER_CACHE_TYPE = s3 # ==================================================================================================== tilerServerCacheCleaner: enabled: false replicaCount: 1 - command: './tile_cache_downloader.sh & ./expire-watcher.sh' + command: "./tile_cache_downloader.sh & ./expire-watcher.sh" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" env: KILL_PROCESS: manually MAX_NUM_PS: 3 @@ -498,8 +498,8 @@ tilerServerCacheCleaner: tilerVisor: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} replicaCount: 1 env: @@ -508,11 +508,11 @@ tilerVisor: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false label_key: nodegroup_type @@ -524,48 +524,47 @@ tilerVisor: tmApi: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" nodeSelector: enabled: false replicaCount: 1 serviceAnnotations: {} env: - POSTGRES_HOST: 'tm' - POSTGRES_DB: 'tm' - POSTGRES_PASSWORD: 'tm' - POSTGRES_USER: 'postgres' + POSTGRES_HOST: "tm" + POSTGRES_DB: "tm" + POSTGRES_PASSWORD: "tm" + POSTGRES_USER: "postgres" POSTGRES_PORT: 5432 - TM_ORG_NAME: 'OpenStreetMap' - TM_ORG_CODE: 'OSM' - TM_ORG_URL: 'openstreetmap.org' - TM_ORG_PRIVACY_POLICY_URL: 'openstreetmap.org/copyright' - TM_ORG_GITHUB: 'github.com/openstreetmap' - OSM_SERVER_URL: 'https://openstreetmap.org' - OSM_NOMINATIM_SERVER_URL: 'https://nominatim.openstreetmap.org' - OSM_REGISTER_URL: 'https://openstreetmap.org/user/new' - ID_EDITOR_URL: 'https://openstreetmap.org/edit?editor=id' - POTLATCH2_EDITOR_URL: 'https://openstreetmap.org/edit?editor=potlatch2' - TM_SECRET: 's0m3l0ngr4nd0mstr1ng-b3cr34tiv3' - TM_CONSUMER_KEY: 'example' - TM_CONSUMER_SECRET: 'example' - TM_EMAIL_FROM_ADDRESS: 'no-reply@openstreetmap.org' - TM_SMTP_HOST: 'email-smtp.us-east-1.amazonaws.com' + TM_ORG_NAME: "OpenStreetMap" + TM_ORG_CODE: "OSM" + TM_ORG_URL: "openstreetmap.org" + TM_ORG_PRIVACY_POLICY_URL: "openstreetmap.org/copyright" + TM_ORG_GITHUB: "github.com/openstreetmap" + OSM_SERVER_URL: "https://openstreetmap.org" + OSM_NOMINATIM_SERVER_URL: "https://nominatim.openstreetmap.org" + OSM_REGISTER_URL: "https://openstreetmap.org/user/new" + ID_EDITOR_URL: "https://openstreetmap.org/edit?editor=id" + POTLATCH2_EDITOR_URL: "https://openstreetmap.org/edit?editor=potlatch2" + TM_SECRET: "s0m3l0ngr4nd0mstr1ng-b3cr34tiv3" + TM_CONSUMER_KEY: "example" + TM_CONSUMER_SECRET: "example" + TM_EMAIL_FROM_ADDRESS: "no-reply@openstreetmap.org" + TM_SMTP_HOST: "email-smtp.us-east-1.amazonaws.com" TM_SMTP_PORT: 25 - TM_SMTP_USER: '' - TM_SMTP_PASSWORD: '' - TM_DEFAULT_LOCALE: 'en' - TM_APP_API_URL: '' - TM_APP_BASE_URL: '' + TM_SMTP_USER: "" + TM_SMTP_PASSWORD: "" + TM_DEFAULT_LOCALE: "en" + TM_APP_API_URL: "" + TM_APP_BASE_URL: "" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' - + memory: "2Gi" + cpu: "2" # ==================================================================================================== # Variables for nominatim api @@ -573,8 +572,8 @@ tmApi: nominatimApi: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} replicaCount: 1 env: @@ -596,11 +595,11 @@ nominatimApi: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" persistenceDisk: enabled: true accessMode: ReadWriteOnce @@ -624,17 +623,16 @@ nominatimApi: overpassApi: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} env: - OVERPASS_META: 'yes' + OVERPASS_META: "yes" OVERPASS_MODE: init OVERPASS_PLANET_URL: http://download.geofabrik.de/europe/monaco-latest.osm.bz2 OVERPASS_DIFF_URL: http://download.openstreetmap.fr/replication/europe/monaco/minute/ OVERPASS_RULES_LOAD: 10 - #OVERPASS_PLANET_PREPROCESS: 'mv /db/planet.osm.bz2 /db/planet.osm.pbf && osmium cat -o /db/planet.osm.bz2 /db/planet.osm.pbf && rm /db/planet.osm.pbf' # it is in case we pass planet files as PBF file uncommment this line - OVERPASS_PLANET_PREPROCESS: 'ls' + OVERPASS_PLANET_PREPROCESS: 'mv /db/planet.osm.bz2 /db/planet.osm.pbf && osmium cat -o /db/planet.osm.bz2 /db/planet.osm.pbf && rm /db/planet.osm.pbf' OVERPASS_REPLICATION_SEQUENCE_NUMBER: 5201000 persistenceDisk: enabled: false @@ -652,11 +650,11 @@ overpassApi: resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false # ==================================================================================================== @@ -665,8 +663,8 @@ overpassApi: taginfo: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" serviceAnnotations: {} env: URL_PLANET_FILE_STATE: https://planet.osm.org/pbf/state.txt @@ -679,16 +677,16 @@ taginfo: INSTANCE_ICON: /img/logo/world.png INSTANCE_CONTACT: Anonymous TAGINFO_PROJECT_REPO: https://github.com/taginfo/taginfo-projects.git - DOWNLOAD_DB: 'languages wiki' - CREATE_DB: 'db projects chronology' + DOWNLOAD_DB: "languages wiki" + CREATE_DB: "db projects chronology" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false # ==================================================================================================== @@ -697,17 +695,17 @@ taginfo: osmSimpleMetrics: enabled: false image: - name: '' - tag: '' - schedule: '* */1 * * *' + name: "" + tag: "" + schedule: "* */1 * * *" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false @@ -717,20 +715,20 @@ osmSimpleMetrics: monitoringReplication: enabled: false image: - name: 'developmentseed/osmseed-replication-job' - tag: '0.1.0-n770.hd15c9f2' - schedule: '* */1 * * *' + name: "developmentseed/osmseed-replication-job" + tag: "0.1.0-n770.hd15c9f2" + schedule: "* */1 * * *" env: CREATE_MISSING_FILES: empty REPLICATION_SEQUENCE_NUMBER: "109789" resources: enabled: false requests: - memory: '1Gi' - cpu: '2' + memory: "1Gi" + cpu: "2" limits: - memory: '2Gi' - cpu: '2' + memory: "2Gi" + cpu: "2" nodeSelector: enabled: false @@ -740,92 +738,87 @@ monitoringReplication: changesetReplicationJob: enabled: false image: - name: '' - tag: '' + name: "" + tag: "" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" nodeSelector: enabled: false - - # ==================================================================================================== -# Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day +# Variables for osmcha web builder # ==================================================================================================== osmchaWeb: - enabled: false + enabled: true + image: + name: "" + tag: "" +# ==================================================================================================== +# Variables for osmcha Api +# ==================================================================================================== +osmchaApi: + enabled: true image: - name: ghcr.io/willemarcel/osmcha-django - tag: b1f4e6afc90e08707cadc4d74580632ca3b93dd2 + name: "ghcr.io/willemarcel/osmcha-django" + tag: "10d7f6748d9e55d439339ed2c35509db15f782a6" env: - OSM_SERVER_URL: 'https://staging.openstreetmap.org' - DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" - DJANGO_DEBUG: "False" + DJANGO_SETTINGS_MODULE: "config.settings.production" + OSMCHA_FRONTEND_VERSION: "v0.86.0-production" DJANGO_SECRET_KEY: abc - DJANGO_SECURE_BROWSER_XSS_FILTER: "True" - DJANGO_SECURE_SSL_REDIRECT: "True" - DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" - DJANGO_SECURE_FRAME_DENY: "True" - DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True" - DJANGO_SESSION_COOKIE_HTTPONLY: "True" - DJANGO_SESSION_COOKIE_SECURE: "True" - DJANGO_DEFAULT_FROM_EMAIL: "webmaster@openstreetmap.org" - DJANGO_SERVER_EMAIL: "server@openstreetmap.org" - DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" - DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" + ## ohm OAUTH_OSM_KEY: abc - OAUTH_OSM_SECRET: abc - DJANGO_ANON_USER_THROTTLE_RATE: "10/h" - DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" - DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" - OAUTH_REDIRECT_URI: https://osmcha.openstreetmap.org/oauth-landing.html - OSMCHA_FRONTEND_VERSION: "v0.86.0-production" - DJANGO_ENABLE_CHANGESET_COMMENTS: "True" - DJANGO_OSM_COMMENTS_API_KEY: "test" - DJANGO_ROOT: /app - DJANGO_SETTINGS_MODULE: config.settings.local - C_FORCE_ROOT: true - REACT_APP_OSM_URL: https://staging.openstreetmap.org - REACT_APP_OSM_API: https://staging.openstreetmap.org/api/0.6 - REACT_APP_OVERPASS_BASE: https://overpass-api-staging.openstreetmap.org/api/interpreter - REACT_APP_NOMINATIM_URL: https://nominatim-api-staging.openstreetmap.org - OSMCHA_URL: https://osmcha-staging.openstreetmap.org + OAUTH_OSM_SECRET: bdc + DJANGO_SECURE_SSL_REDIRECT: "False" + OSM_SERVER_URL: https://www.osmseed.org + OAUTH_REDIRECT_URI: https://osmcha-staging.osmseed.org/oauth-landing.html + OSM_PLANET_BASE_URL: https://s3.amazonaws.com/planet.osmseed.org/replication/changesets/ + + ## frontend + OSMCHA_URL: https://osmcha-staging.osmseed.org + OSMCHA_API_URL: www.osmseed.org + REACT_APP_OSM_URL: https://www.osmseed.org + REACT_APP_OSM_API: https://www.osmseed.org/api/0.6 + REACT_APP_OVERPASS_BASE: //overpass-api.osmseed.org/api/interpreter + REACT_APP_DISABLE_REAL_CHANGESETS: 1 + REACT_APP_MAPBOX_ACCESS_TOKEN: "pk.abc.df" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "512Mi" + cpu: "1" limits: - memory: '24Gi' - cpu: '10' + memory: "512Mi" + cpu: "1" nodeSelector: enabled: false + label_key: nodegroup_type + label_value: db # ==================================================================================================== -# Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day +# Variables for osmcha DB # ==================================================================================================== osmchaDb: - enabled: false + enabled: true image: - name: '' - tag: '' + name: "" + tag: "" env: POSTGRES_DB: osmcha-db POSTGRES_USER: postgres - POSTGRES_PASSWORD: '1234' + POSTGRES_PASSWORD: "1234" resources: enabled: false requests: - memory: '20Gi' - cpu: '8' + memory: "20Gi" + cpu: "8" limits: - memory: '24Gi' - cpu: '10' + memory: "24Gi" + cpu: "10" persistenceDisk: enabled: false accessMode: ReadWriteOnce