Move tiler-db to hetzner server

DEPLOY.md

@@ -0,0 +1,16 @@
+# OHM Site Deployment and Migration Guide
+
+Follow these steps to perform a migration, especially during upstream updates:
+
+- 1.	Set the OHM site to read-only mode
+This prevents users from making changes during the migration process.
+
+- 2.	Take an EBS backup on AWS
+Ensure the backup is fully completed before proceeding with the migration.
+
+- 3.	Adjust the web autoscaling configuration to 1 
+Make sure only one container is running to avoid potential migration conflicts.
+
+### Post-Migration
+
+Once the migration is complete, verify that everything has been successfully applied and bring the site back online.

hetzner/config/postgresql.config

@@ -0,0 +1,81 @@
+#------------------------------------------------------------------------------
+# CONNECTIONS AND AUTHENTICATION
+#------------------------------------------------------------------------------
+
+listen_addresses = '*'                  # Allow connections from all addresses
+max_connections = 100                    # Set max connections to 120
+superuser_reserved_connections = 3       # Reserve for superusers
+
+#------------------------------------------------------------------------------
+# RESOURCE USAGE
+#------------------------------------------------------------------------------
+
+# - Memory Configuration -
+
+shared_buffers = 14GB                    # Allocate 25% of total memory
+work_mem = 512MB                           # Increase per-query memory for sorting, hashing
+maintenance_work_mem = 2GB                # For maintenance operations like VACUUM, ALTER
+effective_cache_size = 41GB                # 75% of total memory for caching
+
+# - Disk Optimization for SSD -
+
+random_page_cost = 1.1                     # Lower for SSD performance
+seq_page_cost = 1.0                         # Default for sequential scans
+
+#------------------------------------------------------------------------------
+# WRITE-AHEAD LOG (WAL)
+#------------------------------------------------------------------------------
+
+wal_level = replica                        # Minimal for read-heavy workloads
+checkpoint_timeout = 15min                  # Reduce checkpoint frequency
+max_wal_size = 4GB                           # Increase to reduce writes
+min_wal_size = 1GB                           # Set a lower boundary for WAL size
+wal_compression = on                         # Enable compression for WAL files
+synchronous_commit = off                     # Speed up transactions by avoiding full durability
+
+#------------------------------------------------------------------------------
+# AUTOVACUUM SETTINGS
+#------------------------------------------------------------------------------
+
+autovacuum_max_workers = 4                   # Run multiple autovacuum workers
+autovacuum_naptime = 30s                      # Frequency of autovacuum runs
+autovacuum_vacuum_cost_limit = 2000           # Increase vacuum efficiency for large data
+
+#------------------------------------------------------------------------------
+# QUERY TUNING
+#------------------------------------------------------------------------------
+
+effective_io_concurrency = 300                # Higher for SSD disks
+parallel_tuple_cost = 0.1                      # Lower to encourage parallelism
+parallel_setup_cost = 500                       # Lower setup cost for parallel processing
+max_worker_processes = 28                      # Utilize CPU cores efficiently
+max_parallel_workers_per_gather = 8            # Allow parallel processing for queries
+max_parallel_workers = 14                       # Limit the total parallel workers
+
+#------------------------------------------------------------------------------
+# LOGGING
+#------------------------------------------------------------------------------
+
+logging_collector = on                  # Enable logging to file
+log_directory = 'pg_log'                 # Directory to store log files
+log_filename = 'postgresql-%Y-%m-%d.log'  # Log file naming pattern
+log_rotation_size = 100MB                 # Rotate logs at 100MB
+log_truncate_on_rotation = on             # Overwrite old logs with the same name
+
+log_min_duration_statement = 20000        # Log queries that take longer than 20 seconds (in milliseconds)
+
+log_statement = 'none'                    # Disable logging of all statements
+log_duration = off                         # Disable duration logging for all queries
+
+#------------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#------------------------------------------------------------------------------
+
+datestyle = 'iso, mdy'
+timezone = 'Etc/UTC'
+lc_messages = 'en_US.utf8'
+lc_monetary = 'en_US.utf8'
+lc_numeric = 'en_US.utf8'
+lc_time = 'en_US.utf8'
+
+default_text_search_config = 'pg_catalog.english'

hetzner/env.production.example

@@ -0,0 +1,28 @@
+
+# #######################################
+# db env variables
+# #######################################
+
+POSTGRES_HOST=db_production
+POSTGRES_DB=tiler_osm_production
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=mysecretpassword
+POSTGRES_PORT=5432
+
+# #######################################
+# imposm env variables
+# #######################################
+
+TILER_IMPORT_FROM=osm
+TILER_IMPORT_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250117_0002.osm.pbf
+REPLICATION_URL=http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute/
+SEQUENCE_NUMBER=1690000
+OVERWRITE_STATE=false
+UPLOAD_EXPIRED_FILES=true
+IMPORT_NATURAL_EARTH=true
+IMPORT_OSM_LAND=true
+IMPOSM3_IMPORT_LAYERS=all
+CLOUDPROVIDER=aws
+AWS_S3_BUCKET=s3://planet.openhistoricalmap.org
+AWS_ACCESS_KEY_ID=xxx
+AWS_SECRET_ACCESS_KEY=xxx

hetzner/tiler.production.yml

@@ -4,7 +4,7 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      - ../data/tiler_production_db_v3:/var/lib/postgresql/data
+      - /data/tiler_production_db_20250123:/var/lib/postgresql/data
     env_file:
       - .env.production
     restart: always
@@ -18,7 +18,7 @@ services:
   imposm_production:
     image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.2087.h94d548c
     volumes:
-      - ../data/tiler_production_imposm_v3:/mnt/data
+      - /data/tiler_production_imposm:/mnt/data
     command:
       - sh
       - -c
@@ -38,6 +38,20 @@ services:
           memory: 3G
     networks:
       - tiler_network
+  node_exporter:
+    image: quay.io/prometheus/node-exporter
+    container_name: node-exporter
+    restart: always
+    ports:
+      - "9100:9100"
+    networks:
+      - tiler_network
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M
 networks:
   tiler_network:
-    driver: bridge
+    driver: bridge
+

hetzner/tiler.staging.yml

@@ -2,7 +2,7 @@ services:
   db:
     image: ghcr.io/openhistoricalmap/tiler-db:0.0.1-0.dev.git.1737.h9b9ef9b
     volumes:
-          - /data/tiler_production_db_v3:/var/lib/postgresql/data
+      - /data/tiler_production_db_v3:/var/lib/postgresql/data
     ports:
       - "5433:5432"
     env_file:

images/tiler-server/start.sh

@@ -1,41 +1,6 @@
 #!/usr/bin/env bash
 export PGPASSWORD=$POSTGRES_PASSWORD
 
-## Merge providers files
-python build_config.py \
-  --output=/opt/tegola_config/config.toml \
-  --provider_names \
-"land,\
-admin_boundaries_lines,\
-admin_boundaries.centroids,\
-admin_boundaries_maritime,\
-place_areas,\
-place_points,\
-water_areas,\
-water_areas.centroids,\
-water_line,\
-transport_areas,\
-transport_associated_streets,\
-transport_lines,\
-transport_multilines,\
-transport_points,\
-route_lines,\
-amenity_areas,\
-amenity_areas.centroids,\
-amenity_points,\
-buildings,\
-buildings.centroids,\
-buildings_points,\
-landuse_areas,\
-landuse_areas.centroids,\
-landuse_lines,\
-landuse_points,\
-other_areas,\
-other_areas.centroids,\
-other_lines,\
-other_points
-"
-
 flag=true
 while [ "$flag" = true ]; do
   # Wait until PostgreSQL is ready

values.production.template.yaml

@@ -362,7 +362,7 @@ osm-seed:
   tilerDb:
     enabled: true
     useExternalHost: # When we are using useExternalHost.enabled= true other variables are giong to be disable ans use the external host config
-      enabled: false
+      enabled: true
     nodeSelector:
       enabled: true
       label_key: nodegroup_type
@@ -455,7 +455,7 @@ osm-seed:
   # ====================================================================================================
 
   tilerImposm:
-    enabled: true
+    enabled: false
     nodeSelector:
       enabled: true
       label_key: nodegroup_type
@@ -509,7 +509,7 @@ osm-seed:
       TILER_CACHE_BUCKET: tiler-cache-production
       TILER_CACHE_AWS_ACCESS_KEY_ID: {{PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID}}
       TILER_CACHE_AWS_SECRET_ACCESS_KEY: {{PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY}}
-      EXECUTE_REINDEX: false
+      EXECUTE_REINDEX: true
       EXECUTE_VACUUM_ANALYZE: false
     # In case you use TILER_CACHE_TYPE: file with  persistenceDisk 
     persistenceDisk: