diff --git a/scripts/population/README.md b/scripts/population/README.md index 8d5e38db..19f5c23a 100644 --- a/scripts/population/README.md +++ b/scripts/population/README.md @@ -15,6 +15,19 @@ $ docker-compose run --rm app bash /app$ scripts/population/load-regions URY Uruguay ``` +### Geometry simplification + +The script simplifies the regions using mapshaper by a distance between points. +We can adjust this parameter with the environment `INTERVAL_DISTANCE`. + +For example to simplify with 120 meters distance + +```sh +$ INTERVAL_DISTANCE=120 ./load-regions -f URY Uruguay +``` + +Here `-f` is used to force updating an existing region. + ## Clip friction layer for new regions The friction raster is clipped to the country-level regions for quicker access @@ -46,4 +59,3 @@ INSERT INTO source_set (name, type, unit, raster_file) VALUES ('Uruguay PPP v2b The important bit of information is the last field. That should be a relative path from the `DATA_PATH` directory to the downloaded raster file. - diff --git a/scripts/population/load-regions b/scripts/population/load-regions index 06c64608..ea1ea91b 100755 --- a/scripts/population/load-regions +++ b/scripts/population/load-regions @@ -4,12 +4,35 @@ set -euo pipefail export PGPASSWORD=$POSTGRES_PASSWORD; if [ $# -lt 2 ]; then - echo "Usage $0 " + echo "Usage $0 [-f] " + echo "The option -f forces updating an existing country geometry" exit 1 fi -COUNTRY_CODE=$1 -COUNTRY_NAME=$2 +FORCE= +COUNTRY_CODE= +COUNTRY_NAME= + +while [ $# -gt 0 ]; do + case $1 in + -f|--force) + FORCE=1 + ;; + *) + if [ ! -z "$COUNTRY_NAME" ]; then + echo Unknown arguments specified + exit 1; + elif [ ! -z "$COUNTRY_CODE" ]; then + COUNTRY_NAME=$1 + else + COUNTRY_CODE=$1 + fi + ;; + esac + shift +done + + #LEVELS=${2:-2,4} LEVELS=(0,1) echo " -> Importing $COUNTRY_NAME at levels $LEVELS" @@ -18,7 +41,7 @@ echo " -> Importing $COUNTRY_NAME at levels $LEVELS" REGIONS=$(psql -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT -t -A -c "SELECT id, country FROM regions WHERE country = '$COUNTRY_NAME';") REGIONS_LEN=$(echo $REGIONS | wc -w | tr -d '[[:space:]]') -if [ ${REGIONS_LEN} -gt 0 ]; then +if [ ${REGIONS_LEN} -gt 0 -a -z $FORCE ]; then echo " -> Regions for country $COUNTRY_NAME already imported." exit 0 fi @@ -37,9 +60,12 @@ fi # From mapshaper wiki https://github.com/mbloch/mapshaper/wiki/Command-Reference#-simplify # # interval= Specify simplification amount in units of distance. -# Uses meters when simplifying unprojected datasets in 3D space (see planar option below), +# Uses meters when simplifying unprojected datasets in 3D space, # otherwise uses the same units as the source data. -INTERVAL_DISTANCE=${INTERVAL_DISTANCE-1000} +# In our case since our GeoJSON are not projected (ie. use WGS84 coordinates) +# then this means the resolution for the simplified shape will be in meters. + +INTERVAL_DISTANCE=${INTERVAL_DISTANCE-100} IFS=','; for i in $LEVELS; do @@ -52,23 +78,49 @@ for i in $LEVELS; do OFILE=${DATA_PATH}/${COUNTRY_CODE}/${COUNTRY_CODE}_adm${i}_${INTERVAL_DISTANCE}.geojson mapshaper $FILE -simplify interval=$INTERVAL_DISTANCE -o $OFILE + + SIZE=$(stat --printf="%s" $FILE) + OSIZE=$(stat --printf="%s" $OFILE) + + echo " -> Finished file size went from $SIZE to $OSIZE" fi - psql -q -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT << SQL_SCRIPT + if [ "$FORCE" = "1" ]; then + psql -q -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT << SQL_SCRIPT + + WITH data AS (SELECT \$$`cat $OFILE`\$$::json AS fc) + UPDATE "regions" SET the_geom=f.the_geom + FROM ( + SELECT + '${COUNTRY_NAME}' as country, + feat#>>'{properties,name}' AS name, + ${i} as admin_level, + ST_SetSRID(ST_CollectionExtract(ST_Multi(ST_GeomFromGeoJSON(feat->>'geometry')), 3), 4326) as the_geom + FROM ( + SELECT json_array_elements(fc->'features') AS feat + FROM data + ) AS dummy + ) AS f + WHERE regions.country=f.country AND regions.name=f.name AND regions.admin_level=f.admin_level; - WITH data AS (SELECT \$$`cat $OFILE`\$$::json AS fc) - INSERT INTO "regions" (country, name, admin_level, the_geom) - SELECT - '${COUNTRY_NAME}', - feat#>>'{properties,name}' AS name, - ${i}, - ST_SetSRID(ST_CollectionExtract(ST_Multi(ST_GeomFromGeoJSON(feat->>'geometry')), 3), 4326) as the_geom - FROM ( - SELECT json_array_elements(fc->'features') AS feat - FROM data - ) AS f; +SQL_SCRIPT + else + psql -q -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT << SQL_SCRIPT + + WITH data AS (SELECT \$$`cat $OFILE`\$$::json AS fc) + INSERT INTO "regions" (country, name, admin_level, the_geom) + SELECT + '${COUNTRY_NAME}', + feat#>>'{properties,name}' AS name, + ${i}, + ST_SetSRID(ST_CollectionExtract(ST_Multi(ST_GeomFromGeoJSON(feat->>'geometry')), 3), 4326) as the_geom + FROM ( + SELECT json_array_elements(fc->'features') AS feat + FROM data + ) AS f; SQL_SCRIPT + fi # TODO: print errors echo " done!" done;