From 66c1de685caf0f7f902ccff25188b484501c2d0c Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 15 Aug 2024 14:45:04 +0300 Subject: [PATCH 1/3] dont require local data api dependency for development --- README.md | 26 +++++++++++++++++++++ docker-compose-local-data-api.dev.yml | 33 +++++++++++++++++++++++++++ docker-compose.dev.yml | 12 ++++++---- 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 README.md create mode 100644 docker-compose-local-data-api.dev.yml diff --git a/README.md b/README.md new file mode 100644 index 00000000..f2a086bf --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# GFW Tile Service +Raster and vector tile API for datasets in the [GFW Data API](https://github.com/wri/gfw-data-api) and Titiler dynamic tiling for raster assets in publicly accessible cloud storage. + +## Developing +### Option 1: Developing against the Data API postgres database in one of the cloud environments (dev, staging or production): +* Make sure you have ssh access to the bastion host of the aws account (contact a Data API Engineering team member to get help with this). + +* Open ssh tunnel connection to the database you'd like to connect to. For example, for the `staging` environment: + + ```ssh -i ~/.ssh/id_rsa -N -L 5432:application-autoscaling-698c9c01-db99-4430-a97a-6baaae853dc6.cljrqsduwhdo.us-east-1.rds.amazonaws.com:5432 ec2-user@gfw-staging``` + +* Set the environment variables for the **read only** credentials of the above database. The environment variables are `GFW_DB_NAME`, `GFW_DB_USER_RO` and `GFW_DB_PASSWORD_RO`. These are also listed in the `docker-compose.dev.yml` file. + +* In `docker-compose.dev.yml`, set `DATA_LAKE_BUCKET` to the desired environment's bucket name. By default, the `staging` environment bucket (`gfw-data-lake-staging`) will be used. + +* In `docker-compose.dev.yml`, set `AWS_DEFAULT_PROFILE` to your aws profile in `~/.aws` that will grant your dev instance access to the aws resources including the data lake bucket above in the aws account of the interest (contact a Data API Engineering team member to get an account). + +* Run the start up script from the root directory: +```./scripts/develop``` + +### Option 2: Developing against a local instance of Data API database + +* Start dev instance of Data API locally using the instructions [here](https://github.com/wri/gfw-data-api?tab=readme-ov-file#run-locally-with-docker) + +* Run the start up script from the root directory with the option to point to the local Data API: +```./scripts/develop --local_data_api``` \ No newline at end of file diff --git a/docker-compose-local-data-api.dev.yml b/docker-compose-local-data-api.dev.yml new file mode 100644 index 00000000..dc50f805 --- /dev/null +++ b/docker-compose-local-data-api.dev.yml @@ -0,0 +1,33 @@ +version: "3.5" + +services: + app: + build: + context: . + args: + - ENV=dev + container_name: tile-cache_dev + restart: on-failure + volumes: + - $HOME/.aws:/root/.aws:ro + - ./app:/app/app + networks: + - gfw-data-api + environment: + - DB_HOST_RO=database_12 + - DATABASE_RO=geostore + - DB_USER_RO=gfw_readonly + - DB_PASSWORD_RO=readonly # pragma: allowlist secret + - DB_PORT_RO=5432 + - AWS_DEFAULT_PROFILE=gfw-dev + - LOG_LEVEL=debug + - RASTER_TILER_LAMBDA_NAME=test + - ENV=dev + - PLANET_API_KEY + ports: + - 8088:80 + entrypoint: wait_for_postgres.sh /start-reload.sh + +networks: + gfw-data-api: + name: gfw-data-api_dev_default \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dc50f805..82b5f00a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -14,16 +14,18 @@ services: networks: - gfw-data-api environment: - - DB_HOST_RO=database_12 - - DATABASE_RO=geostore - - DB_USER_RO=gfw_readonly - - DB_PASSWORD_RO=readonly # pragma: allowlist secret + - DB_HOST_RO=host.docker.internal + - DATABASE_RO=${GFW_DB_NAME} + - DB_USER_RO=${GFW_DB_USER_RO} + - DB_PASSWORD_RO=${GFW_DB_PASSWORD_RO} - DB_PORT_RO=5432 - - AWS_DEFAULT_PROFILE=gfw-dev + - AWS_DEFAULT_PROFILE=gfw-staging + - DATA_LAKE_BUCKET=gfw-data-lake-staging - LOG_LEVEL=debug - RASTER_TILER_LAMBDA_NAME=test - ENV=dev - PLANET_API_KEY + - AWS_REQUEST_PAYER=requester ports: - 8088:80 entrypoint: wait_for_postgres.sh /start-reload.sh From 2869e15b171519da5fae44cd30a8244941b57c28 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 29 Aug 2024 19:59:45 +0300 Subject: [PATCH 2/3] add develop script option for running without data api --- scripts/develop | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/develop b/scripts/develop index beff86de..0a3375dc 100755 --- a/scripts/develop +++ b/scripts/develop @@ -2,11 +2,11 @@ set -e - - # Default values POSITIONAL=() BUILD=true +COMPOSE_FILE="docker-compose.dev.yml" + # extracting cmd line arguments while [[ $# -gt 0 ]] do @@ -17,6 +17,10 @@ do BUILD=false shift # past argument ;; + --local_data_api) + COMPOSE_FILE="docker-compose-local-data-api.dev.yml" + shift # past argument + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -26,8 +30,7 @@ done set -- "${POSITIONAL[@]}" # restore positional parameters if [ "${BUILD}" = true ]; then - docker-compose -f docker-compose.dev.yml up --abort-on-container-exit --remove-orphans --build + docker-compose -f "${COMPOSE_FILE}" up --abort-on-container-exit --remove-orphans --build else - docker-compose -f docker-compose.dev.yml up --abort-on-container-exit --remove-orphans + docker-compose -f "${COMPOSE_FILE}" up --abort-on-container-exit --remove-orphans fi - From 73735b6c145763b6c6be4bb1af18ca1e754bdc2b Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Thu, 29 Aug 2024 20:04:09 +0300 Subject: [PATCH 3/3] fix docker-compose -> docker compose --- scripts/develop | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/develop b/scripts/develop index 0a3375dc..37273121 100755 --- a/scripts/develop +++ b/scripts/develop @@ -30,7 +30,7 @@ done set -- "${POSITIONAL[@]}" # restore positional parameters if [ "${BUILD}" = true ]; then - docker-compose -f "${COMPOSE_FILE}" up --abort-on-container-exit --remove-orphans --build + docker compose -f "${COMPOSE_FILE}" up --abort-on-container-exit --remove-orphans --build else - docker-compose -f "${COMPOSE_FILE}" up --abort-on-container-exit --remove-orphans + docker compose -f "${COMPOSE_FILE}" up --abort-on-container-exit --remove-orphans fi