From c54cd168c85cc3fb6a1590ad513578bc62f7919b Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Fri, 12 Jul 2024 17:43:21 -0600 Subject: [PATCH 1/2] Remove test broken by osm2pgsql #2212. Add related docs. --- Dockerfile | 6 ++++-- Makefile | 39 --------------------------------------- docs/src/docker-build.md | 26 ++++++++++++++++++++++++++ docs/src/replication.md | 6 ++++++ 4 files changed, 36 insertions(+), 41 deletions(-) diff --git a/Dockerfile b/Dockerfile index 062a691f..35972daa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,10 @@ FROM postgis/postgis:16-3.4 LABEL maintainer="PgOSM Flex - https://github.com/rustprooflabs/pgosm-flex" -ARG OSM2PGSQL_BRANCH=master -ARG OSM2PGSQL_REPO=https://github.com/osm2pgsql-dev/osm2pgsql.git +#ARG OSM2PGSQL_BRANCH=master +ARG OSM2PGSQL_BRANCH=check-date-on-replication-init +#ARG OSM2PGSQL_REPO=https://github.com/osm2pgsql-dev/osm2pgsql.git +ARG OSM2PGSQL_REPO=https://github.com/lonvia/osm2pgsql.git RUN apt-get update \ diff --git a/Makefile b/Makefile index 6f26579e..21e3c894 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,6 @@ RAM=2 # to make unit test results visible at the end. .PHONY: all all: docker-exec-region docker-exec-input-file \ - docker-exec-replication-w-input-file \ docker-exec-default unit-tests .PHONY: docker-clean @@ -115,44 +114,6 @@ docker-exec-input-file: build-run-docker -.PHONE: docker-exec-replication-w-input-file -docker-exec-replication-w-input-file: build-run-docker - # NOTE: This step tests --replication file for an initial load. - # It does **NOT** test the actual replication process for updating data - # using replication mode. Testing actual replication over time in this format - # will not be trivial. The historic file used (2021-01-13) cannot be used - # to seed a replication process, there is a time limit in upstream software - # that requires more recency to the source data. This also cannot simply - # download a file from Geofabrik, as the "latest" file will not have a diff - # to apply so also will not test the actual replication. - # - # Open a PR, Issue, discussion on https://github.com/rustprooflabs/pgosm-flex - # if you have an idea on how to implement this testing functionality. - - # copy with arbitrary file name to test --input-file - docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \ - pgosm:/app/output/$(INPUT_FILE_NAME) - - # allow files created in later step to be created - docker exec -it pgosm \ - chown $(CURRENT_UID):$(CURRENT_GID) /app/output/ - # Needed for unit-tests - docker exec -it pgosm \ - chown $(CURRENT_UID):$(CURRENT_GID) /app/docker/ - - # process it, this time without providing the region but directly the filename - docker exec -it \ - -e POSTGRES_PASSWORD=mysecretpassword \ - -e POSTGRES_USER=postgres \ - -u $(CURRENT_UID):$(CURRENT_GID) \ - pgosm python3 docker/pgosm_flex.py \ - --layerset=minimal \ - --ram=$(RAM) \ - --replication \ - --input-file=/app/output/$(INPUT_FILE_NAME) \ - --skip-qgis-style --skip-nested # Make this test run faster - - .PHONE: docker-exec-region docker-exec-region: build-run-docker # copy for simulating region diff --git a/docs/src/docker-build.md b/docs/src/docker-build.md index 5962052e..993ffb26 100644 --- a/docs/src/docker-build.md +++ b/docs/src/docker-build.md @@ -89,3 +89,29 @@ docker pull postgis/postgis:16-3.4 docker build --no-cache -t rustprooflabs/pgosm-flex:dev . ``` + +## Building PgOSM Flex from an `osm2pgsql` feature branch + +There are times it is helpful to build the PgOSM Flex Docker image with a +specific feature branch. To do this, change the `OSM2PGSQL_BRANCH` +and/or `OSM2PGSQL_REPO` arguments as necessary at the beginning of the Dockerfile. +The production setup looks like the following example. + + +```Dockerfile +ARG OSM2PGSQL_BRANCH=master +ARG OSM2PGSQL_REPO=https://github.com/osm2pgsql-dev/osm2pgsql.git +``` + +To test the feature branch associated with +[osm2pgsql #2212](https://github.com/osm2pgsql-dev/osm2pgsql/pull/2212) +the updated version was set like the following example. +This changes the `OSM2PGSQL_BRANCH` to `check-date-on-replication-init` +and changes the username in the `OSM2PGSQL_REPO` to `lonvia`. + + +```Dockerfile +ARG OSM2PGSQL_BRANCH=check-date-on-replication-init +ARG OSM2PGSQL_REPO=https://github.com/lonvia/osm2pgsql.git +``` + diff --git a/docs/src/replication.md b/docs/src/replication.md index d751ad54..88db77cf 100644 --- a/docs/src/replication.md +++ b/docs/src/replication.md @@ -34,6 +34,12 @@ your specific database and process.** ---- +## Not tested by `make` + +The function exposed by `--replication` is not tested via PgOSM's `Makefile`. + + + ## Max connections The other important change when using replication is to increase Postgres' `max_connections`. From 6bd6726b5323ec60cdf4b7686021c98a0a2b4220 Mon Sep 17 00:00:00 2001 From: Ryan Lambert Date: Tue, 6 Aug 2024 11:03:00 -0600 Subject: [PATCH 2/2] Setting branch/repo back to prod values --- Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 35972daa..062a691f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,8 @@ FROM postgis/postgis:16-3.4 LABEL maintainer="PgOSM Flex - https://github.com/rustprooflabs/pgosm-flex" -#ARG OSM2PGSQL_BRANCH=master -ARG OSM2PGSQL_BRANCH=check-date-on-replication-init -#ARG OSM2PGSQL_REPO=https://github.com/osm2pgsql-dev/osm2pgsql.git -ARG OSM2PGSQL_REPO=https://github.com/lonvia/osm2pgsql.git +ARG OSM2PGSQL_BRANCH=master +ARG OSM2PGSQL_REPO=https://github.com/osm2pgsql-dev/osm2pgsql.git RUN apt-get update \