From c6b05ffd0dc9dac60f33cc00243463373f896192 Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 08:37:51 +0200 Subject: [PATCH 1/4] bugfix: avoid killing replication process on shutdown if it was not started --- .github/workflows/ci.yml | 17 ++++++++++++++++- 4.4/start.sh | 5 ++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5683569..4847209c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,7 +215,22 @@ jobs: sleep 120 ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" - + + - name: Check for clean shutdown + commands: |- + docker run --detach --rm \ + -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -p 8009:8080 \ + --name nominatim \ + nominatim + sleep 120 + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" + # Shutdown the container + docker stop -t 60 nominatim + # Verify that the exit code is zero + CONTAINER_EXIT_CODE=$(docker inspect nominatim --format='{{.State.ExitCode}}') + test "$CONTAINER_EXIT_CODE" -eq 0 + steps: - name: Download artifact uses: actions/download-artifact@v4 diff --git a/4.4/start.sh b/4.4/start.sh index c59a4c85..dedab409 100755 --- a/4.4/start.sh +++ b/4.4/start.sh @@ -6,7 +6,10 @@ replicationpid=0 stopServices() { service apache2 stop service postgresql stop - kill $replicationpid + if [ $replicationpid -ne 0 ]; then + echo "Shutting down replication process" + kill $replicationpid + fi kill $tailpid } trap stopServices SIGTERM TERM INT From 227361b027c45489e5e5dc80c933474a8779d0ff Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 10:19:14 +0200 Subject: [PATCH 2/4] Dockerfile: use cache for apt and pip install start.sh: Add comments in stopServices() and exit code 0 --- 4.4/Dockerfile | 10 +++++----- 4.4/start.sh | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/4.4/Dockerfile b/4.4/Dockerfile index fc11891f..80494fe1 100644 --- a/4.4/Dockerfile +++ b/4.4/Dockerfile @@ -8,9 +8,10 @@ ENV LANG=C.UTF-8 WORKDIR /app -RUN true \ +RUN \ + --mount=type=cache,target=/var/cache/apt \ # Do not start daemons after installation. - && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ && chmod +x /usr/sbin/policy-rc.d \ # Install all required packages. && apt-get -y update -qq \ @@ -75,7 +76,8 @@ RUN true \ && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf # Osmium install to run continuous updates. -RUN pip3 install osmium +RUN --mount=type=cache,target=/root/.cache/pip \ + pip3 install osmium # Nominatim install. ARG NOMINATIM_VERSION @@ -107,12 +109,10 @@ RUN true \ liblua*-dev \ postgresql-server-dev-14 \ nlohmann-json3-dev \ - && apt-get clean \ # Clear temporary files and directories. && rm -rf \ /tmp/* \ /var/tmp/* \ - /root/.cache \ /app/src/.git \ /var/lib/apt/lists/* \ # Remove nominatim source and build directories diff --git a/4.4/start.sh b/4.4/start.sh index dedab409..0fdd86cf 100755 --- a/4.4/start.sh +++ b/4.4/start.sh @@ -6,11 +6,14 @@ replicationpid=0 stopServices() { service apache2 stop service postgresql stop + # Check if the replication process is active if [ $replicationpid -ne 0 ]; then echo "Shutting down replication process" kill $replicationpid fi kill $tailpid + # Force exit code 0 to signal a succesfull shutdown to Docker + exit 0 } trap stopServices SIGTERM TERM INT From 1992f00038fdd1c42bb1a2e7ca699df871946d0b Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 11:03:30 +0200 Subject: [PATCH 3/4] Remove forced sleep in CI --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4847209c..160cb932 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -70,7 +70,7 @@ jobs: -v nominatim-data:/var/lib/postgresql/14/main \ -p 8002:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Import with bind-mount, container restart & update @@ -84,7 +84,7 @@ jobs: -p 8003:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" # Stop container @@ -98,7 +98,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 25 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once @@ -116,7 +116,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -133,7 +133,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -146,7 +146,7 @@ jobs: -e IMPORT_STYLE=full \ -p 8005:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Import admin style @@ -157,7 +157,7 @@ jobs: -e IMPORT_STYLE=admin \ -p 8006:8080 \ nominatim & - sleep 120 + ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Import with PBF_PATH @@ -171,7 +171,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" - name: REPLICATION_URL is blank @@ -180,7 +180,7 @@ jobs: -e PBF_URL=http://download.geofabrik.de/europe/monaco-latest.osm.pbf \ -p 8008:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: FREEZE @@ -190,7 +190,7 @@ jobs: -e FREEZE="true" \ -p 8009:8080 \ nominatim & - sleep 120 + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - name: GB postcode import @@ -200,7 +200,7 @@ jobs: -e IMPORT_GB_POSTCODES="true" \ -p 8010:8080 \ nominatim & - sleep 600 + ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=LE15+8TX" ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+3SY" ./assert-non-empty-json "http://localhost:8010/search.php?postalcode=PE9+4ES" @@ -212,7 +212,7 @@ jobs: -e REVERSE_ONLY="true" \ -p 8011:8080 \ nominatim & - sleep 120 + ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" @@ -223,14 +223,14 @@ jobs: -p 8009:8080 \ --name nominatim \ nominatim - sleep 120 + ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" # Shutdown the container docker stop -t 60 nominatim # Verify that the exit code is zero CONTAINER_EXIT_CODE=$(docker inspect nominatim --format='{{.State.ExitCode}}') test "$CONTAINER_EXIT_CODE" -eq 0 - + steps: - name: Download artifact uses: actions/download-artifact@v4 From 04466a2b1d754fa3c0bf6f1e991d3741ec240800 Mon Sep 17 00:00:00 2001 From: Carlo Mion Date: Tue, 16 Apr 2024 11:19:41 +0200 Subject: [PATCH 4/4] ci: use tail with grep apply fix also to version 4.3 --- .github/workflows/ci.yml | 4 ++-- 4.3/start.sh | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 160cb932..d1c326c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,7 +119,7 @@ jobs: ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" - docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log + docker exec nominatim /bin/bash -c 'tail -f /var/log/replication.log | grep -m 1 -c "Update completed."' - name: UPDATE_MODE=continuous with bind-mount commands: |- @@ -136,7 +136,7 @@ jobs: ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo -n "check replication log for Update completed. Count:" - docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log + docker exec nominatim /bin/bash -c 'tail -f /var/log/replication.log | grep -m 1 -c "Update completed."' - name: Import full style commands: |- diff --git a/4.3/start.sh b/4.3/start.sh index c59a4c85..0fdd86cf 100755 --- a/4.3/start.sh +++ b/4.3/start.sh @@ -6,8 +6,14 @@ replicationpid=0 stopServices() { service apache2 stop service postgresql stop - kill $replicationpid + # Check if the replication process is active + if [ $replicationpid -ne 0 ]; then + echo "Shutting down replication process" + kill $replicationpid + fi kill $tailpid + # Force exit code 0 to signal a succesfull shutdown to Docker + exit 0 } trap stopServices SIGTERM TERM INT