From 9546efc036d5a6208ab020eee1b1559ec0614a28 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 11:18:55 +0100 Subject: [PATCH 01/22] First stuff -trying to get bake to work in gha and local testing --- src/docker/Dockerfile | 12 ++-- src/docker/docker-compose-base.yml | 4 +- src/docker/docker-compose-dev.yml | 5 +- src/docker/docker-compose-prod.yml | 8 +-- src/docker/docker-compose-testing.yml | 80 +++++++++++++-------------- 5 files changed, 55 insertions(+), 54 deletions(-) diff --git a/src/docker/Dockerfile b/src/docker/Dockerfile index aed63a6e..f5778e61 100644 --- a/src/docker/Dockerfile +++ b/src/docker/Dockerfile @@ -5,7 +5,7 @@ ARG GROUP_ID RUN addgroup --system --gid $GROUP_ID kantele RUN adduser --system --uid $USER_ID --gid $GROUP_ID kantele RUN apt-get update && apt-get install -y libpq-dev python3-psycopg2 zip -COPY backend/requirements.txt / +COPY src/backend/requirements.txt / RUN pip install -r requirements.txt RUN mkdir /assets RUN chdir /assets && wget https://files.pythonhosted.org/packages/ca/91/6d9b8ccacd0412c08820f72cebaa4f0c0441b5cda699c90f618b6f8a1b42/requests-2.28.1-py3-none-any.whl @@ -14,10 +14,10 @@ RUN chdir /assets && wget https://files.pythonhosted.org/packages/60/ef/76811343 RUN chdir /assets && wget https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl RUN chdir /assets && wget https://files.pythonhosted.org/packages/db/51/a507c856293ab05cdc1db77ff4bc1268ddd39f29e7dc4919aa497f0adbec/charset_normalizer-2.1.1-py3-none-any.whl RUN chdir /assets && wget https://files.pythonhosted.org/packages/6f/de/5be2e3eed8426f871b170663333a0f627fc2924cc386cd41be065e7ea870/urllib3-1.26.12-py2.py3-none-any.whl -COPY docker/setup.bat /assets/ +COPY src/docker/setup.bat /assets/ RUN cd /assets && zip producer.zip *.whl setup.bat && chown kantele:kantele producer.zip USER kantele -COPY --chown=kantele backend /kantele +COPY --chown=kantele src/backend /kantele WORKDIR /kantele CMD uwsgi --socket :8009 --processes 3 --module kantele.wsgi @@ -36,7 +36,7 @@ RUN python manage.py collectstatic --no-input # Compile node frontend stuff for prod container FROM node:18 AS build_frontend -COPY ./frontend /src/frontend +COPY src/frontend /src/frontend RUN cd /src/frontend/analysis && npm install && npm run build RUN cd /src/frontend/dashboard && npm install && npm run build RUN cd /src/frontend/datasets && npm install && npm run build @@ -50,8 +50,8 @@ RUN cd /src/frontend/staffpage && npm install && npm run build FROM nginx:1.21 AS nginx_prod COPY --from=build_frontend /static /static COPY --from=django_static /kantele/static/ /static/ -COPY docker/nginx.conf /etc/nginx/nginx.conf -COPY docker/uwsgi_params /etc/nginx/uwsgi_params +COPY src/docker/nginx.conf /etc/nginx/nginx.conf +COPY src/docker/uwsgi_params /etc/nginx/uwsgi_params # Rsync SSH host for testing (SSHD would run on the web host machine outside of container diff --git a/src/docker/docker-compose-base.yml b/src/docker/docker-compose-base.yml index 7ccdbd36..d58b24c0 100644 --- a/src/docker/docker-compose-base.yml +++ b/src/docker/docker-compose-base.yml @@ -4,8 +4,8 @@ services: web: image: kantele_web build: - context: ../ - dockerfile: ./docker/Dockerfile + context: . + dockerfile: ./src/docker/Dockerfile target: django args: USER_ID: "${USER_ID:-You must run export USER_ID}" diff --git a/src/docker/docker-compose-dev.yml b/src/docker/docker-compose-dev.yml index 491ea3d9..05e357a4 100644 --- a/src/docker/docker-compose-dev.yml +++ b/src/docker/docker-compose-dev.yml @@ -5,8 +5,9 @@ services: storage: image: django_test build: - context: "${BUILD_CONTEXT_DEV:-../}" - dockerfile: ./docker/Dockerfile + #context: "${BUILD_CONTEXT_DEV:-./}" + context: ../../ + dockerfile: ./src/docker/Dockerfile target: django_test args: USER_ID: "${USER_ID:-You must run export USER_ID}" diff --git a/src/docker/docker-compose-prod.yml b/src/docker/docker-compose-prod.yml index f0b83ac6..f9eba563 100644 --- a/src/docker/docker-compose-prod.yml +++ b/src/docker/docker-compose-prod.yml @@ -5,7 +5,7 @@ services: - db - mq env_file: - - ./prod-container.env + - ./src/docker/prod-container.env logging: driver: syslog options: @@ -24,8 +24,8 @@ services: - web image: kantele_nginx build: - context: ../ - dockerfile: ./docker/Dockerfile + context: ./ + dockerfile: ./src/docker/Dockerfile target: nginx_prod args: USER_ID: "${USER_ID:-You must run export USER_ID}" @@ -66,7 +66,7 @@ services: - mq network_mode: host env_file: - - ./prod-container.env + - ./src/docker/prod-container.env logging: driver: syslog options: diff --git a/src/docker/docker-compose-testing.yml b/src/docker/docker-compose-testing.yml index cbcdc907..b6a55062 100644 --- a/src/docker/docker-compose-testing.yml +++ b/src/docker/docker-compose-testing.yml @@ -1,25 +1,25 @@ services: web: - depends_on: - - db - - mq - - storage_mvfiles - - storage_downloads - - tulos_ingester - - upload_bay_rsync - extends: - file: docker-compose-base.yml - service: web + # depends_on: + # - db + # - mq + # - storage_mvfiles + # - storage_downloads + # - tulos_ingester + # - upload_bay_rsync + # extends: + # file: src/docker/docker-compose-base.yml + # service: web image: django_test build: - context: ../ - dockerfile: ./docker/Dockerfile + context: ./ + dockerfile: ./src/docker/Dockerfile target: django_test args: USER_ID: "${USER_ID:-You must run export USER_ID}" GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" env_file: - - testing.env + - src/docker/testing.env volumes: - type: bind source: ../backend @@ -53,17 +53,17 @@ services: volumes: - "./create_kantele_db.sh:/docker-entrypoint-initdb.d/create_kantele_db.sh" - mq: - extends: - file: docker-compose-base.yml - service: mq + #mq: + # extends: + # file: src/docker/docker-compose-base.yml + # service: mq storage_mvfiles: extends: - file: docker-compose-dev.yml + file: src/docker/docker-compose-dev.yml service: storage env_file: - - testing.env + - src/docker/testing.env command: celery -A kantele worker -Q mv_md5_storage -n storage_mv.%h --concurrency=1 volumes: # Overwrite dev storage here @@ -79,9 +79,9 @@ services: tulos_ingester: env_file: - - testing.env + - src/docker/testing.env extends: - file: docker-compose-dev.yml + file: src/docker/docker-compose-dev.yml service: storage volumes: # Overwrite dev storage here @@ -97,10 +97,10 @@ services: storage_downloads: extends: - file: docker-compose-dev.yml + file: src/docker/docker-compose-dev.yml service: storage env_file: - - testing.env + - src/docker/testing.env command: celery -A kantele worker -Q file_download -n storage_downloads.%h --concurrency=1 volumes: # Overwrite dev storage here @@ -117,27 +117,27 @@ services: RSYNC_SSHPORT: "${RSYNC_SSHPORT}" HOST_UPLOADDIR: /storage/uploads - analysis: - extends: - file: docker-compose-base.yml - service: web - depends_on: - - web - command: celery -A kantele worker -Q test_analysis -n analysis.%h --concurrency=1 - env_file: - - testing.env - volumes: - - type: bind - source: ../backend - target: /kantele - environment: - KANTELEHOST: web + # analysis: + # # extends: + # # file: src/docker/docker-compose-base.yml + # # service: web + # depends_on: + # - web + # command: celery -A kantele worker -Q test_analysis -n analysis.%h --concurrency=1 + # env_file: + # - src/docker/testing.env + # volumes: + # - type: bind + # source: ../backend + # target: /kantele + # environment: + # KANTELEHOST: web upload_bay_rsync: image: rsync_stor build: - context: ../ - dockerfile: ./docker/Dockerfile + context: ./ + dockerfile: ./src/docker/Dockerfile target: rsync_storage args: USER_ID: "${USER_ID:-You must run export USER_ID}" From 04002e2802381eda42e37d07e2119c0e307491bc Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 11:44:16 +0100 Subject: [PATCH 02/22] Set uid,gid in gha --- .github/workflows/integration-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 98ca00fe..0635c107 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -14,6 +14,10 @@ jobs: - uses: actions/checkout@v4 - name: Setup buildx uses: docker/setup-buildx-action@v3 + - name: Get USER_ID to env + run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV + - name: Get GROUP_ID to env + run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - run: docker buildx bake --file src/docker/docker-compose-testing.yml --file src/docker/docker-compose-gha-cache.json - run: bash run_lint.sh From 8416436a9265d728d74d73ce808416cdf5710d14 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 14:23:00 +0100 Subject: [PATCH 03/22] Bake yml separate from run yml --- .github/workflows/integration-tests.yml | 8 +++- src/docker/Dockerfile | 12 +++--- src/docker/docker-compose-base.yml | 4 +- src/docker/docker-compose-dev.yml | 54 ++++++++++++------------- src/docker/docker-compose-testing.yml | 52 ++++++++++++------------ src/docker/prod-bake.yml | 20 +++++++++ src/docker/testing-bake.yml | 20 +++++++++ 7 files changed, 107 insertions(+), 63 deletions(-) create mode 100644 src/docker/prod-bake.yml create mode 100644 src/docker/testing-bake.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0635c107..787d6765 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -18,8 +18,12 @@ jobs: run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - - run: docker buildx bake --file src/docker/docker-compose-testing.yml --file src/docker/docker-compose-gha-cache.json - - run: bash run_lint.sh + - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json + - run: | + docker compose --env-file src/docker/.compose.testing.env \ + -f src/docker/docker-compose-testing.yml run web pylint \ + -E --disable E1101,E0307 --ignore-paths '.*\/migrations\/[0-9]+.*.py' \ + analysis datasets dashboard home jobs kantele rawstatus integration-tests: runs-on: ubuntu-latest diff --git a/src/docker/Dockerfile b/src/docker/Dockerfile index f5778e61..bdd0f950 100644 --- a/src/docker/Dockerfile +++ b/src/docker/Dockerfile @@ -5,7 +5,7 @@ ARG GROUP_ID RUN addgroup --system --gid $GROUP_ID kantele RUN adduser --system --uid $USER_ID --gid $GROUP_ID kantele RUN apt-get update && apt-get install -y libpq-dev python3-psycopg2 zip -COPY src/backend/requirements.txt / +COPY backend/requirements.txt / RUN pip install -r requirements.txt RUN mkdir /assets RUN chdir /assets && wget https://files.pythonhosted.org/packages/ca/91/6d9b8ccacd0412c08820f72cebaa4f0c0441b5cda699c90f618b6f8a1b42/requests-2.28.1-py3-none-any.whl @@ -14,10 +14,10 @@ RUN chdir /assets && wget https://files.pythonhosted.org/packages/60/ef/76811343 RUN chdir /assets && wget https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl RUN chdir /assets && wget https://files.pythonhosted.org/packages/db/51/a507c856293ab05cdc1db77ff4bc1268ddd39f29e7dc4919aa497f0adbec/charset_normalizer-2.1.1-py3-none-any.whl RUN chdir /assets && wget https://files.pythonhosted.org/packages/6f/de/5be2e3eed8426f871b170663333a0f627fc2924cc386cd41be065e7ea870/urllib3-1.26.12-py2.py3-none-any.whl -COPY src/docker/setup.bat /assets/ +COPY docker/setup.bat /assets/ RUN cd /assets && zip producer.zip *.whl setup.bat && chown kantele:kantele producer.zip USER kantele -COPY --chown=kantele src/backend /kantele +COPY --chown=kantele backend /kantele WORKDIR /kantele CMD uwsgi --socket :8009 --processes 3 --module kantele.wsgi @@ -36,7 +36,7 @@ RUN python manage.py collectstatic --no-input # Compile node frontend stuff for prod container FROM node:18 AS build_frontend -COPY src/frontend /src/frontend +COPY frontend /src/frontend RUN cd /src/frontend/analysis && npm install && npm run build RUN cd /src/frontend/dashboard && npm install && npm run build RUN cd /src/frontend/datasets && npm install && npm run build @@ -50,8 +50,8 @@ RUN cd /src/frontend/staffpage && npm install && npm run build FROM nginx:1.21 AS nginx_prod COPY --from=build_frontend /static /static COPY --from=django_static /kantele/static/ /static/ -COPY src/docker/nginx.conf /etc/nginx/nginx.conf -COPY src/docker/uwsgi_params /etc/nginx/uwsgi_params +COPY docker/nginx.conf /etc/nginx/nginx.conf +COPY docker/uwsgi_params /etc/nginx/uwsgi_params # Rsync SSH host for testing (SSHD would run on the web host machine outside of container diff --git a/src/docker/docker-compose-base.yml b/src/docker/docker-compose-base.yml index d58b24c0..7ccdbd36 100644 --- a/src/docker/docker-compose-base.yml +++ b/src/docker/docker-compose-base.yml @@ -4,8 +4,8 @@ services: web: image: kantele_web build: - context: . - dockerfile: ./src/docker/Dockerfile + context: ../ + dockerfile: ./docker/Dockerfile target: django args: USER_ID: "${USER_ID:-You must run export USER_ID}" diff --git a/src/docker/docker-compose-dev.yml b/src/docker/docker-compose-dev.yml index 05e357a4..28ebe98b 100644 --- a/src/docker/docker-compose-dev.yml +++ b/src/docker/docker-compose-dev.yml @@ -2,30 +2,30 @@ # services: - storage: - image: django_test - build: - #context: "${BUILD_CONTEXT_DEV:-./}" - context: ../../ - dockerfile: ./src/docker/Dockerfile - target: django_test - args: - USER_ID: "${USER_ID:-You must run export USER_ID}" - GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" - volumes: - - ../../data/storage:/storage - - ../../data/newstorage:/newstorage - - ../../data/analysisfiles:/analysisfiles - - ../backend:/kantele - environment: - APIKEY: "${APIKEY_STORAGE}" - KANTELEHOST: nginx - PROTOCOL: http - TMPSHARE: '/s3storage/tmp' - ANALYSISSHARE: /analysisfiles - RABBITHOST: mq - RABBIT_VHOST: "${RABBIT_VHOST}" - RABBITUSER: "${RABBITUSER:-guest}" - RABBITPASS: "${RABBITPASS:-guest}" - STORAGESHARES: '/storage,/newstorage' - PRIMARY_STORAGE: s3storage + storage: + image: django_test + build: + context: "${BUILD_CONTEXT_DEV:-../}" + #context: ../../ + dockerfile: ./docker/Dockerfile + target: django_test + args: + USER_ID: "${USER_ID:-You must run export USER_ID}" + GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" + volumes: + - ../../data/storage:/storage + - ../../data/newstorage:/newstorage + - ../../data/analysisfiles:/analysisfiles + - ../backend:/kantele + environment: + APIKEY: "${APIKEY_STORAGE}" + KANTELEHOST: nginx + PROTOCOL: http + TMPSHARE: '/s3storage/tmp' + ANALYSISSHARE: /analysisfiles + RABBITHOST: mq + RABBIT_VHOST: "${RABBIT_VHOST}" + RABBITUSER: "${RABBITUSER:-guest}" + RABBITPASS: "${RABBITPASS:-guest}" + STORAGESHARES: '/storage,/newstorage' + PRIMARY_STORAGE: s3storage diff --git a/src/docker/docker-compose-testing.yml b/src/docker/docker-compose-testing.yml index b6a55062..2fcfaa5e 100644 --- a/src/docker/docker-compose-testing.yml +++ b/src/docker/docker-compose-testing.yml @@ -1,25 +1,25 @@ services: web: - # depends_on: - # - db - # - mq - # - storage_mvfiles - # - storage_downloads - # - tulos_ingester - # - upload_bay_rsync - # extends: - # file: src/docker/docker-compose-base.yml - # service: web + depends_on: + - db + - mq + - storage_mvfiles + - storage_downloads + - tulos_ingester + - upload_bay_rsync + extends: + file: docker-compose-base.yml + service: web image: django_test build: - context: ./ - dockerfile: ./src/docker/Dockerfile + context: ../ + dockerfile: ./docker/Dockerfile target: django_test args: USER_ID: "${USER_ID:-You must run export USER_ID}" GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" env_file: - - src/docker/testing.env + - testing.env volumes: - type: bind source: ../backend @@ -53,17 +53,17 @@ services: volumes: - "./create_kantele_db.sh:/docker-entrypoint-initdb.d/create_kantele_db.sh" - #mq: - # extends: - # file: src/docker/docker-compose-base.yml - # service: mq + mq: + extends: + file: docker-compose-base.yml + service: mq storage_mvfiles: extends: - file: src/docker/docker-compose-dev.yml + file: ./docker-compose-dev.yml service: storage env_file: - - src/docker/testing.env + - testing.env command: celery -A kantele worker -Q mv_md5_storage -n storage_mv.%h --concurrency=1 volumes: # Overwrite dev storage here @@ -79,9 +79,9 @@ services: tulos_ingester: env_file: - - src/docker/testing.env + - testing.env extends: - file: src/docker/docker-compose-dev.yml + file: ./docker-compose-dev.yml service: storage volumes: # Overwrite dev storage here @@ -97,10 +97,10 @@ services: storage_downloads: extends: - file: src/docker/docker-compose-dev.yml + file: ./docker-compose-dev.yml service: storage env_file: - - src/docker/testing.env + - testing.env command: celery -A kantele worker -Q file_download -n storage_downloads.%h --concurrency=1 volumes: # Overwrite dev storage here @@ -119,7 +119,7 @@ services: # analysis: # # extends: - # # file: src/docker/docker-compose-base.yml + # # file: docker-compose-base.yml # # service: web # depends_on: # - web @@ -136,8 +136,8 @@ services: upload_bay_rsync: image: rsync_stor build: - context: ./ - dockerfile: ./src/docker/Dockerfile + context: ../ + dockerfile: ./docker/Dockerfile target: rsync_storage args: USER_ID: "${USER_ID:-You must run export USER_ID}" diff --git a/src/docker/prod-bake.yml b/src/docker/prod-bake.yml new file mode 100644 index 00000000..bd19a419 --- /dev/null +++ b/src/docker/prod-bake.yml @@ -0,0 +1,20 @@ +services: + web: + image: kantele_web + build: + context: ./src + dockerfile: ./docker/Dockerfile + target: django + args: + USER_ID: "${USER_ID:-You must run export USER_ID}" + GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" + + nginx: + image: kantele_nginx + build: + context: ./src + dockerfile: ./docker/Dockerfile + target: nginx_prod + args: + USER_ID: "${USER_ID:-You must run export USER_ID}" + GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" diff --git a/src/docker/testing-bake.yml b/src/docker/testing-bake.yml new file mode 100644 index 00000000..312f2be0 --- /dev/null +++ b/src/docker/testing-bake.yml @@ -0,0 +1,20 @@ +services: + web: + image: django_test + build: + context: ./src + dockerfile: ./docker/Dockerfile + target: django_test + args: + USER_ID: "${USER_ID:-You must run export USER_ID}" + GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" + + upload_bay_rsync: + image: rsync_stor + build: + context: ./src + dockerfile: ./docker/Dockerfile + target: rsync_storage + args: + USER_ID: "${USER_ID:-You must run export USER_ID}" + GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" From 8f4bc4add5c262ddaf5518569ddcc81075f32268 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 14:29:29 +0100 Subject: [PATCH 04/22] Test echo output --- .github/workflows/integration-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 787d6765..bcc2a0fc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -19,6 +19,11 @@ jobs: - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json + - run: | + echo docker compose --env-file src/docker/.compose.testing.env \ + -f src/docker/docker-compose-testing.yml run web pylint \ + -E --disable E1101,E0307 --ignore-paths '.*\/migrations\/[0-9]+.*.py' \ + analysis datasets dashboard home jobs kantele rawstatus - run: | docker compose --env-file src/docker/.compose.testing.env \ -f src/docker/docker-compose-testing.yml run web pylint \ From 8fea47eb9352d934ff673e0019e96ce47898b048 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 14:36:34 +0100 Subject: [PATCH 05/22] Test echo output 2 --- .github/workflows/integration-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index bcc2a0fc..1331af16 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -22,13 +22,13 @@ jobs: - run: | echo docker compose --env-file src/docker/.compose.testing.env \ -f src/docker/docker-compose-testing.yml run web pylint \ - -E --disable E1101,E0307 --ignore-paths '.*\/migrations\/[0-9]+.*.py' \ - analysis datasets dashboard home jobs kantele rawstatus + -E --disable E1101,E0307 --ignore-paths \ + '.*\/migrations\/[0-9]+.*.py' analysis datasets dashboard home jobs kantele rawstatus - run: | docker compose --env-file src/docker/.compose.testing.env \ -f src/docker/docker-compose-testing.yml run web pylint \ - -E --disable E1101,E0307 --ignore-paths '.*\/migrations\/[0-9]+.*.py' \ - analysis datasets dashboard home jobs kantele rawstatus + -E --disable E1101,E0307 --ignore-paths \ + '.*\/migrations\/[0-9]+.*.py' analysis datasets dashboard home jobs kantele rawstatus integration-tests: runs-on: ubuntu-latest From 1bf21977512bb592b589356b0a8c59f74313ac09 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 14:50:07 +0100 Subject: [PATCH 06/22] Test if using buildx bake in other job leads to caching from prev job where it is also used --- .github/workflows/integration-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1331af16..85ce280a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -35,6 +35,13 @@ jobs: needs: linting steps: - uses: actions/checkout@v4 + - name: Setup buildx + uses: docker/setup-buildx-action@v3 + - name: Get USER_ID to env + run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV + - name: Get GROUP_ID to env + run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV + - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - run: bash .github/run_tests_integration.sh unit-tests: From 072862228cf4f938975a337090264b860318fab8 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 15:03:13 +0100 Subject: [PATCH 07/22] Try registry instead of gha for cache --- src/docker/docker-compose-gha-cache.json | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/docker/docker-compose-gha-cache.json b/src/docker/docker-compose-gha-cache.json index 28bb4b2a..982ad8ed 100644 --- a/src/docker/docker-compose-gha-cache.json +++ b/src/docker/docker-compose-gha-cache.json @@ -1,37 +1,37 @@ {"target": { "web": { - "cache-from": ["type=gha"], - "cache-to": ["type=gha"], + "cache-from": ["type=registry,ref=ghcr.io/glormph/kantele-web:cache"], + "cache-to": ["type=registry,ref=ghcr.io/glormph/kantele-web:cache"], "output": ["type=docker"] }, "db": { - "cache-from": ["type=gha"], - "cache-to": ["type=gha"], + "cache-from": ["type=registry,ref=ghcr.io/glormph/kantele-db:cache"], + "cache-to": ["type=registry,ref=ghcr.io/glormph/kantele-db:cache"], "output": ["type=docker"] }, "mq": { - "cache-from": ["type=gha"], - "cache-to": ["type=gha"], + "cache-from": ["type=registry,ref=ghcr.io/glormph/kantele-mq:cache"], + "cache-to": ["type=registry,ref=ghcr.io/glormph/kantele-mq:cache"], "output": ["type=docker"] }, "storage_mvfiles": { - "cache-from": ["type=gha"], - "cache-to": ["type=gha"], + "cache-from": ["type=registry,ref=ghcr.io/glormph/kantele-stor:cache"], + "cache-to": ["type=registry,ref=ghcr.io/glormph/kantele-stor:cache"], "output": ["type=docker"] }, "tulos_ingester": { - "cache-from": ["type=gha"], - "cache-to": ["type=gha"], + "cache-from": ["type=registry,ref=ghcr.io/glormph/kantele-stor:cache"], + "cache-to": ["type=registry,ref=ghcr.io/glormph/kantele-stor:cache"], "output": ["type=docker"] }, "storage_downloads": { - "cache-from": ["type=gha"], - "cache-to": ["type=gha"], + "cache-from": ["type=registry,ref=ghcr.io/glormph/kantele-stor:cache"], + "cache-to": ["type=registry,ref=ghcr.io/glormph/kantele-stor:cache"], "output": ["type=docker"] }, "upload_bay_rsync": { - "cache-from": ["type=gha"], - "cache-to": ["type=gha"], + "cache-from": ["type=registry,ref=ghcr.io/glormph/kantele-rsync:cache"], + "cache-to": ["type=registry,ref=ghcr.io/glormph/kantele-rsync:cache"], "output": ["type=docker"] } } From ee0fcede2fb709d90062516faa701ba5504a3ac6 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 15:07:22 +0100 Subject: [PATCH 08/22] Login to ghcr first --- .github/workflows/integration-tests.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 85ce280a..a64e2c10 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -12,12 +12,23 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Setup buildx uses: docker/setup-buildx-action@v3 + - name: Get USER_ID to env run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV + - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV + + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - run: | echo docker compose --env-file src/docker/.compose.testing.env \ @@ -41,6 +52,12 @@ jobs: run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - run: bash .github/run_tests_integration.sh From 60564a4f50652a619686e928740af5c81f0cfdd2 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 15:10:23 +0100 Subject: [PATCH 09/22] Login to ghcr first2 --- .github/workflows/integration-tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a64e2c10..cfc1eca2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -22,12 +22,12 @@ jobs: - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - - name: Login to Docker Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - run: | @@ -52,12 +52,12 @@ jobs: run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - - name: Login to Docker Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - run: bash .github/run_tests_integration.sh From d322cb735a8af9f316c4518c96ea12cdfe4e3200 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 15:23:02 +0100 Subject: [PATCH 10/22] Also use cache in unit tests, write docker cmds in gha yml --- .github/workflows/integration-tests.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index cfc1eca2..b1d7ff6c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -59,11 +59,28 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - - run: bash .github/run_tests_integration.sh + - run: | + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + up --detach db mq + sleep 5 # DB needs to be up or app crashes trying to connect + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + run --use-aliases web python manage.py test --tag slow --exclude-tag mstulos unit-tests: runs-on: ubuntu-latest needs: linting steps: + - name: Login to Docker Registry to get bake cache + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - uses: actions/checkout@v4 - - run: bash .github/run_tests_unit.sh + - run: | + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + up --detach db mq + sleep 5 # DB needs to be up or app crashes trying to connect + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + run --use-aliases web python manage.py test --exclude-tag slow From cce731e40a822fdd3816ccdc48b151c68074c3bd Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 15:24:42 +0100 Subject: [PATCH 11/22] Also use cache in unit tests, write docker cmds in gha yml - typo --- .github/workflows/integration-tests.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b1d7ff6c..e4aef3bf 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -60,11 +60,11 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - run: | - docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ - up --detach db mq - sleep 5 # DB needs to be up or app crashes trying to connect - docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ - run --use-aliases web python manage.py test --tag slow --exclude-tag mstulos + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + up --detach db mq + sleep 5 # DB needs to be up or app crashes trying to connect + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + run --use-aliases web python manage.py test --tag slow --exclude-tag mstulos unit-tests: runs-on: ubuntu-latest @@ -79,8 +79,8 @@ jobs: - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - uses: actions/checkout@v4 - run: | - docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ - up --detach db mq - sleep 5 # DB needs to be up or app crashes trying to connect - docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ - run --use-aliases web python manage.py test --exclude-tag slow + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + up --detach db mq + sleep 5 # DB needs to be up or app crashes trying to connect + docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ + run --use-aliases web python manage.py test --exclude-tag slow From 8d468fd8167749ddc8216c16ca5b4aee4f1cac77 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 15:32:37 +0100 Subject: [PATCH 12/22] Fix unittest bake thing --- .github/workflows/integration-tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e4aef3bf..66c4d546 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -70,6 +70,13 @@ jobs: runs-on: ubuntu-latest needs: linting steps: + - uses: actions/checkout@v4 + - name: Setup buildx + uses: docker/setup-buildx-action@v3 + - name: Get USER_ID to env + run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV + - name: Get GROUP_ID to env + run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - name: Login to Docker Registry to get bake cache uses: docker/login-action@v3 with: @@ -77,7 +84,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - - uses: actions/checkout@v4 - run: | docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ up --detach db mq From 40355e93b60f5af1e2018504da0af41155e1ba80 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 22:39:59 +0100 Subject: [PATCH 13/22] Clean a bit, try to pull images also in the bake step for caching --- .github/run_tests_integration.sh | 14 -------------- .github/run_tests_unit.sh | 14 -------------- .github/workflows/integration-tests.yml | 5 ----- src/docker/prod-bake.yml | 6 ++++++ src/docker/testing-bake.yml | 6 ++++++ 5 files changed, 12 insertions(+), 33 deletions(-) delete mode 100644 .github/run_tests_integration.sh delete mode 100644 .github/run_tests_unit.sh diff --git a/.github/run_tests_integration.sh b/.github/run_tests_integration.sh deleted file mode 100644 index c3e5b1fe..00000000 --- a/.github/run_tests_integration.sh +++ /dev/null @@ -1,14 +0,0 @@ -DOCKERCMD="docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml" -export GROUP_ID=$(id -g) -export USER_ID=$(id -u) - -echo Prebuilding DB and MQ containers -# Get DB container ready so web doesnt try to connect before it has init'ed -$DOCKERCMD up --detach db mq -echo Created db container and started it -sleep 5 -echo Running tests -# Run tests - -TESTCMD="python manage.py test" -$DOCKERCMD run --use-aliases web $TESTCMD --tag slow --exclude-tag mstulos || ($DOCKERCMD logs web storage_mvfiles storage_downloads tulos_ingester && exit 1) diff --git a/.github/run_tests_unit.sh b/.github/run_tests_unit.sh deleted file mode 100644 index 093e3285..00000000 --- a/.github/run_tests_unit.sh +++ /dev/null @@ -1,14 +0,0 @@ -DOCKERCMD="docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml" -export GROUP_ID=$(id -g) -export USER_ID=$(id -u) - -echo Prebuilding DB and MQ containers -# Get DB container ready so web doesnt try to connect before it has init'ed -$DOCKERCMD up --detach db mq -echo Created db container and started it -sleep 5 -echo Running tests -# Run tests - -TESTCMD="python manage.py test" -$DOCKERCMD run --use-aliases web $TESTCMD --exclude-tag slow || ($DOCKERCMD logs web storage_mvfiles storage_downloads tulos_ingester && exit 1) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 66c4d546..36a10fe9 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,11 +30,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - run: docker buildx bake --file src/docker/testing-bake.yml --file src/docker/docker-compose-gha-cache.json - - run: | - echo docker compose --env-file src/docker/.compose.testing.env \ - -f src/docker/docker-compose-testing.yml run web pylint \ - -E --disable E1101,E0307 --ignore-paths \ - '.*\/migrations\/[0-9]+.*.py' analysis datasets dashboard home jobs kantele rawstatus - run: | docker compose --env-file src/docker/.compose.testing.env \ -f src/docker/docker-compose-testing.yml run web pylint \ diff --git a/src/docker/prod-bake.yml b/src/docker/prod-bake.yml index bd19a419..0bb5ba58 100644 --- a/src/docker/prod-bake.yml +++ b/src/docker/prod-bake.yml @@ -18,3 +18,9 @@ services: args: USER_ID: "${USER_ID:-You must run export USER_ID}" GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" + + db: + image: postgres:14.3 + + mq: + image: rabbitmq:3.10 diff --git a/src/docker/testing-bake.yml b/src/docker/testing-bake.yml index 312f2be0..582c5f39 100644 --- a/src/docker/testing-bake.yml +++ b/src/docker/testing-bake.yml @@ -18,3 +18,9 @@ services: args: USER_ID: "${USER_ID:-You must run export USER_ID}" GROUP_ID: "${GROUP_ID:-You must run export GROUP_ID}" + + db: + image: postgres:14.3 + + mq: + image: rabbitmq:3.10 From 2e7f633fb0557eeb9d2978f2c952e04a3d4c352b Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 22:41:28 +0100 Subject: [PATCH 14/22] Increase sleep time to see if that makes jobs complete on gha --- src/backend/rawstatus/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/rawstatus/tests.py b/src/backend/rawstatus/tests.py index f09fc6af..39d35fb5 100644 --- a/src/backend/rawstatus/tests.py +++ b/src/backend/rawstatus/tests.py @@ -150,7 +150,7 @@ def test_transfer_again(self): 'src_path': os.path.join(settings.TMP_UPLOADPATH, f'{self.f3raw.pk}.{self.f3sf.filetype.filetype}')}, timestamp=timezone.now(), state=jj.Jobstates.DONE) sp = self.run_script(fullp) - sleep(1) + sleep(2) self.f3sf.refresh_from_db() self.assertFalse(self.f3sf.checked) self.run_job() @@ -194,7 +194,7 @@ def test_transfer_same_name(self): self.assertFalse(os.path.exists(os.path.join(tmpdir, 'skipbox', self.f3sf.filename))) sp = self.run_script(False, config=os.path.join(tmpdir, 'config.json'), session=True) - sleep(2) + sleep(4) sp.terminate() # Properly kill children since upload.py uses multiprocessing os.killpg(os.getpgid(sp.pid), signal.SIGTERM) @@ -230,7 +230,7 @@ def test_transfer_file_namechanged(self): producer=self.prod, size=123, date=timezone.now(), claimed=False) lastsf = rm.StoredFile.objects.last() sp = self.run_script(fullp) - sleep(1) + sleep(2) self.run_job() spout, sperr = sp.communicate() newsf = rm.StoredFile.objects.last() From 110dd8feb76b32cbd8ad0e47b4f8239b144005e3 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 22:50:02 +0100 Subject: [PATCH 15/22] Put timeouts in place to see what is wrong on gha --- src/backend/rawstatus/tests.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backend/rawstatus/tests.py b/src/backend/rawstatus/tests.py index 39d35fb5..3a77f5e5 100644 --- a/src/backend/rawstatus/tests.py +++ b/src/backend/rawstatus/tests.py @@ -117,8 +117,10 @@ def test_new_file(self): self.assertFalse(sf.checked) self.run_job() sf.refresh_from_db() + spout, sperr = sp.communicate(timeout=10) + print(sperr.decode('utf-8')) + print(spout.decode('utf-8')) self.assertTrue(sf.checked) - spout, sperr = sp.communicate() explines = ['Registering 1 new file(s)', f'File {new_raw.name} matches remote file {new_raw.name} with ID {new_raw.pk}', f'Checking remote state for file {new_raw.name} with ID {new_raw.pk}', @@ -154,7 +156,7 @@ def test_transfer_again(self): self.f3sf.refresh_from_db() self.assertFalse(self.f3sf.checked) self.run_job() - spout, sperr = sp.communicate() + spout, sperr = sp.communicate(timeout=10) self.f3sf.refresh_from_db() self.assertTrue(self.f3sf.checked) self.assertEqual(self.f3sf.md5, self.f3raw.source_md5) @@ -232,7 +234,7 @@ def test_transfer_file_namechanged(self): sp = self.run_script(fullp) sleep(2) self.run_job() - spout, sperr = sp.communicate() + spout, sperr = sp.communicate(timeout=10) newsf = rm.StoredFile.objects.last() self.assertEqual(newsf.pk, lastsf.pk + 1) self.assertEqual(rawfn.pk, newsf.rawfile_id) @@ -272,7 +274,7 @@ def test_rsync_not_finished_yet(self): job.save() self.f3sf.checked = True self.f3sf.save() - spout, sperr = sp.communicate() + spout, sperr = sp.communicate(timeout=10) explines = [f'Checking remote state for file {self.f3raw.name} with ID {self.f3raw.pk}', f'State for file {self.f3raw.name} with ID {self.f3raw.pk} was: wait', f'Checking remote state for file {self.f3raw.name} with ID {self.f3raw.pk}', From 780a6822965c9171d70f7eb42a16ed5159550ce6 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 23:01:17 +0100 Subject: [PATCH 16/22] More sleep --- src/backend/rawstatus/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/rawstatus/tests.py b/src/backend/rawstatus/tests.py index 3a77f5e5..1c1dfc4f 100644 --- a/src/backend/rawstatus/tests.py +++ b/src/backend/rawstatus/tests.py @@ -107,7 +107,7 @@ def test_new_file(self): old_raw = rm.RawFile.objects.last() sp = self.run_script(fullp) # Give time for running script, so job is created before running it etc - sleep(1) + sleep(4) new_raw = rm.RawFile.objects.last() self.assertEqual(new_raw.pk, old_raw.pk + 1) sf = rm.StoredFile.objects.last() @@ -152,7 +152,7 @@ def test_transfer_again(self): 'src_path': os.path.join(settings.TMP_UPLOADPATH, f'{self.f3raw.pk}.{self.f3sf.filetype.filetype}')}, timestamp=timezone.now(), state=jj.Jobstates.DONE) sp = self.run_script(fullp) - sleep(2) + sleep(4) self.f3sf.refresh_from_db() self.assertFalse(self.f3sf.checked) self.run_job() @@ -232,7 +232,7 @@ def test_transfer_file_namechanged(self): producer=self.prod, size=123, date=timezone.now(), claimed=False) lastsf = rm.StoredFile.objects.last() sp = self.run_script(fullp) - sleep(2) + sleep(4) self.run_job() spout, sperr = sp.communicate(timeout=10) newsf = rm.StoredFile.objects.last() From 8aa2216fb71890e8b1da8bbf20c866be9fd316c3 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Tue, 29 Oct 2024 23:09:24 +0100 Subject: [PATCH 17/22] Try to get stdout for erroring jobs on specific gha --- src/backend/rawstatus/tests.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/backend/rawstatus/tests.py b/src/backend/rawstatus/tests.py index 1c1dfc4f..ab788814 100644 --- a/src/backend/rawstatus/tests.py +++ b/src/backend/rawstatus/tests.py @@ -117,7 +117,13 @@ def test_new_file(self): self.assertFalse(sf.checked) self.run_job() sf.refresh_from_db() - spout, sperr = sp.communicate(timeout=10) + try: + spout, sperr = sp.communicate(timeout=10) + except subprocess.TimeoutExpired: + sp.terminate() + # Properly kill children since upload.py uses multiprocessing + os.killpg(os.getpgid(sp.pid), signal.SIGTERM) + spout, sperr = sp.communicate() print(sperr.decode('utf-8')) print(spout.decode('utf-8')) self.assertTrue(sf.checked) @@ -156,7 +162,15 @@ def test_transfer_again(self): self.f3sf.refresh_from_db() self.assertFalse(self.f3sf.checked) self.run_job() - spout, sperr = sp.communicate(timeout=10) + try: + spout, sperr = sp.communicate(timeout=10) + except subprocess.TimeoutExpired: + sp.terminate() + # Properly kill children since upload.py uses multiprocessing + os.killpg(os.getpgid(sp.pid), signal.SIGTERM) + spout, sperr = sp.communicate() + print(sperr.decode('utf-8')) + print(spout.decode('utf-8')) self.f3sf.refresh_from_db() self.assertTrue(self.f3sf.checked) self.assertEqual(self.f3sf.md5, self.f3raw.source_md5) @@ -234,7 +248,15 @@ def test_transfer_file_namechanged(self): sp = self.run_script(fullp) sleep(4) self.run_job() - spout, sperr = sp.communicate(timeout=10) + try: + spout, sperr = sp.communicate(timeout=10) + except subprocess.TimeoutExpired: + sp.terminate() + # Properly kill children since upload.py uses multiprocessing + os.killpg(os.getpgid(sp.pid), signal.SIGTERM) + spout, sperr = sp.communicate() + print(sperr.decode('utf-8')) + print(spout.decode('utf-8')) newsf = rm.StoredFile.objects.last() self.assertEqual(newsf.pk, lastsf.pk + 1) self.assertEqual(rawfn.pk, newsf.rawfile_id) From 650e2324e3414e785015df84b40a89caa3dd7721 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Wed, 30 Oct 2024 09:14:48 +0100 Subject: [PATCH 18/22] Show logs of tasks to find out if task runs at all --- .github/workflows/integration-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 36a10fe9..cd016f97 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,5 +1,5 @@ -name: integration-tests -run-name: Run integration tests +name: lint-and-test +run-name: Run linting and tests on: push: branches: @@ -22,7 +22,7 @@ jobs: - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - - name: Login to Docker Registry + - name: Login to Docker Registry, bake/cache uses: docker/login-action@v3 with: registry: ghcr.io @@ -47,7 +47,7 @@ jobs: run: echo "USER_ID=$(id -u)" >> $GITHUB_ENV - name: Get GROUP_ID to env run: echo "GROUP_ID=$(id -g)" >> $GITHUB_ENV - - name: Login to Docker Registry + - name: Login to Docker Registry to get bake cache uses: docker/login-action@v3 with: registry: ghcr.io @@ -59,7 +59,7 @@ jobs: up --detach db mq sleep 5 # DB needs to be up or app crashes trying to connect docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ - run --use-aliases web python manage.py test --tag slow --exclude-tag mstulos + run --use-aliases web python manage.py test --tag slow --exclude-tag mstulos || docker compose --env-file src/docker.compose.testing.env -f src/docker/docker-compose-testing.yml logs storage_mvfiles storage_downloads unit-tests: runs-on: ubuntu-latest From 07425bd50c923bf875ceae41e1a9337c8d69415d Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Wed, 30 Oct 2024 09:21:30 +0100 Subject: [PATCH 19/22] stupid typo --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index cd016f97..6a6c145f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -59,7 +59,7 @@ jobs: up --detach db mq sleep 5 # DB needs to be up or app crashes trying to connect docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml \ - run --use-aliases web python manage.py test --tag slow --exclude-tag mstulos || docker compose --env-file src/docker.compose.testing.env -f src/docker/docker-compose-testing.yml logs storage_mvfiles storage_downloads + run --use-aliases web python manage.py test --tag slow --exclude-tag mstulos || docker compose --env-file src/docker/.compose.testing.env -f src/docker/docker-compose-testing.yml logs storage_mvfiles storage_downloads unit-tests: runs-on: ubuntu-latest From 941f1c9147b020a1e04eb50781c49b5c433b91a8 Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Wed, 30 Oct 2024 09:39:50 +0100 Subject: [PATCH 20/22] Put rsync key in docker image so file permissions are correct - for testing --- src/docker/Dockerfile | 4 +++- src/docker/docker-compose-testing.yml | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/docker/Dockerfile b/src/docker/Dockerfile index bdd0f950..b1ea01b3 100644 --- a/src/docker/Dockerfile +++ b/src/docker/Dockerfile @@ -27,7 +27,9 @@ USER root RUN apt-get install -y rsync RUN pip install pylint USER kantele -RUN mkdir /home/kantele/.ssh && chmod 700 /home/kantele/.ssh +RUN mkdir /kantelessh && chmod 700 /kantelessh +COPY docker/test_rsync_sshkey /kantelessh/rsync_key +RUN chmod 600 /kantelessh/rsync_key # Build django static for the nginx prod container FROM django AS django_static diff --git a/src/docker/docker-compose-testing.yml b/src/docker/docker-compose-testing.yml index 2fcfaa5e..117ddea0 100644 --- a/src/docker/docker-compose-testing.yml +++ b/src/docker/docker-compose-testing.yml @@ -107,7 +107,6 @@ services: - type: bind source: ../../data/teststorage target: /storage - - "./test_rsync_sshkey:/kantelessh/rsync_key" environment: KANTELEHOST: web TMPSHARE: '/storage/tmp' From d8ee4169187f1db9410e1b19f2fc139b2d83c2cb Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Wed, 30 Oct 2024 09:48:19 +0100 Subject: [PATCH 21/22] Fix key into right, readable location --- src/docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/docker/Dockerfile b/src/docker/Dockerfile index b1ea01b3..ce05bf73 100644 --- a/src/docker/Dockerfile +++ b/src/docker/Dockerfile @@ -26,10 +26,9 @@ FROM django AS django_test USER root RUN apt-get install -y rsync RUN pip install pylint +RUN mkdir /kantelessh && chmod 700 /kantelessh && chown kantele: /kantelessh USER kantele -RUN mkdir /kantelessh && chmod 700 /kantelessh COPY docker/test_rsync_sshkey /kantelessh/rsync_key -RUN chmod 600 /kantelessh/rsync_key # Build django static for the nginx prod container FROM django AS django_static From 3ae76ba7a7eea91816a412aa638b113a1686946f Mon Sep 17 00:00:00 2001 From: Jorrit Boekel Date: Wed, 30 Oct 2024 10:33:20 +0100 Subject: [PATCH 22/22] Decrease sleep times a bit, catch errors to show and then fail, and explicitly check last line so it wont be a wait in case of slow operation or something --- src/backend/rawstatus/tests.py | 44 ++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/backend/rawstatus/tests.py b/src/backend/rawstatus/tests.py index ab788814..a7301314 100644 --- a/src/backend/rawstatus/tests.py +++ b/src/backend/rawstatus/tests.py @@ -107,7 +107,7 @@ def test_new_file(self): old_raw = rm.RawFile.objects.last() sp = self.run_script(fullp) # Give time for running script, so job is created before running it etc - sleep(4) + sleep(2) new_raw = rm.RawFile.objects.last() self.assertEqual(new_raw.pk, old_raw.pk + 1) sf = rm.StoredFile.objects.last() @@ -124,8 +124,8 @@ def test_new_file(self): # Properly kill children since upload.py uses multiprocessing os.killpg(os.getpgid(sp.pid), signal.SIGTERM) spout, sperr = sp.communicate() - print(sperr.decode('utf-8')) - print(spout.decode('utf-8')) + print(sperr.decode('utf-8')) + raise self.assertTrue(sf.checked) explines = ['Registering 1 new file(s)', f'File {new_raw.name} matches remote file {new_raw.name} with ID {new_raw.pk}', @@ -134,11 +134,14 @@ def test_new_file(self): f'Uploading {fullp} to {self.live_server_url}', f'Succesful transfer of file {fullp}', f'Checking remote state for file {new_raw.name} with ID {new_raw.pk}', - f'State for file with ID {new_raw.pk} was "done"'] - for out, exp in zip(sperr.decode('utf-8').strip().split('\n'), explines): + ] + outlines = sperr.decode('utf-8').strip().split('\n') + for out, exp in zip(outlines, explines): out = re.sub('.* - INFO - .producer.worker - ', '', out) out = re.sub('.* - INFO - root - ', '', out) self.assertEqual(out, exp) + lastexp = f'State for file with ID {new_raw.pk} was "done"' + self.assertEqual(re.sub('.* - INFO - .producer.worker - ', '', outlines[-1]), lastexp) def test_transfer_again(self): '''Transfer already existing file, e.g. overwrites of previously @@ -158,7 +161,7 @@ def test_transfer_again(self): 'src_path': os.path.join(settings.TMP_UPLOADPATH, f'{self.f3raw.pk}.{self.f3sf.filetype.filetype}')}, timestamp=timezone.now(), state=jj.Jobstates.DONE) sp = self.run_script(fullp) - sleep(4) + sleep(1) self.f3sf.refresh_from_db() self.assertFalse(self.f3sf.checked) self.run_job() @@ -169,8 +172,8 @@ def test_transfer_again(self): # Properly kill children since upload.py uses multiprocessing os.killpg(os.getpgid(sp.pid), signal.SIGTERM) spout, sperr = sp.communicate() - print(sperr.decode('utf-8')) - print(spout.decode('utf-8')) + print(sperr.decode('utf-8')) + raise self.f3sf.refresh_from_db() self.assertTrue(self.f3sf.checked) self.assertEqual(self.f3sf.md5, self.f3raw.source_md5) @@ -182,11 +185,14 @@ def test_transfer_again(self): f'Uploading {fullp} to {self.live_server_url}', f'Succesful transfer of file {fullp}', f'Checking remote state for file {self.f3raw.name} with ID {self.f3raw.pk}', - f'State for file with ID {self.f3raw.pk} was "done"'] - for out, exp in zip(sperr.decode('utf-8').strip().split('\n'), explines): + ] + outlines = sperr.decode('utf-8').strip().split('\n') + for out, exp in zip(outlines, explines): out = re.sub('.* - INFO - .producer.worker - ', '', out) out = re.sub('.* - INFO - root - ', '', out) self.assertEqual(out, exp) + lastexp = f'State for file with ID {self.f3raw.pk} was "done"' + self.assertEqual(re.sub('.* - INFO - .producer.worker - ', '', outlines[-1]), lastexp) def test_transfer_same_name(self): # Test trying to upload file with same name/path but diff MD5 @@ -210,7 +216,7 @@ def test_transfer_same_name(self): self.assertFalse(os.path.exists(os.path.join(tmpdir, 'skipbox', self.f3sf.filename))) sp = self.run_script(False, config=os.path.join(tmpdir, 'config.json'), session=True) - sleep(4) + sleep(2) sp.terminate() # Properly kill children since upload.py uses multiprocessing os.killpg(os.getpgid(sp.pid), signal.SIGTERM) @@ -246,7 +252,7 @@ def test_transfer_file_namechanged(self): producer=self.prod, size=123, date=timezone.now(), claimed=False) lastsf = rm.StoredFile.objects.last() sp = self.run_script(fullp) - sleep(4) + sleep(1) self.run_job() try: spout, sperr = sp.communicate(timeout=10) @@ -255,8 +261,8 @@ def test_transfer_file_namechanged(self): # Properly kill children since upload.py uses multiprocessing os.killpg(os.getpgid(sp.pid), signal.SIGTERM) spout, sperr = sp.communicate() - print(sperr.decode('utf-8')) - print(spout.decode('utf-8')) + print(sperr.decode('utf-8')) + raise newsf = rm.StoredFile.objects.last() self.assertEqual(newsf.pk, lastsf.pk + 1) self.assertEqual(rawfn.pk, newsf.rawfile_id) @@ -296,7 +302,15 @@ def test_rsync_not_finished_yet(self): job.save() self.f3sf.checked = True self.f3sf.save() - spout, sperr = sp.communicate(timeout=10) + try: + spout, sperr = sp.communicate(timeout=10) + except subprocess.TimeoutExpired: + sp.terminate() + # Properly kill children since upload.py uses multiprocessing + os.killpg(os.getpgid(sp.pid), signal.SIGTERM) + spout, sperr = sp.communicate() + print(sperr.decode('utf-8')) + raise explines = [f'Checking remote state for file {self.f3raw.name} with ID {self.f3raw.pk}', f'State for file {self.f3raw.name} with ID {self.f3raw.pk} was: wait', f'Checking remote state for file {self.f3raw.name} with ID {self.f3raw.pk}',