From 59e2d07f44bc11a1615f12f7b24a9ec8d00817f2 Mon Sep 17 00:00:00 2001 From: Sampo Tawast Date: Sun, 30 Jul 2023 10:32:11 +0300 Subject: [PATCH] refactor: remove minio --- .env.benefit-backend.example | 7 ---- backend/benefit/README.md | 28 ++++----------- backend/benefit/helsinkibenefit/settings.py | 16 --------- backend/benefit/requirements.in | 1 - backend/benefit/requirements.txt | 14 -------- compose.benefit.yml | 39 ++------------------- 6 files changed, 9 insertions(+), 96 deletions(-) diff --git a/.env.benefit-backend.example b/.env.benefit-backend.example index 3c27188cc0..3e6181b5d8 100644 --- a/.env.benefit-backend.example +++ b/.env.benefit-backend.example @@ -102,10 +102,3 @@ SENTRY_ENVIRONMENT=local # for Mailhog inbox EMAIL_HOST=mailhog EMAIL_PORT=1025 - -# Variables for using a S3 compatible disk in local development environment in upcoming staging / production environments -USE_S3=1 -S3_ENDPOINT_URL="http://minio:9000" -S3_ACCESS_KEY_ID=minio-root -S3_SECRET_ACCESS_KEY=minio-pass -S3_STORAGE_BUCKET_NAME=local-s3-bucket diff --git a/backend/benefit/README.md b/backend/benefit/README.md index 7e2f27accb..7d2a5b6c7a 100644 --- a/backend/benefit/README.md +++ b/backend/benefit/README.md @@ -56,7 +56,6 @@ Set default permissions This creates permissions for the handler's group so they have access to the Terms in the django admin. - ### Configure docker environment In the yjdh project root, set up the .env.benefit-backend file: `cp .env.benefit-backend.example .env.benefit-backend` @@ -86,6 +85,7 @@ The project is now running at [localhost:8000](https://localhost:8000) ### Updating translations In `backend/benefit/`: + - Run `python manage.py makemessages --no-location -l fi -l sv -l en` - Run `python manage.py compilemessages` @@ -98,7 +98,7 @@ DUMMY_COMPANY_FORM_CODE can be set to test with different company_form parameter To seed the database with some mock application data, run `python manage.py seed` , which by default generates 10 applications for each of the seven possible application statuses and one attachment with a .pdf-file for each of them. To generate more applications, use the optional `--number` flag, for example, running `python manage.py seed --number=30` creates 30 applications of each status. **Note that running the command deletes all previous application data from the database and clears the media folder.** -[Mailhog](https://github.com/mailhog) is available for the local development environment (localhost:8025)[http://localhost:8025/] for previewing +[Mailhog](https://github.com/mailhog) is available for the local development environment (localhost:8025)[http://localhost:8025/] for previewing and testing the emails sent by the application after setting the `EMAIL_HOST` and `EMAIL_PORT` as in the `.env.benefit-backend.example`. **Using LOAD_FIXTURES=1 is recommended for local testing** as it loads e.g. default @@ -176,7 +176,7 @@ and redoc documentation at [https://localhost:8000/api_docs/redoc/](https://loca ## Scheduled jobs -Jobs can be scheduled using the Django extensions-package and setting the jobs to run as a cronjob. +Jobs can be scheduled using the Django extensions-package and setting the jobs to run as a cronjob. Currently configured jobs (registered in the `applications/jobs`-directory): - Daily: check applications that have been in the cancelled state for 30 or more days and delete them. @@ -208,24 +208,8 @@ env variables / settings are provided by Azure blob storage: - `AZURE_ACCOUNT_KEY` - `AZURE_CONTAINER` -An AWS S3 compatible disk storage can be configured with the following environment variables. - -- `USE_S3` -- `S3_ENDPOINT_URL` -- `S3_ACCESS_KEY_ID` -- `S3_SECRET_ACCESS_KEY` -- `S3_STORAGE_BUCKET_NAME` - -[MinIO](https://min.io/) can been configured to work as the AWS S3 compatible file storage on the local development environment. The MinIO admin panel can be accessed at (localhost:9090)[http://localhost:9090/]. -See `.env.benefit-backend.example` for the Minio variables and credentials. - -**Note** -As tests freeze time with [freezegun](https://github.com/spulec/freezegun), the MinIO requests fail when running tests with exception `botocore.exceptions.ClientError: An error occurred (RequestTimeTooSkewed) when calling the PutObject operation: The difference between the request time and the server's time is too large.` - -Switch from MinIO to the local disk by setting `USE_S3=0` before running the pytest tests. For now, the only workaround to run tests with S3 enabled is to set host machine date to the date that is used in tests: `2021-06-04 00:00:00 (UTC)`. - - ## Sentry error monitoring -The `local`, `development` and `testing` environments are connected to the Sentry instance at [`https://sentry.test.hel.ninja/`](https://sentry.test.hel.ninja/) under the `yjdh-benefit`-team. -There are separate Sentry projects for the Django api (`yjdh-benefit-api`), handler UI (`yjdh-benefit-handler`) and applicant UI (`yjdh-benefit-applicant`). + +The `local`, `development` and `testing` environments are connected to the Sentry instance at [`https://sentry.test.hel.ninja/`](https://sentry.test.hel.ninja/) under the `yjdh-benefit`-team. +There are separate Sentry projects for the Django api (`yjdh-benefit-api`), handler UI (`yjdh-benefit-handler`) and applicant UI (`yjdh-benefit-applicant`). To limit the amount of possibly sensitive data sent to Sentry, the same configuration as in kesaseteli is used by default, see [`https://github.com/City-of-Helsinki/yjdh/pull/779`](https://github.com/City-of-Helsinki/yjdh/pull/779). diff --git a/backend/benefit/helsinkibenefit/settings.py b/backend/benefit/helsinkibenefit/settings.py index 59beb57e9a..2c9bc7787c 100644 --- a/backend/benefit/helsinkibenefit/settings.py +++ b/backend/benefit/helsinkibenefit/settings.py @@ -157,11 +157,6 @@ SERVICE_BUS_SEARCH_LIMIT=(int, 10), GDPR_API_QUERY_SCOPE=(str, "helsinkibenefit.gdprquery"), GDPR_API_DELETE_SCOPE=(str, "helsinkibenefit.gdprdelete"), - USE_S3=(bool, False), - S3_ENDPOINT_URL=(str, ""), - S3_ACCESS_KEY_ID=(str, ""), - S3_SECRET_ACCESS_KEY=(str, ""), - S3_STORAGE_BUCKET_NAME=(str, ""), ) if os.path.exists(env_file): env.read_env(env_file) @@ -490,19 +485,8 @@ code = compile(fp.read(), local_settings_path, "exec") exec(code, globals(), locals()) -# S3 settings - -USE_S3 = env("USE_S3") - CORS_ALLOW_HEADERS = ( *default_headers, "baggage", "sentry-trace", ) - -if USE_S3: - AWS_S3_ENDPOINT_URL = env("S3_ENDPOINT_URL") - DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" - AWS_ACCESS_KEY_ID = env("S3_ACCESS_KEY_ID") - AWS_SECRET_ACCESS_KEY = env("S3_SECRET_ACCESS_KEY") - AWS_STORAGE_BUCKET_NAME = env("S3_STORAGE_BUCKET_NAME") diff --git a/backend/benefit/requirements.in b/backend/benefit/requirements.in index 22f1ac69d0..9a34aa8507 100644 --- a/backend/benefit/requirements.in +++ b/backend/benefit/requirements.in @@ -1,6 +1,5 @@ -e file:../shared babel -boto3 django-cors-headers django-environ django-extensions diff --git a/backend/benefit/requirements.txt b/backend/benefit/requirements.txt index bea262294e..0a1fdc7808 100644 --- a/backend/benefit/requirements.txt +++ b/backend/benefit/requirements.txt @@ -16,12 +16,6 @@ azure-storage-blob==12.15.0 # via django-storages babel==2.11.0 # via -r requirements.in -boto3==1.26.78 - # via -r requirements.in -botocore==1.29.78 - # via - # boto3 - # s3transfer cachetools==5.3.0 # via django-helusers certifi==2022.12.7 @@ -135,10 +129,6 @@ isodate==0.6.1 # via azure-storage-blob jinja2==3.1.2 # via -r requirements.in -jmespath==1.0.1 - # via - # boto3 - # botocore josepy==1.13.0 # via mozilla-django-oidc jsonschema==4.17.3 @@ -184,7 +174,6 @@ pysaml2==7.4.1 python-dateutil==2.8.2 # via # -r requirements.in - # botocore # faker # pysaml2 python-jose==3.3.0 @@ -214,8 +203,6 @@ requests==2.28.2 # pysaml2 rsa==4.9 # via python-jose -s3transfer==0.6.0 - # via boto3 sentry-sdk==1.15.0 # via -r requirements.in six==1.16.0 @@ -239,7 +226,6 @@ uritemplate==4.1.1 # drf-spectacular urllib3==1.26.14 # via - # botocore # django-auth-adfs # elasticsearch # requests diff --git a/compose.benefit.yml b/compose.benefit.yml index f71e5c60d9..e722903754 100644 --- a/compose.benefit.yml +++ b/compose.benefit.yml @@ -10,8 +10,8 @@ services: POSTGRES_USER: benefit POSTGRES_PASSWORD: benefit POSTGRES_DB: benefit - LC_COLLATE: 'fi_FI.UTF-8' - LC_CTYPE: 'fi_FI.UTF-8' + LC_COLLATE: "fi_FI.UTF-8" + LC_CTYPE: "fi_FI.UTF-8" ports: - 5434:5432 volumes: @@ -33,7 +33,6 @@ services: - ./backend/shared:/shared depends_on: - postgres - - minio container_name: benefit-backend applicant: @@ -85,7 +84,7 @@ services: - 127.0.0.1:3100:3100 mailhog: - image: 'mailhog/mailhog@sha256:8d76a3d4ffa32a3661311944007a415332c4bb855657f4f6c57996405c009bea' + image: "mailhog/mailhog@sha256:8d76a3d4ffa32a3661311944007a415332c4bb855657f4f6c57996405c009bea" ports: - 1025:1025 - 8025:8025 @@ -93,41 +92,9 @@ services: networks: - default - minio: - image: minio/minio@sha256:b6ee4f78beddd690e4b4b0fe95bd88ea93925ef15e4e7f4a9de7312a9fe2e1f6 - ports: - - 9000:9000 - - 9090:9090 - container_name: benefit-minio - volumes: - - s3-volume:/data - environment: - - MINIO_ROOT_USER=minio-root - - MINIO_ROOT_PASSWORD=minio-pass - command: server --console-address :9090 --address :9000 /data - healthcheck: - test: - [ - "CMD", - "curl", - "-f", - "http://localhost:9000/minio/health/live" - ] - retries: 3 - timeout: 5s - # Create an initial bucket for local development - createbucket: - image: minio/mc@sha256:0099b4225101e65c636838bf56be2a717fa71c80a718bee0c6eb6ecf767c41a0 - container_name: benefit-minio-client - depends_on: - - minio - entrypoint: > - /bin/sh -c ' mc config host add s3 http://minio:9000 minio-root minio-pass --api S3v4; [[ ! -z "`mc ls s3 | grep local-s3-bucket`" ]] || mc mb s3/local-s3-bucket; exit 0; ' volumes: pgdata: driver: local - s3-volume: - driver: local networks: default: