Improvements (#83)

* Fix text logged for acceptance commands * Update Varnish and Nginx versions (Fixes #78) * Install and configure npm during image generation (Fixes #77) Add missing variable * Fix Docker build warning about case in Dockerfile * Fix invalid .PHONY start-ps (Fixes #76) * Pass PLONE_VERSION to backend image generation on GHA (Fixes #75) * Add stack-logs-db target to devops Makefile (Fixes #79) * Allow setting ANSIBLE_REMOTE_PORT if the server sshd is not running on port 22 (Fixes #80) * Fix variable interpolation in docker stack (Fixes #81) * Devops: Add Ansible-managed network with external access (Fixes #82) * Variable project_folders should be a list * Devops: Render contents of devops/inventory/group_vars/all/projects.yml --------- Co-authored-by: David Glick <[email protected]>
plone · Oct 29, 2024 · 6e8c38d · 6e8c38d
1 parent d4f986b
commit 6e8c38d
Show file tree

Hide file tree

Showing 15 changed files with 86 additions and 25 deletions.
diff --git a/project/cookiecutter.json b/project/cookiecutter.json
@@ -35,6 +35,7 @@
   "__python_package_name_upper": "{{ cookiecutter.python_package_name | pascal_case }}",
   "__node_version": "{{ cookiecutter.volto_version | node_version_for_volto }}",
   "__version_plone_volto": "{{ cookiecutter.volto_version }}",
+  "__version_pnpm": "9.1.1",
   "__container_registry_prefix": "{{ cookiecutter.container_registry | image_prefix }}",
   "__container_image_prefix": "{{ cookiecutter.__container_registry_prefix }}{{ cookiecutter.github_organization }}/{{ cookiecutter.project_slug }}",
   "__profile_version": "{% now 'utc', '%Y%m%d001' %}",
@@ -57,12 +58,13 @@
   "__devops_host": "{{ cookiecutter.hostname | extract_host }}",
   "__devops_compose_name": "{{ cookiecutter.project_slug | replace('.','-') | replace('_','-') }}",
   "__devops_stack_name": "{{ cookiecutter.hostname | replace('.','-') | replace('_','-') }}",
-  "__devops_swarm_public_network": "nw-internal",
-  "__devops_traefik_version": "v2.10",
+  "__devops_swarm_public_network": "nw-public",
+  "__devops_swarm_stack_network": "nw-internal",
+  "__devops_traefik_version": "v2.11",
   "__devops_traefik_local_include_ui": "yes",
   "__devops_traefik_stack_include_ui": "no",
-  "__devops_traefik_docker_network": "{{ cookiecutter.__devops_stack_name }}_{{ cookiecutter.__devops_swarm_public_network }}",
-  "__devops_varnish_version": "7.4",
+  "__devops_traefik_docker_network": "{{ cookiecutter.__devops_stack_name }}_{{ cookiecutter.__devops_swarm_stack_network }}",
+  "__devops_varnish_version": "7.6",
   "__devops_db_version": "14",
   "__devops_db_password": "{{ random_ascii_string(12) }}",
   "__backend_addon_format": "1",
@@ -117,8 +119,7 @@
     "devops/playbooks",
     "devops/requirements",
     "devops/tasks",
-    "devops/inventory/group_vars/all/users.yml",
-    "devops/inventory/group_vars/all/projects.yml"
+    "devops/inventory/group_vars/all/users.yml"
 
   ],
   "_extensions": [

diff --git a/project/tests/test_project_devops.py b/project/tests/test_project_devops.py
@@ -90,3 +90,10 @@ def test_project_devops_no_gha_deploy(
     folder = cutter_result_devops_no_gha_deploy.project_path
     path = folder / filepath
     assert path.exists() is False
+
+
+def test_ansible_inventory_projects_replacement(cutter_result):
+    """Test GHA deploy files are not present."""
+    folder = cutter_result.project_path
+    path = folder / "devops/inventory/group_vars/all/projects.yml"
+    assert "{{ cookiecutter.hostname }}" not in path.read_text()
diff --git a/project/{{ cookiecutter.__folder_name }}/.github/workflows/backend.yml b/project/{{ cookiecutter.__folder_name }}/.github/workflows/backend.yml
@@ -29,6 +29,8 @@ jobs:
 
   black:
     runs-on: ubuntu-latest
+    needs:
+      - meta
     steps:
       - name: Checkout codebase
         uses: actions/checkout@{{ cookiecutter.__gha_version_checkout }}
@@ -41,6 +43,8 @@ jobs:
 
   flake8:
     runs-on: ubuntu-latest
+    needs:
+      - meta
     steps:
       - name: Checkout codebase
         uses: actions/checkout@{{ cookiecutter.__gha_version_checkout }}
@@ -53,6 +57,8 @@ jobs:
 
   isort:
     runs-on: ubuntu-latest
+    needs:
+      - meta
     steps:
       - name: Checkout codebase
         uses: actions/checkout@{{ cookiecutter.__gha_version_checkout }}
@@ -65,6 +71,8 @@ jobs:
 
   zpretty:
     runs-on: ubuntu-latest
+    needs:
+      - meta
     steps:
       - name: Checkout codebase
         uses: actions/checkout@{{ cookiecutter.__gha_version_checkout }}
@@ -105,6 +113,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     needs:
+      - meta
       - black
       - flake8
       - isort
@@ -162,3 +171,5 @@ jobs:
           push: {{"${{ github.event_name != 'pull_request' }}"}}
           tags: {{"${{ steps.meta.outputs.tags }}"}}
           labels: {{"${{ steps.meta.outputs.labels }}"}}
+          build-args: |
+            PLONE_VERSION={{"${{ needs.meta.outputs.PLONE_VERSION }}"}}
diff --git a/project/{{ cookiecutter.__folder_name }}/Makefile b/project/{{ cookiecutter.__folder_name }}/Makefile
@@ -135,7 +135,7 @@ stack-create-site:  ## Local Stack: Create a new site
 	@echo "Create a new site in the local Docker stack"
 	@docker compose -f docker-compose.yml exec backend ./docker-entrypoint.sh create-site
 
-.PHONY: start-ps
+.PHONY: stack-status
 stack-status:  ## Local Stack: Check Status
 	@echo "Check the status of the local Docker stack"
 	@docker compose -f docker-compose.yml ps

diff --git a/project/{{ cookiecutter.__folder_name }}/devops/.env_dist b/project/{{ cookiecutter.__folder_name }}/devops/.env_dist
@@ -1,3 +1,4 @@
+ANSIBLE_REMOTE_PORT=22
 DEPLOY_ENV=prod
 DEPLOY_HOST={{ cookiecutter.hostname }}
 DEPLOY_PORT=22

diff --git a/project/{{ cookiecutter.__folder_name }}/devops/Makefile b/project/{{ cookiecutter.__folder_name }}/devops/Makefile
@@ -118,6 +118,11 @@ stack-logs-backend: ## Display backend logs
 	@echo "$(GREEN)==> Stack $(STACK_NAME): Logs for backend in context $(DEPLOY_ENV) $(RESET)"
 	${DOCKER_CMD_CONTEXT} service logs $(STACK_NAME)_backend
 
+.PHONY: stack-logs-db
+stack-logs-db: ## Display db logs
+	@echo "$(GREEN)==> Stack $(STACK_NAME): Logs for db in context $(DEPLOY_ENV) $(RESET)"
+	${DOCKER_CMD_CONTEXT} service logs $(STACK_NAME)_db
+
 .PHONY: stack-scale-services
 stack-scale-services: ## Scale services
 	@echo "$(GREEN)==> Stack $(STACK_NAME): Scaling Frontend and Backend to $(DEPLOY_SCALES) in context $(DEPLOY_ENV) $(RESET)"

diff --git a/project/{{ cookiecutter.__folder_name }}/devops/README.md b/project/{{ cookiecutter.__folder_name }}/devops/README.md
@@ -41,10 +41,12 @@ cp .env_dist .env
 Edit the `.env` file to suit your environment. For example:
 
 ```
+ANSIBLE_REMOTE_PORT=22
 DEPLOY_ENV=prod
 DEPLOY_HOST={{ cookiecutter.hostname }}
 DEPLOY_PORT=22
 DEPLOY_USER=plone
+DEPLOY_FOLDER=/srv/{{ cookiecutter.project_slug }}/data
 DOCKER_CONFIG=.docker
 STACK_NAME={{ cookiecutter.__devops_stack_name }}
 ```

diff --git a/project/{{ cookiecutter.__folder_name }}/devops/inventory/group_vars/all/projects.yml b/project/{{ cookiecutter.__folder_name }}/devops/inventory/group_vars/all/projects.yml
@@ -1,7 +1,7 @@
 ---
 project_folders:
   # Path to store {{ cookiecutter.hostname }} postgres data
-    path: "{{ lookup('ansible.builtin.env', 'DEPLOY_FOLDER', default='/srv/plone/data') }}"
+  - path: {{ "\"{{ lookup('ansible.builtin.env', 'DEPLOY_FOLDER', default='/srv/plone/data') }}\"" }}
     owner: 999
     group: 999
     mode: "0750"
diff --git a/project/{{ cookiecutter.__folder_name }}/devops/stacks/{{ cookiecutter.hostname }}.yml b/project/{{ cookiecutter.__folder_name }}/devops/stacks/{{ cookiecutter.hostname }}.yml
@@ -79,6 +79,7 @@ services:
 
     networks:
       - {{ cookiecutter.__devops_swarm_public_network }}
+      - {{ cookiecutter.__devops_swarm_stack_network }}
 
 {%- if cookiecutter.devops_cache == '1' %}
 
@@ -89,7 +90,7 @@ services:
       PURGER_SERVICE_PORT: 80
       PURGER_PUBLIC_SITES: "['{{ cookiecutter.hostname }}']"
     networks:
-      - {{ cookiecutter.__devops_swarm_public_network }}
+      - {{ cookiecutter.__devops_swarm_stack_network }}
     deploy:
       replicas: 2
       update_config:
@@ -111,7 +112,7 @@ services:
       - frontend
       - backend
     networks:
-      - {{ cookiecutter.__devops_swarm_public_network }}
+      - {{ cookiecutter.__devops_swarm_stack_network }}
     deploy:
       replicas: 1
       update_config:
@@ -142,6 +143,7 @@ services:
       - backend
     networks:
       - {{ cookiecutter.__devops_swarm_public_network }}
+      - {{ cookiecutter.__devops_swarm_stack_network }}
     deploy:
       replicas: 2
       update_config:
@@ -177,6 +179,7 @@ services:
       - db
     networks:
       - {{ cookiecutter.__devops_swarm_public_network }}
+      - {{ cookiecutter.__devops_swarm_stack_network }}
     deploy:
       replicas: 2
       update_config:
@@ -239,17 +242,19 @@ services:
     volumes:
       - vol-site-data:/var/lib/postgresql/data
     networks:
-      - {{ cookiecutter.__devops_swarm_public_network }}
+      - {{ cookiecutter.__devops_swarm_stack_network }}
 
 volumes:
   vol-traefik-certs: {}
   vol-site-data:
     driver_opts:
       type: none
-      device: "${DEPLOY_FOLDER:/srv/{{ cookiecutter.project_slug }}/data}"
+      device: ${DEPLOY_FOLDER:-/srv/{{ cookiecutter.project_slug }}/data}
       o: bind
 
 networks:
   {{ cookiecutter.__devops_swarm_public_network }}:
+    external: true
+  {{ cookiecutter.__devops_swarm_stack_network }}:
     internal: true
     driver: overlay
diff --git a/project/{{ cookiecutter.__folder_name }}/devops/tasks/swarm/task_swarm.yml b/project/{{ cookiecutter.__folder_name }}/devops/tasks/swarm/task_swarm.yml
@@ -11,7 +11,31 @@
         advertise_addr: "{{ swarm.advertise_addr }}"
       register: swarm_info
 
+- name: "Docker Swarm: Set public network"
+  become: true
+  become_user: "{{ users.default.name }}"
+  block:
+    - name: "Docker Swarm: Check for public network"
+      community.docker.docker_network_info:
+        name: "{{ cookiecutter.__devops_swarm_public_network }}"
+      register: docker_swarm_net_public
+
+    - name: "Docker Swarm: Create public network"
+      community.docker.docker_network:
+        name: "{{ cookiecutter.__devops_swarm_public_network }}"
+        internal: false
+        attachable: true
+        driver: overlay
+        driver_options:
+          "com.docker.network.driver.mtu": "{{ docker.network.mtu }}"
+      when: not docker_swarm_net_public.exists
+
 - name: "Docker Swarm: Purge data"
+  become: true
+  become_user: "{{ users.default.name }}"
+  tags:
+    - cron
+    - swarm
   block:
     - name: "Docker Swarm: Purge old containers"
       ansible.builtin.cron:
@@ -34,7 +58,3 @@
         hour: "*"
         minute: 15
         state: present
-  become_user: "{{ users.default.name }}"
-  tags:
-    - cron
-    - swarm
diff --git a/sub/cache/cookiecutter.json b/sub/cache/cookiecutter.json
@@ -21,7 +21,7 @@
   "__gha_version_docker_buildx": "v3",
   "__gha_version_docker_login": "v3",
   "__gha_version_docker_build_push": "v4",
-  "__devops_varnish_version": "7.4",
+  "__devops_varnish_version": "7.6",
   "_copy_without_render": [
   ],
   "_extensions": [

diff --git a/sub/project_settings/cookiecutter.json b/sub/project_settings/cookiecutter.json
@@ -19,6 +19,7 @@
   "__package_namespace": "{{ cookiecutter.python_package_name | package_namespace }}",
   "__profile_language": "{{ cookiecutter.language_code|gs_language_code }}",
   "__locales_language": "{{ cookiecutter.language_code|locales_language_code }}",
+  "__version_pnpm": "9.1.1",
   "_copy_without_render": [],
   "_extensions": [
     "cookieplone.filters.use_prerelease_versions",

diff --git a/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Dockerfile b/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 ARG PLONE_VERSION={{ cookiecutter.plone_version }}
-FROM plone/server-builder:${PLONE_VERSION} as builder
+FROM plone/server-builder:${PLONE_VERSION} AS builder
 
 WORKDIR /app
 

diff --git a/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Dockerfile.acceptance b/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Dockerfile.acceptance
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 ARG PLONE_VERSION={{ cookiecutter.plone_version }}
-FROM plone/server-builder:${PLONE_VERSION} as builder
+FROM plone/server-builder:${PLONE_VERSION} AS builder
 
 WORKDIR /app
 

diff --git a/sub/project_settings/{{ cookiecutter.__folder_name }}/frontend/Dockerfile b/sub/project_settings/{{ cookiecutter.__folder_name }}/frontend/Dockerfile
@@ -1,17 +1,18 @@
 # syntax=docker/dockerfile:1
 ARG VOLTO_VERSION
-FROM plone/frontend-builder:${VOLTO_VERSION} as builder
+FROM plone/frontend-builder:${VOLTO_VERSION} AS builder
 
 COPY --chown=node packages/{{cookiecutter.frontend_addon_name}} /app/packages/{{cookiecutter.frontend_addon_name}}
 COPY --chown=node volto.config.js /app/
 COPY --chown=node package.json /app/package.json.temp
 
 RUN --mount=type=cache,id=pnpm,target=/app/.pnpm-store,uid=1000 <<EOT
-     python3 -c "import json; data = json.load(open('package.json.temp')); deps = data['dependencies']; data['dependencies'].update(deps); json.dump(data, open('package.json', 'w'), indent=2)"
-     rm package.json.temp
-     pnpm install && pnpm build:deps
-     pnpm build
-     pnpm install --prod
+    set -e
+    python3 -c "import json; data = json.load(open('package.json.temp')); deps = data['dependencies']; data['dependencies'].update(deps); json.dump(data, open('package.json', 'w'), indent=2)"
+    rm package.json.temp
+    pnpm install && pnpm build:deps
+    pnpm build
+    pnpm install --prod
 EOT
 
 FROM plone/frontend-prod-config:${VOLTO_VERSION}
@@ -22,3 +23,10 @@ LABEL maintainer="{{ cookiecutter.author }} <{{ cookiecutter.email }}>" \
       org.label-schema.vendor="{{ cookiecutter.author }}"
 
 COPY --from=builder /app/ /app/
+
+RUN <<EOT
+    set -e
+    corepack enable pnpm
+    corepack use pnpm@{{ cookiecutter.__version_pnpm }}
+    corepack prepare pnpm@{{ cookiecutter.__version_pnpm }} --activate
+EOT