Merge branch 'main' into graphql-yoga

the-hideout · Aug 12, 2024 · 3b53c53 · 3b53c53
2 parents 560b402 + a351fca
commit 3b53c53
Show file tree

Hide file tree

Showing 17 changed files with 2,371 additions and 2,275 deletions.
diff --git a/.github/workflows/branch-deploy.yml b/.github/workflows/branch-deploy.yml
@@ -4,12 +4,25 @@ on:
   issue_comment:
     types: [ created ]
 
-# Permissions needed for reacting and adding comments for IssueOps commands
+
 permissions:
+  # Permissions needed for reacting and adding comments for IssueOps commands
   pull-requests: write
   deployments: write
   contents: write
   checks: read
+  # Permissions needed for building and deploying docker images
+  packages: write
+  # contents: read
+  attestations: write
+  id-token: write
+
+env:
+  # Environment variables used by docker build and push
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  DEFAULT_PORT: 8788
+  IMAGE_TAG: ghcr.io/${{ github.repository }}:pr-${{ github.event.issue.number }}
 
 jobs:
   deploy:
@@ -26,33 +39,85 @@ jobs:
           environment_targets: production,development
           environment_urls: production|https://api.tarkov.dev/graphql,development|https://dev-api.tarkov.dev/graphql
           sticky_locks: "true"
+          skip_reviews: "development"
 
       - name: checkout
         if: ${{ steps.branch-deploy.outputs.continue == 'true' }}
         uses: actions/checkout@v4
         with:
           ref: ${{ steps.branch-deploy.outputs.ref }}
 
-      - name: setup node
+      - name: Log in to the Container registry
         if: ${{ steps.branch-deploy.outputs.continue == 'true' }}
-        uses: actions/setup-node@v4
+        uses: docker/[email protected]
         with:
-          node-version-file: .node-version
-          cache: npm
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Install dependencies
+      - name: Extract metadata (tags, labels) for Docker
         if: ${{ steps.branch-deploy.outputs.continue == 'true' }}
-        run: npm ci
+        id: docker-meta
+        uses: docker/[email protected]
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Publish - Development
-        if: ${{ steps.branch-deploy.outputs.environment == 'development' &&
-          steps.branch-deploy.outputs.noop != 'true' &&
-          steps.branch-deploy.outputs.continue == 'true' }}
-        uses: cloudflare/wrangler-action@f84a562284fc78278ff9052435d9526f9c718361 # [email protected]
+      # In theory, we could build the container as soon as a commit is pushed, rather than waiting until just before deployment.
+      # This would make deployments faster, but it does mean we would have images in our repository that may not have been reviewed.
+      - name: Build and push Docker image to registry
+        if: ${{ steps.branch-deploy.outputs.continue == 'true' }}
+        id: docker-push
+        uses: docker/[email protected]
         with:
-          wranglerVersion: '2.17.0'
-          apiToken: ${{ secrets.CF_API_TOKEN }}
-          environment: "development"
+          context: .
+          push: true
+          tags: ${{ env.IMAGE_TAG }}
+          labels: ${{ steps.docker-meta.outputs.labels }}
+
+      - name: Generate artifact attestation
+        if: ${{ steps.branch-deploy.outputs.continue == 'true' }}
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.docker-push.outputs.digest }}
+          push-to-registry: true
+
+      # Setup SSH agent
+      - name: Setup Deployment Agent
+        if: ${{ steps.branch-deploy.outputs.continue == 'true' &&
+          steps.branch-deploy.outputs.noop != 'true' }}
+        id: setup-ssh
+        env:
+          # Sets up the ssh agent to be used in future steps for connecting to the deployment environment
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+          # Sets the ssh host address based on the desired environment if they differ, falling back to SSH_HOST
+          ENV_HOST: ${{ steps.branch-deploy.outputs.environment == 'production' && secrets.SSH_HOST_PROD || secrets.SSH_HOST_DEV || secrets.SSH_HOST }}
+        run: |
+          mkdir -p /home/runner/.ssh
+          ssh-keyscan $ENV_HOST >> /home/runner/.ssh/known_hosts
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > /home/runner/.ssh/api-server-key
+          chmod 600 /home/runner/.ssh/api-server-key
+          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+          ssh-add /home/runner/.ssh/api-server-key
+          echo "ssh-host=$ENV_HOST" >> $GITHUB_OUTPUT
+
+      - name: Deploy
+        if: ${{ steps.branch-deploy.outputs.continue == 'true' &&
+          steps.branch-deploy.outputs.noop != 'true'}}
+        env:
+            # Uses the ssh agent set up in the previous step to connect to the deployment environment
+            SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+            # Allows for the API to be deployed to the same system on different ports based on the environment
+            ENV_PORT: ${{ steps.branch-deploy.outputs.environment == 'production' && '80' || env.DEFAULT_PORT }}
+            # The name of the container to be run
+            CONTAINER_NAME: tarkov-api-${{ steps.branch-deploy.outputs.environment }}
+        run: |
+          # Stop and remove any existing container matching our container name variable
+          ssh ${{ secrets.SSH_USER }}@${{ steps.setup-ssh.outputs.ssh-host }} "docker rm -f ${{ env.CONTAINER_NAME }} || true"
+          # Pull the latest version of the image published to the registry
+          ssh ${{ secrets.SSH_USER }}@${{ steps.setup-ssh.outputs.ssh-host }} "docker pull ${{ env.IMAGE_TAG }}"
+          # Run the image as a container
+          ssh ${{ secrets.SSH_USER }}@${{ steps.setup-ssh.outputs.ssh-host }} "docker run -d --name ${{ env.CONTAINER_NAME }} --restart unless-stopped -p ${{ env.ENV_PORT }}:${{ env.DEFAULT_PORT }} -e ENVIRONMENT='${{ steps.branch-deploy.outputs.environment }}' -e SENTRY_ENV='${{ steps.branch-deploy.outputs.environment }}' -e SENTRY_DSN='${{ secrets.SENTRY_DSN || '' }}' -e SENTRY_TRACE_RATE=${{ vars.SENTRY_TRACE_RATE || 0 }} -e SENTRY_PROFILE_RATE=${{ vars.SENTRY_PROFILE_RATE || 0 }} -e DEPLOY_REF='${{ github.event.pull_request.head.sha || github.sha }}' -e CLOUDFLARE_TOKEN='${{ secrets.CLOUDFLARE_TOKEN }}' -e CACHE_BASIC_AUTH='${{ secrets.CACHE_BASIC_AUTH }}' ${{ env.IMAGE_TAG }}"
 
       # Post comment on PR with development deploy info
       - uses: GrantBirki/[email protected]
@@ -71,15 +136,6 @@ jobs:
 
             > Pusher: @${{ github.actor }}, Action: `${{ github.event_name }}`, Workflow: `${{ github.workflow }}`;
 
-      - name: Publish - Production
-        if: ${{ steps.branch-deploy.outputs.continue == 'true' &&
-          steps.branch-deploy.outputs.noop != 'true' &&
-          steps.branch-deploy.outputs.environment == 'production' }}
-        uses: cloudflare/wrangler-action@f84a562284fc78278ff9052435d9526f9c718361 # [email protected]
-        with:
-          wranglerVersion: '2.17.0'
-          apiToken: ${{ secrets.CF_API_TOKEN }}
-
       # Post comment on PR with production deploy info
       - uses: GrantBirki/[email protected]
         if: ${{ steps.branch-deploy.outputs.continue == 'true' &&

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -7,8 +7,57 @@ on:
 
 permissions:
   contents: read
+  # Permissions needed for building and deploying docker images
+  packages: write
+  # contents: read
+  attestations: write
+  id-token: write
+
+env:
+  # Environment variables used by docker build and push
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+  DEFAULT_PORT: 8788
+  IMAGE_LATEST_TAG: ghcr.io/${{ github.repository }}:latest
+  IMAGE_MAIN_TAG: ghcr.io/${{ github.repository }}:main
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/[email protected]
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: docker-meta
+        uses: docker/[email protected]
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image to registry
+        id: docker-push
+        uses: docker/[email protected]
+        with:
+          context: .
+          push: true
+          tags: ${{ env.IMAGE_LATEST_TAG }},${{ env.IMAGE_MAIN_TAG }}
+          labels: ${{ steps.docker-meta.outputs.labels }}
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.docker-push.outputs.digest }}
+          push-to-registry: true
+
   deployment-check:
     runs-on: ubuntu-latest
     outputs: # set outputs for use in downstream jobs
@@ -25,25 +74,45 @@ jobs:
 
   deploy:
     if: ${{ needs.deployment-check.outputs.continue == 'true' }}
-    needs: deployment-check
+    needs: 
+      - deployment-check
+      - build
     environment: production
     runs-on: ubuntu-latest
 
     steps:
       - name: checkout
         uses: actions/checkout@v4
 
-      - name: setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: .node-version
-          cache: npm
-
-      - name: install dependencies
-        run: npm ci
+      # Setup SSH agent
+      - name: Setup Deployment Agent
+        id: setup-ssh
+        env:
+          # Sets up the ssh agent to be used in future steps for connecting to the deployment environment
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+          # Sets the ssh host address based on the desired environment if they differ, falling back to SSH_HOST
+          ENV_HOST: ${{ secrets.SSH_HOST_PROD || secrets.SSH_HOST }}
+        run: |
+          mkdir -p /home/runner/.ssh
+          ssh-keyscan $ENV_HOST >> /home/runner/.ssh/known_hosts
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > /home/runner/.ssh/api-server-key
+          chmod 600 /home/runner/.ssh/api-server-key
+          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+          ssh-add /home/runner/.ssh/api-server-key
+          echo "ssh-host=$ENV_HOST" >> $GITHUB_OUTPUT
 
-      - name: Publish - Production
-        uses: cloudflare/wrangler-action@f84a562284fc78278ff9052435d9526f9c718361 # [email protected]
-        with:
-          wranglerVersion: '2.17.0'
-          apiToken: ${{ secrets.CF_API_TOKEN }}
+      - name: Deploy
+        env:
+            # Uses the ssh agent set up in the previous step to connect to the deployment environment
+            SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+            # Allows for the API to be deployed to the same system on different ports based on the environment
+            ENV_PORT: '80'
+            # The name of the container to be run
+            CONTAINER_NAME: tarkov-api-production
+        run: |
+          # Stop and remove any existing container matching our container name variable
+          ssh ${{ secrets.SSH_USER }}@${{ steps.setup-ssh.outputs.ssh-host }} "docker rm -f ${{ env.CONTAINER_NAME }} || true"
+          # Pull the latest version of the image published to the registry
+          ssh ${{ secrets.SSH_USER }}@${{ steps.setup-ssh.outputs.ssh-host }} "docker pull ${{ env.IMAGE_MAIN_TAG }}"
+          # Run the image as a container
+          ssh ${{ secrets.SSH_USER }}@${{ steps.setup-ssh.outputs.ssh-host }} "docker run -d --name ${{ env.CONTAINER_NAME }} --restart unless-stopped -p ${{ env.ENV_PORT }}:${{ env.DEFAULT_PORT }} -e ENVIRONMENT='production' -e SENTRY_ENV='production' -e SENTRY_DSN='${{ secrets.SENTRY_DSN || '' }}' -e SENTRY_TRACE_RATE=${{ vars.SENTRY_TRACE_RATE || 0 }} -e SENTRY_PROFILE_RATE=${{ vars.SENTRY_PROFILE_RATE || 0 }} -e DEPLOY_REF='${{ github.sha }}'  -e CLOUDFLARE_TOKEN='${{ secrets.CLOUDFLARE_TOKEN }}' -e CACHE_BASIC_AUTH='${{ secrets.CACHE_BASIC_AUTH }}' ${{ env.IMAGE_MAIN_TAG }}"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -12,20 +12,25 @@ permissions:
 
 jobs:
   test:
+    environment: secrets
     runs-on: ubuntu-latest
     steps:
-      - name: checkout
+      - name: Checkout
         uses: actions/checkout@v4
 
-      - name: setup node
+      - name: Setup Node Environment
         uses: actions/setup-node@v4
         with:
           node-version-file: .node-version
           cache: npm
 
-      - run: npm ci
+      - name: Install Dependencies
+        run: |
+          npm ci
+          cd http && npm ci
 
-      - name: test
+      - name: Execute Tests
         env:
-          CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+          CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }}
+          CACHE_BASIC_AUTH: ${{ secrets.CACHE_BASIC_AUTH }}
         run: script/test
diff --git a/.node-version b/.node-version
@@ -1 +1 @@
-20.11.0
+20.15.1
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,25 @@
+FROM node:20-slim
+
+# Copy the files from the host to the container
+COPY . /app
+
+# Setup base package
+WORKDIR /app
+RUN npm ci
+
+# Setup HTTP package
+WORKDIR /app/http
+RUN npm ci
+
+ENV CLOUDFLARE_TOKEN=placeholder
+ENV CACHE_BASIC_AUTH=placeholder
+ENV PORT=8788
+ENV ENVIRONMENT=dev
+ENV SENTRY_DSN=
+ENV SENTRY_TRACE_RATE=0
+ENV SENTRY_PROFILE_RATE=0
+
+EXPOSE $PORT
+
+# Run the application
+ENTRYPOINT ["npm", "run", "start"]
diff --git a/docs/maintainer-notes.md b/docs/maintainer-notes.md
@@ -14,6 +14,16 @@ This is a simple run down of how review / deploy a pull request.
 
 > It should be noted that the approval step can come before the deployment steps if that suits the situation better.
 
+## Required Secrets
+
+To deploy the API, you will need the following secrets. To deploy via GitHub Actions, you will need to add these secrets to the `secrets` and `production` environments in the repository settings.
+
+* `CLOUDFLARE_TOKEN` - The Cloudflare API token used for accessing the KV store until such a time as we have an independent database for the new express hosted API.
+* `CACHE_BASIC_AUTH` - The basic auth string to use with the cache server.
+* `SSH_HOST` - The host (ip or DNS record) of the server to deploy the Docker container.
+* `SSH_USER` - The user to use when connecting to the deployment server.
+* `SSH_PRIVATE_KEY` - The private key to use when connecting to the deployment server.
+
 ## CI Failures
 
 A known issue (I am not sure of the cause) for CI failures is when dependabot opens a pull request. For some very strange reason, the necessary secrets are not injected into the Actions workflow when the pull request comes from dependabot. This causes the wrangler environment in CI to fail because it lacks the proper credentials to authenticate with Cloudflare.

diff --git a/http/README.md b/http/README.md
@@ -0,0 +1,16 @@
+This folder is for running the API as a standalonen application, without using Cloudflare workers.
+
+## Setup
+1. Install [Node.js](https://nodejs.org/en/download/)
+2. Run `npm install` from the parent directory to install the base dependencies
+3. Run `npm install` from this directory to install the dependencies for the HTTP server
+
+## Enviroment Variables
+* `PORT` - The port the server will listen on. Default is `8088`.
+* `ENVIRONMENT` - The environment to run in. Either `production` or `dev`. Default is `dev`.
+* `CACHE_BASIC_AUTH` - The basic auth string to use for caching. Default is `placeholder`.
+* `CLOUDFLARE_TOKEN` - The Cloudflare token to use for accessing the KV store. Default is `placeholder`.
+* `SENTRY_DSN` - The Sentry DSN to use for error reporting. Defaults to empty.
+* `SENTRY_TRACE_RATE` - The Sentry trace sample rate to use for error reporting. Default is `0`.
+* `SENTRY_PROFILE_RATE` - The Sentry profile sample rate to use for error reporting. Default is `0`.
+* `SENTRY_ENV` - The Sentry environment to use for error reporting. Default is `unknown`.