Integration tests

.github/workflows/pipeline.yaml

@@ -99,6 +99,48 @@ jobs:
         if: always()
         run: docker stop ${{ steps.start_container.outputs.container_id }}
 
+
+  test-with-k3s:
+    if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
+      # We only run this action if all the prior test actions succeed
+    needs:
+      - test-general-edge-endpoint
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Install k3s
+        run: |
+          curl -sfL https://get.k3s.io | sh -
+          sudo chmod 644 /etc/rancher/k3s/k3s.yaml
+          sudo chown $USER /etc/rancher/k3s/k3s.yaml
+          # symlink to kubeconfig
+          mkdir -p ~/.kube
+          ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_GL_PUBLIC_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_GL_PUBLIC_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install edge-endpoint's python dependencies
+        run: |
+          poetry install --no-interaction --no-root
+
+      - name: Run tests with k3s
+        run: make test-with-k3s
+
   # Run Groundlight SDK tests against the edge proxy endpoint
   test-sdk:
     runs-on: ubuntu-22.04
@@ -180,13 +222,13 @@ jobs:
         if: always()
         run: docker stop ${{ steps.start_container.outputs.container_id }}
 
-
   build-push-edge-endpoint-multiplatform:
-    if: (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch')
+    if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
     # We only run this action if all the prior test actions succeed
     needs:
       - test-general-edge-endpoint
       - test-sdk
+      - test-with-k3s
     runs-on: ubuntu-22.04
     steps:
       - name: Configure AWS credentials
@@ -204,54 +246,10 @@ jobs:
         uses: actions/checkout@v4
       - name: Build and Push Multiplatform edge-endpoint Image to ECR
         timeout-minutes: 45
-        run:
-          ./deploy/bin/build-push-edge-endpoint-image.sh
-
-  test-in-k3s:
-    needs:
-      # In order to test in k3s, we pull the latest edge-endpoint image from ECR. Which means that
-      # we need to build and push the image first or, we're just testing against a stale image.
-      - build-push-edge-endpoint-multiplatform
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v3
-
-      - name: Install k3s
-        run: |
-          curl -sfL https://get.k3s.io | sh -
-          sudo chmod 644 /etc/rancher/k3s/k3s.yaml
-          sudo chown $USER /etc/rancher/k3s/k3s.yaml
-          # symlink to kubeconfig
-          mkdir -p ~/.kube
-          ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_GL_PUBLIC_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_GL_PUBLIC_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-
-      - name: Deploy the edge-endpoint yaml
-        run: |
-          set -ex
-          # Once there's a fix for disk-pressure issues on GHA's runner, we can uncomment the
-          # following command in order to set up the environment to test out edge inference
-          # source test/setup_inference_test_env.sh
-          export INFERENCE_FLAVOR="CPU"
-          export DEPLOYMENT_NAMESPACE="default"
-          bash deploy/bin/setup-ee.sh
-          kubectl describe deployment
-
-      - name: Wait for edge-endpoint pod to be ready
-        run: |
-          set -ex
-          kubectl rollout status deployment edge-endpoint --timeout=5m
-          kubectl describe pod edge-endpoint
+        run: ./deploy/bin/build-push-edge-endpoint-image.sh
 
   update-glhub:
-    needs: test-in-k3s
+    needs: test-with-k3s
     runs-on: ubuntu-latest
     environment: live
 

Makefile

@@ -1,4 +1,5 @@
 .PHONY: install install-lint install-pre-commit test test-with-docker test-all lint format
+SHELL := /bin/bash
 
 install:
 	poetry install --no-root
@@ -18,7 +19,9 @@ test-with-docker: install  ## Run tests that require a live edge-endpoint server
 test-all: test test-with-docker  ## Run all tests in one make command
 	@echo "All tests completed."
 
-
+test-with-k3s:
+	. test/integration/setup_and_run_tests.sh
+
 # Adjust which paths we lint
 LINT_PATHS="app test"
 

deploy/bin/git-tag-name.sh

@@ -5,6 +5,8 @@
 #     including the branch name, and a consistent hash of the uncommitted changes
 
 set -e
+cd "$(dirname "$0")"
+
 
 fail() {
     echo $1

deploy/bin/setup-ee.sh

@@ -14,7 +14,6 @@
 # - DEPLOYMENT_NAMESPACE: The namespace for deployment. Defaults to the current namespace.
 # - RUN_EDGE_ENDPOINT: Controls the launch of edge endpoint pods.
 #   - If set, the edge-endpoint pods will be launched. If not set, pods will not be launched.
-
 set -ex
 
 # If we're in a container, sudo won't be available. But otherwise there are commands where we want sudo.
@@ -67,13 +66,23 @@ K=${KUBECTL_CMD:-"kubectl"}
 INFERENCE_FLAVOR=${INFERENCE_FLAVOR:-"GPU"}
 DEPLOY_LOCAL_VERSION=${DEPLOY_LOCAL_VERSION:-1}
 DEPLOYMENT_NAMESPACE=${DEPLOYMENT_NAMESPACE:-$($K config view -o json | jq -r '.contexts[] | select(.name == "'$($K config current-context)'") | .context.namespace // "default"')}
+IMAGE_TAG=${IMAGE_TAG:-"latest"}
 
 # Update K to include the deployment namespace
 K="$K -n $DEPLOYMENT_NAMESPACE"
 
 # move to the root directory of the repo
 cd "$(dirname "$0")"/../..
 
+
+# Replace some configurable values in our deployment manifests,
+# if they are provided in environment variables.
+
+# Most users do not need to think about these.
+
+PERSISTENT_VOLUME_NAME=${PERSISTENT_VOLUME_NAME:-"edge-endpoint-pv"}
+EDGE_ENDPOINT_PORT=${EDGE_ENDPOINT_PORT:-30101}
+
 # Create Secrets
 if ! ./deploy/bin/make-aws-secret.sh; then
     echo "Failed to execute make-aws-secret.sh successfully. Exiting."
@@ -137,28 +146,33 @@ $K get service -o custom-columns=":metadata.name" --no-headers=true | \
 
 # Check if DEPLOY_LOCAL_VERSION is set. If so, use a local volume instead of an EFS volume
 if [[ "${DEPLOY_LOCAL_VERSION}" == "1" ]]; then
-    if ! check_pv_conflict "edge-endpoint-pv" "local-sc"; then
-        fail "PersistentVolume edge-endpoint-pv conflicts with the existing resource."
+    if ! check_pv_conflict "$PERSISTENT_VOLUME_NAME" "local-sc"; then
+        fail "PersistentVolume $PERSISTENT_VOLUME_NAME conflicts with the existing resource."
     fi
 
-    $K apply -f deploy/k3s/local_persistent_volume.yaml
+    # Use envsubst to replace the PERSISTENT_VOLUME_NAME, PERSISTENT_VOLUME_NAME in the local_persistent_volume.yaml template
+    envsubst < deploy/k3s/local_persistent_volume.yaml > deploy/k3s/local_persistentvolume.yaml
+    $K apply -f deploy/k3s/local_persistentvolume.yaml
+    rm deploy/k3s/local_persistentvolume.yaml
+
 else
     # If environment variable EFS_VOLUME_ID is not set, exit
     if [[ -z "${EFS_VOLUME_ID}" ]]; then
         fail "EFS_VOLUME_ID environment variable not set"
     fi
 
-    if ! check_pv_conflict "edge-endpoint-pv" "efs-sc"; then
-        fail "PersistentVolume edge-endpoint-pv conflicts with the existing resource."
+    if ! check_pv_conflict "$PERSISTENT_VOLUME_NAME" "efs-sc"; then
+        fail "PersistentVolume $PERSISTENT_VOLUME_NAME conflicts with the existing resource."
     fi
 
-    # Use envsubst to replace the EFS_VOLUME_ID in the persistentvolumeclaim.yaml template
+    # Use envsubst to replace the EFS_VOLUME_ID, PERSISTENT_VOLUME_NAME, PERSISTENT_PERSISTENT_VOLUME_NAMEVOLUME_CLAIM_NAME
+    # in the persistentvolumeclaim.yaml template
     envsubst < deploy/k3s/efs_persistent_volume.yaml > deploy/k3s/persistentvolume.yaml
     $K apply -f deploy/k3s/persistentvolume.yaml
     rm deploy/k3s/persistentvolume.yaml
 fi
 
-# Check if the edge-endpoint-pvc exists. If not, create it
+# Check if the persistent volume claim exists. If not, create it
 if ! $K get pvc edge-endpoint-pvc; then
     # If environment variable EFS_VOLUME_ID is not set, exit
     if [[ -z "${EFS_VOLUME_ID}" ]]; then
@@ -179,6 +193,10 @@ $K apply -f deploy/k3s/service_account.yaml.tmp
 rm deploy/k3s/service_account.yaml.tmp
 
 $K apply -f deploy/k3s/inference_deployment/warmup_inference_model.yaml
-$K apply -f deploy/k3s/edge_deployment/edge_deployment.yaml
+
+# Substitutes the EDGE_ENDPOINT_PORT
+envsubst < deploy/k3s/edge_deployment/edge_deployment.yaml > deploy/k3s/edge_deployment/edge_deployment.yaml.tmp
+$K apply -f deploy/k3s/edge_deployment/edge_deployment.yaml.tmp
+rm deploy/k3s/edge_deployment/edge_deployment.yaml.tmp
 
 $K describe deployment edge-endpoint

deploy/k3s/edge_deployment/edge_deployment.yaml

@@ -16,7 +16,7 @@ spec:
   - protocol: TCP
     # Service port for NGINX
     port: 30101
-    nodePort: 30101
+    nodePort: ${EDGE_ENDPOINT_PORT}
   type: NodePort
 ---
 apiVersion: apps/v1
@@ -40,7 +40,7 @@ spec:
       serviceAccountName: edge-endpoint-service-account
       initContainers:
       - name: database-prep
-        image: &edgeEndpointImage 767397850842.dkr.ecr.us-west-2.amazonaws.com/edge-endpoint:latest
+        image: &edgeEndpointImage 767397850842.dkr.ecr.us-west-2.amazonaws.com/edge-endpoint:${IMAGE_TAG}
         imagePullPolicy: Always
         volumeMounts:
         - name: edge-endpoint-persistent-volume

deploy/k3s/efs_persistent_volume.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: PersistentVolume
 metadata:
-  name: edge-endpoint-pv
+  name: ${PERSISTENT_VOLUME_NAME}
 spec:
   capacity:
     storage: 2Gi

deploy/k3s/local_persistent_volume.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1 
 kind: PersistentVolume 
 metadata:
-  name: edge-endpoint-pv
+  name: ${PERSISTENT_VOLUME_NAME}
 spec:
   capacity: 
     storage: 2Gi