Merge pull request #1637 from gchq/feature/BAI-1503-create-a-helm-cha…

…rt-for-model-scan

Feature/bai 1503 create a helm chart for model scan

.github/workflows/build.js.yml

@@ -63,6 +63,34 @@ jobs:
           name: bailo_frontend
           path: /tmp/bailo_frontend.tar
 
+  build_modelscan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      # Clone repository
+      - uses: actions/checkout@v3
+
+      # Setup BuildX
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build modelscan image
+        uses: docker/build-push-action@v3
+        with:
+          context: lib/modelscan_api
+          file: lib/modelscan_api/Dockerfile
+          tags: bailo_modelscan:latest
+          cache-from: type=gha,scope=modelscan
+          cache-to: type=gha,mode=max,scope=modelscan
+          outputs: type=docker,dest=/tmp/bailo_modelscan.tar
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bailo_modelscan
+          path: /tmp/bailo_modelscan.tar
+
   unit_testing:
     runs-on: ubuntu-latest
 
@@ -107,7 +135,7 @@ jobs:
       - run: npm run lint
 
   kubernetes:
-    needs: [build_backend, build_frontend]
+    needs: [build_backend, build_frontend, build_modelscan]
     runs-on: ubuntu-latest
 
     steps:
@@ -134,6 +162,11 @@ jobs:
         with:
           name: bailo_frontend
           path: /tmp
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: bailo_modelscan
+          path: /tmp
 
       - name: Start minikube
         uses: medyagh/setup-minikube@master
@@ -146,6 +179,7 @@ jobs:
           eval $(minikube -p minikube docker-env)
           docker load -i /tmp/bailo_backend.tar
           docker load -i /tmp/bailo_frontend.tar
+          docker load -i /tmp/bailo_modelscan.tar
 
       # Install dependencies
       - run: npm ci
@@ -193,6 +227,8 @@ jobs:
             frontendTag: latest
             backendRepository: "bailo_backend"
             backendTag: latest
+            modelscanRepository: "bailo_modelscan"
+            modelscanTag: latest
           # Used for k8s not openshift
           ingress:
             enabled: false
@@ -259,7 +295,7 @@ jobs:
         run: kubectl logs -l app.kubernetes.io/instance=bailo --tail=-1
 
   end_to_end:
-    needs: [build_backend, build_frontend]
+    needs: [build_backend, build_frontend, build_modelscan]
     runs-on: ubuntu-latest
 
     steps:
@@ -287,14 +323,20 @@ jobs:
         with:
           name: bailo_frontend
           path: /tmp
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: bailo_modelscan
+          path: /tmp
 
       - name: Load image
         run: |
           docker load --input /tmp/bailo_frontend.tar
           docker load --input /tmp/bailo_backend.tar
+          docker load --input /tmp/bailo_modelscan.tar
           docker image ls -a
 
-          rm -rf /tmp/bailo_frontend.tar /tmp/bailo_backend.tar
+          rm -rf /tmp/bailo_frontend.tar /tmp/bailo_backend.tar /tmp/bailo_modelscan.tar
 
       # Create logs directory
       - run: mkdir logs

.gitlab-ci.yml

@@ -73,9 +73,35 @@ os-frontend-build:
     - docker tag frontend-ci:$OS_TAG $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/frontend-ci:$OS_TAG
     - docker push $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/frontend-ci:$OS_TAG
 
+os-modelscan-build:
+  stage: os_build
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      when: always
+      allow_failure: true
+    - when: manual
+      allow_failure: true
+  image:
+    name: docker:latest
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_TLS_CERTDIR: ''
+  services:
+    - name: docker:stable-dind
+      command:
+        - '--tls=false'
+        - '--host=tcp://0.0.0.0:2375'
+  before_script:
+    - docker login -u $OPENSHIFT_USER -p $OPENSHIFT_TOKEN $REGISTRY_URL
+    - cd lib/modelscan_api
+  script:
+    - DOCKER_BUILDKIT=1 docker build -t modelscan-ci:$OS_TAG .
+    - docker tag modelscan-ci:$OS_TAG $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/modelscan-ci:$OS_TAG
+    - docker push $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/modelscan-ci:$OS_TAG
+
 openshift-deploy:
   stage: os_build
-  needs: ['os-frontend-build', 'os-backend-build']
+  needs: ['os-frontend-build', 'os-backend-build', 'os-modelscan-build']
   rules:
     - when: manual
       allow_failure: true
@@ -111,6 +137,8 @@ openshift-deploy:
         frontendTag: $OS_TAG
         backendRepository: "$BACKEND_REPO"
         backendTag: $OS_TAG
+        modelscanRepository: "$BACKEND_REPO"
+        modelscanTag: $OS_TAG
       # Used for openshift
       route:
         enabled: true
@@ -189,3 +217,35 @@ aws-frontend-build:
     - docker tag $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$AWS_TAG $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$OS_TAG
     - docker push $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$AWS_TAG
     - docker push $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$OS_TAG
+
+aws-modelscan-build:
+  stage: aws_build
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      when: always
+      allow_failure: true
+    - when: manual
+      allow_failure: true
+  image:
+    name: docker:latest
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_TLS_CERTDIR: ''
+  services:
+    - name: docker:stable-dind
+      command:
+        - '--tls=false'
+        - '--host=tcp://0.0.0.0:2375'
+  before_script:
+    - apk add --no-cache curl jq python3 py3-pip
+    - pip install awscli --break-system-packages
+    - aws ecr get-login-password | docker login --username AWS --password-stdin $DOCKER_REGISTRY
+    - aws --version
+    - docker info
+    - docker --version
+    - cd lib/modelscan_api
+  script:
+    - DOCKER_BUILDKIT=1 docker build -t $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG .
+    - docker tag $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$OS_TAG
+    - docker push $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG
+    - docker push $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$OS_TAG

frontend/pages/docs/administration/helm/configuration.mdx

@@ -34,6 +34,8 @@ image:
   frontendTag: 'v2.1.2'
   backendRepository: '<docker registry>'
   backendTag: 'v2.1.2'
+  modelscanRepository: '<docker registry>'
+  modelscanTag: 'v1.0.0'
 ```
 
 To create these images, follow the instructions in [Building Bailo Images](../building-the-bailo-image).

frontend/pages/docs/v1/administration/getting-started/building-the-bailo-image.mdx

@@ -9,6 +9,7 @@ frontend and one for the backend.
 # Bailo uses a monorepo, so images should be built from the root directory
 $ docker build -t "bailo-backend" -f ./backend/Dockerfile .
 $ docker build -t "bailo-frontend" -f ./frontend/Dockerfile .
+$ docker build -t "bailo-modelscan" -f ./lib/modelscan_api/Dockerfile .
 ```
 
 Ensure you do not use the development Dockerfiles (called `Dockerfile.dev`) as they expect the application to be mounted

infrastructure/helm/README.md

@@ -49,6 +49,8 @@ image:
   frontendTag: tag
   backendRepository: 'image-registry-openshift-imagestreams'
   backendTag: tag
+  modelscanRepository: 'image-registry-openshift-imagestreams'
+  modelscanTag: tag
 
 route:
   enabled: true
@@ -73,6 +75,8 @@ image:
   frontendTag: tag
   backendRepository: 'aws-elastic-container-registry'
   backendTag: tag
+  modelscanRepository: 'aws-elastic-container-registry'
+  modelscanTag: tag
 
 ingress:
   enabled: true

infrastructure/helm/bailo/templates/modelscan/modelscan.deployment.yaml

@@ -0,0 +1,52 @@
+{{- if .Values.modelscan.enabled }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "bailo.fullname" . }}-modelscan
+  labels:
+    {{- include "bailo.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: modelscan
+  template:
+    metadata:
+      labels:
+        name: modelscan
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsUser: {{ .Values.modelscan.runAsUser }}
+      containers:
+      - name: {{ include "bailo.fullname" . }}-modelscan
+        ports:
+          - name: modelscanport
+            containerPort: 3311
+            protocol: TCP
+        image: "{{ .Values.image.modelscanRepository }}:{{ .Values.image.modelscanTag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        resources:
+          requests:
+            memory: "100Mi"
+            ephemeral-storage: "100Mi"
+            cpu: "200m"
+          limits:
+            memory: "2Gi"
+            ephemeral-storage: "300Mi"
+            cpu: "1"
+        volumeMounts:
+        - name: tmp
+          mountPath: /tmp
+        securityContext:
+          allowPrivilegeEscalation: false
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: modelscan-log
+          emptyDir: {}
+{{- end }}

infrastructure/helm/bailo/templates/modelscan/modelscan.service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.modelscan.enabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "bailo.fullname" . }}-modelscan
+  labels:
+    {{- include "bailo.labels" . | nindent 4 }}
+spec:
+  ports:
+    - port: {{ .Values.modelscan.port }}
+      targetPort: {{ .Values.modelscan.port }}
+      protocol: TCP
+      name: {{ .Values.modelscan.protocol }}
+  selector:
+    name: modelscan
+{{- end }}

infrastructure/helm/bailo/values.yaml

@@ -8,6 +8,8 @@ image:
   frontendTag: imageTag
   backendRepository: "backend-image-registry.location"
   backendTag: imageTag
+  modelscanRepository: "modelscan-image-registry.location"
+  modelscanTag: imageTag
   pullPolicy: IfNotPresent
 imagePullSecrets: []
 
@@ -63,7 +65,7 @@ route:
   enabled: false # false for k8s | aws
   appPublicRoute: "route-url" # can also be used for aws loadbalancer url
 
-# Oauth | Congnito
+# Oauth | Cognito
 # using bash for generating cookie secret example
 # dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 | tr -d -- '\n' | tr -- '+/' '-_'; echo
 cookie:
@@ -76,7 +78,7 @@ oauth:
     secret: "cognito app client secret"
     dynamic: [] # example 'scope'
     response: [] #example 'tokens', 'raw', 'jwt'
-    callback: "" # exmaple /
+    callback: "" # example /
     subdomain: "cognito domain"
   identityProviderClient:
     userPoolId: "region_id"
@@ -118,11 +120,11 @@ mongodb:
   enabled: true
   ssl: false
   sslValidate: false
-  host: "" # serviceName:mongoport/collectioname or serviceName-headless:mongoport/collectionname for mongodb replicaset. see template/_helper.tpl and templates/bailo/bailo.configmap.yaml.
+  host: "" # serviceName:mongoport/collectionname or serviceName-headless:mongoport/collectionname for mongodb replicaset. see template/_helper.tpl and templates/bailo/bailo.configmap.yaml.
   livenessProbe:
     enabled: false
   readinessProbe:
-    enabled: false #Rediness probe failed: /bitnami/scripts/readiness-probe.sh: line 9: mongosh: command not found
+    enabled: false #Readiness probe failed: /bitnami/scripts/readiness-probe.sh: line 9: mongosh: command not found
   auth:
     rootUser: ""
     rootPassword: ""
@@ -135,7 +137,7 @@ mongodb:
     port: 27017
   collectionName: bailo # Collection name in Mongo
   persistence:
-    #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Stoage is not removed when deployed mongodb is removed.
+    #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Storage is not removed when deployed mongodb is removed.
     size: 10Gi
   containerSecurityContext:
     enabled: false
@@ -165,7 +167,7 @@ minio:
       api: 31111 # default 9000
   useSSL: false
   persistence:
-    #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Stoage is not removed when deployed minio is removed.
+    #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Storage is not removed when deployed minio is removed.
     size: 20Gi
   containerSecurityContext:
     enabled: false
@@ -196,7 +198,7 @@ registry:
 
 # Nginx Dependencies
 nginxAuth:
-  repository: nginxinc/nginx-unprivileged # runs Nginx as non-root unrivileged user.
+  repository: nginxinc/nginx-unprivileged # runs Nginx as non-root unprivileged user.
   tag: latest
   #host: "bailo-nginx" # service name
   port: 8080
@@ -299,3 +301,11 @@ clamav:
   accessModes:
     - ReadWriteOnce
   size: 10G
+
+modelscan:
+  enabled: false
+  protocol: http
+  port: 3311
+  accessModes:
+    - ReadWriteOnce
+  size: 20G  # modelscan saves each incoming file to the disk so needs plenty of storage

lib/modelscan_api/bailo_modelscan_api/config.py

@@ -24,6 +24,7 @@ class Settings(BaseSettings):
     Bailo ModelScan API allows for easy programmatic interfacing with ProtectAI's ModelScan package to scan and detect potential threats within files stored in Bailo.
 
     You can upload files and view modelscan's result."""
+    # Update frontend/pages/docs/administration/helm/configuration.mdx if bumping this.
     app_version: str = "1.0.0"
     # download_dir is used if it evaluates, otherwise a temporary directory is used.
     download_dir: Optional[str] = None