diff --git a/.github/workflows/build.js.yml b/.github/workflows/build.js.yml index ed0053156..95ec5960e 100644 --- a/.github/workflows/build.js.yml +++ b/.github/workflows/build.js.yml @@ -63,6 +63,34 @@ jobs: name: bailo_frontend path: /tmp/bailo_frontend.tar + build_modelscan: + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + # Clone repository + - uses: actions/checkout@v3 + + # Setup BuildX + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build modelscan image + uses: docker/build-push-action@v3 + with: + context: lib/modelscan_api + file: lib/modelscan_api/Dockerfile + tags: bailo_modelscan:latest + cache-from: type=gha,scope=modelscan + cache-to: type=gha,mode=max,scope=modelscan + outputs: type=docker,dest=/tmp/bailo_modelscan.tar + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bailo_modelscan + path: /tmp/bailo_modelscan.tar + unit_testing: runs-on: ubuntu-latest @@ -107,7 +135,7 @@ jobs: - run: npm run lint kubernetes: - needs: [build_backend, build_frontend] + needs: [build_backend, build_frontend, build_modelscan] runs-on: ubuntu-latest steps: @@ -134,6 +162,11 @@ jobs: with: name: bailo_frontend path: /tmp + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: bailo_modelscan + path: /tmp - name: Start minikube uses: medyagh/setup-minikube@master @@ -146,6 +179,7 @@ jobs: eval $(minikube -p minikube docker-env) docker load -i /tmp/bailo_backend.tar docker load -i /tmp/bailo_frontend.tar + docker load -i /tmp/bailo_modelscan.tar # Install dependencies - run: npm ci @@ -193,6 +227,8 @@ jobs: frontendTag: latest backendRepository: "bailo_backend" backendTag: latest + modelscanRepository: "bailo_modelscan" + modelscanTag: latest # Used for k8s not openshift ingress: enabled: false @@ -259,7 +295,7 @@ jobs: run: kubectl logs -l app.kubernetes.io/instance=bailo --tail=-1 end_to_end: - needs: [build_backend, build_frontend] + needs: [build_backend, build_frontend, build_modelscan] runs-on: ubuntu-latest steps: @@ -287,14 +323,20 @@ jobs: with: name: bailo_frontend path: /tmp + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: bailo_modelscan + path: /tmp - name: Load image run: | docker load --input /tmp/bailo_frontend.tar docker load --input /tmp/bailo_backend.tar + docker load --input /tmp/bailo_modelscan.tar docker image ls -a - rm -rf /tmp/bailo_frontend.tar /tmp/bailo_backend.tar + rm -rf /tmp/bailo_frontend.tar /tmp/bailo_backend.tar /tmp/bailo_modelscan.tar # Create logs directory - run: mkdir logs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b41e6af5..7e63843b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,9 +73,35 @@ os-frontend-build: - docker tag frontend-ci:$OS_TAG $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/frontend-ci:$OS_TAG - docker push $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/frontend-ci:$OS_TAG +os-modelscan-build: + stage: os_build + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + when: always + allow_failure: true + - when: manual + allow_failure: true + image: + name: docker:latest + variables: + DOCKER_HOST: tcp://docker:2375 + DOCKER_TLS_CERTDIR: '' + services: + - name: docker:stable-dind + command: + - '--tls=false' + - '--host=tcp://0.0.0.0:2375' + before_script: + - docker login -u $OPENSHIFT_USER -p $OPENSHIFT_TOKEN $REGISTRY_URL + - cd lib/modelscan_api + script: + - DOCKER_BUILDKIT=1 docker build -t modelscan-ci:$OS_TAG . + - docker tag modelscan-ci:$OS_TAG $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/modelscan-ci:$OS_TAG + - docker push $REGISTRY_URL/$OPENSHIFT_PROJECTNAME/modelscan-ci:$OS_TAG + openshift-deploy: stage: os_build - needs: ['os-frontend-build', 'os-backend-build'] + needs: ['os-frontend-build', 'os-backend-build', 'os-modelscan-build'] rules: - when: manual allow_failure: true @@ -111,6 +137,8 @@ openshift-deploy: frontendTag: $OS_TAG backendRepository: "$BACKEND_REPO" backendTag: $OS_TAG + modelscanRepository: "$BACKEND_REPO" + modelscanTag: $OS_TAG # Used for openshift route: enabled: true @@ -189,3 +217,35 @@ aws-frontend-build: - docker tag $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$AWS_TAG $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$OS_TAG - docker push $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$AWS_TAG - docker push $DOCKER_REGISTRY/$APP_NAME_FRONTEND:$OS_TAG + +aws-modelscan-build: + stage: aws_build + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + when: always + allow_failure: true + - when: manual + allow_failure: true + image: + name: docker:latest + variables: + DOCKER_HOST: tcp://docker:2375 + DOCKER_TLS_CERTDIR: '' + services: + - name: docker:stable-dind + command: + - '--tls=false' + - '--host=tcp://0.0.0.0:2375' + before_script: + - apk add --no-cache curl jq python3 py3-pip + - pip install awscli --break-system-packages + - aws ecr get-login-password | docker login --username AWS --password-stdin $DOCKER_REGISTRY + - aws --version + - docker info + - docker --version + - cd lib/modelscan_api + script: + - DOCKER_BUILDKIT=1 docker build -t $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG . + - docker tag $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$OS_TAG + - docker push $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$AWS_TAG + - docker push $DOCKER_REGISTRY/$APP_NAME_MODELSCAN:$OS_TAG diff --git a/frontend/pages/docs/administration/helm/configuration.mdx b/frontend/pages/docs/administration/helm/configuration.mdx index 2c24f7ff0..f4becc52b 100644 --- a/frontend/pages/docs/administration/helm/configuration.mdx +++ b/frontend/pages/docs/administration/helm/configuration.mdx @@ -34,6 +34,8 @@ image: frontendTag: 'v2.1.2' backendRepository: '' backendTag: 'v2.1.2' + modelscanRepository: '' + modelscanTag: 'v1.0.0' ``` To create these images, follow the instructions in [Building Bailo Images](../building-the-bailo-image). diff --git a/frontend/pages/docs/v1/administration/getting-started/building-the-bailo-image.mdx b/frontend/pages/docs/v1/administration/getting-started/building-the-bailo-image.mdx index 63d60b677..23e4a470d 100644 --- a/frontend/pages/docs/v1/administration/getting-started/building-the-bailo-image.mdx +++ b/frontend/pages/docs/v1/administration/getting-started/building-the-bailo-image.mdx @@ -9,6 +9,7 @@ frontend and one for the backend. # Bailo uses a monorepo, so images should be built from the root directory $ docker build -t "bailo-backend" -f ./backend/Dockerfile . $ docker build -t "bailo-frontend" -f ./frontend/Dockerfile . +$ docker build -t "bailo-modelscan" -f ./lib/modelscan_api/Dockerfile . ``` Ensure you do not use the development Dockerfiles (called `Dockerfile.dev`) as they expect the application to be mounted diff --git a/infrastructure/helm/README.md b/infrastructure/helm/README.md index 7fc5fadab..f6b37b6da 100644 --- a/infrastructure/helm/README.md +++ b/infrastructure/helm/README.md @@ -49,6 +49,8 @@ image: frontendTag: tag backendRepository: 'image-registry-openshift-imagestreams' backendTag: tag + modelscanRepository: 'image-registry-openshift-imagestreams' + modelscanTag: tag route: enabled: true @@ -73,6 +75,8 @@ image: frontendTag: tag backendRepository: 'aws-elastic-container-registry' backendTag: tag + modelscanRepository: 'aws-elastic-container-registry' + modelscanTag: tag ingress: enabled: true diff --git a/infrastructure/helm/bailo/templates/modelscan/modelscan.deployment.yaml b/infrastructure/helm/bailo/templates/modelscan/modelscan.deployment.yaml new file mode 100644 index 000000000..6f44a5335 --- /dev/null +++ b/infrastructure/helm/bailo/templates/modelscan/modelscan.deployment.yaml @@ -0,0 +1,52 @@ +{{- if .Values.modelscan.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "bailo.fullname" . }}-modelscan + labels: + {{- include "bailo.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + name: modelscan + template: + metadata: + labels: + name: modelscan + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsUser: {{ .Values.modelscan.runAsUser }} + containers: + - name: {{ include "bailo.fullname" . }}-modelscan + ports: + - name: modelscanport + containerPort: 3311 + protocol: TCP + image: "{{ .Values.image.modelscanRepository }}:{{ .Values.image.modelscanTag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + requests: + memory: "100Mi" + ephemeral-storage: "100Mi" + cpu: "200m" + limits: + memory: "2Gi" + ephemeral-storage: "300Mi" + cpu: "1" + volumeMounts: + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: tmp + emptyDir: {} + - name: modelscan-log + emptyDir: {} +{{- end }} diff --git a/infrastructure/helm/bailo/templates/modelscan/modelscan.service.yaml b/infrastructure/helm/bailo/templates/modelscan/modelscan.service.yaml new file mode 100644 index 000000000..10efebb44 --- /dev/null +++ b/infrastructure/helm/bailo/templates/modelscan/modelscan.service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.modelscan.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "bailo.fullname" . }}-modelscan + labels: + {{- include "bailo.labels" . | nindent 4 }} +spec: + ports: + - port: {{ .Values.modelscan.port }} + targetPort: {{ .Values.modelscan.port }} + protocol: TCP + name: {{ .Values.modelscan.protocol }} + selector: + name: modelscan +{{- end }} diff --git a/infrastructure/helm/bailo/values.yaml b/infrastructure/helm/bailo/values.yaml index f0dac1160..60a782e9e 100644 --- a/infrastructure/helm/bailo/values.yaml +++ b/infrastructure/helm/bailo/values.yaml @@ -8,6 +8,8 @@ image: frontendTag: imageTag backendRepository: "backend-image-registry.location" backendTag: imageTag + modelscanRepository: "modelscan-image-registry.location" + modelscanTag: imageTag pullPolicy: IfNotPresent imagePullSecrets: [] @@ -63,7 +65,7 @@ route: enabled: false # false for k8s | aws appPublicRoute: "route-url" # can also be used for aws loadbalancer url -# Oauth | Congnito +# Oauth | Cognito # using bash for generating cookie secret example # dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 | tr -d -- '\n' | tr -- '+/' '-_'; echo cookie: @@ -76,7 +78,7 @@ oauth: secret: "cognito app client secret" dynamic: [] # example 'scope' response: [] #example 'tokens', 'raw', 'jwt' - callback: "" # exmaple / + callback: "" # example / subdomain: "cognito domain" identityProviderClient: userPoolId: "region_id" @@ -118,11 +120,11 @@ mongodb: enabled: true ssl: false sslValidate: false - host: "" # serviceName:mongoport/collectioname or serviceName-headless:mongoport/collectionname for mongodb replicaset. see template/_helper.tpl and templates/bailo/bailo.configmap.yaml. + host: "" # serviceName:mongoport/collectionname or serviceName-headless:mongoport/collectionname for mongodb replicaset. see template/_helper.tpl and templates/bailo/bailo.configmap.yaml. livenessProbe: enabled: false readinessProbe: - enabled: false #Rediness probe failed: /bitnami/scripts/readiness-probe.sh: line 9: mongosh: command not found + enabled: false #Readiness probe failed: /bitnami/scripts/readiness-probe.sh: line 9: mongosh: command not found auth: rootUser: "" rootPassword: "" @@ -135,7 +137,7 @@ mongodb: port: 27017 collectionName: bailo # Collection name in Mongo persistence: - #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Stoage is not removed when deployed mongodb is removed. + #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Storage is not removed when deployed mongodb is removed. size: 10Gi containerSecurityContext: enabled: false @@ -165,7 +167,7 @@ minio: api: 31111 # default 9000 useSSL: false persistence: - #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Stoage is not removed when deployed minio is removed. + #enabled: false # Required for aws. EKS is not able to handle dynamic creation of PVC using this helm chart. Instead /templates/storage.yaml handles eks storage. Storage is not removed when deployed minio is removed. size: 20Gi containerSecurityContext: enabled: false @@ -196,7 +198,7 @@ registry: # Nginx Dependencies nginxAuth: - repository: nginxinc/nginx-unprivileged # runs Nginx as non-root unrivileged user. + repository: nginxinc/nginx-unprivileged # runs Nginx as non-root unprivileged user. tag: latest #host: "bailo-nginx" # service name port: 8080 @@ -299,3 +301,11 @@ clamav: accessModes: - ReadWriteOnce size: 10G + +modelscan: + enabled: false + protocol: http + port: 3311 + accessModes: + - ReadWriteOnce + size: 20G # modelscan saves each incoming file to the disk so needs plenty of storage diff --git a/lib/modelscan_api/bailo_modelscan_api/config.py b/lib/modelscan_api/bailo_modelscan_api/config.py index ee1d75e6b..b6f6435e6 100644 --- a/lib/modelscan_api/bailo_modelscan_api/config.py +++ b/lib/modelscan_api/bailo_modelscan_api/config.py @@ -24,6 +24,7 @@ class Settings(BaseSettings): Bailo ModelScan API allows for easy programmatic interfacing with ProtectAI's ModelScan package to scan and detect potential threats within files stored in Bailo. You can upload files and view modelscan's result.""" + # Update frontend/pages/docs/administration/helm/configuration.mdx if bumping this. app_version: str = "1.0.0" # download_dir is used if it evaluates, otherwise a temporary directory is used. download_dir: Optional[str] = None