mlflow-template.yaml

apiVersion: template.openshift.io/v1
kind: Template
labels:
  app: mlflow
  template: mlflow
message: |-
  The following components have been scheduled for creation in your project:
  - MLflow Tracking server
  - MLflow Models service
  - PostgreSQL pod for backend database
  - MinIO S3 compatible private object storage for default artifact store
  Go to project overview page and once the creation of all the components are finished, use the following links -
  To access UI: https://${APP_NAME}.${APPLICATION_DOMAIN_SUFFIX}
  URL for model serving: https://${APP_NAME}-models.${APPLICATION_DOMAIN_SUFFIX}
  Artifact store: https://${APP_NAME}-minio.${APPLICATION_DOMAIN_SUFFIX}
  Artifact store key: ${AWS_ACCESS_KEY_ID}
  Artifact store secret key: ${AWS_SECRET_ACCESS_KEY}
  Installation instructions and user guide can be found from https://github.com/CSCfi/mlflow-openshift
metadata:
  annotations:
    description: Deploys MLflow Lifecycle management service.
      The configuration for your pods can be changed according to the Openshift Quota(Limit Range).
      To get more quota, you should contact Openshift admins.
    iconClass: icon-other-unknown
    openshift.io/display-name: MLflow
    openshift.io/documentation-url: https://mlflow.org
    openshift.io/support-url: https://www.csc.fi/contact-info
    openshift.io/long-description: MLflow is tool for Machine Learning lifecycle management. https://mlflow.org/
      f
    openshift.io/provider-display-name: CSC
    tags: mlflow
    template.openshift.io/bindable: "false"
  name: mlflow

objects:
  - apiVersion: "v1"
    kind: "PersistentVolumeClaim"
    metadata:
      name: ${APP_NAME}-pvc
    spec:
      accessModes:
        - "ReadWriteOnce"
      resources:
        requests:
          storage: ${STORAGE_SIZE}

  - apiVersion: "v1"
    kind: "PersistentVolumeClaim"
    metadata:
      name: postgre-db-pvc
    spec:
      accessModes:
        - "ReadWriteOnce"
      resources:
        requests:
          storage: ${DB_PVC_SIZE}

  - apiVersion: "v1"
    kind: "PersistentVolumeClaim"
    metadata:
      name: minio-pvc
    spec:
      accessModes:
        - "ReadWriteOnce"
      resources:
        requests:
          storage: ${MINIO_PVC_SIZE}

  - apiVersion: v1
    kind: Secret
    metadata:
      name: mlflow-ui-secret
    type: Opaque
    stringData:
      secret.env: |-
        USER=${USERNAME}
        PASS=${PASSWORD}
  - apiVersion: v1
    kind: Secret
    metadata:
      name: mlflow-s3-credentials
    type: Opaque
    stringData:
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}

  - apiVersion: v1
    stringData:
      database-name: ${POSTGRESQL_DATABASE}
      database-password: ${POSTGRESQL_PASSWORD}
      database-user: ${POSTGRESQL_USER}
      connection-string: postgresql+psycopg2://${POSTGRESQL_USER}:${POSTGRESQL_PASSWORD}@postgresql:5432/${POSTGRESQL_DATABASE}
      result-backend: db+postgresql://${POSTGRESQL_USER}:${POSTGRESQL_PASSWORD}@postgresql:5432/${POSTGRESQL_DATABASE}
    kind: Secret
    metadata:
      labels:
        app: ${APP_NAME}
        template: postgresql-ephemeral-template
      name: postgresql
    type: Opaque

  - apiVersion: v1
    kind: ConfigMap
    metadata:
      name: mlflow-cfg
    data:
      LOCAL_STORAGE_PATH: postgresql+psycopg2://${POSTGRESQL_USER}:${POSTGRESQL_PASSWORD}@postgresql:5432/${POSTGRESQL_DATABASE}
      DEFAULT_ARTIFACT_ROOT: s3://default
      MLFLOW_S3_ENDPOINT_URL: https://${APP_NAME}-minio.${APPLICATION_DOMAIN_SUFFIX}

  - apiVersion: v1
    kind: ConfigMap
    metadata:
      name: mlflow-ui-startup-cfg
    data:
      startup.sh: |
        #!/bin/sh
        if [ "$DEFAULT_ARTIFACT_ROOT" ];
        then
            # start MLflow with user defined default artifact store
            mlflow server --backend-store-uri=$LOCAL_STORAGE_PATH --default-artifact-root=$DEFAULT_ARTIFACT_ROOT --host=0.0.0.0
        else
            # start MLflow without artifact store path set
            mlflow server --backend-store-uri=$LOCAL_STORAGE_PATH --host=0.0.0.0
        fi
  - apiVersion: v1
    kind: ConfigMap
    metadata:
      name: nginx-config-mlflow
    data:
      default.conf: |
        upstream node {
          server localhost:5000;
        }
        server {
            server_name             _;
            listen                  8080;
            location / {
                proxy_set_header X-Real-IP \$remote_addr;
                proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
                proxy_set_header Host ${APP_NAME}.${APPLICATION_DOMAIN_SUFFIX};
                proxy_pass http://node;
                proxy_redirect off;
                port_in_redirect off;
                auth_basic "MLflow UI Login";
                auth_basic_user_file /etc/nginx/secrets/passwd;
            }
        }
  - apiVersion: v1
    kind: ConfigMap
    metadata:
      name: models-cfg
    data:
      MODELS_URI:

  - apiVersion: v1
    kind: ConfigMap
    metadata:
      name: mlflow-models-startup-cfg
    data:
      startup.sh: |
        #!/bin/sh
        if [ "$MODELS_URI" ];
        then
            mlflow models serve --model-uri=$MODELS_URI --port=5001 --host=0.0.0.0 --no-conda
        else
            echo "Model URI not specified to config map. Cannot startup serving."
        fi
  - apiVersion: v1
    kind: Service
    metadata:
      annotations:
        description: PostgreSQL service for backend store
      name: postgresql
      labels:
        app: postgresql
    spec:
      ports:
        - name: postgresql-service
          port: 5432
          protocol: TCP
          targetPort: 5432
      selector:
        name: postgresql
      sessionAffinity: None
      type: ClusterIP
    status:
      loadBalancer: { }

  - apiVersion: v1
    kind: Service
    metadata:
      annotations:
        description: Minio service for artifact store
      name: minio-service
      labels:
        app: minio
    spec:
      ports:
        - name: minio-service
          port: 9000
          protocol: TCP
          targetPort: 9000
      selector:
        app: minio

  - apiVersion: v1
    kind: Service
    metadata:
      annotations:
        description: Exposes and load balances the API pods
      name: mlflow-models-service
      labels:
        app: mlflow-models
    spec:
      ports:
        - name: mlflow-models-service
          port: 80
          targetPort: 5001
      selector:
        name: mlflow-models

  - apiVersion: v1
    kind: Service
    metadata:
      annotations:
        description: Exposes and load balances the Tracking Server pod
      name: mlflow-ui-service
    spec:
      ports:
        - name: mlflow-ui-service
          port: 80
          targetPort: 8080
      selector:
        name: mlflow-ui

  - apiVersion: route.openshift.io/v1
    kind: Route
    metadata:
      name: mlflow-models-route
      annotations:
        haproxy.router.openshift.io/ip_whitelist: ${ROUTEWHITELIST}
    spec:
      host: ${APP_NAME}-models.${APPLICATION_DOMAIN_SUFFIX}
      path: /
      to:
        kind: Service
        name: mlflow-models-service
        weight: 100
      port:
        targetPort: mlflow-models-service
      tls:
        insecureEdgeTerminationPolicy: Redirect
        termination: edge

  - apiVersion: route.openshift.io/v1
    kind: Route
    metadata:
      name: mlflow-ui-route
      annotations:
        haproxy.router.openshift.io/ip_whitelist: ${ROUTEWHITELIST}
    spec:
      host: ${APP_NAME}.${APPLICATION_DOMAIN_SUFFIX}
      path: /
      to:
        kind: Service
        name: mlflow-ui-service
        weight: 100
      port:
        targetPort: mlflow-ui-service
      tls:
        insecureEdgeTerminationPolicy: Redirect
        termination: edge

  - apiVersion: route.openshift.io/v1
    kind: Route
    metadata:
      labels:
        app: minio
      name: minio-route
      annotations:
        haproxy.router.openshift.io/ip_whitelist: ${ROUTEWHITELIST}
    spec:
      host: ${APP_NAME}-minio.${APPLICATION_DOMAIN_SUFFIX}
      path: /
      to:
        kind: Service
        name: minio-service
        weight: 100
      port:
        targetPort: minio-service
      tls:
        termination: edge
        insecureEdgeTerminationPolicy: Redirect

  - apiVersion: apps/v1
    kind: Deployment
    metadata:
      annotations:
        description: MLflow Models
      name: mlflow-models
    spec:
      replicas: 0
      selector:
        matchLabels:
          name: mlflow-models
      strategy:
        type: RollingUpdate
        triggers:
          - type: ConfigChange
      template:
        metadata:
          labels:
            name: mlflow-models
            app: mlflow
          name: mlflow
        spec:
          containers:
            - name: mlflow-models
              image: ${MLFLOW_IMAGE}
              imagePullPolicy: Always
              command: ["sh", "./startup/startup.sh"]
              resources:
                requests:
                  cpu: 200m
                  memory: 200Mi
                limits:
                  cpu: 1600m
                  memory: 1Gi
              ports:
                - containerPort: 5001
              volumeMounts:
                - name: ${APP_NAME}-vol
                  mountPath: /mnt/${APP_NAME}-pvc
                - name: mlflow-models-startup-vol
                  mountPath: "/startup"
              env:
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: mlflow-s3-credentials
                      key: AWS_ACCESS_KEY_ID
                      optional: false
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: mlflow-s3-credentials
                      key: AWS_SECRET_ACCESS_KEY
                      optional: false
              envFrom:
                - configMapRef:
                    name: models-cfg
                    optional: true
                - configMapRef:
                    name: mlflow-cfg
                    key: MLFLOW_S3_ENDPOINT_URL
          volumes:
            - name: mlflow-models-startup-vol
              configMap:
                name: mlflow-models-startup-cfg
                items:
                  - key: startup.sh
                    path: startup.sh
            - name: ${APP_NAME}-vol
              persistentVolumeClaim:
                claimName: ${APP_NAME}-pvc

  - apiVersion: v1
    kind: DeploymentConfig
    metadata:
      labels:
        app: ${APP_NAME}
        template: postgresql-ephemeral-template
      name: postgresql
    spec:
      replicas: 1
      selector:
        name: postgresql
      strategy:
        type: Recreate
      template:
        metadata:
          labels:
            name: postgresql
        spec:
          containers:
          - env:
            - name: POSTGRESQL_USER
              valueFrom:
                secretKeyRef:
                  key: database-user
                  name: postgresql
            - name: POSTGRESQL_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: database-password
                  name: postgresql
            - name: POSTGRESQL_DATABASE
              valueFrom:
                secretKeyRef:
                  key: database-name
                  name: postgresql
            image: centos/postgresql-12-centos7:1
            imagePullPolicy: IfNotPresent
            livenessProbe:
              failureThreshold: 3
              initialDelaySeconds: 30
              periodSeconds: 10
              successThreshold: 1
              tcpSocket:
                port: 5432
              timeoutSeconds: 1
            name: postgresql
            ports:
            - containerPort: 5432
              protocol: TCP
            readinessProbe:
              exec:
                command:
                - /bin/sh
                - -i
                - -c
                - psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c
                  'SELECT 1'
              failureThreshold: 3
              initialDelaySeconds: 20
              periodSeconds: 10
              successThreshold: 1
              timeoutSeconds: 1
            resources:
              limits:
                memory: 1Gi
            securityContext:
              capabilities: {}
              privileged: false
            volumeMounts:
            - mountPath: /var/lib/pgsql/data
              name: postgresql-data
          dnsPolicy: ClusterFirst
          restartPolicy: Always
          schedulerName: default-scheduler
          securityContext: {}
          terminationGracePeriodSeconds: 30
          volumes:
          - name: postgresql-data
            persistentVolumeClaim:
              claimName: postgre-db-pvc
      triggers:
      - imageChangeParams:
          automatic: true
          containerNames:
          - postgresql
          from:
            kind: ImageStreamTag
            name: postgresql:12
            namespace: openshift
        type: ImageChange
      - type: ConfigChange

  - apiVersion: v1
    kind: DeploymentConfig
    metadata:
      labels:
        app: minio
      name: minio
    spec:
      replicas: 1
      template:
        metadata:
          labels:
            app: minio
        spec:
          containers:
          - image: minio/minio:RELEASE.2021-04-18T19-26-29Z
            name: minio
            ports:
            - containerPort: 9000
              protocol: TCP
            resources:
              limits:
                cpu: 500m
                memory: 512Mi
              requests:
                cpu: 200m
                memory: 256Mi
            readinessProbe:
              httpGet:
                path: /minio/health/ready
                port: 9000
                scheme: HTTP
            livelinessProbe:
              httpGet:
                path: /health/live
                port: 9000
                scheme: HTTP
            volumeMounts:
            - mountPath: /data/files
              name: minio-vol
            command: ["/bin/sh","-c","mkdir -p /data/files/default && minio server /data/files"]
            env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: mlflow-s3-credentials
                  key: AWS_ACCESS_KEY_ID
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: mlflow-s3-credentials
                  key: AWS_SECRET_ACCESS_KEY
          volumes:
          - name: minio-vol
            persistentVolumeClaim:
              claimName: minio-pvc

  - apiVersion: apps/v1
    kind: Deployment
    metadata:
      annotations:
        description: MLflow Tracking Server API
      name: mlflow-ui
    spec:
      replicas: 1
      selector:
        matchLabels:
          name: mlflow-ui
      strategy:
        type: RollingUpdate
      template:
        metadata:
          labels:
            name: mlflow-ui
            app: mlflow
          name: mlflow
        spec:
          initContainers:
            - image: docker-registry.rahti.csc.fi/da-images/alpine-htpasswd:latest
              command: ["/bin/sh","-c","source /tmp/secret-env/secret.env && htpasswd -bc /tmp/secret-file/passwd $USER $PASS"]
              name: htpasswd-generator
              resources:
                requests:
                  cpu: 200m
                  memory: 128Mi
                limits:
                  cpu: 300m
                  memory: 512Mi
              volumeMounts:
                - name: mlflow-htpasswd-vol
                  mountPath: "/tmp/secret-file"
                - name: mlflow-secret-vol
                  mountPath: "/tmp/secret-env"
          containers:
            - name: proxy-rewriter
              image: docker-registry.rahti.csc.fi/spark-images/nginx:latest
              imagePullPolicy: Always
              resources:
                requests:
                  cpu: 200m
                  memory: 128Mi
                limits:
                  cpu: 300m
                  memory: 512Mi
              ports:
                - containerPort: 8080
              volumeMounts:
                - name: mlflow-nginx-config-vol
                  mountPath: /opt/bitnami/nginx/conf/server_blocks
                - name: mlflow-htpasswd-vol
                  mountPath: /etc/nginx/secrets
            - name: mlflow
              image: ${MLFLOW_IMAGE}
              imagePullPolicy: Always
              command: ["sh", "./startup/startup.sh"]
              resources:
                requests:
                  cpu: 200m
                  memory: 200Mi
                limits:
                  cpu: 1600m
                  memory: 2Gi
              ports:
                - containerPort: 5000
              livenessProbe:
                httpGet:
                  path: /
                  port: 5000
                initialDelaySeconds: 30
                timeoutSeconds: 3
              readinessProbe:
                httpGet:
                  path: /
                  port: 5000
                initialDelaySeconds: 3
                timeoutSeconds: 3
              volumeMounts:
                - name: ${APP_NAME}-vol
                  mountPath: /mnt/${APP_NAME}-pvc
                - name: mlflow-ui-startup-vol
                  mountPath: "/startup"
              env:
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: mlflow-s3-credentials
                      key: AWS_ACCESS_KEY_ID
                      optional: false
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: mlflow-s3-credentials
                      key: AWS_SECRET_ACCESS_KEY
                      optional: false
              envFrom:
                - configMapRef:
                    name: mlflow-cfg
                    key: LOCAL_STORAGE_PATH
                - configMapRef:
                    name: mlflow-cfg
                    key: MLFLOW_S3_ENDPOINT_URL
          volumes:
            - name: mlflow-nginx-config-vol
              configMap:
                name: nginx-config-mlflow
                items:
                  - key: default.conf
                    path: default.conf
            - name: mlflow-ui-startup-vol
              configMap:
                name: mlflow-ui-startup-cfg
                items:
                  - key: startup.sh
                    path: startup.sh
            - name: mlflow-secret-vol
              secret:
                secretName: mlflow-ui-secret
            - name: mlflow-htpasswd-vol
              emptyDir: {}
            - name: ${APP_NAME}-vol
              persistentVolumeClaim:
                claimName: ${APP_NAME}-pvc

parameters:
- description: Unique identifier for your MLflow-app. Recommended value - your username
  displayName: App Name
  name: APP_NAME
  required: true

- description: Create a new username for logging into Tracking server UI
  displayName: Username
  name: USERNAME
  required: true

- description: Create a new password for logging into Tracking server UI
  displayName: Password
  name: PASSWORD
  required: true

- description: Persistent application storage size
  displayName: Storage Size
  name: STORAGE_SIZE
  required: true
  value: 1Gi

- description: Size of the backend storage persistent volume storage
  displayName: Backend storage size
  name: DB_PVC_SIZE
  required: true
  value: 1Gi

- description: Object storage persistent volume size
  displayName: Object storage size
  name: MINIO_PVC_SIZE
  required: true
  value: 1Gi

- description: By default, application is accessible from anywhere. To restrict access, add your workstation IP or network range to whitelist.
  displayName: Route whitelist
  name: ROUTEWHITELIST
  required: false
  value:

- description: Docker Image for the MLflow
  displayName: MLflow image
  name: MLFLOW_IMAGE
  required: true
  value: docker-registry.rahti.csc.fi/mlflow-images/mlflow:latest

- description: Access key for Artifact store
  displayName: Artifact store access key
  name: AWS_ACCESS_KEY_ID
  required: true
  from: '[A-Z0-9]{15}'
  generate: expression

- description: Secret Access key for Artifact store
  displayName: Artifact store secret access key
  name: AWS_SECRET_ACCESS_KEY
  required: true
  from: '[A-Z0-9]{15}'
  generate: expression

- description: Username for PostgreSQL user that will be used for accessing the database
  displayName: PostgreSQL connection username
  from: 'user[A-Z0-9]{5}'
  generate: expression
  name: POSTGRESQL_USER
  required: true

- description: Password for the PostgreSQL connection user
  displayName: PostgreSQL connection password
  from: '[a-zA-Z0-9]{16}'
  generate: expression
  name: POSTGRESQL_PASSWORD
  required: true

- description: Database name for PostgreSQL database
  displayName: PostgreSQL connection database
  from: '[A-Z0-9]{5}'
  generate: expression
  name: POSTGRESQL_DATABASE
  required: true

- description: The exposed hostname suffix that will be used to create routes for Tracking Server UI and Models API
  displayName: Application Hostname Suffix
  name: APPLICATION_DOMAIN_SUFFIX
  value: rahtiapp.fi