added ui button to allow manual file re scan and included migration s…

…cript

backend/config/default.cjs

@@ -148,6 +148,12 @@ module.exports = {
       host: '127.0.0.1',
       port: 3310,
     },
+
+    modelscan: {
+      protocol: 'http',
+      host: '127.0.0.1',
+      port: 3311,
+    },
   },
 
   // These settings are PUBLIC and shared with the UI
@@ -195,6 +201,10 @@ module.exports = {
       text: '',
       startTimestamp: '',
     },
+
+    helpPopoverText: {
+      manualEntryAccess: '',
+    },
   },
 
   connectors: {

backend/config/docker_compose.cjs

@@ -65,11 +65,15 @@ module.exports = {
     clamdscan: {
       host: 'clamd',
     },
+
+    modelscan: {
+      host: 'modelscan',
+    },
   },
 
   connectors: {
     fileScanners: {
-      kinds: ['clamAV'],
+      kinds: ['clamAV', 'modelScan'],
     },
   },
 }

backend/src/clients/modelScan.ts

@@ -0,0 +1,93 @@
+import fetch, { Response } from 'node-fetch'
+
+import config from '../utils/config.js'
+import { BadReq, InternalError } from '../utils/error.js'
+
+interface ModelScanInfoResponse {
+  apiName: string
+  apiVersion: string
+  scannerName: string
+  modelscanVersion: string
+}
+
+interface ModelScanResponse {
+  summary: {
+    total_issues: number
+    total_issues_by_severity: {
+      LOW: number
+      MEDIUM: number
+      HIGH: number
+      CRITICAL: number
+    }
+    input_path: string
+    absolute_path: string
+    modelscan_version: string
+    timestamp: string
+    scanned: {
+      total_scanned: number
+      scanned_files: string[]
+    }
+    skipped: {
+      total_skipped: number
+      skipped_files: string[]
+    }
+  }
+  issues: [
+    {
+      description: string
+      operator: string
+      module: string
+      source: string
+      scanner: string
+      severity: string
+    },
+  ]
+  // TODO: currently unknown what this might look like
+  errors: object[]
+}
+
+export async function getModelScanInfo() {
+  const url = `${config.avScanning.modelscan.protocol}://${config.avScanning.modelscan.host}:${config.avScanning.modelscan.port}`
+  let res: Response
+
+  try {
+    res = await fetch(`${url}/info`, {
+      method: 'GET',
+      headers: { 'Content-Type': 'application/json' },
+    })
+  } catch (err) {
+    throw InternalError('Unable to communicate with the ModelScan service.', { err })
+  }
+  if (!res.ok) {
+    throw BadReq('Unrecognised response returned by the ModelScan service.')
+  }
+
+  return (await res.json()) as ModelScanInfoResponse
+}
+
+export async function scanFile(file: Blob, file_name: string) {
+  const url = `${config.avScanning.modelscan.protocol}://${config.avScanning.modelscan.host}:${config.avScanning.modelscan.port}`
+  let res: Response
+
+  try {
+    const formData = new FormData()
+    formData.append('in_file', file, file_name)
+
+    res = await fetch(`${url}/scan/file`, {
+      method: 'POST',
+      headers: {
+        accept: 'application/json',
+      },
+      body: formData,
+    })
+  } catch (err) {
+    throw InternalError('Unable to communicate with the ModelScan service.', { err })
+  }
+  if (!res.ok) {
+    throw BadReq('Unrecognised response returned by the ModelScan service.', {
+      body: JSON.stringify(await res.json()),
+    })
+  }
+
+  return (await res.json()) as ModelScanResponse
+}

backend/src/connectors/fileScanning/clamAv.ts

@@ -25,7 +25,7 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector {
       av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan })
     } catch (error) {
       throw ConfigurationError('Could not scan file as Clam AV is not running.', {
-        clamAvConfig: config.avScanning,
+        clamAvConfig: config.avScanning.clamdscan,
       })
     }
   }
@@ -35,11 +35,13 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector {
       throw ConfigurationError(
         'Clam AV does not look like it is running. Check that it has been correctly initialised by calling the init function.',
         {
-          clamAvConfig: config.avScanning,
+          clamAvConfig: config.avScanning.clamdscan,
         },
       )
     }
     const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable
+    const scannerVersion = await av.getVersion()
+    const modifiedVersion = scannerVersion.substring(scannerVersion.indexOf(' ') + 1, scannerVersion.indexOf('/'))
     try {
       const { isInfected, viruses } = await av.scanStream(s3Stream)
       log.info(
@@ -50,7 +52,7 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector {
         {
           toolName: clamAvToolName,
           state: ScanState.Complete,
-          scannerVersion: await av.getVersion(),
+          scannerVersion: modifiedVersion,
           isInfected,
           viruses,
           lastRunAt: new Date(),
@@ -62,7 +64,7 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector {
         {
           toolName: clamAvToolName,
           state: ScanState.Error,
-          scannerVersion: await av.getVersion(),
+          scannerVersion: modifiedVersion,
           lastRunAt: new Date(),
         },
       ]

backend/src/connectors/fileScanning/index.ts

@@ -2,10 +2,12 @@ import config from '../../utils/config.js'
 import { ConfigurationError } from '../../utils/error.js'
 import { BaseFileScanningConnector } from './Base.js'
 import { ClamAvFileScanningConnector } from './clamAv.js'
+import { ModelScanFileScanningConnector } from './modelScan.js'
 import { FileScanningWrapper } from './wrapper.js'
 
 export const FileScanKind = {
   ClamAv: 'clamAV',
+  ModelScan: 'modelScan',
 } as const
 export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind]
 
@@ -26,6 +28,15 @@ export function runFileScanners(cache = true) {
           throw ConfigurationError('Could not configure or initialise Clam AV')
         }
         break
+      case FileScanKind.ModelScan:
+        try {
+          const scanner = new ModelScanFileScanningConnector()
+          await scanner.ping()
+          fileScanConnectors.push(scanner)
+        } catch (error) {
+          throw ConfigurationError('Could not configure or initialise ModelScan')
+        }
+        break
       default:
         throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, {
           validKinds: Object.values(FileScanKind),

backend/src/connectors/fileScanning/modelScan.ts

@@ -0,0 +1,81 @@
+import { Response } from 'node-fetch'
+import { Readable } from 'stream'
+
+import { getModelScanInfo, scanFile } from '../../clients/modelScan.js'
+import { getObjectStream } from '../../clients/s3.js'
+import { FileInterfaceDoc, ScanState } from '../../models/File.js'
+import log from '../../services/log.js'
+import config from '../../utils/config.js'
+import { ConfigurationError } from '../../utils/error.js'
+import { BaseFileScanningConnector, FileScanResult } from './Base.js'
+
+export const modelScanToolName = 'ModelScan'
+
+export class ModelScanFileScanningConnector extends BaseFileScanningConnector {
+  constructor() {
+    super()
+  }
+
+  info() {
+    return [modelScanToolName]
+  }
+
+  async ping() {
+    try {
+      // discard the results as we only want to know if the endpoint is reachable
+      await getModelScanInfo()
+    } catch (error) {
+      throw ConfigurationError(
+        'ModelScan does not look like it is running. Check that the service configuration is correct.',
+        {
+          modelScanConfig: config.avScanning.modelscan,
+        },
+      )
+    }
+  }
+
+  async scan(file: FileInterfaceDoc): Promise<FileScanResult[]> {
+    this.ping()
+
+    const { modelscanVersion } = await getModelScanInfo()
+
+    const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable
+    try {
+      // TODO: see if it's possible to directly send the Readable stream rather than a blob
+      const fileBlob = await new Response(s3Stream).blob()
+      const scanResults = await scanFile(fileBlob, file.name)
+
+      const issues = scanResults.summary.total_issues
+      const isInfected = issues > 0
+      const viruses: string[] = []
+      if (isInfected) {
+        for (const issue of scanResults.issues) {
+          viruses.push(`${issue.severity}: ${issue.description}. ${issue.scanner}`)
+        }
+      }
+      log.info(
+        { modelId: file.modelId, fileId: file._id, name: file.name, result: { isInfected, viruses } },
+        'Scan complete.',
+      )
+      return [
+        {
+          toolName: modelScanToolName,
+          state: ScanState.Complete,
+          scannerVersion: modelscanVersion,
+          isInfected,
+          viruses,
+          lastRunAt: new Date(),
+        },
+      ]
+    } catch (error) {
+      log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.')
+      return [
+        {
+          toolName: modelScanToolName,
+          state: ScanState.Error,
+          lastRunAt: new Date(),
+        },
+      ]
+    }
+  }
+}

backend/src/migrations/011_find_and_remove_invalid_users.ts

@@ -0,0 +1,44 @@
+import authentication from '../connectors/authentication/index.js'
+import { MigrationMetadata } from '../models/Migration.js'
+import ModelModel from '../models/Model.js'
+
+/**
+ * As we now do backend validation for users being added to model access lists, we
+ * added this script to find and remove all existing users that do not pass the
+ * "getUserInformation" call in the authentication connector. You can find a
+ * list of removed users for all affected models by looking at the "metadata"
+ * property of this migration's database object.
+ **/
+
+export async function up() {
+  const models = await ModelModel.find({})
+  const metadata: MigrationMetadata[] = []
+  for (const model of models) {
+    const invalidUsers: string[] = []
+    await Promise.all(
+      model.collaborators.map(async (collaborator) => {
+        if (collaborator.entity !== '') {
+          try {
+            await authentication.getUserInformation(collaborator.entity)
+          } catch (err) {
+            invalidUsers.push(collaborator.entity)
+          }
+        }
+      }),
+    )
+    if (invalidUsers.length > 0) {
+      const invalidUsersForModel = { modelId: model.id, invalidUsers: invalidUsers }
+      const invalidUsersRemoved = model.collaborators.filter(
+        (collaborator) => !invalidUsers.includes(collaborator.entity),
+      )
+      model.collaborators = invalidUsersRemoved
+      await model.save()
+      metadata.push(invalidUsersForModel)
+    }
+  }
+  return metadata
+}
+
+export async function down() {
+  /* NOOP */
+}

backend/src/migrations/012_add_avscan_lastRanAt_property.ts

@@ -0,0 +1,17 @@
+import FileModel from '../models/File.js'
+
+export async function up() {
+  const files = await FileModel.find({})
+  for (const file of files) {
+    for (const avResult of file.avScan) {
+      if (avResult.lastRunAt === undefined) {
+        avResult.lastRunAt = file.createdAt
+      }
+    }
+    await file.save()
+  }
+}
+
+export async function down() {
+  /* NOOP */
+}

backend/src/models/Migration.ts

@@ -1,7 +1,12 @@
 import { Document, model, Schema } from 'mongoose'
 
+export interface MigrationMetadata {
+  [key: string]: any
+}
+
 export interface Migration {
   name: string
+  metadata?: MigrationMetadata
 
   createdAt: Date
   updatedAt: Date
@@ -12,6 +17,7 @@ export type MigrationDoc = Migration & Document<any, any, Migration>
 const MigrationSchema = new Schema<Migration>(
   {
     name: { type: String, required: true },
+    metadata: { type: Schema.Types.Mixed },
   },
   {
     timestamps: true,

backend/src/services/file.ts

@@ -58,14 +58,14 @@ async function updateFileWithResults(_id: Schema.Types.ObjectId, results: FileSc
     const updateExistingResult = await FileModel.updateOne(
       { _id, 'avScan.toolName': result.toolName },
       {
-        $set: { 'avScan.$': { lastRanAt: new Date(), ...result } },
+        $set: { 'avScan.$': { ...result } },
       },
     )
     if (updateExistingResult.modifiedCount === 0) {
       await FileModel.updateOne(
-        { _id },
+        { _id, avScan: { $exists: true } },
         {
-          $set: { avScan: { toolName: result.toolName, state: result.state, lastRanAt: new Date() } },
+          $push: { avScan: { toolName: result.toolName, state: result.state, lastRunAt: new Date() } },
         },
       )
     }