Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into feature/BAI-1627-migr…
Browse files Browse the repository at this point in the history
…ate-scans-to-have-their-own-mongo-collection
  • Loading branch information
PE39806 committed Mar 6, 2025
2 parents 9c2c0d1 + 1785d71 commit 73d2bcc
Show file tree
Hide file tree
Showing 46 changed files with 1,895 additions and 4,778 deletions.
10 changes: 5 additions & 5 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,14 @@ repos:
# Pydantic fails check so ignore the one problematic file
exclude: lib/modelscan_api/bailo_modelscan_api/config.py
- id: pyupgrade
name: pyupgrade 3.9 (pedantic)
name: pyupgrade 3.9 (pydantic)
args: ['--py39-plus', '--keep-runtime-typing']
# scan problematic separately with extra arg to workaround problem
# scan problematic file(s) separately with extra arg to workaround problem
files: lib/modelscan_api/bailo_modelscan_api/config.py
- id: pyupgrade
name: pyupgrade 3.10 (main)
args: ['--py310-plus']
# backend/docs/ has the minimum python version 3.10
name: pyupgrade 3.11 (main)
args: ['--py311-plus']
# backend/docs/ has the minimum python version 3.11
files: backend/docs/

- repo: https://github.com/hadialqattan/pycln
Expand Down
2 changes: 1 addition & 1 deletion backend/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ Documentation is rendered with Sphinx and served [here](https://gchq.github.io/B

<!-- prettier-ignore-start -->
> [!IMPORTANT]
> Python 3.10 or higher is required
> Python 3.11 or higher is required
<!-- prettier-ignore-end -->
#### Prerequisites
Expand Down
4 changes: 2 additions & 2 deletions backend/docs/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
ipython==8.32.0
ipython==9.0.1
myst_parser==4.0.1
nbsphinx==0.9.6
sphinx==8.1.3
sphinx==8.2.3
sphinx-copybutton==0.5.2
sphinx-rtd-theme==3.0.2
1,258 changes: 722 additions & 536 deletions backend/package-lock.json

Large diffs are not rendered by default.

28 changes: 14 additions & 14 deletions backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@
"@asteasolutions/zod-to-openapi": "^7.3.0",
"@aws-sdk/client-s3": "3.666.0",
"@aws-sdk/lib-storage": "3.666.0",
"@opentelemetry/auto-instrumentations-node": "^0.56.0",
"@opentelemetry/exporter-logs-otlp-proto": "^0.57.1",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.57.1",
"@opentelemetry/auto-instrumentations-node": "^0.56.1",
"@opentelemetry/exporter-logs-otlp-proto": "^0.57.2",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.57.2",
"@opentelemetry/host-metrics": "^0.35.5",
"@opentelemetry/sdk-metrics": "^1.30.0",
"@smithy/node-http-handler": "^4.0.2",
"@smithy/node-http-handler": "^4.0.3",
"@types/express-session": "^1.18.1",
"app-root-path": "^3.1.0",
"archiver": "^7.0.1",
Expand All @@ -52,11 +52,11 @@
"jsrsasign": "^11.1.0",
"lodash-es": "^4.17.21",
"mjml": "5.0.0-alpha.6",
"mongodb": "^6.13.0",
"mongodb": "^6.14.0",
"mongoose": "7.8.6",
"mongoose-delete": "^1.0.2",
"morgan": "^1.10.0",
"nanoid": "^5.0.9",
"nanoid": "^5.1.2",
"node-fetch": "^3.3.2",
"nodemailer": "^6.10.0",
"outdent": "^0.8.0",
Expand All @@ -65,11 +65,11 @@
"semver": "^7.7.0",
"shelljs": "^0.8.5",
"showdown": "^2.1.0",
"tsx": "^4.19.2",
"tsx": "^4.19.3",
"utility-types": "^3.11.0",
"uuid": "^11.0.5",
"uuid": "^11.1.0",
"yargs": "^17.7.2",
"zod": "^3.24.1",
"zod": "^3.24.2",
"zod-error": "^1.5.0"
},
"devDependencies": {
Expand All @@ -83,7 +83,7 @@
"@types/cls-hooked": "^4.3.9",
"@types/config": "^3.3.5",
"@types/express": "^4.17.21",
"@types/jsonwebtoken": "^9.0.8",
"@types/jsonwebtoken": "^9.0.9",
"@types/lodash-es": "^4.17.12",
"@types/mjml": "^4.7.4",
"@types/mongoose-delete": "^1.0.6",
Expand All @@ -94,10 +94,10 @@
"@types/shelljs": "^0.8.15",
"@types/supertest": "^6.0.2",
"@types/uuid": "^10.0.0",
"@typescript-eslint/eslint-plugin": "^8.22.0",
"@typescript-eslint/parser": "^8.22.0",
"@vitest/coverage-v8": "^3.0.5",
"eslint": "^9.19.0",
"@typescript-eslint/eslint-plugin": "^8.25.0",
"@typescript-eslint/parser": "^8.25.0",
"@vitest/coverage-v8": "^3.0.7",
"eslint": "^9.21.0",
"eslint-plugin-prettier": "^5.2.1",
"eslint-plugin-simple-import-sort": "^12.1.1",
"nodemon": "^3.1.9",
Expand Down
18 changes: 14 additions & 4 deletions backend/src/models/Inference.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { Document, model, Schema } from 'mongoose'
import { model, Schema } from 'mongoose'
import MongooseDelete, { SoftDeleteDocument } from 'mongoose-delete'

export interface InferenceSetting {
processorType: string
Expand All @@ -15,14 +16,16 @@ export interface InferenceInterface {

settings: InferenceSetting

deleted: boolean

createdBy: string
createdAt: Date
updatedAt: Date
}

export type InferenceDoc = InferenceInterface & Document<any, any, InferenceInterface>
export type InferenceDoc = InferenceInterface & SoftDeleteDocument

const InferenceSchema = new Schema<InferenceInterface>(
const InferenceSchema = new Schema<InferenceDoc>(
{
modelId: { type: String, required: true },
image: { type: String, required: true },
Expand Down Expand Up @@ -55,8 +58,15 @@ const InferenceSchema = new Schema<InferenceInterface>(
},
)

InferenceSchema.plugin(MongooseDelete, {
overrideMethods: 'all',
deletedBy: true,
deletedByType: String,
deletedAt: true,
})

InferenceSchema.index({ modelId: 1, image: 1, tag: 1 }, { unique: true })

const InferenceModel = model<InferenceInterface>('v2_Model_Inference', InferenceSchema)
const InferenceModel = model<InferenceDoc>('v2_Model_Inference', InferenceSchema)

export default InferenceModel
17 changes: 13 additions & 4 deletions backend/src/models/ModelCardRevision.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { Document, model, Schema } from 'mongoose'
import { model, Schema } from 'mongoose'
import MongooseDelete, { SoftDeleteDocument } from 'mongoose-delete'

import { ModelCardInterface } from './Model.js'

Expand All @@ -7,14 +8,15 @@ import { ModelCardInterface } from './Model.js'
// client.
export interface ModelCardRevisionInterface extends ModelCardInterface {
modelId: string
deleted: boolean
}

// The doc type includes all values in the plain interface, as well as all the
// properties and functions that Mongoose provides. If a function takes in an
// object from Mongoose it should use this interface
export type ModelCardRevisionDoc = ModelCardRevisionInterface & Document<any, any, ModelCardRevisionInterface>
export type ModelCardRevisionDoc = ModelCardRevisionInterface & SoftDeleteDocument

const ModelCardRevisionSchema = new Schema<ModelCardRevisionInterface>(
const ModelCardRevisionSchema = new Schema<ModelCardRevisionDoc>(
{
modelId: { type: String, required: true },
schemaId: { type: String, required: true },
Expand All @@ -36,6 +38,13 @@ const ModelCardRevisionSchema = new Schema<ModelCardRevisionInterface>(
// to learn more.
ModelCardRevisionSchema.index({ modelId: 1, version: 1 }, { unique: true })

const ModelCardRevisionModel = model<ModelCardRevisionInterface>('v2_Model_Card_Revision', ModelCardRevisionSchema)
ModelCardRevisionSchema.plugin(MongooseDelete, {
overrideMethods: 'all',
deletedBy: true,
deletedByType: String,
deletedAt: true,
})

const ModelCardRevisionModel = model<ModelCardRevisionDoc>('v2_Model_Card_Revision', ModelCardRevisionSchema)

export default ModelCardRevisionModel
3 changes: 3 additions & 0 deletions backend/src/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import { patchModel } from './routes/v2/model/patchModel.js'
import { postModel } from './routes/v2/model/postModel.js'
import { postRequestExportToS3 } from './routes/v2/model/postRequestExport.js'
import { postRequestImportFromS3 } from './routes/v2/model/postRequestImport.js'
import { getAllModelReviewRoles } from './routes/v2/model/roles/getAllModelReviewRoles.js'
import { getModelCurrentUserRoles } from './routes/v2/model/roles/getModelCurrentUserRoles.js'
import { getModelRoles } from './routes/v2/model/roles/getModelRoles.js'
import { deleteWebhook } from './routes/v2/model/webhook/deleteWebhook.js'
Expand Down Expand Up @@ -178,6 +179,8 @@ server.get('/api/v2/model/:modelId/roles', ...getModelRoles)
server.get('/api/v2/model/:modelId/roles/mine', ...getModelCurrentUserRoles)
server.get('/api/v2/model/:modelId/permissions/mine', ...getModelCurrentUserPermissions)

server.get('/api/v2/roles/review', ...getAllModelReviewRoles)

server.get('/api/v2/entities', ...getEntities)
server.get('/api/v2/entities/me', ...getCurrentUser)
server.get('/api/v2/entity/:dn/lookup', ...getEntityLookup)
Expand Down
3 changes: 1 addition & 2 deletions backend/src/routes/v2/model/getModelsSearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import audit from '../../../connectors/audit/index.js'
import { EntryKind, EntryKindKeys } from '../../../models/Model.js'
import { searchModels } from '../../../services/model.js'
import { registerPath } from '../../../services/specification.js'
import { GetModelFilters } from '../../../types/enums.js'
import { coerceArray, parse, strictCoerceBoolean } from '../../../utils/validate.js'

export const getModelsSearchSchema = z.object({
Expand All @@ -16,7 +15,7 @@ export const getModelsSearchSchema = z.object({
kind: z.string(z.nativeEnum(EntryKind)).optional(),
task: z.string().optional(),
libraries: coerceArray(z.array(z.string()).optional().default([])),
filters: coerceArray(z.array(z.nativeEnum(GetModelFilters)).optional().default([])),
filters: coerceArray(z.array(z.string()).optional().default([])),
search: z.string().optional().default(''),
allowTemplating: strictCoerceBoolean(z.boolean().optional()),
schemaId: z.string().optional(),
Expand Down
8 changes: 6 additions & 2 deletions backend/src/routes/v2/model/postRequestExport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ export const postRequestExportToS3Schema = z.object({
}),
body: z.object({
disclaimerAgreement: z.boolean(),
semvers: z
.array(z.string())
.openapi({ example: ['1.0.0'] })
.optional(),
}),
})

Expand Down Expand Up @@ -48,10 +52,10 @@ export const postRequestExportToS3 = [
req.audit = AuditInfo.CreateExport
const {
params: { modelId },
body: { disclaimerAgreement },
body: { disclaimerAgreement, semvers },
} = parse(req, postRequestExportToS3Schema)

await exportModel(req.user, modelId, disclaimerAgreement)
await exportModel(req.user, modelId, disclaimerAgreement, semvers)
await audit.onCreateS3Export(req, modelId)

return res.json({
Expand Down
32 changes: 32 additions & 0 deletions backend/src/routes/v2/model/roles/getAllModelReviewRoles.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import bodyParser from 'body-parser'
import { Request, Response } from 'express'

import { Role, RoleKind } from '../../../../types/types.js'

interface GetModelCurrentUserRolesResponse {
roles: Array<Role>
}

export const getAllModelReviewRoles = [
bodyParser.json(),
async (_req: Request, res: Response<GetModelCurrentUserRolesResponse>) => {
return res.json({
roles: [
{
id: 'msro',
name: 'Model Senior Responsible Officer',
short: 'MSRO',
kind: RoleKind.SCHEMA,
description: 'This role is specified by the schema in accordance with its policy.',
},
{
id: 'mtr',
name: 'Model Technical Reviewer',
short: 'MTR',
kind: RoleKind.SCHEMA,
description: 'This role is specified by the schema in accordance with its policy.',
},
],
})
},
]
79 changes: 79 additions & 0 deletions backend/src/scripts/modelSoftDelete.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import AccessRequestModel from '../models/AccessRequest.js'
import FileModel from '../models/File.js'
import InferenceModel from '../models/Inference.js'
import ModelModel from '../models/Model.js'
import ModelCardRevisionModel from '../models/ModelCardRevision.js'
import ReleaseModel from '../models/Release.js'
import ResponseModel from '../models/Response.js'
import ReviewModel from '../models/Review.js'
import WebhookModel from '../models/Webhook.js'
import log from '../services/log.js'
import { connectToMongoose } from '../utils/database.js'

async function script() {
const modelId = process.argv.slice(2)

if (!modelId || !process.argv[2]) {
log.error('No model ID option. Please use format "npm run script -- modelSoftDelete <model-id>"')
return
}

await connectToMongoose()

const model = await ModelModel.findOne({ id: modelId })
if (!model) {
log.error(`Cannot find model using ID ${modelId}`)
return
}
const releases = await ReleaseModel.find({ modelId })
log.info(`Deleting ${releases.length} releases`)
for (const release of releases) {
await release.delete()
}

const accesses = await AccessRequestModel.find({ modelId })
log.info(`Deleting ${accesses.length} access requests`)
for (const access of accesses) {
await access.delete()
}

const revisions = await ModelCardRevisionModel.find({ modelId })
log.info(`Deleting ${revisions.length} model card revisions`)
for (const revision of revisions) {
await revision.delete()
}

const reviews = await ReviewModel.find({ modelId })
log.info(`Deleting ${reviews.length} reviews`)
for (const review of reviews) {
const responses = await ResponseModel.find({ parentId: review._id })
log.info(`Deleting ${responses.length} responses from review ${review._id} `)
for (const response of responses) {
await response.delete()
}
await review.delete()
}

const files = await FileModel.find({ modelId })
log.info(`Deleting ${accesses.length} files`)
for (const file of files) {
await file.delete()
}

const webhooks = await WebhookModel.find({ modelId })
log.info(`Deleting ${webhooks.length} webhooks`)
for (const webhook of webhooks) {
await webhook.delete()
}

const inferences = await InferenceModel.find({ modelId })
log.info(`Deleting ${inferences.length} inferences`)
for (const inference of inferences) {
await inference.delete()
}

await model.delete()
return
}

script()
5 changes: 3 additions & 2 deletions backend/src/scripts/runScript.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ const { exec } = shelljs
// npx tsx src/scripts/uploadExampleModel.ts
export default async function runScript() {
const argv = await yargs(hideBin(process.argv)).usage('Usage: $0 [script]').argv
const script = argv._
const script = argv._[0]
const args = argv._.slice(1)

exec(`npx tsx src/scripts/${script}.ts`)
exec(`npx tsx src/scripts/${script}.ts ${args}`)
}

runScript()
6 changes: 5 additions & 1 deletion backend/src/services/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ export function isFileInterfaceDoc(data: unknown): data is FileInterfaceDoc {
return true
}

export const createFilePath = (modelId: string, fileId: string) => {
return `beta/model/${modelId}/files/${fileId}`
}

export async function uploadFile(user: UserInterface, modelId: string, name: string, mime: string, stream: Readable) {
const model = await getModelById(user, modelId)
if (model.settings.mirror.sourceModelId) {
Expand All @@ -49,7 +53,7 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str
const fileId = longId()

const bucket = config.s3.buckets.uploads
const path = `beta/model/${modelId}/files/${fileId}`
const path = createFilePath(modelId, fileId)

const file: FileInterfaceDoc = new FileModel({ modelId, name, mime, bucket, path, complete: true })

Expand Down
Loading

0 comments on commit 73d2bcc

Please sign in to comment.