Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored how file paths are constructed/exported #1972

Merged
merged 3 commits into from
Mar 4, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion backend/src/services/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ export function isFileInterfaceDoc(data: unknown): data is FileInterfaceDoc {
return true
}

export const createFilePath = (modelId: string, fileId: string) => {
return `beta/model/${modelId}/files/${fileId}`
}

export async function uploadFile(user: UserInterface, modelId: string, name: string, mime: string, stream: Readable) {
const model = await getModelById(user, modelId)
if (model.settings.mirror.sourceModelId) {
Expand All @@ -49,7 +53,7 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str
const fileId = longId()

const bucket = config.s3.buckets.uploads
const path = `beta/model/${modelId}/files/${fileId}`
const path = createFilePath(modelId, fileId)

const file = new FileModel({ modelId, name, mime, bucket, path, complete: true })

Expand Down
20 changes: 9 additions & 11 deletions backend/src/services/mirroredModel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { UserInterface } from '../models/User.js'
import config from '../utils/config.js'
import { BadReq, Forbidden, InternalError } from '../utils/error.js'
import {
createFilePath,
downloadFile,
getFilesByIds,
getTotalFileSize,
Expand Down Expand Up @@ -166,13 +167,15 @@ export async function importModel(

switch (importKind) {
case ImportKind.Documents: {
log.info({ mirroredModelId, payloadUrl }, 'Importing colection of documents.')
return await importDocuments(user, res, mirroredModelId, sourceModelId, payloadUrl)
}
case ImportKind.File: {
log.info({ mirroredModelId, payloadUrl }, 'Importing file data.')
if (!filePath) {
throw BadReq('Missing File Path.', { mirroredModelId, sourceModelIdMeta: sourceModelId })
}
const result = await importModelFile(res, filePath, mirroredModelId, sourceModelId)
const result = await importModelFile(res, filePath, mirroredModelId)
return {
mirroredModel,
importResult: {
Expand Down Expand Up @@ -297,14 +300,9 @@ async function importDocuments(
}
}

async function importModelFile(
content: Response,
importedPath: string,
mirroredModelId: string,
sourceModelId: string,
) {
async function importModelFile(content: Response, importedPath: string, mirroredModelId: string) {
const bucket = config.s3.buckets.uploads
const updatedPath = importedPath.replace(sourceModelId, mirroredModelId)
const updatedPath = createFilePath(mirroredModelId, importedPath)
await putObjectStream(bucket, updatedPath, content.body as Readable)
log.debug({ bucket, path: updatedPath }, 'Imported file successfully uploaded to S3.')
await markFileAsCompleteAfterImport(updatedPath)
Expand Down Expand Up @@ -362,7 +360,7 @@ async function parseFile(fileJson: string, mirroredModelId: string, sourceModelI

const modelId = file.modelId
file.modelId = mirroredModelId
file.path = file.path.replace(modelId, mirroredModelId)
file.path = createFilePath(mirroredModelId, file.id)
if (sourceModelId !== modelId) {
throw InternalError('Zip file contains files from an invalid model.', { modelIds: [sourceModelId, modelId] })
}
Expand Down Expand Up @@ -555,13 +553,13 @@ async function addReleaseToZip(
for (const file of files) {
zip.append(JSON.stringify(file.toJSON()), { name: `files/${file._id}.json` })
await uploadToS3(
file.path,
file.id,
(await downloadFile(user, file._id)).Body as stream.Readable,
{
exporter: user.dn,
sourceModelId: model.id,
mirroredModelId,
filePath: file.path,
filePath: file.id,
importKind: ImportKind.File,
},
{
Expand Down
1 change: 1 addition & 0 deletions backend/test/services/mirroredModel.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ const fileMocks = vi.hoisted(() => ({
downloadFile: vi.fn(() => ({ Body: 'test' })),
markFileAsCompleteAfterImport: vi.fn(),
isFileInterfaceDoc: vi.fn(() => true),
createFilePath: vi.fn(() => 'file/path'),
}))
vi.mock('../../src/services/file.js', () => fileMocks)

Expand Down
Loading