From a49ce537b70cb4d53783e78af40cc84ba67c9465 Mon Sep 17 00:00:00 2001 From: Denis Bykhov Date: Thu, 19 Sep 2024 13:18:50 +0500 Subject: [PATCH] Improve init Signed-off-by: Denis Bykhov --- common/config/rush/pnpm-lock.yaml | 17 +- dev/tool/package.json | 4 +- server/client/src/account.ts | 29 +-- server/tool/package.json | 2 + server/tool/src/index.ts | 3 +- server/tool/src/initializer.ts | 289 +++++++++++++++++++++++------- 6 files changed, 261 insertions(+), 83 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 2155209fa83..157ba254fc0 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -25376,7 +25376,7 @@ packages: dev: false file:projects/chunter.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-7CCSMrDbX8Mb8drPtpIVbQbvx1fsXW6YAqceu/rhsE7b9+i3LRaNGlyggJ1V1Kg32FTEHK2Q4jkKrn34aLL+3A==, tarball: file:projects/chunter.tgz} + resolution: {integrity: sha512-30IiBYbNohYn7c6fiy/QTHMy6qkpRv3ZvghD595O/is4lFkDfaBLP18Zd90fgyQNg/G/2HtGMGXunlWEr/OAoQ==, tarball: file:projects/chunter.tgz} id: file:projects/chunter.tgz name: '@rush-temp/chunter' version: 0.0.0 @@ -25535,7 +25535,7 @@ packages: dev: false file:projects/collaborator.tgz(@tiptap/pm@2.6.6)(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-protocols@1.0.6): - resolution: {integrity: sha512-ILQCgoJM6kiRT3YNphnywB42nFA5VZJkRXae0PiKrm+RJwm9/Qf0Berkrj+SDy32iC9fw6PI/+yES11MeqFeSA==, tarball: file:projects/collaborator.tgz} + resolution: {integrity: sha512-ljGlQulKneGos0pwJDjmKkgFZVizNJ4CMyGUp14orVTc40IKNK4szU09WskcwoZ6MGbBnUJttMVuGf3tb0/0iA==, tarball: file:projects/collaborator.tgz} id: file:projects/collaborator.tgz name: '@rush-temp/collaborator' version: 0.0.0 @@ -27495,7 +27495,7 @@ packages: dev: false file:projects/love-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2): - resolution: {integrity: sha512-e49VrhVkk6U8nj/ZyKfYVj/gK/3rTSrVCicDM5Luxw/f/8pPEVf9uEzmdAV11uiXuXxmj5t0BmHdG5eeVhiK6A==, tarball: file:projects/love-resources.tgz} + resolution: {integrity: sha512-6dbzkDxFSSCZr7b3Cm7AFvESRax8JpNYiIvI/ScUNPN+M7L2a3PKWKlfdcD/ZvWOAKmQW1l0j5GPH/ucHHu8sQ==, tarball: file:projects/love-resources.tgz} id: file:projects/love-resources.tgz name: '@rush-temp/love-resources' version: 0.0.0 @@ -27543,7 +27543,7 @@ packages: dev: false file:projects/love.tgz(@types/node@20.11.19)(esbuild@0.20.1)(svelte@4.2.12)(ts-node@10.9.2): - resolution: {integrity: sha512-IA1DOwfYE00ACQ47zgXq3yOcNghk1oU1nPcq15bqdjOxJeKaE5sa4w/yJ7WoGn1Dm5F0TmxKc67GGDxcTT6kDQ==, tarball: file:projects/love.tgz} + resolution: {integrity: sha512-D9QBf+/PREVaj/zqju1EnET+45ImzmEpqYThK+I2fbGxZ2t128DbfspBi3Lf2pUure8yF2PSTlTJSzKmUtgvpA==, tarball: file:projects/love.tgz} id: file:projects/love.tgz name: '@rush-temp/love' version: 0.0.0 @@ -29813,7 +29813,7 @@ packages: dev: false file:projects/pod-github.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(y-prosemirror@1.2.12): - resolution: {integrity: sha512-WPkKYn8Uz2hTqWrzT7vjrfyFyOZctsvFVeixkSDH2RPydyXaB6tE2qIm0UFa5VgCnuBu9Rsc7nN62m2Ac7mL1w==, tarball: file:projects/pod-github.tgz} + resolution: {integrity: sha512-eyMBtVIlmdGMh+T89SuihlOHLsLA6czYiSY5SiZ06YL+VOb/TrBiluqvbWyMWUThZ1gFQCpUH/jm+0nkDUZg+Q==, tarball: file:projects/pod-github.tgz} id: file:projects/pod-github.tgz name: '@rush-temp/pod-github' version: 0.0.0 @@ -33304,13 +33304,15 @@ packages: dev: false file:projects/server-tool.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-laY4rfz2M3gZw+L9hEHz3Ew3kVSUXEbiQNawxzPvnWXd1QfazHMNBqf0odz6OfqKlRDBCqtb+qx/F7c45wDP5Q==, tarball: file:projects/server-tool.tgz} + resolution: {integrity: sha512-1q+NoWPMYF3GDh8hYG9HpkAiKqNqIFcubqeCe3tWtfr5tmVDdEkAOTb4fFqREmPR/WtF8DCO14quDLZ4ZywpxQ==, tarball: file:projects/server-tool.tgz} id: file:projects/server-tool.tgz name: '@rush-temp/server-tool' version: 0.0.0 dependencies: '@types/jest': 29.5.12 '@types/js-yaml': 4.0.9 + '@types/mime-types': 2.1.4 + '@types/node-fetch': 2.6.11 '@types/uuid': 8.3.4 '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) @@ -33323,7 +33325,9 @@ packages: fast-equals: 5.0.1 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) js-yaml: 4.1.0 + mime-types: 2.1.35 mongodb: 6.9.0 + node-fetch: 2.7.0 prettier: 3.2.5 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 @@ -33338,6 +33342,7 @@ packages: - babel-jest - babel-plugin-macros - bufferutil + - encoding - esbuild - gcp-metadata - kerberos diff --git a/dev/tool/package.json b/dev/tool/package.json index 34f8c58a314..af4f2223651 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -19,9 +19,9 @@ "docker:tbuild": "docker build -t hardcoreeng/tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool", - "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --max-old-space-size=18000 ./bundle/bundle.js", + "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret DB_URL=postgresql://postgres:example@localhost:5432 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --max-old-space-size=18000 ./bundle/bundle.js", "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost MONGO_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service ELASTIC_URL=http://localhost:9200 REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js", - "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js", + "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=fai4uugae9Xuuse5aelo ACCOUNTS_URL=https://account.huly.net TRANSACTOR_URL=wss://transactor.huly.net node --max-old-space-size=8000 ./bundle/bundle.js", "upgrade": "rushx run-local upgrade", "format": "format src", "test": "jest --passWithNoTests --silent --forceExit", diff --git a/server/client/src/account.ts b/server/client/src/account.ts index 0d20f61a342..d41a36b8136 100644 --- a/server/client/src/account.ts +++ b/server/client/src/account.ts @@ -115,19 +115,24 @@ export async function updateWorkspaceInfo ( progress: number, message?: string ): Promise { - const accountsUrl = getAccoutsUrlOrFail() - await ( - await fetch(accountsUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - method: 'updateWorkspaceInfo', - params: [token, workspaceId, event, version, progress, message] + try { + const accountsUrl = getAccoutsUrlOrFail() + console.log('accountsUrl', accountsUrl) + await ( + await fetch(accountsUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + method: 'updateWorkspaceInfo', + params: [token, workspaceId, event, version, progress, message] + }) }) - }) - ).json() + ).json() + } catch (err) { + console.log('error', err) + } } export async function workerHandshake ( diff --git a/server/tool/package.json b/server/tool/package.json index e3e3e350227..366b62c025a 100644 --- a/server/tool/package.json +++ b/server/tool/package.json @@ -37,6 +37,7 @@ "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", + "@types/mime-types": "~2.1.1", "@types/js-yaml": "^4.0.9" }, "dependencies": { @@ -60,6 +61,7 @@ "@hcengineering/minio": "^0.6.0", "fast-equals": "^5.0.1", "@hcengineering/text": "^0.6.5", + "mime-types": "~2.1.34", "js-yaml": "^4.1.0" } } diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 23aefeba97a..73685548020 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -236,7 +236,8 @@ export async function initializeWorkspace ( return } - const initializer = new WorkspaceInitializer(ctx, storageAdapter, wsUrl, client) + const baseUrl = scriptUrl.substring(0, scriptUrl.lastIndexOf('/')) + const initializer = new WorkspaceInitializer(ctx, storageAdapter, wsUrl, client, baseUrl) await initializer.processScript(script, logger, progress) } catch (err: any) { ctx.error('Failed to initialize workspace', { error: err }) diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts index 4a15468c744..34e8ee3475c 100644 --- a/server/tool/src/initializer.ts +++ b/server/tool/src/initializer.ts @@ -18,6 +18,8 @@ import { makeRank } from '@hcengineering/rank' import { AggregatorStorageAdapter } from '@hcengineering/server-core' import { jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text' import { v4 as uuid } from 'uuid' +import { contentType } from 'mime-types' +import * as yaml from 'js-yaml' const fieldRegexp = /\${\S+?}/ @@ -25,6 +27,7 @@ export interface InitScript { name: string lang?: string default: boolean + from?: string steps: InitStep[] } @@ -35,6 +38,8 @@ export type InitStep = | UpdateStep | FindStep | UploadStep + | VarsStep + | CreateFrom export interface CreateStep { type: 'create' @@ -45,10 +50,19 @@ export interface CreateStep { resultVariable?: string } -export interface DefaultStep { +export interface CreateFrom { + type: 'createFrom' + fromUrl: string +} + +export interface VarsStep { + type: 'vars' + vars: Record +} + +export interface DefaultStep extends Defaults { type: 'default' _class: Ref> - data: Props } export interface MixinStep { @@ -82,45 +96,95 @@ export interface UploadStep { resultVariable?: string } +type PostOp = (id: Ref, clazz: Ref>) => Promise + +interface Defaults { + markdownFields?: string[] + collabFields?: string[] + data: Props +} + +function concatArrs (a: string[] | undefined, b: string[] | undefined): string[] { + return a !== undefined && b !== undefined ? [...a, ...b] : a ?? b ?? [] +} + export type Props = Data & Partial & { space: Ref } export class WorkspaceInitializer { private readonly imageUrl = 'image://' private readonly nextRank = '#nextRank' private readonly now = '#now' + private readonly vars: Record = {} + private readonly defaults = new Map>, Defaults>() constructor ( private readonly ctx: MeasureContext, private readonly storageAdapter: AggregatorStorageAdapter, private readonly wsUrl: WorkspaceIdWithUrl, - private readonly client: TxOperations + private readonly client: TxOperations, + private readonly baseUrl: string ) {} + async getBase (logger: ModelLogger): Promise[]> { + try { + const req = await fetch(`${this.baseUrl}/base.yaml`) + const text = await req.text() + const script = yaml.load(text) as any as InitStep[] + return script + } catch (err) { + logger.error('Error getting base script', err) + return [] + } + } + + async getScript (url: string, logger: ModelLogger): Promise[]> { + try { + const req = await fetch(url) + const text = await req.text() + const script = yaml.load(text) as any as InitStep[] + return script + } catch (err) { + logger.error('Error getting base script', err) + throw err + } + } + async processScript ( script: InitScript, logger: ModelLogger, progress: (value: number) => Promise ): Promise { - const vars: Record = {} - const defaults = new Map>, Props>() - for (let index = 0; index < script.steps.length; index++) { + this.defaults.clear() + const base = await this.getBase(logger) + if (script.from !== undefined) { + script.steps = [...(await this.getScript(script.from, logger)), ...script.steps] + } + const steps = [...base, ...script.steps] + for (let index = 0; index < steps.length; index++) { try { - const step = script.steps[index] - if (step.type === 'default') { - await this.processDefault(step, defaults) + const step = steps[index] + if (step.type === 'vars') { + for (const key in step.vars) { + const value = step.vars[key] + this.vars[`\${${key}}`] = value + } + } else if (step.type === 'default') { + this.processDefault(step) + } else if (step.type === 'createFrom') { + await this.processCreateFrom(step) } else if (step.type === 'create') { - await this.processCreate(step, vars, defaults) + await this.processCreate(step) } else if (step.type === 'update') { - await this.processUpdate(step, vars) + await this.processUpdate(step) } else if (step.type === 'mixin') { - await this.processMixin(step, vars) + await this.processMixin(step) } else if (step.type === 'find') { - await this.processFind(step, vars) + await this.processFind(step) } else if (step.type === 'upload') { - await this.processUpload(step, vars, logger) + await this.processUpload(step, logger) } - await progress(Math.round(((index + 1) * 100) / script.steps.length)) + // await progress(Math.round(((index + 1) * 100) / steps.length)) } catch (error) { logger.error(`Error in script on step ${index}`, error) throw error @@ -128,23 +192,37 @@ export class WorkspaceInitializer { } } - private async processDefault( - step: DefaultStep, - defaults: Map>, Props> - ): Promise { - const obj = defaults.get(step._class) ?? {} - defaults.set(step._class, { ...obj, ...step.data }) + private processDefault(step: DefaultStep): void { + const _class = this.vars[`\${${step._class}}`] ?? step._class + const obj = this.defaults.get(_class) + if (obj === undefined) { + this.defaults.set(_class, { + data: step.data, + collabFields: step.collabFields, + markdownFields: step.markdownFields + }) + } else { + const data = { ...obj.data, ...step.data } + const collabFields = concatArrs(obj.collabFields, step.collabFields) + const markdownFields = concatArrs(obj.markdownFields, step.markdownFields) + this.defaults.set(_class, { data, collabFields, markdownFields }) + } + } + + private getUrl (url: string): string { + return url.startsWith('./') ? `${this.baseUrl}/${url.substring(2)}` : url } - private async processUpload (step: UploadStep, vars: Record, logger: ModelLogger): Promise { + private async processUpload (step: UploadStep, logger: ModelLogger): Promise { try { const id = uuid() - const resp = await fetch(step.fromUrl) + const url = this.getUrl(step.fromUrl) + const resp = await fetch(url) const buffer = Buffer.from(await resp.arrayBuffer()) await this.storageAdapter.put(this.ctx, this.wsUrl, id, buffer, step.contentType, buffer.length) if (step.resultVariable !== undefined) { - vars[`\${${step.resultVariable}}`] = id - vars[`\${${step.resultVariable}_size}`] = buffer.length + this.vars[`\${${step.resultVariable}}`] = id + this.vars[`\${${step.resultVariable}_size}`] = buffer.length } } catch (error) { logger.error('Upload failed', error) @@ -152,52 +230,95 @@ export class WorkspaceInitializer { } } - private async processFind(step: FindStep, vars: Record): Promise { - const query = this.fillProps(step.query, vars) - const res = await this.client.findOne(step._class, { ...(query as any) }) + private async processFind(step: FindStep): Promise { + const _class = this.vars[step._class] ?? step._class + const query = this.fillProps(step.query) + const res = await this.client.findOne(_class, { ...(query as any) }) if (res === undefined) { throw new Error(`Document not found: ${JSON.stringify(query)}`) } if (step.resultVariable !== undefined) { - vars[`\${${step.resultVariable}}`] = res + this.vars[`\${${step.resultVariable}}`] = res } } - private async processMixin(step: MixinStep, vars: Record): Promise { - const data = await this.fillPropsWithMarkdown(step.data, vars, step.markdownFields) + private async processMixin(step: MixinStep): Promise { + const _class = this.vars[`\${${step._class}}`] ?? step._class + const markdownFields = concatArrs(this.defaults.get(_class)?.markdownFields, step.markdownFields) + const data = await this.fillPropsWithMarkdown(step.data, markdownFields) const { _id, space, ...props } = data if (_id === undefined || space === undefined) { throw new Error('Mixin step must have _id and space') } - await this.client.createMixin(_id, step._class, space, step.mixin, props) + await this.client.createMixin(_id, _class, space, step.mixin, props) } - private async processUpdate(step: UpdateStep, vars: Record): Promise { - const data = await this.fillPropsWithMarkdown(step.data, vars, step.markdownFields) + private async processUpdate(step: UpdateStep): Promise { + const _class = this.vars[`\${${step._class}}`] ?? step._class + const markdownFields = concatArrs(this.defaults.get(_class)?.markdownFields, step.markdownFields) + const data = await this.fillPropsWithMarkdown(step.data, markdownFields) const { _id, space, ...props } = data if (_id === undefined || space === undefined) { throw new Error('Update step must have _id and space') } - await this.client.updateDoc(step._class, space, _id as Ref, props) + await this.client.updateDoc(_class, space, _id as Ref, props) } - private async processCreate( - step: CreateStep, - vars: Record, - defaults: Map>, Props> - ): Promise { + private async processCreate(step: CreateStep): Promise { const _id = generateId() if (step.resultVariable !== undefined) { - vars[`\${${step.resultVariable}}`] = _id + this.vars[`\${${step.resultVariable}}`] = _id + } + const postOps: PostOp[] = [] + const _class = this.vars[`\${${step._class}}`] ?? step._class + const markdownFields = concatArrs(this.defaults.get(_class)?.markdownFields, step.markdownFields) + const collabFields = concatArrs(this.defaults.get(_class)?.collabFields, step.collabFields) + + const data = await this.fillPropsWithMarkdown( + { ...(this.defaults.get(_class)?.data ?? {}), ...step.data }, + markdownFields, + postOps + ) + + for (const field of collabFields) { + if ((data as any)[field] !== undefined) { + const res = await this.createCollab((data as any)[field], field, _id) + ;(data as any)[field] = res + } + } + + await this.create(_class, data, _id) + for (const op of postOps) { + await op(_id, _class) + } + } + + private async processCreateFrom(step: CreateFrom): Promise { + const url = this.getUrl(step.fromUrl) + const resp = await fetch(url) + const text = await resp.text() + const script = yaml.load(text) as any as Props + const { _class, ...props } = script + if (_class === undefined) { + throw new Error('CreateFrom step must have _class') } + const _id = generateId() + const resultVariable = url.substring(url.lastIndexOf('/') + 1, url.lastIndexOf('.')) + this.vars[`\${${resultVariable}}`] = _id + + const clazz = this.vars[`\${${_class}}`] ?? _class + const markdownFields = this.defaults.get(clazz)?.markdownFields + const collabFields = this.defaults.get(clazz)?.collabFields + const postOps: PostOp[] = [] + const data = await this.fillPropsWithMarkdown( - { ...(defaults.get(step._class) ?? {}), ...step.data }, - vars, - step.markdownFields + { ...(this.defaults.get(clazz)?.data ?? {}), ...(props as Props) }, + markdownFields, + postOps ) - if (step.collabFields !== undefined) { - for (const field of step.collabFields) { + if (collabFields !== undefined) { + for (const field of collabFields) { if ((data as any)[field] !== undefined) { const res = await this.createCollab((data as any)[field], field, _id) ;(data as any)[field] = res @@ -205,7 +326,10 @@ export class WorkspaceInitializer { } } - await this.create(step._class, data, _id) + await this.create(clazz, data, _id) + for (const op of postOps) { + await op(_id, clazz) + } } private parseMarkdown (text: string): string { @@ -246,10 +370,10 @@ export class WorkspaceInitializer { private async fillPropsWithMarkdown | Props>( data: P, - vars: Record, - markdownFields?: string[] + markdownFields?: string[], + postOps?: PostOp[] ): Promise

{ - data = await this.fillProps(data, vars) + data = await this.fillProps(data, postOps) if (markdownFields !== undefined) { for (const field of markdownFields) { if ((data as any)[field] !== undefined) { @@ -276,28 +400,25 @@ export class WorkspaceInitializer { return collabId } - private async fillProps | Props>( - data: P, - vars: Record - ): Promise

{ + private async fillProps | Props>(data: P, postOps?: PostOp[]): Promise

{ for (const key in data) { const value = (data as any)[key] - ;(data as any)[key] = await this.fillValue(value, vars) + ;(data as any)[key] = await this.fillValue(value, postOps) } return data } - private async fillValue (value: any, vars: Record): Promise { + private async fillValue (value: any, postOps?: PostOp[]): Promise { if (typeof value === 'object') { if (Array.isArray(value)) { - return await Promise.all(value.map(async (v) => await this.fillValue(v, vars))) + return await Promise.all(value.map(async (v) => await this.fillValue(v, postOps))) } else { - return await this.fillProps(value, vars) + return await this.fillProps(value, postOps) } } else if (typeof value === 'string') { if (value === this.nextRank) { - const rank = makeRank(vars[this.nextRank], undefined) - vars[this.nextRank] = rank + const rank = makeRank(this.vars[this.nextRank], undefined) + this.vars[this.nextRank] = rank return rank } else if (value === this.now) { return new Date().getTime() @@ -305,9 +426,22 @@ export class WorkspaceInitializer { while (true) { const matched = fieldRegexp.exec(value) if (matched === null) break - const result = vars[matched[0]] + const result = this.vars[matched[0]] if (result === undefined) { - throw new Error(`Variable ${matched[0]} not found`) + if (matched[0].startsWith('${file://')) { + const val = matched[0].substring(9, matched[0].length - 1) + const fileUrl = this.getUrl(val) + const name = fileUrl.substring(fileUrl.lastIndexOf('/') + 1) + const res = await this.getFile(fileUrl, name) + if (postOps !== undefined) { + postOps?.push(async (id, clazz) => { + await this.createAttachment(id, clazz, name) + }) + } + value = value.replaceAll(matched[0], res) + } else { + throw new Error(`Variable ${matched[0]} not found`) + } } else { value = value.replaceAll(matched[0], result) fieldRegexp.lastIndex = 0 @@ -318,4 +452,35 @@ export class WorkspaceInitializer { } return value } + + private async getFile (url: string, name: string): Promise { + const id = uuid() + const resp = await fetch(url) + const buffer = Buffer.from(await resp.arrayBuffer()) + const parsedType = contentType(name) + const type = parsedType === false ? 'application/octet-stream' : parsedType + await this.storageAdapter.put(this.ctx, this.wsUrl, id, buffer, type, buffer.length) + this.vars[`\${${name}}`] = id + this.vars[`\${${name}_size}`] = buffer.length + this.vars[`\${${name}_type}`] = type + return id + } + + private async createAttachment (attachedTo: Ref, _class: Ref>, fileName: string): Promise { + const clazz = 'attachment:class:Attachment' as Ref> + const file = this.vars[`\${${fileName}}`] + const size = this.vars[`\${${fileName}_size}`] + const type = this.vars[`\${${fileName}_type}`] + const props = { + ...(this.defaults.get(clazz)?.data ?? {}), + attachedTo, + attachedToClass: _class, + file, + name: fileName, + size, + type + } + const data = await this.fillProps(props as Props) + await this.create(_class, data) + } }