From 38be696a7bf132e831af5ec1b279744396d672fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Thu, 9 Jan 2025 16:28:49 +0100 Subject: [PATCH 01/18] inital grantee handlers with pss --- package-lock.json | 4 +- src/constants.ts | 1 + src/fileManager.ts | 173 +++++++++++++++++++++++++++++++++++++++++++-- src/types.ts | 13 ++++ src/utils.ts | 34 +++++++++ 5 files changed, 215 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 230ad09..9cbdfaf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,10 +42,8 @@ "node": ">=14" } }, - "../../../../ujvaribalint/repos/solarpunk/mantaray-js": {}, "../mantaray-js": { "version": "1.0.4", - "extraneous": true, "license": "BSD-3-Clause", "dependencies": { "get-random-values": "^1.2.2", @@ -7257,7 +7255,7 @@ } }, "node_modules/mantaray-js": { - "resolved": "../../../../ujvaribalint/repos/solarpunk/mantaray-js", + "resolved": "../mantaray-js", "link": true }, "node_modules/math-intrinsics": { diff --git a/src/constants.ts b/src/constants.ts index 5b67eab..c93e203 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,3 +2,4 @@ const feedTypes = ['sequence', 'epoch'] as const; export type FeedType = (typeof feedTypes)[number]; export const DEFAULT_FEED_TYPE: FeedType = 'sequence'; export const STAMP_LIST_TOIC = 'stamps'; +export const SHARED_INBOX_TOPIC = 'shared-inbox'; diff --git a/src/fileManager.ts b/src/fileManager.ts index 71adfab..de21e20 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -1,14 +1,25 @@ -import { BatchId, Bee, PostageBatch, Reference, Utils } from '@ethersphere/bee-js'; +import { + BatchId, + Bee, + Data, + PostageBatch, + Reference, + Utils, + REFERENCE_HEX_LENGTH, + GranteesResult, + PssSubscription, +} from '@ethersphere/bee-js'; import { readFileSync } from 'fs'; import { MantarayNode, MetadataMapping, Reference as MantarayRef } from 'mantaray-js'; import path from 'path'; -import { DEFAULT_FEED_TYPE, STAMP_LIST_TOIC } from './constants'; -import { FileWithMetadata, StampList, StampWithMetadata } from './types'; -import { encodePathToBytes, getContentType } from './utils'; +import { DEFAULT_FEED_TYPE, STAMP_LIST_TOIC, SHARED_INBOX_TOPIC } from './constants'; +import { FileWithMetadata, GranteeList, StampList, SharedMessage, StampWithMetadata } from './types'; +import { assertSharedMessage, encodePathToBytes, getContentType } from './utils'; export class FileManager { // TODO: private vars + // TODO: store shared refs and own files in the same array ? public bee: Bee; public mantaray: MantarayNode; public importedFiles: FileWithMetadata[]; @@ -16,6 +27,9 @@ export class FileManager { private stampList: StampWithMetadata[]; private nextStampFeedIndex: string; private privateKey: string; + private granteeList: GranteeList; + private sharedWithMe: SharedMessage[]; + private sharedSubscription: PssSubscription; constructor(beeUrl: string, privateKey: string) { if (!beeUrl) { @@ -24,6 +38,7 @@ export class FileManager { if (!privateKey) { throw new Error('privateKey is required for initializing the FileManager.'); } + console.log('Initializing Bee client...'); this.bee = new Bee(beeUrl); this.stampList = []; @@ -31,12 +46,20 @@ export class FileManager { this.privateKey = privateKey; this.mantaray = new MantarayNode(); this.importedFiles = []; - - // Create personalized feed + this.granteeList = { filesSharedWith: new Map() }; + this.sharedWithMe = []; + this.sharedSubscription = {} as PssSubscription; } // TODO: use allSettled for file fetching and only save the ones that are successful async initialize(items: any | undefined) { + try { + // TODO: is await needed ? + this.sharedSubscription = await this.subscribeToSharedInbox(); + } catch (error: any) { + console.log('Error during shared inbox subscription: ', error); + } + console.log('Importing stamps and references...'); try { await this.initStamps(); @@ -223,7 +246,8 @@ export class FileManager { this.addToMantaray(undefined, reference, metadata); // Track imported files - this.importedFiles.push({ reference: reference, name: fileName, batchId: batchId || '' }); + // TODO: shared flag + this.importedFiles.push({ reference: reference, name: fileName, batchId: batchId || '', shared: false }); } catch (error: any) { console.error(`[ERROR] Failed to process reference ${reference}: ${error.message}`); } @@ -568,4 +592,139 @@ export class FileManager { return contents; } + + // fetches the list of grantees under the given reference + async getGrantees(eGlRef: string | Reference): Promise { + if (eGlRef.length !== REFERENCE_HEX_LENGTH) { + console.error('Invalid reference: ', eGlRef); + return; + } + + try { + // TODO: parse data as ref array + const grantResult = await this.bee.getGrantees(eGlRef); + const grantees = grantResult.data; + console.log('Grantees fetched: ', grantees); + return grantees; + } catch (error: any) { + console.error(`Failed to get share grantee list: ${error}`); + return undefined; + } + } + // TODO: cache and search locally or on swarm ? + // async getGranteesForFile(fileRef: string | Reference): Promise { + // try { + // const file = this.importedFiles.find((f) => f.reference === fileRef); + // if (!file) { + // console.error('File not found: ', fileRef); + // return undefined; + // } + + // const fileGrantees = this.granteeList.find((g) => g.filesSharedWith[fileRef]); + // if (!fileGrantees) { + // console.error('No grantees found for file: ', fileRef); + // return undefined; + // } + + // const grantees = fileGrantees.filesSharedWith[fileRef]; + // console.log('Grantees for file fetched: ', grantees); + // return grantees; + // } catch (error: any) { + // console.error(`Failed to get share grantee list for file${fileRef}\n: ${error}`); + // return undefined; + // } + // } + + // TODO: separate revoke function or frontend will handle it by creating a new act ? + // TODO: create a feed just like for the stamps to store the grantee list refs + // TODO: create a feed for the share access that can be read by each grantee + // TODO: notify user if it has been granted access by someone else + // TODO: history handling ? + // updates the list of grantees who can access the file reference under the history reference + async handleGrantees( + batchId: string | BatchId, + fileRef: string, + grantees: { + add?: string[]; + revoke?: string[]; + }, + historyRef: string | Reference, + eGlRef?: string | Reference, + ): Promise { + console.log('Allowing grantees to share files with me'); + + try { + let grantResult: GranteesResult; + if (eGlRef !== undefined && eGlRef.length === REFERENCE_HEX_LENGTH) { + grantResult = await this.bee.patchGrantees(batchId, eGlRef, historyRef, grantees); + console.log('Access patched, grantee list reference: ', grantResult.ref); + } else { + if (grantees.add === undefined || grantees.add.length === 0) { + console.error('No grantees specified.'); + return undefined; + } + + grantResult = await this.bee.createGrantees(batchId, grantees.add); + console.log('Access granted, new grantee list reference: ', grantResult.ref); + } + + const currentGrantees = this.granteeList.filesSharedWith.get(fileRef) || []; + const newGrantees = [...new Set([...currentGrantees, ...(grantees.add || []), ...(grantees.revoke || [])])]; + this.granteeList.filesSharedWith.set(fileRef, newGrantees); + return grantResult; + } catch (error: any) { + console.error(`Failed to grant share access: ${error}`); + return undefined; + } + } + + async subscribeToSharedInbox(): Promise { + const subscription = this.bee.pssSubscribe(SHARED_INBOX_TOPIC, { + onMessage: async (message) => { + console.log('Received shared inbox message: ', message); + assertSharedMessage(message); + this.sharedWithMe.push(message); + }, + onError: (e) => { + console.log('Error received in shared inbox: ', e.message); + throw e; + }, + }); + + return subscription; + } + + // recipient is optional, if not provided the message will be broadcasted == pss public key + async shareItem(batchId: string, targetOverlay: string, message: SharedMessage, recipient: string): Promise { + try { + const target = Utils.makeMaxTarget(targetOverlay); + const msgData = new Uint8Array(Buffer.from(JSON.stringify(message))); + await this.bee.pssSend(batchId, SHARED_INBOX_TOPIC, target, msgData, recipient); + } catch (error: any) { + console.log('Failed to share item: ', error); + } + } + + async downloadSharedItem(reference: string): Promise { + if (!this.sharedWithMe.find((msg) => msg.references.includes(reference))) { + console.log('Cannot find reference in shared messages: ', reference); + return undefined; + } + + try { + const data = await this.bee.downloadFile(reference); + return data.data; + } catch (error: any) { + console.error(`Failed to download shared file ${reference}\n: ${error}`); + return undefined; + } + } + + // TODO: do we need to cancel sub at shutdown ? + unsubscribeFromSharedInbox() { + if (this.sharedSubscription) { + console.log('Unsubscribed from shared inbox, topic: ', this.sharedSubscription.topic); + this.sharedSubscription.cancel(); + } + } } diff --git a/src/types.ts b/src/types.ts index 7bfb221..8193bc5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -4,6 +4,7 @@ export interface FileWithMetadata { reference: string | Reference; name: string; batchId: string | BatchId; + shared: boolean; timestamp?: number; uploader?: string; } @@ -18,3 +19,15 @@ export interface StampWithMetadata { export interface StampList { filesOfStamps: Map; } + +export interface GranteeList { + filesSharedWith: Map; +} + +// TODO: unify own files with shared and add stamp data potentially +export interface SharedMessage { + owner: string; + references: string[]; + timestamp?: number; + message?: string; +} diff --git a/src/utils.ts b/src/utils.ts index fddb73a..72feb0c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,8 @@ import { Bytes } from 'mantaray-js'; import path from 'path'; +import { SharedMessage } from './types'; + export function getContentType(filePath: string) { const ext = path.extname(filePath).toLowerCase(); const contentTypes: Map = new Map([ @@ -33,3 +35,35 @@ export function encodePathToBytes(pathString: string) { export function decodeBytesToPath(bytes: Bytes<32>) { return new TextDecoder().decode(bytes); } + +export function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object'; +} + +export function isStrictlyObject(value: unknown): value is Record { + return isObject(value) && !Array.isArray(value); +} + +export function assertSharedMessage(value: unknown): asserts value is SharedMessage { + if (!isStrictlyObject(value)) { + throw new TypeError('SharedMessage has to be object!'); + } + + const message = value as unknown as SharedMessage; + + if (typeof message.owner !== 'string') { + throw new TypeError('owner property of SharedMessage has to be string!'); + } + + if (!Array.isArray(message.references)) { + throw new TypeError('references property of SharedMessage has to be array!'); + } + + if (message.timestamp !== undefined && typeof message.timestamp !== 'number') { + throw new TypeError('timestamp property of SharedMessage has to be number!'); + } + + if (message.message !== undefined && typeof message.message !== 'string') { + throw new TypeError('message property of SharedMessage has to be string!'); + } +} From db86cf52f913954be798e4b01d9a074042fb73ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Fri, 10 Jan 2025 19:04:06 +0100 Subject: [PATCH 02/18] improving on grantee and file assignments --- .eslintrc.cjs | 1 + src/fileManager.ts | 168 +++++++++++++++++++++++-------------------- src/index.ts | 2 +- src/types.ts | 13 ++-- src/utils.ts | 23 +----- tests/mockHelpers.ts | 15 ++-- 6 files changed, 103 insertions(+), 119 deletions(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 7579c3f..46aeb85 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -19,6 +19,7 @@ module.exports = { plugins: ['@typescript-eslint', 'simple-import-sort'], rules: { '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/explicit-function-return-type': 'error', '@typescript-eslint/no-non-null-assertion': 'off', '@typescript-eslint/no-unused-vars': [ 'warn', diff --git a/src/fileManager.ts b/src/fileManager.ts index dc51800..8418cc1 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -15,8 +15,8 @@ import { readFileSync } from 'fs'; import path from 'path'; import { DEFAULT_FEED_TYPE, SHARED_INBOX_TOPIC, STAMP_LIST_TOPIC } from './constants'; -import { FileWithMetadata, GranteeList, SharedMessage, StampList, StampWithMetadata } from './types'; -import { assertSharedMessage, encodePathToBytes, getContentType } from './utils'; +import { FileWithMetadata, SharedMessage, StampList, StampWithMetadata } from './types'; +import { assertSharedMessage, getContentType } from './utils'; export class FileManager { // TODO: private vars @@ -28,7 +28,7 @@ export class FileManager { private stampList: StampWithMetadata[]; private nextStampFeedIndex: string; private privateKey: string; - private granteeList: GranteeList; + private granteeLists: string[]; private sharedWithMe: SharedMessage[]; private sharedSubscription: PssSubscription; private address: string; @@ -50,16 +50,14 @@ export class FileManager { this.mantaray = new MantarayNode(); this.importedFiles = []; - this.granteeList = { filesSharedWith: new Map() }; this.sharedWithMe = []; this.sharedSubscription = {} as PssSubscription; } // TODO: use allSettled for file fetching and only save the ones that are successful - async initialize(items: any | undefined) { + async initialize(items: any | undefined): Promise { try { - // TODO: is await needed ? - this.sharedSubscription = await this.subscribeToSharedInbox(); + this.sharedSubscription = this.subscribeToSharedInbox(); } catch (error: any) { console.log('Error during shared inbox subscription: ', error); } @@ -70,8 +68,8 @@ export class FileManager { if (this.stampList.length > 0) { console.log('Using stamp list for initialization.'); for (const elem of this.stampList) { - if (elem.fileReferences !== undefined && elem.fileReferences.length > 0) { - await this.importReferences(elem.fileReferences as Reference[], elem.stamp.batchID); + if (elem.references !== undefined && elem.references.length > 0) { + await this.importReferences(elem.references as Reference[], elem.stamp.batchID); } } } @@ -94,22 +92,23 @@ export class FileManager { throw error; } } - - async intializeMantarayUsingFeed() { + // TODO: is this method necessary ? + async intializeMantarayUsingFeed(): Promise { // } - async loadMantaray(manifestReference: Reference) { + async loadMantaray(manifestReference: Reference): Promise { const loadFunction = async (address: MantarayRef): Promise => { return this.bee.downloadData(Utils.bytesToHex(address)); }; - this.mantaray.load(loadFunction, Utils.hexToBytes(manifestReference)); + await this.mantaray.load(loadFunction, Utils.hexToBytes(manifestReference)); } // TODO: method to list new stamp with files // TODO: encrypt // TODO: how and how long to store the stamps feed data ? + // TODO: it seems inefficient to update always with the whole fileref array async updateStampData(stamp: string | BatchId, privateKey: string): Promise { const topicHex = this.bee.makeFeedTopic(STAMP_LIST_TOPIC); const feedWriter = this.bee.makeFeedWriter( @@ -118,8 +117,10 @@ export class FileManager { privateKey /*, { headers: { encrypt: "true" } }*/, ); try { - const data = JSON.stringify({ filesOfStamps: this.stampList.map((s) => [s.stamp.batchID, s.fileReferences]) }); - const stampListDataRef = await this.bee.uploadData(stamp, data); + const stampData = { + filesOfStamps: this.stampList.map((s) => [s.stamp.batchID, s.references]), + } as unknown as StampList; + const stampListDataRef = await this.bee.uploadData(stamp, JSON.stringify(stampData)); const writeResult = await feedWriter.upload(stamp, stampListDataRef.reference, { index: this.nextStampFeedIndex, }); @@ -157,7 +158,7 @@ export class FileManager { const stampIx = this.stampList.findIndex((s) => s.stamp.batchID === batchId); if (stampIx !== -1) { if (fileRefs.length > 0) { - this.stampList[stampIx].fileReferences = [...fileRefs]; + this.stampList[stampIx].references = [...fileRefs]; } } } @@ -221,7 +222,7 @@ export class FileManager { return this.stampList; } - async importReferences(referenceList: Reference[], batchId?: string, isLocal = false) { + async importReferences(referenceList: Reference[], batchId?: string, isLocal = false): Promise { const processPromises = referenceList.map(async (item: any) => { const reference: Reference = isLocal ? item.hash : item; try { @@ -252,7 +253,7 @@ export class FileManager { this.addToMantaray(undefined, reference, metadata); // Track imported files - // TODO: shared flag + // TODO: eglref, shared, timestamp -> mantaray root metadata this.importedFiles.push({ reference: reference, name: fileName, batchId: batchId || '', shared: false }); } catch (error: any) { console.error(`[ERROR] Failed to process reference ${reference}: ${error.message}`); @@ -262,16 +263,16 @@ export class FileManager { await Promise.all(processPromises); // Wait for all references to be processed } - async importPinnedReferences() { + async importPinnedReferences(): Promise { const allPins = await this.bee.getAllPins(); await this.importReferences(allPins); } - async importLocalReferences(items: any) { + async importLocalReferences(items: any): Promise { await this.importReferences(items, undefined, true); } - async downloadFile(mantaray: MantarayNode, filePath: string) { + async downloadFile(mantaray: MantarayNode, filePath: string): Promise { mantaray = mantaray || this.mantaray; console.log(`Downloading file: ${filePath}`); const normalizedPath = path.normalize(filePath); @@ -279,7 +280,7 @@ export class FileManager { let currentNode = mantaray; for (const segment of segments) { - const segmentBytes = encodePathToBytes(segment); + const segmentBytes = Utils.hexToBytes(segment); const fork = Object.values(currentNode.forks || {}).find((f) => Buffer.compare(f.prefix, segmentBytes) === 0); if (!fork) throw new Error(`Path segment not found: ${segment}`); @@ -308,7 +309,7 @@ export class FileManager { } } - async downloadFiles(mantaray: MantarayNode) { + async downloadFiles(mantaray: MantarayNode): Promise { mantaray = mantaray || this.mantaray; console.log('Downloading all files from Mantaray...'); const forks = mantaray.forks; @@ -359,6 +360,8 @@ export class FileManager { return validResults; // Return successful download results } + // TODO: always upload with ACT, only adding the publisher as grantee first, (by defualt) then when shared add the grantees + // TODO: store filerefs with the historyrefs async uploadFile( file: string, mantaray: MantarayNode | undefined, @@ -366,7 +369,7 @@ export class FileManager { customMetadata = {}, redundancyLevel = '1', save = true, - ) { + ): Promise { mantaray = mantaray || this.mantaray; console.log(`Uploading file: ${file}`); const fileData = readFileSync(file); @@ -383,6 +386,7 @@ export class FileManager { const uploadHeaders = { contentType, + act: true, headers: { 'swarm-redundancy-level': redundancyLevel, }, @@ -401,16 +405,16 @@ export class FileManager { const stampIx = this.stampList.findIndex((s) => s.stamp.batchID === stamp); if (stampIx === -1) { const newStamp = await this.fetchStamp(stamp); - // TODO: what to do here ? batch should alreade be usable + // TODO: what to do here ? batch should already be usable if (newStamp === undefined) { throw new Error(`Stamp not found: ${stamp}`); } - this.stampList.push({ stamp: newStamp, fileReferences: [uploadResponse.reference] }); - } else if (this.stampList[stampIx].fileReferences === undefined) { - this.stampList[stampIx].fileReferences = [uploadResponse.reference]; + this.stampList.push({ stamp: newStamp, references: [uploadResponse.reference] }); + } else if (this.stampList[stampIx].references === undefined) { + this.stampList[stampIx].references = [uploadResponse.reference]; } else { - this.stampList[stampIx].fileReferences.push(uploadResponse.reference); + this.stampList[stampIx].references.push(uploadResponse.reference); } await this.updateStampData(stamp, this.privateKey); @@ -423,13 +427,13 @@ export class FileManager { } } - addToMantaray(mantaray: MantarayNode | undefined, reference: string, metadata: MetadataMapping = {}) { + addToMantaray(mantaray: MantarayNode | undefined, reference: string, metadata: MetadataMapping = {}): void { mantaray = mantaray || this.mantaray; const filePath = metadata.fullPath || metadata.Filename || 'file'; const originalFileName = metadata.originalFileName || path.basename(filePath); - const bytesPath = encodePathToBytes(filePath); + const bytesPath = Utils.hexToBytes(filePath); const metadataWithOriginalName = { ...metadata, @@ -439,14 +443,17 @@ export class FileManager { mantaray.addFork(bytesPath, Utils.hexToBytes(reference), metadataWithOriginalName); } - async saveMantaray(mantaray: MantarayNode | undefined, stamp: string | BatchId) { + async saveMantaray(mantaray: MantarayNode | undefined, stamp: string | BatchId): Promise { mantaray = mantaray || this.mantaray; console.log('Saving Mantaray manifest...'); const saveFunction = async (data: Uint8Array): Promise => { const fileName = 'manifest'; const contentType = 'application/json'; - const uploadResponse = await this.bee.uploadFile(stamp, data, fileName, { contentType }); + const uploadResponse = await this.bee.uploadFile(stamp, data, fileName, { + contentType, + act: true, + }); return Utils.hexToBytes(uploadResponse.reference); }; @@ -457,7 +464,7 @@ export class FileManager { return hexReference; } - listFiles(mantaray: MantarayNode | undefined, includeMetadata = false) { + listFiles(mantaray: MantarayNode | undefined, includeMetadata = false): any { mantaray = mantaray || this.mantaray; console.log('Listing files in Mantaray...'); @@ -504,7 +511,7 @@ export class FileManager { return fileList; } - getDirectoryStructure(mantaray: MantarayNode | undefined, rootDirName: string) { + getDirectoryStructure(mantaray: MantarayNode | undefined, rootDirName: string): any { mantaray = mantaray || this.mantaray; console.log('Building directory structure from Mantaray...'); @@ -517,7 +524,7 @@ export class FileManager { return wrappedStructure; } - buildDirectoryStructure(mantaray: MantarayNode) { + buildDirectoryStructure(mantaray: MantarayNode): any { mantaray = mantaray || this.mantaray; console.log('Building raw directory structure...'); @@ -549,7 +556,7 @@ export class FileManager { return structure; } - getContentsOfDirectory(targetPath: string, mantaray: MantarayNode | undefined, rootDirName: string) { + getContentsOfDirectory(targetPath: string, mantaray: MantarayNode | undefined, rootDirName: string): any { mantaray = mantaray || this.mantaray; const directoryStructure: { [key: string]: any } = this.getDirectoryStructure(mantaray, rootDirName); @@ -610,6 +617,10 @@ export class FileManager { // TODO: parse data as ref array const grantResult = await this.bee.getGrantees(eGlRef); const grantees = grantResult.data; + const granteeList = this.granteeLists.find((glref) => glref === eGlRef); + if (granteeList !== undefined) { + this.granteeLists.push(eGlRef); + } console.log('Grantees fetched: ', grantees); return grantees; } catch (error: any) { @@ -617,29 +628,16 @@ export class FileManager { return undefined; } } - // TODO: cache and search locally or on swarm ? - // async getGranteesForFile(fileRef: string | Reference): Promise { - // try { - // const file = this.importedFiles.find((f) => f.reference === fileRef); - // if (!file) { - // console.error('File not found: ', fileRef); - // return undefined; - // } - - // const fileGrantees = this.granteeList.find((g) => g.filesSharedWith[fileRef]); - // if (!fileGrantees) { - // console.error('No grantees found for file: ', fileRef); - // return undefined; - // } - - // const grantees = fileGrantees.filesSharedWith[fileRef]; - // console.log('Grantees for file fetched: ', grantees); - // return grantees; - // } catch (error: any) { - // console.error(`Failed to get share grantee list for file${fileRef}\n: ${error}`); - // return undefined; - // } - // } + + // fetches the list of grantees who can access the file reference + async getGranteesOfFile(fileRef: string | Reference): Promise { + const file = this.importedFiles.find((f) => f.reference === fileRef); + if (file === undefined || file.eGlRef === undefined) { + console.error('File or grantee ref not found for reference: ', fileRef); + return undefined; + } + return await this.getGrantees(file.eGlRef); + } // TODO: separate revoke function or frontend will handle it by creating a new act ? // TODO: create a feed just like for the stamps to store the grantee list refs @@ -649,7 +647,7 @@ export class FileManager { // updates the list of grantees who can access the file reference under the history reference async handleGrantees( batchId: string | BatchId, - fileRef: string, + file: FileWithMetadata, grantees: { add?: string[]; revoke?: string[]; @@ -674,9 +672,23 @@ export class FileManager { console.log('Access granted, new grantee list reference: ', grantResult.ref); } - const currentGrantees = this.granteeList.filesSharedWith.get(fileRef) || []; - const newGrantees = [...new Set([...currentGrantees, ...(grantees.add || []), ...(grantees.revoke || [])])]; - this.granteeList.filesSharedWith.set(fileRef, newGrantees); + // TODO: how to handle sharing: base fileref remains but the latest & encrypted ref that is shared changes -> versioning ?? + const currentGranteesIx = this.granteeLists.findIndex((glref) => glref === file.eGlRef); + if (currentGranteesIx === -1) { + this.granteeLists.push(grantResult.ref); + } else { + this.granteeLists[currentGranteesIx] = grantResult.ref; + // TODO: maybe don't need to check if upload + patch happens at the same time -> add to import ? + const fIx = this.importedFiles.findIndex((f) => f.reference === file.reference); + if (fIx === -1) { + console.log('Provided file reference not found in imported files: ', file.reference); + return undefined; + } else { + this.importedFiles[fIx].eGlRef = grantResult.ref; + } + + // const newGrantees = [...new Set([...currentGrantees, ...(grantees.add || []), ...(grantees.revoke || [])])]; + } return grantResult; } catch (error: any) { console.error(`Failed to grant share access: ${error}`); @@ -684,9 +696,9 @@ export class FileManager { } } - async subscribeToSharedInbox(): Promise { - const subscription = this.bee.pssSubscribe(SHARED_INBOX_TOPIC, { - onMessage: async (message) => { + subscribeToSharedInbox(): PssSubscription { + return this.bee.pssSubscribe(SHARED_INBOX_TOPIC, { + onMessage: (message) => { console.log('Received shared inbox message: ', message); assertSharedMessage(message); this.sharedWithMe.push(message); @@ -696,8 +708,14 @@ export class FileManager { throw e; }, }); + } - return subscription; + // TODO: do we need to cancel sub at shutdown ? + unsubscribeFromSharedInbox(): void { + if (this.sharedSubscription) { + console.log('Unsubscribed from shared inbox, topic: ', this.sharedSubscription.topic); + this.sharedSubscription.cancel(); + } } // recipient is optional, if not provided the message will be broadcasted == pss public key @@ -708,9 +726,10 @@ export class FileManager { await this.bee.pssSend(batchId, SHARED_INBOX_TOPIC, target, msgData, recipient); } catch (error: any) { console.log('Failed to share item: ', error); + return undefined; } } - + // TODO: maybe store only the encrypted refs for security and use async downloadSharedItem(reference: string): Promise { if (!this.sharedWithMe.find((msg) => msg.references.includes(reference))) { console.log('Cannot find reference in shared messages: ', reference); @@ -718,19 +737,12 @@ export class FileManager { } try { - const data = await this.bee.downloadFile(reference); + // TODO: publisher and history headers + const data = await this.bee.downloadFile(reference, undefined, { headers: { 'swarm-act': 'true' } }); return data.data; } catch (error: any) { console.error(`Failed to download shared file ${reference}\n: ${error}`); return undefined; } } - - // TODO: do we need to cancel sub at shutdown ? - unsubscribeFromSharedInbox() { - if (this.sharedSubscription) { - console.log('Unsubscribed from shared inbox, topic: ', this.sharedSubscription.topic); - this.sharedSubscription.cancel(); - } - } } diff --git a/src/index.ts b/src/index.ts index 3d8df8c..f5c5525 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,7 @@ const wallet = { privateKey: 'af829bfc5c90818776f0b3d587c14c8e7a4222b781e9f7ebc6e527f4dfd117c2', }; -async function main() { +async function main(): Promise { try { console.log('### Simulation: FileManager Operations ###'); diff --git a/src/types.ts b/src/types.ts index 8193bc5..69a32ca 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,16 +2,17 @@ import { BatchId, PostageBatch, Reference } from '@ethersphere/bee-js'; export interface FileWithMetadata { reference: string | Reference; - name: string; batchId: string | BatchId; - shared: boolean; + shared?: boolean; + name?: string; + owner?: string; + eGlRef?: string | Reference; timestamp?: number; - uploader?: string; } export interface StampWithMetadata { stamp: PostageBatch; - fileReferences?: string[] | Reference[]; + references?: string[] | Reference[]; feedReference?: string | Reference; nextIndex?: number; } @@ -20,10 +21,6 @@ export interface StampList { filesOfStamps: Map; } -export interface GranteeList { - filesSharedWith: Map; -} - // TODO: unify own files with shared and add stamp data potentially export interface SharedMessage { owner: string; diff --git a/src/utils.ts b/src/utils.ts index 7f2b33a..0fe0577 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,9 +1,8 @@ -import { Bytes } from '@solarpunkltd/mantaray-js'; import path from 'path'; import { SharedMessage } from './types'; -export function getContentType(filePath: string) { +export function getContentType(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); const contentTypes: Map = new Map([ ['.txt', 'text/plain'], @@ -16,26 +15,6 @@ export function getContentType(filePath: string) { return contentTypes.get(ext) || 'application/octet-stream'; } -export function pathToBytes(s: string) { - return new TextEncoder().encode(s); -} - -export function hexStringToReference(reference: string) { - const bytes = new Uint8Array(Buffer.from(reference, 'hex')); - if (bytes.length !== 32 && bytes.length !== 64) { - throw new Error('Invalid reference length'); - } - return bytes; -} - -export function encodePathToBytes(pathString: string) { - return new TextEncoder().encode(pathString); -} - -export function decodeBytesToPath(bytes: Bytes<32>) { - return new TextDecoder().decode(bytes); -} - export function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object'; } diff --git a/tests/mockHelpers.ts b/tests/mockHelpers.ts index 81aba09..b6d1c8b 100644 --- a/tests/mockHelpers.ts +++ b/tests/mockHelpers.ts @@ -1,5 +1,4 @@ -import { Bee, Reference, Topic } from '@ethersphere/bee-js'; -import { TextDecoder, TextEncoder } from 'util'; +import { Bee, Reference, Topic, Utils } from '@ethersphere/bee-js'; export function createMockBee(): Partial { return { @@ -67,11 +66,11 @@ export function createMockBee(): Partial { export function createMockMantarayNode(customForks: Record = {}): any { const defaultForks: { [key: string]: any } = { file: { - prefix: encodePathToBytes('file'), + prefix: Utils.hexToBytes('file'), node: { forks: { '1.txt': { - prefix: encodePathToBytes('1.txt'), + prefix: Utils.hexToBytes('1.txt'), node: { isValueType: () => true, getEntry: new Uint8Array(Buffer.from('a'.repeat(64), 'hex')), @@ -79,7 +78,7 @@ export function createMockMantarayNode(customForks: Record = {}): a }, }, '2.txt': { - prefix: encodePathToBytes('2.txt'), + prefix: Utils.hexToBytes('2.txt'), node: { isValueType: () => true, getEntry: new Uint8Array(Buffer.from('b'.repeat(64), 'hex')), @@ -95,7 +94,7 @@ export function createMockMantarayNode(customForks: Record = {}): a return { forks: customForks || defaultForks, addFork: jest.fn((path: Uint8Array, reference: Uint8Array) => { - const decodedPath = new TextDecoder().decode(path); + const decodedPath = Utils.bytesToHex(path); console.log(`Mock addFork called with path: ${decodedPath}`); defaultForks[decodedPath] = { prefix: path, @@ -109,7 +108,3 @@ export function createMockMantarayNode(customForks: Record = {}): a }), }; } - -function encodePathToBytes(path: string): Uint8Array { - return path ? new TextEncoder().encode(path) : new Uint8Array(); -} From 819debfda85d6327a6cf19a3d94230ad8a188f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Sat, 11 Jan 2025 22:36:47 +0100 Subject: [PATCH 03/18] improve sharing and act handling --- src/fileManager.ts | 119 +++++++++++++++++++++++++++++++++++++-------- src/types.ts | 3 +- 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/src/fileManager.ts b/src/fileManager.ts index 8418cc1..371dee0 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -1,7 +1,9 @@ import { BatchId, Bee, + BeeRequestOptions, Data, + ENCRYPTED_REFERENCE_HEX_LENGTH, GranteesResult, PostageBatch, PssSubscription, @@ -32,7 +34,7 @@ export class FileManager { private sharedWithMe: SharedMessage[]; private sharedSubscription: PssSubscription; private address: string; - + // TODO: is this.mantaray needed ? always a new mantaray instance is created when wokring on an item constructor(beeUrl: string, privateKey: string) { if (!beeUrl) { throw new Error('Bee URL is required for initializing the FileManager.'); @@ -222,6 +224,7 @@ export class FileManager { return this.stampList; } + // TODO: only download metadata files for listing -> only download the whole file on demand async importReferences(referenceList: Reference[], batchId?: string, isLocal = false): Promise { const processPromises = referenceList.map(async (item: any) => { const reference: Reference = isLocal ? item.hash : item; @@ -229,7 +232,25 @@ export class FileManager { console.log(`Processing reference: ${reference}`); // Download the file to extract its metadata - const fileData = await this.bee.downloadFile(reference); + + // TODO: act headers + const options: BeeRequestOptions = {}; + // if (file.reference.length === ENCRYPTED_REFERENCE_HEX_LENGTH) { + // if (file.historyRef !== undefined) { + // options.headers = { 'swarm-act-history-address': file.historyRef }; + // } + // if (file.owner !== undefined) { + // options.headers = { + // ...options.headers, + // 'swarm-act-publisher': file.owner, + // }; + // } + // if (file.timestamp !== undefined) { + // options.headers = { ...options.headers, 'swarm-act-timestamp': file.timestamp.toString() }; + // } + // } + + const fileData = await this.bee.downloadFile(reference, undefined, options); const content = Buffer.from(fileData.data.toString() || ''); const fileName = fileData.name || `pinned-${reference.substring(0, 6)}`; const contentType = fileData.contentType || 'application/octet-stream'; @@ -360,7 +381,7 @@ export class FileManager { return validResults; // Return successful download results } - // TODO: always upload with ACT, only adding the publisher as grantee first, (by defualt) then when shared add the grantees + // TODO: always upload with ACT, only adding the publisher as grantee first (by defualt), then when shared, add the grantees // TODO: store filerefs with the historyrefs async uploadFile( file: string, @@ -443,7 +464,8 @@ export class FileManager { mantaray.addFork(bytesPath, Utils.hexToBytes(reference), metadataWithOriginalName); } - async saveMantaray(mantaray: MantarayNode | undefined, stamp: string | BatchId): Promise { + // TODO: problem: mantary impl. is old and does not return the history address + async saveMantaray(mantaray: MantarayNode | undefined, stamp: string | BatchId): Promise { mantaray = mantaray || this.mantaray; console.log('Saving Mantaray manifest...'); @@ -457,11 +479,10 @@ export class FileManager { return Utils.hexToBytes(uploadResponse.reference); }; - const manifestReference = await mantaray.save(saveFunction); + const manifestReference = Utils.bytesToHex(await mantaray.save(saveFunction)); - const hexReference = Buffer.from(manifestReference).toString('hex'); - console.log(`Mantaray manifest saved with reference: ${hexReference}`); - return hexReference; + console.log(`Mantaray manifest saved with reference: ${manifestReference}`); + return { eRef: manifestReference /*hRef: uploadResponse.historyAddress */ }; } listFiles(mantaray: MantarayNode | undefined, includeMetadata = false): any { @@ -643,7 +664,7 @@ export class FileManager { // TODO: create a feed just like for the stamps to store the grantee list refs // TODO: create a feed for the share access that can be read by each grantee // TODO: notify user if it has been granted access by someone else - // TODO: history handling ? + // TODO: stamp of the file vs grantees stamp? // updates the list of grantees who can access the file reference under the history reference async handleGrantees( batchId: string | BatchId, @@ -718,30 +739,88 @@ export class FileManager { } } - // recipient is optional, if not provided the message will be broadcasted == pss public key - async shareItem(batchId: string, targetOverlay: string, message: SharedMessage, recipient: string): Promise { + // TODO: allsettled + // TODO: history handling ? -> bee-js: is historyref mandatory ? patch can create a granteelist and update it in place + async shareItems( + batchId: string, + targetOverlays: string[], + item: SharedMessage, + recipients: string[], + ): Promise { try { - const target = Utils.makeMaxTarget(targetOverlay); - const msgData = new Uint8Array(Buffer.from(JSON.stringify(message))); - await this.bee.pssSend(batchId, SHARED_INBOX_TOPIC, target, msgData, recipient); + for (const ref of item.references) { + const file = this.importedFiles.find((f) => f.reference === ref); + if (file === undefined) { + console.log('File not found for reference: ', ref); + continue; + } + if (file.historyRef === undefined) { + console.log('History not found for reference: ', ref); + continue; + } + + await this.handleGrantees(batchId, { reference: ref }, { add: recipients }, file.historyRef, file.eGlRef); + } + + await this.sendShareMessage(batchId, targetOverlays, item, recipients); } catch (error: any) { - console.log('Failed to share item: ', error); + console.log('Failed to share items: ', error); + return undefined; + } + } + + // recipient is optional, if not provided the message will be broadcasted == pss public key + async sendShareMessage( + batchId: string, + targetOverlays: string[], + item: SharedMessage, + recipients: string[], + ): Promise { + // TODO: valid length check of recipient and target + if (recipients.length === 0 || recipients.length !== targetOverlays.length) { + console.log('Invalid recipients or targetoverlays specified for sharing.'); return undefined; } + + for (let i = 0; i < recipients.length; i++) { + try { + const target = Utils.makeMaxTarget(targetOverlays[i]); + const msgData = new Uint8Array(Buffer.from(JSON.stringify(item))); + await this.bee.pssSend(batchId, SHARED_INBOX_TOPIC, target, msgData, recipients[i]); + } catch (error: any) { + console.log(`Failed to share item with recipient: ${recipients[i]}\n `, error); + } + } } // TODO: maybe store only the encrypted refs for security and use - async downloadSharedItem(reference: string): Promise { - if (!this.sharedWithMe.find((msg) => msg.references.includes(reference))) { - console.log('Cannot find reference in shared messages: ', reference); + async downloadSharedItem(file: FileWithMetadata, path?: string): Promise { + if (!this.sharedWithMe.find((msg) => msg.references.includes(file.reference))) { + console.log('Cannot find reference in shared messages: ', file.reference); return undefined; } + const options: BeeRequestOptions = {}; + if (file.reference.length === ENCRYPTED_REFERENCE_HEX_LENGTH) { + if (file.historyRef !== undefined) { + options.headers = { 'swarm-act-history-address': file.historyRef }; + } + if (file.owner !== undefined) { + options.headers = { + ...options.headers, + 'swarm-act-publisher': file.owner, + }; + } + if (file.timestamp !== undefined) { + options.headers = { ...options.headers, 'swarm-act-timestamp': file.timestamp.toString() }; + } + } + try { // TODO: publisher and history headers - const data = await this.bee.downloadFile(reference, undefined, { headers: { 'swarm-act': 'true' } }); + const data = await this.bee.downloadFile(file.reference, path, options); return data.data; } catch (error: any) { - console.error(`Failed to download shared file ${reference}\n: ${error}`); + console.error(`Failed to download shared file ${file.reference}\n: ${error}`); return undefined; } } diff --git a/src/types.ts b/src/types.ts index 69a32ca..4ad9ed2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,11 +2,12 @@ import { BatchId, PostageBatch, Reference } from '@ethersphere/bee-js'; export interface FileWithMetadata { reference: string | Reference; - batchId: string | BatchId; + batchId?: string | BatchId; shared?: boolean; name?: string; owner?: string; eGlRef?: string | Reference; + historyRef?: string | Reference; timestamp?: number; } From cfc25cac73fea15f01179f376ac81cc3b09aff47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Sun, 12 Jan 2025 22:16:22 +0100 Subject: [PATCH 04/18] update metadata feed during share --- src/constants.ts | 1 + src/fileManager.ts | 118 +++++++++++++++++++++++++++++++-------------- 2 files changed, 84 insertions(+), 35 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 2dbe1af..538c34c 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,4 +2,5 @@ const feedTypes = ['sequence', 'epoch'] as const; export type FeedType = (typeof feedTypes)[number]; export const DEFAULT_FEED_TYPE: FeedType = 'sequence'; export const STAMP_LIST_TOPIC = 'stamps'; +export const METADATA_TOPIC = 'filemetadata'; export const SHARED_INBOX_TOPIC = 'shared-inbox'; diff --git a/src/fileManager.ts b/src/fileManager.ts index 371dee0..6644f8d 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -16,7 +16,7 @@ import { Wallet } from 'ethers'; import { readFileSync } from 'fs'; import path from 'path'; -import { DEFAULT_FEED_TYPE, SHARED_INBOX_TOPIC, STAMP_LIST_TOPIC } from './constants'; +import { DEFAULT_FEED_TYPE, METADATA_TOPIC, SHARED_INBOX_TOPIC, STAMP_LIST_TOPIC } from './constants'; import { FileWithMetadata, SharedMessage, StampList, StampWithMetadata } from './types'; import { assertSharedMessage, getContentType } from './utils'; @@ -108,27 +108,22 @@ export class FileManager { } // TODO: method to list new stamp with files - // TODO: encrypt // TODO: how and how long to store the stamps feed data ? // TODO: it seems inefficient to update always with the whole fileref array - async updateStampData(stamp: string | BatchId, privateKey: string): Promise { + async updateStampData(stamp: string | BatchId): Promise { const topicHex = this.bee.makeFeedTopic(STAMP_LIST_TOPIC); - const feedWriter = this.bee.makeFeedWriter( - DEFAULT_FEED_TYPE, - topicHex, - privateKey /*, { headers: { encrypt: "true" } }*/, - ); + const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); try { const stampData = { filesOfStamps: this.stampList.map((s) => [s.stamp.batchID, s.references]), } as unknown as StampList; - const stampListDataRef = await this.bee.uploadData(stamp, JSON.stringify(stampData)); - const writeResult = await feedWriter.upload(stamp, stampListDataRef.reference, { + const uploadResult = await this.bee.uploadData(stamp, JSON.stringify(stampData), { encrypt: true }); + const writeResult = await feedWriter.upload(stamp, uploadResult.reference, { index: this.nextStampFeedIndex, }); console.log('Stamp feed updated: ', writeResult.reference); } catch (error: any) { - console.error(`Failed to download feed update: ${error}`); + console.error(`Failed to update stamp feed: ${error}`); return; } } @@ -164,9 +159,9 @@ export class FileManager { } } } - console.log('File referene list fetched from feed.'); + console.log('Stamps fetched from feed.'); } catch (error: any) { - console.error(`Failed to fetch file reference list from feed: ${error}`); + console.error(`Failed to fetch stamps from feed: ${error}`); return; } } @@ -174,7 +169,6 @@ export class FileManager { async getUsableStamps(): Promise { try { const stamps = (await this.bee.getAllPostageBatch()).filter((s) => s.usable); - // TOOD: files as importedFiles return stamps.map((s) => ({ stamp: s, files: [] })); } catch (error: any) { console.error(`Failed to get usable stamps: ${error}`); @@ -201,7 +195,7 @@ export class FileManager { }); } - async getLocalStamp(batchId: string | BatchId): Promise { + async getCachedStamp(batchId: string | BatchId): Promise { return this.stampList.find((s) => s.stamp.batchID === batchId); } @@ -249,8 +243,9 @@ export class FileManager { // options.headers = { ...options.headers, 'swarm-act-timestamp': file.timestamp.toString() }; // } // } - - const fileData = await this.bee.downloadFile(reference, undefined, options); + // TODO: maybe use path to get the rootmetadata and store it locally + const path = '/rootmetadata.json'; + const fileData = await this.bee.downloadFile(reference, path, options); const content = Buffer.from(fileData.data.toString() || ''); const fileName = fileData.name || `pinned-${reference.substring(0, 6)}`; const contentType = fileData.contentType || 'application/octet-stream'; @@ -275,7 +270,7 @@ export class FileManager { // Track imported files // TODO: eglref, shared, timestamp -> mantaray root metadata - this.importedFiles.push({ reference: reference, name: fileName, batchId: batchId || '', shared: false }); + this.importedFiles.push({ reference: reference, name: fileName, batchId: batchId, shared: undefined }); } catch (error: any) { console.error(`[ERROR] Failed to process reference ${reference}: ${error.message}`); } @@ -417,10 +412,10 @@ export class FileManager { const uploadResponse = await this.bee.uploadFile(stamp, fileData, fileName, uploadHeaders); this.addToMantaray(mantaray, uploadResponse.reference, metadata); - if (save) { - console.log('Saving Mantaray node...'); - await this.saveMantaray(mantaray, stamp); - } + // if (save) { + console.log('Saving Mantaray node...'); + const { eRef, hRef } = await this.saveMantaray(mantaray, stamp); + // } // TODO: handle stamplist and filelist here const stampIx = this.stampList.findIndex((s) => s.stamp.batchID === stamp); @@ -431,17 +426,17 @@ export class FileManager { throw new Error(`Stamp not found: ${stamp}`); } - this.stampList.push({ stamp: newStamp, references: [uploadResponse.reference] }); + this.stampList.push({ stamp: newStamp, references: [eRef] }); } else if (this.stampList[stampIx].references === undefined) { - this.stampList[stampIx].references = [uploadResponse.reference]; + this.stampList[stampIx].references = [eRef]; } else { - this.stampList[stampIx].references.push(uploadResponse.reference); + this.stampList[stampIx].references.push(eRef); } - await this.updateStampData(stamp, this.privateKey); + await this.updateStampData(stamp); - console.log(`File uploaded successfully: ${file}, Reference: ${uploadResponse.reference}`); - return uploadResponse.reference; + console.log(`File uploaded successfully: ${file}, Reference: ${eRef}`); + return eRef; } catch (error: any) { console.error(`[ERROR] Failed to upload file ${file}: ${error.message}`); throw error; @@ -465,7 +460,7 @@ export class FileManager { } // TODO: problem: mantary impl. is old and does not return the history address - async saveMantaray(mantaray: MantarayNode | undefined, stamp: string | BatchId): Promise { + async saveMantaray(mantaray: MantarayNode | undefined, stamp: string | BatchId): Promise { mantaray = mantaray || this.mantaray; console.log('Saving Mantaray manifest...'); @@ -482,7 +477,7 @@ export class FileManager { const manifestReference = Utils.bytesToHex(await mantaray.save(saveFunction)); console.log(`Mantaray manifest saved with reference: ${manifestReference}`); - return { eRef: manifestReference /*hRef: uploadResponse.historyAddress */ }; + return { eRef: manifestReference, hRef: saveResult.historyAddress }; } listFiles(mantaray: MantarayNode | undefined, includeMetadata = false): any { @@ -660,6 +655,29 @@ export class FileManager { return await this.getGrantees(file.eGlRef); } + async updateFileMetadata(file: FileWithMetadata): Promise { + if (!file.batchId) { + console.error('No batchId provided for file metadata update.'); + return; + } + + const topicHex = this.bee.makeFeedTopic(METADATA_TOPIC + file.reference); + const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); + try { + const uploadResult = await this.bee.uploadData(file.batchId, JSON.stringify(file), { + encrypt: true, + }); + const writeResult = await feedWriter.upload(file.batchId, uploadResult.reference, { + index: undefined, // todo: keep track of the latest index ?? + }); + console.log('File metadata feed updated: ', writeResult.reference); + return writeResult.reference; + } catch (error: any) { + console.error(`Failed to update file metadata feed: ${error}`); + return undefined; + } + } + // TODO: separate revoke function or frontend will handle it by creating a new act ? // TODO: create a feed just like for the stamps to store the grantee list refs // TODO: create a feed for the share access that can be read by each grantee @@ -707,9 +725,9 @@ export class FileManager { } else { this.importedFiles[fIx].eGlRef = grantResult.ref; } - - // const newGrantees = [...new Set([...currentGrantees, ...(grantees.add || []), ...(grantees.revoke || [])])]; } + + console.log('Grantees updated: ', grantResult); return grantResult; } catch (error: any) { console.error(`Failed to grant share access: ${error}`); @@ -743,12 +761,15 @@ export class FileManager { // TODO: history handling ? -> bee-js: is historyref mandatory ? patch can create a granteelist and update it in place async shareItems( batchId: string, + references: Reference[], targetOverlays: string[], - item: SharedMessage, recipients: string[], + message?: string, ): Promise { try { - for (const ref of item.references) { + const historyRefs = new Array(references.length); + for (let i = 0; i < references.length; i++) { + const ref = references[i]; const file = this.importedFiles.find((f) => f.reference === ref); if (file === undefined) { console.log('File not found for reference: ', ref); @@ -758,10 +779,37 @@ export class FileManager { console.log('History not found for reference: ', ref); continue; } + // TODO: how to update file metadata with new eglref ? -> filemetadata = /feed/decryptedref/metadata + const grantResult = await this.handleGrantees( + batchId, + { reference: ref }, + { add: recipients }, + file.historyRef, + file.eGlRef, + ); - await this.handleGrantees(batchId, { reference: ref }, { add: recipients }, file.historyRef, file.eGlRef); + if (grantResult !== undefined) { + const feedMetadatRef = await this.updateFileMetadata({ + ...file, + eGlRef: grantResult.ref, + historyRef: grantResult.historyref, + }); + + if (feedMetadatRef !== undefined) { + historyRefs[i] = grantResult.historyref; + } else { + console.log('Failed to update file metadata: ', ref); + } + } } + const item = { + owner: this.address, + references: historyRefs, + timestamp: Date.now(), + message: message, + } as SharedMessage; + await this.sendShareMessage(batchId, targetOverlays, item, recipients); } catch (error: any) { console.log('Failed to share items: ', error); From 321f55ca8453039d5899b4986e679dbf1d929355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Wed, 15 Jan 2025 00:57:13 +0100 Subject: [PATCH 05/18] start to rework metadata and feed handling --- src/constants.ts | 5 +- src/fileManager.ts | 333 +++++++++++++++++++++++++-------------------- src/types.ts | 22 +-- 3 files changed, 201 insertions(+), 159 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 538c34c..17ec754 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,7 @@ const feedTypes = ['sequence', 'epoch'] as const; export type FeedType = (typeof feedTypes)[number]; export const DEFAULT_FEED_TYPE: FeedType = 'sequence'; -export const STAMP_LIST_TOPIC = 'stamps'; -export const METADATA_TOPIC = 'filemetadata'; +export const REFERENCE_LIST_TOPIC = 'reference-list'; +export const METADATA_TOPIC = 'metadata'; export const SHARED_INBOX_TOPIC = 'shared-inbox'; +export const OWNER_FEED_STAMP_LABEL = 'owner-stamp'; diff --git a/src/fileManager.ts b/src/fileManager.ts index 99d817b..4f5c5ba 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -4,11 +4,14 @@ import { BeeRequestOptions, Data, ENCRYPTED_REFERENCE_HEX_LENGTH, + FileUploadOptions, GranteesResult, PostageBatch, PssSubscription, + RedundancyLevel, Reference, REFERENCE_HEX_LENGTH, + UploadRedundancyOptions, Utils, } from '@ethersphere/bee-js'; import { MantarayNode, MetadataMapping, Reference as MantarayRef } from '@solarpunkltd/mantaray-js'; @@ -16,8 +19,14 @@ import { Wallet } from 'ethers'; import { readFileSync } from 'fs'; import path from 'path'; -import { DEFAULT_FEED_TYPE, METADATA_TOPIC, SHARED_INBOX_TOPIC, STAMP_LIST_TOPIC } from './constants'; -import { FileWithMetadata, SharedMessage, StampList, StampWithMetadata } from './types'; +import { + DEFAULT_FEED_TYPE, + METADATA_TOPIC, + OWNER_FEED_STAMP_LABEL, + REFERENCE_LIST_TOPIC, + SHARED_INBOX_TOPIC, +} from './constants'; +import { MetadataFile, OwnerFeedData, SharedMessage } from './types'; import { assertSharedMessage, decodeBytesToPath, encodePathToBytes, getContentType } from './utils'; export class FileManager { @@ -25,10 +34,11 @@ export class FileManager { // TODO: store shared refs and own files in the same array ? public bee: Bee; public mantaray: MantarayNode; - public importedFiles: FileWithMetadata[]; + public importedFiles: MetadataFile[]; - private stampList: StampWithMetadata[]; - private nextStampFeedIndex: string; + private stampList: PostageBatch[]; + private metadataFileList: MetadataFile[]; + private nextOwnerFeedIndex: string; private wallet: Wallet; private privateKey: string; private granteeLists: string[]; @@ -49,18 +59,21 @@ export class FileManager { console.log('Initializing Bee client...'); this.bee = new Bee(beeUrl); this.stampList = []; - this.nextStampFeedIndex = ''; + this.metadataFileList = []; + this.nextOwnerFeedIndex = ''; this.privateKey = privateKey; this.wallet = new Wallet(privateKey); this.address = this.wallet.address; - this.topic = Utils.bytesToHex(Utils.keccak256Hash(STAMP_LIST_TOPIC)); + this.topic = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); this.mantaray = new MantarayNode(); this.importedFiles = []; + this.granteeLists = []; this.sharedWithMe = []; this.sharedSubscription = {} as PssSubscription; } + // Start init methods // TODO: use allSettled for file fetching and only save the ones that are successful async initialize(items: any | undefined): Promise { try { @@ -69,22 +82,30 @@ export class FileManager { console.log('Error during shared inbox subscription: ', error); } - console.log('Importing stamps and references...'); try { + console.log('Importing stamps...'); await this.initStamps(); - if (this.stampList.length > 0) { - console.log('Using stamp list for initialization.'); - for (const elem of this.stampList) { - if (elem.references !== undefined && elem.references.length > 0) { - await this.importReferences(elem.references as Reference[], elem.stamp.batchID); - } - } - } } catch (error: any) { console.error(`[ERROR] Failed to initialize stamps: ${error.message}`); throw error; } + try { + console.log('Importing metadata of files...'); + await this.initMetadataFileList(); + } catch (error: any) { + console.error(`[ERROR] Failed to initialize file metadata: ${error.message}`); + throw error; + } + + // if stamp is not found than the file cannot be downloaded? is this necessary ?? + for (const stamp of this.stampList) { + const mtdtIx = this.metadataFileList.findIndex((f) => stamp.batchID === f.batchId); + if (mtdtIx === undefined) { + this.metadataFileList.splice(mtdtIx, 1); + } + } + try { if (items) { console.log('Using provided items for initialization.'); @@ -99,10 +120,47 @@ export class FileManager { throw error; } } - // TODO: is this method necessary ? - async intializeMantarayUsingFeed(): Promise { - // + + // TODO: import other stamps in order to topup: owner(s) ? + async initStamps(): Promise { + try { + this.stampList = await this.getUsableStamps(); + console.log('Usable stamps fetched successfully.'); + } catch (error: any) { + console.error(`Failed to fetch stamps: ${error}`); + throw error; + } + } + + // TODO: shared file feed similarly + // TODO: util func to make options for act headers + async initMetadataFileList(): Promise { + const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); + const feedReader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topicHex, this.address); + try { + const latestFeedData = await feedReader.download(); + this.nextOwnerFeedIndex = latestFeedData.feedIndexNext; + + const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); + const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as OwnerFeedData; + const options: BeeRequestOptions = { + headers: { 'swarm-act-history-address': ownerFeedData.history, 'swarm-act-publisher': this.address }, + } as const; + // TODO: act encrpyt the metadatalist refs?? + const metadataList = JSON.parse( + (await this.bee.downloadData(ownerFeedData.metadataListReference, options)).text(), + ) as MetadataFile[]; + for (const mtdt of metadataList) { + const metadata = JSON.parse((await this.bee.downloadData(mtdt.reference)).text()) as MetadataFile; + this.metadataFileList.push(metadata); + } + console.log('Stamps fetched from feed.'); + } catch (error: any) { + console.error(`Failed to fetch stamps from feed: ${error}`); + return; + } } + // End init methods async loadMantaray(manifestReference: Reference): Promise { const loadFunction = async (address: MantarayRef): Promise => { @@ -166,87 +224,28 @@ export class FileManager { } } - // TODO: method to list new stamp with files - // TODO: how and how long to store the stamps feed data ? - // TODO: it seems inefficient to update always with the whole fileref array - async updateStampData(stamp: string | BatchId): Promise { - const topicHex = this.bee.makeFeedTopic(STAMP_LIST_TOPIC); - const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); - try { - const stampData = { - filesOfStamps: this.stampList.map((s) => [s.stamp.batchID, s.references]), - } as unknown as StampList; - const uploadResult = await this.bee.uploadData(stamp, JSON.stringify(stampData), { encrypt: true }); - const writeResult = await feedWriter.upload(stamp, uploadResult.reference, { - index: this.nextStampFeedIndex, - }); - console.log('Stamp feed updated: ', writeResult.reference); - } catch (error: any) { - console.error(`Failed to update stamp feed: ${error}`); - return; - } - } - - // TODO: fetch usable stamps or read from feed - // TODO: import other stamps in order to topup: owner(s) ? - async initStamps(): Promise { - try { - this.stampList = await this.getUsableStamps(); - console.log('Usable stamps fetched successfully.'); - } catch (error: any) { - console.error(`Failed to update stamps: ${error}`); - throw error; - } - - // TODO: stamps of other users -> feature to fetch other nodes' stamp data - - const topicHex = this.bee.makeFeedTopic(STAMP_LIST_TOPIC); - const feedReader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topicHex, this.address); + // Start stamp methods + async getUsableStamps(): Promise { try { - const latestFeedData = await feedReader.download(); - this.nextStampFeedIndex = latestFeedData.feedIndexNext; - const stampListData = (await this.bee.downloadData(latestFeedData.reference)).text(); - const stampList = JSON.parse(stampListData) as StampList; - for (const [batchId, fileRefs] of stampList.filesOfStamps) { - // if (this.stampList.find((s) => s.stamp.batchID === stamp.stamp.batchID) === undefined) { - // await this.fetchStamp(stamp.stamp.batchID); - // } - const stampIx = this.stampList.findIndex((s) => s.stamp.batchID === batchId); - if (stampIx !== -1) { - if (fileRefs.length > 0) { - this.stampList[stampIx].references = [...fileRefs]; - } - } - } - console.log('Stamps fetched from feed.'); - } catch (error: any) { - console.error(`Failed to fetch stamps from feed: ${error}`); - return; - } - } - - async getUsableStamps(): Promise { - try { - const stamps = (await this.bee.getAllPostageBatch()).filter((s) => s.usable); - return stamps.map((s) => ({ stamp: s, files: [] })); + return (await this.bee.getAllPostageBatch()).filter((s) => s.usable); } catch (error: any) { console.error(`Failed to get usable stamps: ${error}`); return []; } } - async filterBatches(ttl?: number, utilization?: number, capacity?: number): Promise { + async filterBatches(ttl?: number, utilization?: number, capacity?: number): Promise { // TODO: clarify depth vs capacity return this.stampList.filter((s) => { - if (utilization !== undefined && s.stamp.utilization <= utilization) { + if (utilization !== undefined && s.utilization <= utilization) { return false; } - if (capacity !== undefined && s.stamp.depth <= capacity) { + if (capacity !== undefined && s.depth <= capacity) { return false; } - if (ttl !== undefined && s.stamp.batchTTL <= ttl) { + if (ttl !== undefined && s.batchTTL <= ttl) { return false; } @@ -254,8 +253,16 @@ export class FileManager { }); } - async getCachedStamp(batchId: string | BatchId): Promise { - return this.stampList.find((s) => s.stamp.batchID === batchId); + async getStamps(): Promise { + return this.stampList; + } + + async getOwnerFeedStamp(): Promise { + return this.stampList.find((s) => s.label === OWNER_FEED_STAMP_LABEL); + } + + async getCachedStamp(batchId: string | BatchId): Promise { + return this.stampList.find((s) => s.batchID === batchId); } async fetchStamp(batchId: string | { batchID: string }): Promise { @@ -263,7 +270,7 @@ export class FileManager { const id = typeof batchId === 'string' ? batchId : batchId.batchID; const newStamp = await this.bee.getPostageBatch(id); if (newStamp?.exists && newStamp.usable) { - this.stampList.push({ stamp: newStamp }); + this.stampList.push(newStamp); return newStamp; } return undefined; @@ -272,10 +279,7 @@ export class FileManager { return undefined; } } - - async getStamps(): Promise { - return this.stampList; - } + // End stamp methods // TODO: only download metadata files for listing -> only download the whole file on demand async importReferences(referenceList: Reference[], batchId?: string, isLocal = false): Promise { @@ -449,10 +453,9 @@ export class FileManager { async uploadFile( file: string, mantaray: MantarayNode | undefined, - stamp: string | BatchId, + batchId: string | BatchId, customMetadata = {}, - redundancyLevel = '1', - save = true, + redundancyLevel = RedundancyLevel.MEDIUM, ): Promise { mantaray = mantaray || this.mantaray; console.log(`Uploading file: ${file}`); @@ -469,39 +472,60 @@ export class FileManager { }; const uploadHeaders = { - contentType, + contentType: contentType, act: true, - headers: { - 'swarm-redundancy-level': redundancyLevel, - }, - }; + redundancyLevel: redundancyLevel, + } as FileUploadOptions & UploadRedundancyOptions; try { - const uploadResponse = await this.bee.uploadFile(stamp, fileData, fileName, uploadHeaders); + const uploadResponse = await this.bee.uploadFile(batchId, fileData, fileName, uploadHeaders); this.addToMantaray(mantaray, uploadResponse.reference, metadata); - // if (save) { - console.log('Saving Mantaray node...'); - const { eRef, hRef } = await this.saveMantaray(mantaray, stamp); - // } - - // TODO: handle stamplist and filelist here - const stampIx = this.stampList.findIndex((s) => s.stamp.batchID === stamp); - if (stampIx === -1) { - const newStamp = await this.fetchStamp(stamp); - // TODO: what to do here ? batch should already be usable - if (newStamp === undefined) { - throw new Error(`Stamp not found: ${stamp}`); - } + const metadataFile = { + reference: uploadResponse.reference, + batchId: batchId, + name: fileName, + owner: this.address, + historyRef: uploadResponse.historyAddress, + timestamp: new Date().getTime(), + eGlRef: undefined, + } as MetadataFile; - this.stampList.push({ stamp: newStamp, references: [eRef] }); - } else if (this.stampList[stampIx].references === undefined) { - this.stampList[stampIx].references = [eRef]; - } else { - this.stampList[stampIx].references.push(eRef); + // TODO: always update everything with the same redundancy level ?? + try { + const uploadMetadataResponse = await this.bee.uploadFile( + batchId, + JSON.stringify(metadataFile), + 'metadata.json', + { + contentType: 'application/json', + act: true, + redundancyLevel: redundancyLevel, + }, + ); + const topicHex = this.bee.makeFeedTopic(uploadMetadataResponse.reference); + const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); + + const uploadResult = await this.bee.uploadData(batchId, uploadMetadataResponse.historyAddress); + const feedWriteResult = await feedWriter.upload(batchId, uploadResult.reference, { + index: undefined, // todo: keep track of the latest index ?? + act: true, + }); + + console.log('File metadata feed updated: ', feedWriteResult.reference); + this.addToMantaray(mantaray, feedWriteResult.reference, metadata); + } catch (error: any) { + console.error(`Failed to update file metadata feed: ${error}`); } - await this.updateStampData(stamp); + // TODO: wrapped mantaray + await this.updateWrappedMantaray(); + this.metadataFileList.push(metadataFile); + + console.log('Saving Mantaray node...'); + const { eRef, hRef } = await this.saveMantaray(mantaray, batchId); + + await this.updateMetadataFileList(hRef); console.log(`File uploaded successfully: ${file}, Reference: ${eRef}`); return eRef; @@ -751,6 +775,42 @@ export class FileManager { return contents; } + // Start feed handler methods + async updateWrappedMantaray(): Promise {} + + async updateMetadataFileList(): Promise { + const ownerFeedStamp = await this.getOwnerFeedStamp(); + if (!ownerFeedStamp) { + console.error('Owner feed stamp is not found.'); + return; + } + + const ownerFeedTopicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); + const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, ownerFeedTopicHex, this.privateKey); + try { + const uploadResult = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(this.metadataFileList), { + act: true, + }); + const ownerFeedData = { + history: uploadResult.historyAddress, + metadataListReference: uploadResult.reference, + } as OwnerFeedData; + const ownerFeedRawDataUploadResult = await this.bee.uploadData( + ownerFeedStamp.batchID, + JSON.stringify(ownerFeedData), + ); + const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawDataUploadResult.reference, { + index: this.nextOwnerFeedIndex, + }); + console.log('Metdata and owner feed updated: ', writeResult.reference); + } catch (error: any) { + console.error(`Failed to update metdata and owner feed: ${error}`); + return; + } + } + // Start feed handler methods + + // Start grantee methods // fetches the list of grantees under the given reference async getGrantees(eGlRef: string | Reference): Promise { if (eGlRef.length !== REFERENCE_HEX_LENGTH) { @@ -774,9 +834,10 @@ export class FileManager { } } + // TODO: do not store encrypted grantee ref in the wrapedd mantaray just in the owner mtdt feed // fetches the list of grantees who can access the file reference async getGranteesOfFile(fileRef: string | Reference): Promise { - const file = this.importedFiles.find((f) => f.reference === fileRef); + const file = this.metadataFileList.find((f) => f.reference === fileRef); if (file === undefined || file.eGlRef === undefined) { console.error('File or grantee ref not found for reference: ', fileRef); return undefined; @@ -784,29 +845,6 @@ export class FileManager { return await this.getGrantees(file.eGlRef); } - async updateFileMetadata(file: FileWithMetadata): Promise { - if (!file.batchId) { - console.error('No batchId provided for file metadata update.'); - return; - } - - const topicHex = this.bee.makeFeedTopic(METADATA_TOPIC + file.reference); - const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); - try { - const uploadResult = await this.bee.uploadData(file.batchId, JSON.stringify(file), { - encrypt: true, - }); - const writeResult = await feedWriter.upload(file.batchId, uploadResult.reference, { - index: undefined, // todo: keep track of the latest index ?? - }); - console.log('File metadata feed updated: ', writeResult.reference); - return writeResult.reference; - } catch (error: any) { - console.error(`Failed to update file metadata feed: ${error}`); - return undefined; - } - } - // TODO: separate revoke function or frontend will handle it by creating a new act ? // TODO: create a feed just like for the stamps to store the grantee list refs // TODO: create a feed for the share access that can be read by each grantee @@ -815,7 +853,7 @@ export class FileManager { // updates the list of grantees who can access the file reference under the history reference async handleGrantees( batchId: string | BatchId, - file: FileWithMetadata, + file: MetadataFile, grantees: { add?: string[]; revoke?: string[]; @@ -863,7 +901,9 @@ export class FileManager { return undefined; } } + // End grantee methods + // Start share methods subscribeToSharedInbox(): PssSubscription { return this.bee.pssSubscribe(SHARED_INBOX_TOPIC, { onMessage: (message) => { @@ -970,7 +1010,7 @@ export class FileManager { } } // TODO: maybe store only the encrypted refs for security and use - async downloadSharedItem(file: FileWithMetadata, path?: string): Promise { + async downloadSharedItem(file: MetadataFile, path?: string): Promise { if (!this.sharedWithMe.find((msg) => msg.references.includes(file.reference))) { console.log('Cannot find reference in shared messages: ', file.reference); return undefined; @@ -1001,4 +1041,5 @@ export class FileManager { return undefined; } } + // End share methods } diff --git a/src/types.ts b/src/types.ts index 4ad9ed2..c1d896a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,8 +1,9 @@ -import { BatchId, PostageBatch, Reference } from '@ethersphere/bee-js'; +import { BatchId, Reference } from '@ethersphere/bee-js'; -export interface FileWithMetadata { +// TODO: maybe rename to FileInfo +export interface MetadataFile { reference: string | Reference; - batchId?: string | BatchId; + batchId: string | BatchId; shared?: boolean; name?: string; owner?: string; @@ -11,16 +12,15 @@ export interface FileWithMetadata { timestamp?: number; } -export interface StampWithMetadata { - stamp: PostageBatch; - references?: string[] | Reference[]; - feedReference?: string | Reference; - nextIndex?: number; +export interface OwnerFeedData { + history: string; + metadataListReference: string; } -export interface StampList { - filesOfStamps: Map; -} +// export interface MetadataFile { +// history: string; +// metadataReference: string; +// } // TODO: unify own files with shared and add stamp data potentially export interface SharedMessage { From a9b69562431847ca8a1f24ea5ba580348df5c076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Wed, 15 Jan 2025 18:19:28 +0100 Subject: [PATCH 06/18] upload file saves wrapped mantaray --- src/constants.ts | 6 +- src/fileManager.ts | 313 ++++++++++++++++++++++---------------- src/types.ts | 15 +- src/utils.ts | 19 +++ tests/fileManager.test.ts | 2 +- 5 files changed, 214 insertions(+), 141 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 17ec754..e6e1495 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,6 +2,10 @@ const feedTypes = ['sequence', 'epoch'] as const; export type FeedType = (typeof feedTypes)[number]; export const DEFAULT_FEED_TYPE: FeedType = 'sequence'; export const REFERENCE_LIST_TOPIC = 'reference-list'; -export const METADATA_TOPIC = 'metadata'; +export const FILEINFO_TOPIC = 'fileinfo'; export const SHARED_INBOX_TOPIC = 'shared-inbox'; export const OWNER_FEED_STAMP_LABEL = 'owner-stamp'; +export const INVALID_STMAP = '0'.repeat(64); +export const FILEIINFO_NAME = 'fileinfo.json'; +export const FILEINFO_HISTORY_NAME = 'history.json'; +export const ROOT_PATH = '/'; diff --git a/src/fileManager.ts b/src/fileManager.ts index 0359daf..ad134bb 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -3,7 +3,6 @@ import { Bee, BeeRequestOptions, Data, - ENCRYPTED_REFERENCE_HEX_LENGTH, FileUploadOptions, GranteesResult, PostageBatch, @@ -12,32 +11,42 @@ import { Reference, REFERENCE_HEX_LENGTH, UploadRedundancyOptions, + UploadResultWithCid, Utils, } from '@ethersphere/bee-js'; -import { MantarayNode, MetadataMapping, Reference as MantarayRef } from '@solarpunkltd/mantaray-js'; +import { initManifestNode, MantarayNode, MetadataMapping, Reference as MantarayRef } from '@solarpunkltd/mantaray-js'; import { Wallet } from 'ethers'; import { readFileSync } from 'fs'; import path from 'path'; import { DEFAULT_FEED_TYPE, - METADATA_TOPIC, + FILEIINFO_NAME, + FILEINFO_HISTORY_NAME, + INVALID_STMAP, OWNER_FEED_STAMP_LABEL, REFERENCE_LIST_TOPIC, + ROOT_PATH, SHARED_INBOX_TOPIC, } from './constants'; -import { MetadataFile, OwnerFeedData, SharedMessage } from './types'; -import { assertSharedMessage, decodeBytesToPath, encodePathToBytes, getContentType } from './utils'; +import { FileInfo, FileInfoHistory, OwnerFeedData, SharedMessage } from './types'; +import { + assertSharedMessage, + decodeBytesToPath, + encodePathToBytes, + getContentType, + makeBeeRequestOptions, +} from './utils'; export class FileManager { // TODO: private vars // TODO: store shared refs and own files in the same array ? public bee: Bee; public mantaray: MantarayNode; - public importedFiles: MetadataFile[]; private stampList: PostageBatch[]; - private metadataFileList: MetadataFile[]; + private ownerFileInfoFeedData: object; + private fileInfoList: FileInfo[]; private nextOwnerFeedIndex: string; private wallet: Wallet; private privateKey: string; @@ -59,15 +68,15 @@ export class FileManager { console.log('Initializing Bee client...'); this.bee = new Bee(beeUrl); this.stampList = []; - this.metadataFileList = []; + this.fileInfoList = []; + this.ownerFileInfoFeedData = {}; this.nextOwnerFeedIndex = ''; this.privateKey = privateKey; this.wallet = new Wallet(privateKey); this.address = this.wallet.address; this.topic = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); - this.mantaray = new MantarayNode(); - this.importedFiles = []; + this.mantaray = initManifestNode(); this.granteeLists = []; this.sharedWithMe = []; this.sharedSubscription = {} as PssSubscription; @@ -100,9 +109,9 @@ export class FileManager { // if stamp is not found than the file cannot be downloaded? is this necessary ?? for (const stamp of this.stampList) { - const mtdtIx = this.metadataFileList.findIndex((f) => stamp.batchID === f.batchId); + const mtdtIx = this.fileInfoList.findIndex((f) => stamp.batchID === f.batchId); if (mtdtIx === undefined) { - this.metadataFileList.splice(mtdtIx, 1); + this.fileInfoList.splice(mtdtIx, 1); } } @@ -144,15 +153,15 @@ export class FileManager { const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as OwnerFeedData; const options: BeeRequestOptions = { - headers: { 'swarm-act-history-address': ownerFeedData.history, 'swarm-act-publisher': this.address }, + headers: { 'swarm-act-history-address': ownerFeedData.historyRef, 'swarm-act-publisher': this.address }, } as const; - // TODO: act encrpyt the metadatalist refs?? - const metadataList = JSON.parse( - (await this.bee.downloadData(ownerFeedData.metadataListReference, options)).text(), - ) as MetadataFile[]; - for (const mtdt of metadataList) { - const metadata = JSON.parse((await this.bee.downloadData(mtdt.reference)).text()) as MetadataFile; - this.metadataFileList.push(metadata); + // TODO: act encrpyt the fileInfoList refs?? + const fileInfoList = JSON.parse( + (await this.bee.downloadData(ownerFeedData.wrappedFeedListRef, options)).text(), + ) as FileInfo[]; + for (const fi of fileInfoList) { + const metadata = JSON.parse((await this.bee.downloadData(fi.fileRef)).text()) as FileInfo; + this.fileInfoList.push(metadata); } console.log('Stamps fetched from feed.'); } catch (error: any) { @@ -285,27 +294,14 @@ export class FileManager { async importReferences(referenceList: Reference[], batchId?: string, isLocal = false): Promise { const processPromises = referenceList.map(async (item: any) => { const reference: Reference = isLocal ? item.hash : item; + const fileInfo = { fileRef: reference, batchId: batchId || INVALID_STMAP } as FileInfo; try { console.log(`Processing reference: ${reference}`); // Download the file to extract its metadata // TODO: act headers - const options: BeeRequestOptions = {}; - // if (file.reference.length === ENCRYPTED_REFERENCE_HEX_LENGTH) { - // if (file.historyRef !== undefined) { - // options.headers = { 'swarm-act-history-address': file.historyRef }; - // } - // if (file.owner !== undefined) { - // options.headers = { - // ...options.headers, - // 'swarm-act-publisher': file.owner, - // }; - // } - // if (file.timestamp !== undefined) { - // options.headers = { ...options.headers, 'swarm-act-timestamp': file.timestamp.toString() }; - // } - // } + const options = makeBeeRequestOptions(fileInfo.historyRef, fileInfo.owner, fileInfo.timestamp); // TODO: maybe use path to get the rootmetadata and store it locally const path = '/rootmetadata.json'; const fileData = await this.bee.downloadFile(reference, path, options); @@ -333,7 +329,13 @@ export class FileManager { // Track imported files // TODO: eglref, shared, timestamp -> mantaray root metadata - this.importedFiles.push({ reference: reference, name: fileName, batchId: batchId, shared: undefined }); + // TODO: it shall be reverted to importedReferences + this.importedReferences.push({ + fileRef: reference, + fileName: fileName, + batchId: batchId || INVALID_STMAP, + shared: undefined, + }); } catch (error: any) { console.error(`[ERROR] Failed to process reference ${reference}: ${error.message}`); } @@ -351,7 +353,12 @@ export class FileManager { await this.importReferences(items, undefined, true); } - async downloadFile(mantaray: MantarayNode, filePath: string, onlyMetadata = false): Promise { + async downloadFile( + mantaray: MantarayNode, + filePath: string, + onlyMetadata = false, + options?: BeeRequestOptions, + ): Promise { mantaray = mantaray || this.mantaray; console.log(`Downloading file: ${filePath}`); const normalizedPath = path.normalize(filePath); @@ -386,7 +393,7 @@ export class FileManager { console.log(`Downloading file with reference: ${hexReference}`); try { - const fileData = await this.bee.downloadFile(hexReference); + const fileData = await this.bee.downloadFile(hexReference, 'encryptedfilepath', options); return { data: fileData.data ? Buffer.from(fileData.data).toString('utf-8').trim() : '', metadata, @@ -469,70 +476,113 @@ export class FileManager { 'Time-Uploaded': new Date().toISOString(), Filename: fileName, 'Custom-Metadata': JSON.stringify(customMetadata), - }; + } as MetadataMapping; - const uploadHeaders = { - contentType: contentType, + const defaultOptions = { act: true, redundancyLevel: redundancyLevel, } as FileUploadOptions & UploadRedundancyOptions; + let uploadFileRes: UploadResultWithCid; try { - const uploadResponse = await this.bee.uploadFile(batchId, fileData, fileName, uploadHeaders); - this.addToMantaray(mantaray, uploadResponse.reference, metadata); + uploadFileRes = await this.bee.uploadFile(batchId, fileData, fileName, { + ...defaultOptions, + contentType: contentType, + }); + // this.addToMantaray(mantaray, uploadFileRes.reference, metadata); + mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadFileRes.reference), metadata); + } catch (error: any) { + console.error(`Failed to upload file ${file}: ${error.message}`); + throw error; + } - const metadataFile = { - reference: uploadResponse.reference, - batchId: batchId, - name: fileName, - owner: this.address, - historyRef: uploadResponse.historyAddress, - timestamp: new Date().getTime(), - eGlRef: undefined, - } as MetadataFile; + const fileInfo = { + fileRef: uploadFileRes.reference, + batchId: batchId, + fileName: fileName, + owner: this.address, + shared: false, + historyRef: uploadFileRes.historyAddress, + timestamp: new Date().getTime(), + eGlRef: undefined, + } as FileInfo; + + let historyAddress: string; + try { + const uploadInfoRes = await this.bee.uploadFile(batchId, JSON.stringify(fileInfo), FILEIINFO_NAME, { + ...defaultOptions, + contentType: 'application/json', + }); - // TODO: always update everything with the same redundancy level ?? - try { - const uploadMetadataResponse = await this.bee.uploadFile( - batchId, - JSON.stringify(metadataFile), - 'metadata.json', - { - contentType: 'application/json', - act: true, - redundancyLevel: redundancyLevel, - }, - ); - const topicHex = this.bee.makeFeedTopic(uploadMetadataResponse.reference); - const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); + historyAddress = uploadInfoRes.historyAddress; + console.log('Fileinfo updated: ', uploadInfoRes.reference); + // this.addToMantaray(mantaray, uploadInfoRes.reference, {}); + mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadInfoRes.reference), { + 'Content-Type': 'application/json', + Filename: FILEIINFO_NAME, + }); + this.fileInfoList.push(fileInfo); + } catch (error: any) { + console.error(`Failed to save fileinfo: ${error}`); + throw error; + } - const uploadResult = await this.bee.uploadData(batchId, uploadMetadataResponse.historyAddress); - const feedWriteResult = await feedWriter.upload(batchId, uploadResult.reference, { - index: undefined, // todo: keep track of the latest index ?? - act: true, - }); + // TODO: do not even save separately fileInfoHistory just let it be the feed data + const fileInfoHistory = { + fileInfoHistoryRef: historyAddress, + } as FileInfoHistory; - console.log('File metadata feed updated: ', feedWriteResult.reference); - this.addToMantaray(mantaray, feedWriteResult.reference, metadata); - } catch (error: any) { - console.error(`Failed to update file metadata feed: ${error}`); - } + try { + const uploadHistoryRes = await this.bee.uploadFile( + batchId, + JSON.stringify(fileInfoHistory), + FILEINFO_HISTORY_NAME, + { + redundancyLevel: redundancyLevel, + contentType: 'application/json', + }, + ); - // TODO: wrapped mantaray - await this.updateWrappedMantaray(); - this.metadataFileList.push(metadataFile); + console.log('Fileinfo history updated: ', uploadHistoryRes.reference); + // this.addToMantaray(mantaray, uploadHistoryRes.reference, {}); + mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadHistoryRes.reference), { + 'Content-Type': 'application/json', + Filename: FILEIINFO_NAME, + }); + } catch (error: any) { + console.error(`Failed to save fileinfo history: ${error}`); + throw error; + } - console.log('Saving Mantaray node...'); - const { eRef, hRef } = await this.saveMantaray(mantaray, batchId); + let wrappedMantarayRef: string; + try { + // TODO: wrapped mantaray + wrappedMantarayRef = await this.saveMantaray(mantaray, batchId); + } catch (error: any) { + console.error(`Failed to save wrapped mantaray: ${error}`); + throw error; + } - await this.updateMetadataFileList(hRef); + try { + //TODO: test if feed ACT up/down actually works !!! + const topicHex = this.bee.makeFeedTopic(wrappedMantarayRef); + const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); + const wrappedFeedRes = await this.bee.uploadData(batchId, fileInfoHistory.fileInfoHistoryRef); + const feedWriteResult = await feedWriter.upload(batchId, wrappedFeedRes.reference, { + index: undefined, // todo: keep track of the latest index ?? + act: true, + }); + // TODO: properly handle feed data of wrapped feed addresses and histories + this.ownerFileInfoFeedData.feedWriteResult = feedWriteResult.historyAddress; - console.log(`File uploaded successfully: ${file}, Reference: ${eRef}`); - return eRef; + await this.saveWrappedMantarayList(); } catch (error: any) { - console.error(`[ERROR] Failed to upload file ${file}: ${error.message}`); + console.error(`Failed to save owner info feed: ${error}`); throw error; } + + console.log(`File uploaded successfully: ${file}, Reference: ${wrappedMantarayRef}`); + return wrappedMantarayRef; } addToMantaray(mantaray: MantarayNode | undefined, reference: string, metadata: MetadataMapping = {}): void { @@ -541,7 +591,7 @@ export class FileManager { const filePath = metadata.fullPath || metadata.Filename || 'file'; const originalFileName = metadata.originalFileName || path.basename(filePath); - const bytesPath = Utils.hexToBytes(filePath); + const bytesPath = encodePathToBytes(filePath); const metadataWithOriginalName = { ...metadata, @@ -552,24 +602,18 @@ export class FileManager { } // TODO: problem: mantary impl. is old and does not return the history address - async saveMantaray(mantaray: MantarayNode | undefined, stamp: string | BatchId): Promise { + async saveMantaray(mantaray: MantarayNode | undefined, batchId: string | BatchId): Promise { mantaray = mantaray || this.mantaray; console.log('Saving Mantaray manifest...'); const saveFunction = async (data: Uint8Array): Promise => { - const fileName = 'manifest'; - const contentType = 'application/json'; - const uploadResponse = await this.bee.uploadFile(stamp, data, fileName, { - contentType, - act: true, - }); + const uploadResponse = await this.bee.uploadData(batchId, data); return Utils.hexToBytes(uploadResponse.reference); }; const manifestReference = Utils.bytesToHex(await mantaray.save(saveFunction)); - - console.log(`Mantaray manifest saved with reference: ${manifestReference}`); - return { eRef: manifestReference, hRef: saveResult.historyAddress }; + console.log(`Mantaray manifest saved, reference: ${manifestReference}`); + return manifestReference; } searchFilesByName(fileNameQuery: string, includeMetadata = false): any { @@ -599,7 +643,7 @@ export class FileManager { extension?: string; }, includeMetadata = false, - ) { + ): any { let results = this.listFiles(this.mantaray, true); if (fileName) { @@ -684,6 +728,12 @@ export class FileManager { return fileList; } + // list: + // metadata 1 + // metadata 2 --> root mantary hash -> download -> getDirectoryStructure + // + // + getDirectoryStructure(mantaray: MantarayNode | undefined, rootDirName: string): any { mantaray = mantaray || this.mantaray; console.log('Building directory structure from Mantaray...'); @@ -780,9 +830,16 @@ export class FileManager { } // Start feed handler methods - async updateWrappedMantaray(): Promise {} + async updateWrappedMantaray(batchId: string, mantaray: MantarayNode): Promise { + try { + return this.saveMantaray(mantaray, batchId); + } catch (error: any) { + console.error("Couldn't save wrapped mantaray:", error); + return undefined; + } + } - async updateMetadataFileList(): Promise { + async saveWrappedMantarayList(): Promise { const ownerFeedStamp = await this.getOwnerFeedStamp(); if (!ownerFeedStamp) { console.error('Owner feed stamp is not found.'); @@ -792,12 +849,16 @@ export class FileManager { const ownerFeedTopicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, ownerFeedTopicHex, this.privateKey); try { - const uploadResult = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(this.metadataFileList), { - act: true, - }); + const uploadResult = await this.bee.uploadData( + ownerFeedStamp.batchID, + JSON.stringify(this.ownerFileInfoFeedData), + { + act: true, + }, + ); const ownerFeedData = { - history: uploadResult.historyAddress, - metadataListReference: uploadResult.reference, + wrappedFeedListRef: uploadResult.reference, + historyRef: uploadResult.historyAddress, } as OwnerFeedData; const ownerFeedRawDataUploadResult = await this.bee.uploadData( ownerFeedStamp.batchID, @@ -841,7 +902,7 @@ export class FileManager { // TODO: do not store encrypted grantee ref in the wrapedd mantaray just in the owner mtdt feed // fetches the list of grantees who can access the file reference async getGranteesOfFile(fileRef: string | Reference): Promise { - const file = this.metadataFileList.find((f) => f.reference === fileRef); + const file = this.fileInfoList.find((f) => f.fileRef === fileRef); if (file === undefined || file.eGlRef === undefined) { console.error('File or grantee ref not found for reference: ', fileRef); return undefined; @@ -857,7 +918,7 @@ export class FileManager { // updates the list of grantees who can access the file reference under the history reference async handleGrantees( batchId: string | BatchId, - file: MetadataFile, + file: FileInfo, grantees: { add?: string[]; revoke?: string[]; @@ -889,12 +950,12 @@ export class FileManager { } else { this.granteeLists[currentGranteesIx] = grantResult.ref; // TODO: maybe don't need to check if upload + patch happens at the same time -> add to import ? - const fIx = this.importedFiles.findIndex((f) => f.reference === file.reference); + const fIx = this.fileInfoList.findIndex((f) => f.fileRef === file.fileRef); if (fIx === -1) { - console.log('Provided file reference not found in imported files: ', file.reference); + console.log('Provided file reference not found in imported files: ', file.fileRef); return undefined; } else { - this.importedFiles[fIx].eGlRef = grantResult.ref; + this.fileInfoList[fIx].eGlRef = grantResult.ref; } } @@ -943,7 +1004,7 @@ export class FileManager { const historyRefs = new Array(references.length); for (let i = 0; i < references.length; i++) { const ref = references[i]; - const file = this.importedFiles.find((f) => f.reference === ref); + const file = this.fileInfoList.find((f) => f.fileRef === ref); if (file === undefined) { console.log('File not found for reference: ', ref); continue; @@ -952,17 +1013,17 @@ export class FileManager { console.log('History not found for reference: ', ref); continue; } - // TODO: how to update file metadata with new eglref ? -> filemetadata = /feed/decryptedref/metadata + // TODO: how to update fileinfo with new eglref, href and not separate params const grantResult = await this.handleGrantees( batchId, - { reference: ref }, + { fileRef: ref, batchId: batchId }, { add: recipients }, file.historyRef, file.eGlRef, ); if (grantResult !== undefined) { - const feedMetadatRef = await this.updateFileMetadata({ + const feedMetadatRef = await this.updateWrappedMantaray({ ...file, eGlRef: grantResult.ref, historyRef: grantResult.historyref, @@ -1014,34 +1075,20 @@ export class FileManager { } } // TODO: maybe store only the encrypted refs for security and use - async downloadSharedItem(file: MetadataFile, path?: string): Promise { - if (!this.sharedWithMe.find((msg) => msg.references.includes(file.reference))) { - console.log('Cannot find reference in shared messages: ', file.reference); + async downloadSharedItem(file: FileInfo, path?: string): Promise { + if (!this.sharedWithMe.find((msg) => msg.references.includes(file.fileRef))) { + console.log('Cannot find file reference in shared messages: ', file.fileRef); return undefined; } - const options: BeeRequestOptions = {}; - if (file.reference.length === ENCRYPTED_REFERENCE_HEX_LENGTH) { - if (file.historyRef !== undefined) { - options.headers = { 'swarm-act-history-address': file.historyRef }; - } - if (file.owner !== undefined) { - options.headers = { - ...options.headers, - 'swarm-act-publisher': file.owner, - }; - } - if (file.timestamp !== undefined) { - options.headers = { ...options.headers, 'swarm-act-timestamp': file.timestamp.toString() }; - } - } + const options = makeBeeRequestOptions(file.historyRef, file.owner, file.timestamp); try { // TODO: publisher and history headers - const data = await this.bee.downloadFile(file.reference, path, options); + const data = await this.bee.downloadFile(file.fileRef, path, options); return data.data; } catch (error: any) { - console.error(`Failed to download shared file ${file.reference}\n: ${error}`); + console.error(`Failed to download shared file ${file.fileRef}\n: ${error}`); return undefined; } } diff --git a/src/types.ts b/src/types.ts index c1d896a..fa2e57b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,20 +1,23 @@ import { BatchId, Reference } from '@ethersphere/bee-js'; -// TODO: maybe rename to FileInfo -export interface MetadataFile { - reference: string | Reference; +export interface FileInfo { + fileRef: string | Reference; batchId: string | BatchId; shared?: boolean; - name?: string; + fileName?: string; owner?: string; eGlRef?: string | Reference; historyRef?: string | Reference; timestamp?: number; } +export interface FileInfoHistory { + fileInfoHistoryRef: string; +} + export interface OwnerFeedData { - history: string; - metadataListReference: string; + wrappedFeedListRef: string; + historyRef: string; } // export interface MetadataFile { diff --git a/src/utils.ts b/src/utils.ts index dc56724..f297ba4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,4 @@ +import { BeeRequestOptions } from '@ethersphere/bee-js'; import path from 'path'; import { SharedMessage } from './types'; @@ -59,3 +60,21 @@ export function decodeBytesToPath(bytes: Uint8Array): string { export function encodePathToBytes(pathString: string): Uint8Array { return new TextEncoder().encode(pathString); } + +export function makeBeeRequestOptions(historyRef?: string, publisher?: string, timestamp?: number): BeeRequestOptions { + const options: BeeRequestOptions = {}; + if (historyRef !== undefined) { + options.headers = { 'swarm-act-history-address': historyRef }; + } + if (publisher !== undefined) { + options.headers = { + ...options.headers, + 'swarm-act-publisher': publisher, + }; + } + if (timestamp !== undefined) { + options.headers = { ...options.headers, 'swarm-act-timestamp': timestamp.toString() }; + } + + return options; +} diff --git a/tests/fileManager.test.ts b/tests/fileManager.test.ts index 5661dc9..2e9dda9 100644 --- a/tests/fileManager.test.ts +++ b/tests/fileManager.test.ts @@ -39,7 +39,7 @@ describe('FileManager - Initialize', () => { beforeEach(() => { mockBee = createMockBee(); fileManager = new FileManager('http://localhost:1633', privateKey); - fileManager.importedFiles = []; + fileManager.importedFileInfoList = []; jest.clearAllMocks(); }); From 2f53d1f6b1c95608617c482835d5e7d5c1a74c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Sat, 18 Jan 2025 22:52:17 +0100 Subject: [PATCH 07/18] impoving fileinfo ref list handling --- src/fileManager.ts | 94 ++++++++++++++++++++-------------------------- src/types.ts | 6 +-- src/utils.ts | 6 +-- 3 files changed, 46 insertions(+), 60 deletions(-) diff --git a/src/fileManager.ts b/src/fileManager.ts index ad134bb..70ae998 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -29,7 +29,7 @@ import { ROOT_PATH, SHARED_INBOX_TOPIC, } from './constants'; -import { FileInfo, FileInfoHistory, OwnerFeedData, SharedMessage } from './types'; +import { FileInfo, FileInfoFeedRef, FileInfoHistory, ShareItem } from './types'; import { assertSharedMessage, decodeBytesToPath, @@ -44,14 +44,15 @@ export class FileManager { public bee: Bee; public mantaray: MantarayNode; + private importedReferences: string[]; private stampList: PostageBatch[]; - private ownerFileInfoFeedData: object; + private fileInfoFeedRefList: FileInfoFeedRef[]; private fileInfoList: FileInfo[]; private nextOwnerFeedIndex: string; private wallet: Wallet; private privateKey: string; private granteeLists: string[]; - private sharedWithMe: SharedMessage[]; + private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; private address: string; // TODO: is this.mantaray needed ? always a new mantaray instance is created when wokring on an item @@ -67,9 +68,10 @@ export class FileManager { console.log('Initializing Bee client...'); this.bee = new Bee(beeUrl); + this.importedReferences = []; this.stampList = []; this.fileInfoList = []; - this.ownerFileInfoFeedData = {}; + this.fileInfoFeedRefList = []; this.nextOwnerFeedIndex = ''; this.privateKey = privateKey; this.wallet = new Wallet(privateKey); @@ -151,16 +153,13 @@ export class FileManager { this.nextOwnerFeedIndex = latestFeedData.feedIndexNext; const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); - const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as OwnerFeedData; - const options: BeeRequestOptions = { - headers: { 'swarm-act-history-address': ownerFeedData.historyRef, 'swarm-act-publisher': this.address }, - } as const; - // TODO: act encrpyt the fileInfoList refs?? - const fileInfoList = JSON.parse( - (await this.bee.downloadData(ownerFeedData.wrappedFeedListRef, options)).text(), - ) as FileInfo[]; - for (const fi of fileInfoList) { - const metadata = JSON.parse((await this.bee.downloadData(fi.fileRef)).text()) as FileInfo; + const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as FileInfoFeedRef; + const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.address); + this.fileInfoFeedRefList = JSON.parse( + (await this.bee.downloadData(ownerFeedData.fileInfoFeedRef, options)).text(), + ) as FileInfoFeedRef[]; + for (const fi of this.fileInfoFeedRefList) { + const metadata = JSON.parse((await this.bee.downloadData(fi.fileInfoFeedRef)).text()) as FileInfo; this.fileInfoList.push(metadata); } console.log('Stamps fetched from feed.'); @@ -330,12 +329,7 @@ export class FileManager { // Track imported files // TODO: eglref, shared, timestamp -> mantaray root metadata // TODO: it shall be reverted to importedReferences - this.importedReferences.push({ - fileRef: reference, - fileName: fileName, - batchId: batchId || INVALID_STMAP, - shared: undefined, - }); + this.importedReferences.push(reference); } catch (error: any) { console.error(`[ERROR] Failed to process reference ${reference}: ${error.message}`); } @@ -463,6 +457,7 @@ export class FileManager { batchId: string | BatchId, customMetadata = {}, redundancyLevel = RedundancyLevel.MEDIUM, + refAsTopicHex?: string, ): Promise { mantaray = mantaray || this.mantaray; console.log(`Uploading file: ${file}`); @@ -507,14 +502,14 @@ export class FileManager { eGlRef: undefined, } as FileInfo; - let historyAddress: string; + let fileInfoHistoryRef: string; try { const uploadInfoRes = await this.bee.uploadFile(batchId, JSON.stringify(fileInfo), FILEIINFO_NAME, { ...defaultOptions, contentType: 'application/json', }); - historyAddress = uploadInfoRes.historyAddress; + fileInfoHistoryRef = uploadInfoRes.historyAddress; console.log('Fileinfo updated: ', uploadInfoRes.reference); // this.addToMantaray(mantaray, uploadInfoRes.reference, {}); mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadInfoRes.reference), { @@ -529,7 +524,7 @@ export class FileManager { // TODO: do not even save separately fileInfoHistory just let it be the feed data const fileInfoHistory = { - fileInfoHistoryRef: historyAddress, + fileInfoHistoryRef: fileInfoHistoryRef, } as FileInfoHistory; try { @@ -556,7 +551,6 @@ export class FileManager { let wrappedMantarayRef: string; try { - // TODO: wrapped mantaray wrappedMantarayRef = await this.saveMantaray(mantaray, batchId); } catch (error: any) { console.error(`Failed to save wrapped mantaray: ${error}`); @@ -564,18 +558,23 @@ export class FileManager { } try { - //TODO: test if feed ACT up/down actually works !!! - const topicHex = this.bee.makeFeedTopic(wrappedMantarayRef); + // TODO: test if feed ACT up/down actually works !!! + // TODO: uploadfile should accept a topic if a new version is uploaded + const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); - const wrappedFeedRes = await this.bee.uploadData(batchId, fileInfoHistory.fileInfoHistoryRef); - const feedWriteResult = await feedWriter.upload(batchId, wrappedFeedRes.reference, { + const wrappedUploadRes = await this.bee.uploadData(batchId, wrappedMantarayRef); + const feedWriteResult = await feedWriter.upload(batchId, wrappedUploadRes.reference, { index: undefined, // todo: keep track of the latest index ?? act: true, }); // TODO: properly handle feed data of wrapped feed addresses and histories - this.ownerFileInfoFeedData.feedWriteResult = feedWriteResult.historyAddress; + this.fileInfoFeedRefList.push({ + // feedWriteResultRef: feedWriteResult.reference, ??? TODO: check if this is needed + fileInfoFeedRef: topicHex, + historyRef: feedWriteResult.historyAddress, + }); - await this.saveWrappedMantarayList(); + await this.saveFileInfoListFeed(); } catch (error: any) { console.error(`Failed to save owner info feed: ${error}`); throw error; @@ -611,7 +610,8 @@ export class FileManager { return Utils.hexToBytes(uploadResponse.reference); }; - const manifestReference = Utils.bytesToHex(await mantaray.save(saveFunction)); + const saveRes = await mantaray.save(saveFunction); + const manifestReference = Utils.bytesToHex(saveRes); console.log(`Mantaray manifest saved, reference: ${manifestReference}`); return manifestReference; } @@ -728,12 +728,6 @@ export class FileManager { return fileList; } - // list: - // metadata 1 - // metadata 2 --> root mantary hash -> download -> getDirectoryStructure - // - // - getDirectoryStructure(mantaray: MantarayNode | undefined, rootDirName: string): any { mantaray = mantaray || this.mantaray; console.log('Building directory structure from Mantaray...'); @@ -830,16 +824,8 @@ export class FileManager { } // Start feed handler methods - async updateWrappedMantaray(batchId: string, mantaray: MantarayNode): Promise { - try { - return this.saveMantaray(mantaray, batchId); - } catch (error: any) { - console.error("Couldn't save wrapped mantaray:", error); - return undefined; - } - } - - async saveWrappedMantarayList(): Promise { + // TODO: probably a missing bee-js impl.: history upload option if act is true + async saveFileInfoListFeed(): Promise { const ownerFeedStamp = await this.getOwnerFeedStamp(); if (!ownerFeedStamp) { console.error('Owner feed stamp is not found.'); @@ -849,17 +835,17 @@ export class FileManager { const ownerFeedTopicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, ownerFeedTopicHex, this.privateKey); try { - const uploadResult = await this.bee.uploadData( + const fileInfoFeedUploadRes = await this.bee.uploadData( ownerFeedStamp.batchID, - JSON.stringify(this.ownerFileInfoFeedData), + JSON.stringify(this.fileInfoFeedRefList), { act: true, }, ); const ownerFeedData = { - wrappedFeedListRef: uploadResult.reference, - historyRef: uploadResult.historyAddress, - } as OwnerFeedData; + fileInfoFeedRef: fileInfoFeedUploadRes.reference, + historyRef: fileInfoFeedUploadRes.historyAddress, + } as FileInfoFeedRef; const ownerFeedRawDataUploadResult = await this.bee.uploadData( ownerFeedStamp.batchID, JSON.stringify(ownerFeedData), @@ -1042,7 +1028,7 @@ export class FileManager { references: historyRefs, timestamp: Date.now(), message: message, - } as SharedMessage; + } as ShareItem; await this.sendShareMessage(batchId, targetOverlays, item, recipients); } catch (error: any) { @@ -1055,7 +1041,7 @@ export class FileManager { async sendShareMessage( batchId: string, targetOverlays: string[], - item: SharedMessage, + item: ShareItem, recipients: string[], ): Promise { // TODO: valid length check of recipient and target diff --git a/src/types.ts b/src/types.ts index fa2e57b..d529068 100644 --- a/src/types.ts +++ b/src/types.ts @@ -15,8 +15,8 @@ export interface FileInfoHistory { fileInfoHistoryRef: string; } -export interface OwnerFeedData { - wrappedFeedListRef: string; +export interface FileInfoFeedRef { + fileInfoFeedRef: string; historyRef: string; } @@ -26,7 +26,7 @@ export interface OwnerFeedData { // } // TODO: unify own files with shared and add stamp data potentially -export interface SharedMessage { +export interface ShareItem { owner: string; references: string[]; timestamp?: number; diff --git a/src/utils.ts b/src/utils.ts index f297ba4..f4bf1ad 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,7 +1,7 @@ import { BeeRequestOptions } from '@ethersphere/bee-js'; import path from 'path'; -import { SharedMessage } from './types'; +import { ShareItem } from './types'; export function getContentType(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); @@ -24,12 +24,12 @@ export function isStrictlyObject(value: unknown): value is Record Date: Mon, 20 Jan 2025 15:40:15 +0100 Subject: [PATCH 08/18] improving mantaray feed list upload and download with fileinfo --- package-lock.json | 133 ++++++++++++++++++---------------- package.json | 3 +- src/constants.ts | 8 +- src/fileManager.ts | 177 ++++++++++++++++++++++++++++++--------------- src/types.ts | 31 ++++++-- src/utils.ts | 37 +++++++++- 6 files changed, 255 insertions(+), 134 deletions(-) diff --git a/package-lock.json b/package-lock.json index 52bc10b..f720185 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "license": "MIT", "dependencies": { "@ethersphere/bee-js": "^8.3.1", - "@solarpunkltd/mantaray-js": "^1.0.5" + "@solarpunkltd/mantaray-js": "^1.0.5", + "cafe-utility": "^21.5.0" }, "devDependencies": { "@babel/preset-env": "^7.26.0", @@ -79,9 +80,9 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.3.tgz", - "integrity": "sha512-nHIxvKPniQXpmQLb0vhY3VaFb3S0YrTAwpOWJZh1wn3oJPjJk9Asva204PsBdmAE8vpzfHudT8DB0scYvy9q0g==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.5.tgz", + "integrity": "sha512-XvcZi1KWf88RVbF9wn8MN6tYFloU5qX8KjuF3E1PVBmJ9eypXfs4GRiJwLuTZL0iSnJUKn1BFPa5BPZZJyFzPg==", "dev": true, "license": "MIT", "engines": { @@ -120,14 +121,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.3.tgz", - "integrity": "sha512-6FF/urZvD0sTeO7k6/B15pMLC4CHUv1426lzr3N01aHJTl046uCAh9LXW/fzeXXjPNCJ6iABW5XaWOsIZB93aQ==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.5.tgz", + "integrity": "sha512-2caSP6fN9I7HOe6nqhtft7V4g7/V/gfDsC3Ag4W7kEzzvRGKqiv0pu0HogPiZ3KaVSoNDhUws6IJjDjpfmYIXw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.26.3", - "@babel/types": "^7.26.3", + "@babel/parser": "^7.26.5", + "@babel/types": "^7.26.5", "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.25", "jsesc": "^3.0.2" @@ -150,13 +151,13 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", - "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.26.5.tgz", + "integrity": "sha512-IXuyn5EkouFJscIDuFF5EsiSolseme1s0CZB+QxVugqJLYmKdxI1VfIBOst0SUu4rnk2Z7kqTwmoO1lp3HIfnA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.25.9", + "@babel/compat-data": "^7.26.5", "@babel/helper-validator-option": "^7.25.9", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", @@ -283,9 +284,9 @@ } }, "node_modules/@babel/helper-plugin-utils": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz", - "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.26.5.tgz", + "integrity": "sha512-RS+jZcRdZdRFzMyr+wcsaqOmld1/EqTghfaBGQQd/WnRdzdlvSZ//kF7U8VQTxf1ynZ4cjUcYgjVGx13ewNPMg==", "dev": true, "license": "MIT", "engines": { @@ -311,15 +312,15 @@ } }, "node_modules/@babel/helper-replace-supers": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.25.9.tgz", - "integrity": "sha512-IiDqTOTBQy0sWyeXyGSC5TBJpGFXBkRynjBeXsvbhQFKj2viwJC76Epz35YLU1fpe/Am6Vppb7W7zM4fPQzLsQ==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.26.5.tgz", + "integrity": "sha512-bJ6iIVdYX1YooY2X7w1q6VITt+LnUILtNk7zT78ykuwStx8BauCzxvFqFaHjOpW1bVnSUM1PN1f0p5P21wHxvg==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-member-expression-to-functions": "^7.25.9", "@babel/helper-optimise-call-expression": "^7.25.9", - "@babel/traverse": "^7.25.9" + "@babel/traverse": "^7.26.5" }, "engines": { "node": ">=6.9.0" @@ -402,13 +403,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.3.tgz", - "integrity": "sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.5.tgz", + "integrity": "sha512-SRJ4jYmXRqV1/Xc+TIVG84WjHBXKlxO9sHQnA2Pf12QQEAp1LOh6kDzNHXcUnbH1QI0FDoPPVOt+vyUDucxpaw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.26.3" + "@babel/types": "^7.26.5" }, "bin": { "parser": "bin/babel-parser.js" @@ -839,13 +840,13 @@ } }, "node_modules/@babel/plugin-transform-block-scoped-functions": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.25.9.tgz", - "integrity": "sha512-toHc9fzab0ZfenFpsyYinOX0J/5dgJVA2fm64xPewu7CoYHWEivIWKxkK2rMi4r3yQqLnVmheMXRdG+k239CgA==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.26.5.tgz", + "integrity": "sha512-chuTSY+hq09+/f5lMj8ZSYgCFpppV2CbYrhNFJ1BFoXpiWPnnAb7R0MqrafCpN8E1+YRrtM1MXZHJdIx8B6rMQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" + "@babel/helper-plugin-utils": "^7.26.5" }, "engines": { "node": ">=6.9.0" @@ -1259,13 +1260,13 @@ } }, "node_modules/@babel/plugin-transform-nullish-coalescing-operator": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.25.9.tgz", - "integrity": "sha512-ENfftpLZw5EItALAD4WsY/KUWvhUlZndm5GC7G3evUsVeSJB6p0pBeLQUnRnBCBx7zV0RKQjR9kCuwrsIrjWog==", + "version": "7.26.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.26.6.tgz", + "integrity": "sha512-CKW8Vu+uUZneQCPtXmSBUC6NCAUdya26hWCElAWh5mVSlSRsmiCPUUDKb3Z0szng1hiAJa098Hkhg9o4SE35Qw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-plugin-utils": "^7.25.9" + "@babel/helper-plugin-utils": "^7.26.5" }, "engines": { "node": ">=6.9.0" @@ -1557,15 +1558,15 @@ } }, "node_modules/@babel/plugin-transform-typescript": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.26.3.tgz", - "integrity": "sha512-6+5hpdr6mETwSKjmJUdYw0EIkATiQhnELWlE3kJFBwSg/BGIVwVaVbX+gOXBCdc7Ln1RXZxyWGecIXhUfnl7oA==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.26.5.tgz", + "integrity": "sha512-GJhPO0y8SD5EYVCy2Zr+9dSZcEgaSmq5BLR0Oc25TOEhC+ba49vUAGZFjy8v79z9E1mdldq4x9d1xgh4L1d5dQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-annotate-as-pure": "^7.25.9", "@babel/helper-create-class-features-plugin": "^7.25.9", - "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-plugin-utils": "^7.26.5", "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9", "@babel/plugin-syntax-typescript": "^7.25.9" }, @@ -1791,17 +1792,17 @@ } }, "node_modules/@babel/traverse": { - "version": "7.26.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.4.tgz", - "integrity": "sha512-fH+b7Y4p3yqvApJALCPJcwb0/XaOSgtK4pzV6WVjPR5GLFQBRI7pfoX2V2iM48NXvX07NUxxm1Vw98YjqTcU5w==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.5.tgz", + "integrity": "sha512-rkOSPOw+AXbgtwUga3U4u8RpoK9FEFWBNAlTpcnkLFjL5CT+oyHNuUUC/xx6XefEJ16r38r8Bc/lfp6rYuHeJQ==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.26.2", - "@babel/generator": "^7.26.3", - "@babel/parser": "^7.26.3", + "@babel/generator": "^7.26.5", + "@babel/parser": "^7.26.5", "@babel/template": "^7.25.9", - "@babel/types": "^7.26.3", + "@babel/types": "^7.26.5", "debug": "^4.3.1", "globals": "^11.1.0" }, @@ -1810,9 +1811,9 @@ } }, "node_modules/@babel/types": { - "version": "7.26.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz", - "integrity": "sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==", + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.5.tgz", + "integrity": "sha512-L6mZmwFDK6Cjh1nRCLXpa6no13ZIioJDz7mdkzHv399pThrTa/k0nUlNaenOeh2kWu/iaOQYElEpKPUswUa9Vg==", "dev": true, "license": "MIT", "dependencies": { @@ -1989,6 +1990,12 @@ "beeApiVersion": "7.1.0" } }, + "node_modules/@ethersphere/bee-js/node_modules/cafe-utility": { + "version": "23.12.0", + "resolved": "https://registry.npmjs.org/cafe-utility/-/cafe-utility-23.12.0.tgz", + "integrity": "sha512-B8MHryv6dDTw8GRfJxHLy4zzhewEEYulPAXiSRqkNCeqXFoQAk8THhlU00Yk7dvc8bppnHoS7FaQ468dfGfe6A==", + "license": "MIT" + }, "node_modules/@ethersphere/bee-js/node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -2779,9 +2786,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.10.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.5.tgz", - "integrity": "sha512-F8Q+SeGimwOo86fiovQh8qiXfFEh2/ocYv7tU5pJ3EXMSSxk1Joj5wefpFK2fHTf/N6HKGSxIDBT9f3gCxXPkQ==", + "version": "22.10.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.7.tgz", + "integrity": "sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==", "dev": true, "license": "MIT", "dependencies": { @@ -3626,9 +3633,9 @@ "license": "MIT" }, "node_modules/cafe-utility": { - "version": "23.12.0", - "resolved": "https://registry.npmjs.org/cafe-utility/-/cafe-utility-23.12.0.tgz", - "integrity": "sha512-B8MHryv6dDTw8GRfJxHLy4zzhewEEYulPAXiSRqkNCeqXFoQAk8THhlU00Yk7dvc8bppnHoS7FaQ468dfGfe6A==", + "version": "21.5.0", + "resolved": "https://registry.npmjs.org/cafe-utility/-/cafe-utility-21.5.0.tgz", + "integrity": "sha512-5u+9cf7fvcH3j2Q3jrd7nA3bUITUBj8b9Arg4eA6almqeA5+dwQA6NKba4GnW6zS9uL1iVCEQqM3z3tQVs2Xjw==", "license": "MIT" }, "node_modules/call-bind": { @@ -3702,9 +3709,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001692", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001692.tgz", - "integrity": "sha512-A95VKan0kdtrsnMubMKxEKUKImOPSuCpYgxSQBo036P5YYgVIcOYJEgt/txJWqObiRQeISNCfef9nvlQ0vbV7A==", + "version": "1.0.30001695", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001695.tgz", + "integrity": "sha512-vHyLade6wTgI2u1ec3WQBxv+2BrTERV28UXQu9LO6lZ9pYeMk34vjXFLOxo1A4UBA8XTL4njRQZdno/yYaSmWw==", "dev": true, "funding": [ { @@ -4151,9 +4158,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.80", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.80.tgz", - "integrity": "sha512-LTrKpW0AqIuHwmlVNV+cjFYTnXtM9K37OGhpe0ZI10ScPSxqVSryZHIY3WnCS5NSYbBODRTZyhRMS2h5FAEqAw==", + "version": "1.5.83", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.83.tgz", + "integrity": "sha512-LcUDPqSt+V0QmI47XLzZrz5OqILSMGsPFkDYus22rIbgorSvBYEFqq854ltTmUdHkY92FSdAAvsh4jWEULMdfQ==", "dev": true, "license": "ISC" }, @@ -4289,9 +4296,9 @@ } }, "node_modules/es-object-atoms": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", - "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", "dev": true, "license": "MIT", "dependencies": { @@ -4825,9 +4832,9 @@ } }, "node_modules/eslint-plugin-react-refresh": { - "version": "0.4.16", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.16.tgz", - "integrity": "sha512-slterMlxAhov/DZO8NScf6mEeMBBXodFUolijDvrtTxyezyLoTQaa73FyYus/VbTdftd8wBgBxPMRk3poleXNQ==", + "version": "0.4.18", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.18.tgz", + "integrity": "sha512-IRGEoFn3OKalm3hjfolEWGqoF/jPqeEYFp+C8B0WMzwGwBMvlRDQd06kghDhF0C61uJ6WfSDhEZE/sAQjduKgw==", "dev": true, "license": "MIT", "peerDependencies": { diff --git a/package.json b/package.json index 4f6ba97..bb0d3a0 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "license": "MIT", "dependencies": { "@ethersphere/bee-js": "^8.3.1", - "@solarpunkltd/mantaray-js": "^1.0.5" + "@solarpunkltd/mantaray-js": "^1.0.5", + "cafe-utility": "^21.5.0" }, "devDependencies": { "@babel/preset-env": "^7.26.0", diff --git a/src/constants.ts b/src/constants.ts index e6e1495..65d53cb 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,11 +1,13 @@ -const feedTypes = ['sequence', 'epoch'] as const; -export type FeedType = (typeof feedTypes)[number]; +import { FeedType } from './types'; + export const DEFAULT_FEED_TYPE: FeedType = 'sequence'; export const REFERENCE_LIST_TOPIC = 'reference-list'; export const FILEINFO_TOPIC = 'fileinfo'; export const SHARED_INBOX_TOPIC = 'shared-inbox'; export const OWNER_FEED_STAMP_LABEL = 'owner-stamp'; export const INVALID_STMAP = '0'.repeat(64); +export const ROOT_PATH = '/'; export const FILEIINFO_NAME = 'fileinfo.json'; +export const FILEIINFO_PATH = ROOT_PATH + 'fileinfo.json'; export const FILEINFO_HISTORY_NAME = 'history.json'; -export const ROOT_PATH = '/'; +export const FILEINFO_HISTORY_PATH = ROOT_PATH + 'history.json'; diff --git a/src/fileManager.ts b/src/fileManager.ts index 70ae998..8c58758 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -10,11 +10,13 @@ import { RedundancyLevel, Reference, REFERENCE_HEX_LENGTH, + TOPIC_HEX_LENGTH, UploadRedundancyOptions, UploadResultWithCid, Utils, } from '@ethersphere/bee-js'; import { initManifestNode, MantarayNode, MetadataMapping, Reference as MantarayRef } from '@solarpunkltd/mantaray-js'; +import { randomBytes } from 'crypto'; import { Wallet } from 'ethers'; import { readFileSync } from 'fs'; import path from 'path'; @@ -22,40 +24,43 @@ import path from 'path'; import { DEFAULT_FEED_TYPE, FILEIINFO_NAME, + FILEIINFO_PATH, FILEINFO_HISTORY_NAME, + FILEINFO_HISTORY_PATH, INVALID_STMAP, OWNER_FEED_STAMP_LABEL, REFERENCE_LIST_TOPIC, - ROOT_PATH, SHARED_INBOX_TOPIC, } from './constants'; -import { FileInfo, FileInfoFeedRef, FileInfoHistory, ShareItem } from './types'; +import { FileInfo, FileInfoHistory, OwnerFeedData, ShareItem, WrappedMantarayFeed } from './types'; import { assertSharedMessage, decodeBytesToPath, encodePathToBytes, getContentType, makeBeeRequestOptions, + makeNumericIndex, + numberToFeedIndex, } from './utils'; export class FileManager { // TODO: private vars // TODO: store shared refs and own files in the same array ? public bee: Bee; + // TODO: is this.mantaray needed ? always a new mantaray instance is created when wokring on an item public mantaray: MantarayNode; private importedReferences: string[]; private stampList: PostageBatch[]; - private fileInfoFeedRefList: FileInfoFeedRef[]; + private mantarayFeedList: WrappedMantarayFeed[]; private fileInfoList: FileInfo[]; - private nextOwnerFeedIndex: string; + private nextOwnerFeedIndex: number; private wallet: Wallet; private privateKey: string; private granteeLists: string[]; private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; private address: string; - // TODO: is this.mantaray needed ? always a new mantaray instance is created when wokring on an item private topic: string; constructor(beeUrl: string, privateKey: string) { @@ -68,17 +73,16 @@ export class FileManager { console.log('Initializing Bee client...'); this.bee = new Bee(beeUrl); + this.mantaray = initManifestNode(); this.importedReferences = []; this.stampList = []; this.fileInfoList = []; - this.fileInfoFeedRefList = []; - this.nextOwnerFeedIndex = ''; + this.mantarayFeedList = []; + this.nextOwnerFeedIndex = -1; this.privateKey = privateKey; this.wallet = new Wallet(privateKey); this.address = this.wallet.address; - this.topic = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); - - this.mantaray = initManifestNode(); + this.topic = ''; this.granteeLists = []; this.sharedWithMe = []; this.sharedSubscription = {} as PssSubscription; @@ -103,7 +107,7 @@ export class FileManager { try { console.log('Importing metadata of files...'); - await this.initMetadataFileList(); + await this.initFileInfoList(); } catch (error: any) { console.error(`[ERROR] Failed to initialize file metadata: ${error.message}`); throw error; @@ -111,9 +115,9 @@ export class FileManager { // if stamp is not found than the file cannot be downloaded? is this necessary ?? for (const stamp of this.stampList) { - const mtdtIx = this.fileInfoList.findIndex((f) => stamp.batchID === f.batchId); - if (mtdtIx === undefined) { - this.fileInfoList.splice(mtdtIx, 1); + const ix = this.fileInfoList.findIndex((f) => stamp.batchID === f.batchId); + if (ix === undefined) { + this.fileInfoList.splice(ix, 1); } } @@ -132,6 +136,34 @@ export class FileManager { } } + // TODO: save owner feed topic on a separate feed protected by ACT lol + async initOwnerTopic(): Promise { + try { + const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); + const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.address); + const feedData = await fw.download(); + if (feedData.feedIndexNext === undefined) { + const ownerStamp = await this.getOwnerFeedStamp(); + if (ownerStamp === undefined) { + console.log('Owner stamp not found'); + return; + } + // TODO: handle firsrt init -> udpate feed with random data + this.topic = Utils.bytesToHex(randomBytes(TOPIC_HEX_LENGTH), TOPIC_HEX_LENGTH); + const dataRes = await this.bee.uploadData(ownerStamp.batchID, this.topic, { act: true }); + await fw.upload(ownerStamp.batchID, dataRes.reference, { index: numberToFeedIndex(0) }); + } else { + const feedTopic = await this.bee.downloadData(feedData.reference); + const feedTopicData = JSON.parse(JSON.stringify(feedTopic)) as OwnerFeedData; + + const options = makeBeeRequestOptions(feedTopicData.historyRef, this.address); + this.topic = (await this.bee.downloadData(feedTopicData.mantarayListFeedRef, options)).text(); + } + } catch (error: any) { + console.log('error reading owner feed topic: ', error); + } + } + // TODO: import other stamps in order to topup: owner(s) ? async initStamps(): Promise { try { @@ -144,23 +176,52 @@ export class FileManager { } // TODO: shared file feed similarly - // TODO: util func to make options for act headers - async initMetadataFileList(): Promise { + // TODO: heavy refactoring needed + async initFileInfoList(): Promise { const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); const feedReader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topicHex, this.address); try { const latestFeedData = await feedReader.download(); - this.nextOwnerFeedIndex = latestFeedData.feedIndexNext; + this.nextOwnerFeedIndex = makeNumericIndex(latestFeedData.feedIndexNext); const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); - const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as FileInfoFeedRef; + const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as OwnerFeedData; const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.address); - this.fileInfoFeedRefList = JSON.parse( - (await this.bee.downloadData(ownerFeedData.fileInfoFeedRef, options)).text(), - ) as FileInfoFeedRef[]; - for (const fi of this.fileInfoFeedRefList) { - const metadata = JSON.parse((await this.bee.downloadData(fi.fileInfoFeedRef)).text()) as FileInfo; - this.fileInfoList.push(metadata); + const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.mantarayListFeedRef, options)).text(); + this.mantarayFeedList = JSON.parse(mantarayFeedList) as WrappedMantarayFeed[]; + for (const mantaryFeedItem of this.mantarayFeedList) { + const wrappedMantarayFr = this.bee.makeFeedReader( + DEFAULT_FEED_TYPE, + mantaryFeedItem.mantarayFeedTopic, + this.address, + ); + // TODO: donwload encrypted file info data + const wrappedMantarayData = await wrappedMantarayFr.download(); + let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.address); + const wrappedMantarayRef = ( + await this.bee.downloadData(wrappedMantarayData.reference, options) + ).hex() as Reference; + + await this.loadMantaray(wrappedMantarayRef); + const fileInfoFork = this.mantaray.getForkAtPath(encodePathToBytes(FILEIINFO_PATH)); + const refBytes = fileInfoFork?.node.getEntry; + if (refBytes === undefined) { + console.log("object doesn't have a fileinfo entry, ref: ", wrappedMantarayRef); + continue; + } + const fileInfoRef = Utils.bytesToHex(refBytes); + + // TODO: download history data + const histsoryFork = this.mantaray.getForkAtPath(encodePathToBytes(FILEINFO_HISTORY_PATH)); + const historyRefBytes = histsoryFork?.node.getEntry; + if (historyRefBytes === undefined) { + console.log("object doesn't have a history entry, ref: ", wrappedMantarayRef); + continue; + } + const historyRef = Utils.bytesToHex(historyRefBytes); + options = makeBeeRequestOptions(historyRef, this.address); + const fileInfo = JSON.parse(JSON.stringify(await this.bee.downloadData(fileInfoRef, options))) as FileInfo; + this.fileInfoList.push(fileInfo); } console.log('Stamps fetched from feed.'); } catch (error: any) { @@ -221,7 +282,7 @@ export class FileManager { throw new Error('Wallet not initialized. Please call initializeFeed first.'); } - const reader = this.bee.makeFeedReader('sequence', this.topic, this.wallet.address); + const reader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); try { const { reference } = await reader.download(); console.log(`Latest feed reference fetched: ${reference}`); @@ -290,20 +351,18 @@ export class FileManager { // End stamp methods // TODO: only download metadata files for listing -> only download the whole file on demand - async importReferences(referenceList: Reference[], batchId?: string, isLocal = false): Promise { + async importReferences(referenceList: Reference[], isLocal = false): Promise { const processPromises = referenceList.map(async (item: any) => { const reference: Reference = isLocal ? item.hash : item; - const fileInfo = { fileRef: reference, batchId: batchId || INVALID_STMAP } as FileInfo; try { console.log(`Processing reference: ${reference}`); // Download the file to extract its metadata // TODO: act headers - const options = makeBeeRequestOptions(fileInfo.historyRef, fileInfo.owner, fileInfo.timestamp); // TODO: maybe use path to get the rootmetadata and store it locally const path = '/rootmetadata.json'; - const fileData = await this.bee.downloadFile(reference, path, options); + const fileData = await this.bee.downloadFile(reference, path); const content = Buffer.from(fileData.data.toString() || ''); const fileName = fileData.name || `pinned-${reference.substring(0, 6)}`; const contentType = fileData.contentType || 'application/octet-stream'; @@ -327,8 +386,6 @@ export class FileManager { this.addToMantaray(undefined, reference, metadata); // Track imported files - // TODO: eglref, shared, timestamp -> mantaray root metadata - // TODO: it shall be reverted to importedReferences this.importedReferences.push(reference); } catch (error: any) { console.error(`[ERROR] Failed to process reference ${reference}: ${error.message}`); @@ -485,7 +542,7 @@ export class FileManager { contentType: contentType, }); // this.addToMantaray(mantaray, uploadFileRes.reference, metadata); - mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadFileRes.reference), metadata); + // mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadFileRes.reference), metadata); } catch (error: any) { console.error(`Failed to upload file ${file}: ${error.message}`); throw error; @@ -512,7 +569,7 @@ export class FileManager { fileInfoHistoryRef = uploadInfoRes.historyAddress; console.log('Fileinfo updated: ', uploadInfoRes.reference); // this.addToMantaray(mantaray, uploadInfoRes.reference, {}); - mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadInfoRes.reference), { + mantaray.addFork(encodePathToBytes(FILEIINFO_PATH), Utils.hexToBytes(uploadInfoRes.reference), { 'Content-Type': 'application/json', Filename: FILEIINFO_NAME, }); @@ -540,9 +597,10 @@ export class FileManager { console.log('Fileinfo history updated: ', uploadHistoryRes.reference); // this.addToMantaray(mantaray, uploadHistoryRes.reference, {}); - mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadHistoryRes.reference), { + // TODO: this can be simple data + mantaray.addFork(encodePathToBytes(FILEINFO_HISTORY_PATH), Utils.hexToBytes(uploadHistoryRes.reference), { 'Content-Type': 'application/json', - Filename: FILEIINFO_NAME, + Filename: FILEINFO_HISTORY_NAME, }); } catch (error: any) { console.error(`Failed to save fileinfo history: ${error}`); @@ -561,20 +619,18 @@ export class FileManager { // TODO: test if feed ACT up/down actually works !!! // TODO: uploadfile should accept a topic if a new version is uploaded const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); - const feedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); - const wrappedUploadRes = await this.bee.uploadData(batchId, wrappedMantarayRef); - const feedWriteResult = await feedWriter.upload(batchId, wrappedUploadRes.reference, { + const wrappedMantarayFw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); + const wrappedMantarayData = await this.bee.uploadData(batchId, wrappedMantarayRef, { act: true }); + const feedWriteResult = await wrappedMantarayFw.upload(batchId, wrappedMantarayData.reference, { index: undefined, // todo: keep track of the latest index ?? - act: true, }); // TODO: properly handle feed data of wrapped feed addresses and histories - this.fileInfoFeedRefList.push({ - // feedWriteResultRef: feedWriteResult.reference, ??? TODO: check if this is needed - fileInfoFeedRef: topicHex, - historyRef: feedWriteResult.historyAddress, - }); + const feedUpdate = { + mantarayFeedTopic: topicHex, + historyRef: wrappedMantarayData.historyAddress, + } as WrappedMantarayFeed; - await this.saveFileInfoListFeed(); + await this.saveFileInfoListFeed(feedUpdate, topicHex); } catch (error: any) { console.error(`Failed to save owner info feed: ${error}`); throw error; @@ -824,35 +880,39 @@ export class FileManager { } // Start feed handler methods - // TODO: probably a missing bee-js impl.: history upload option if act is true - async saveFileInfoListFeed(): Promise { + async saveFileInfoListFeed(feedUpdate: WrappedMantarayFeed, topicHex: string): Promise { const ownerFeedStamp = await this.getOwnerFeedStamp(); if (!ownerFeedStamp) { console.error('Owner feed stamp is not found.'); return; } + const ix = this.mantarayFeedList.findIndex((f) => f.mantarayFeedTopic === topicHex); + if (ix !== -1) { + this.mantarayFeedList[ix] = feedUpdate; + } else { + this.mantarayFeedList.push(feedUpdate); + } + const ownerFeedTopicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, ownerFeedTopicHex, this.privateKey); try { - const fileInfoFeedUploadRes = await this.bee.uploadData( + const mantarayFeedListData = await this.bee.uploadData( ownerFeedStamp.batchID, - JSON.stringify(this.fileInfoFeedRefList), + JSON.stringify(this.mantarayFeedList), { act: true, }, ); const ownerFeedData = { - fileInfoFeedRef: fileInfoFeedUploadRes.reference, - historyRef: fileInfoFeedUploadRes.historyAddress, - } as FileInfoFeedRef; - const ownerFeedRawDataUploadResult = await this.bee.uploadData( - ownerFeedStamp.batchID, - JSON.stringify(ownerFeedData), - ); - const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawDataUploadResult.reference, { - index: this.nextOwnerFeedIndex, + mantarayListFeedRef: mantarayFeedListData.reference, + historyRef: mantarayFeedListData.historyAddress, + } as OwnerFeedData; + const ownerFeedRawData = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(ownerFeedData)); + const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawData.reference, { + index: numberToFeedIndex(this.nextOwnerFeedIndex), }); + this.nextOwnerFeedIndex += 1; console.log('Metdata and owner feed updated: ', writeResult.reference); } catch (error: any) { console.error(`Failed to update metdata and owner feed: ${error}`); @@ -1008,6 +1068,7 @@ export class FileManager { file.eGlRef, ); + // TODO: create a fileinfo and update a the wrapped mantaray with it if (grantResult !== undefined) { const feedMetadatRef = await this.updateWrappedMantaray({ ...file, diff --git a/src/types.ts b/src/types.ts index d529068..82cde36 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,4 @@ -import { BatchId, Reference } from '@ethersphere/bee-js'; +import { BatchId, Reference, ReferenceResponse } from '@ethersphere/bee-js'; export interface FileInfo { fileRef: string | Reference; @@ -15,15 +15,15 @@ export interface FileInfoHistory { fileInfoHistoryRef: string; } -export interface FileInfoFeedRef { - fileInfoFeedRef: string; +export interface WrappedMantarayFeed { + mantarayFeedTopic: string; historyRef: string; } -// export interface MetadataFile { -// history: string; -// metadataReference: string; -// } +export interface OwnerFeedData { + mantarayListFeedRef: string; + historyRef: string; +} // TODO: unify own files with shared and add stamp data potentially export interface ShareItem { @@ -32,3 +32,20 @@ export interface ShareItem { timestamp?: number; message?: string; } +export interface Bytes extends Uint8Array { + readonly length: Length; +} +export type IndexBytes = Bytes<8>; +export interface Epoch { + time: number; + level: number; +} + +export type Index = number | Epoch | IndexBytes | string; +interface FeedUpdateHeaders { + feedIndex: Index; + feedIndexNext: string; +} +export interface FetchFeedUpdateResponse extends ReferenceResponse, FeedUpdateHeaders {} +const feedTypes = ['sequence', 'epoch'] as const; +export type FeedType = (typeof feedTypes)[number]; diff --git a/src/utils.ts b/src/utils.ts index f4bf1ad..88fa327 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,7 +1,8 @@ -import { BeeRequestOptions } from '@ethersphere/bee-js'; +import { BeeRequestOptions, Utils } from '@ethersphere/bee-js'; +import { Binary } from 'cafe-utility'; import path from 'path'; -import { ShareItem } from './types'; +import { Index, ShareItem } from './types'; export function getContentType(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); @@ -78,3 +79,35 @@ export function makeBeeRequestOptions(historyRef?: string, publisher?: string, t return options; } + +export function numberToFeedIndex(index: number | undefined): string | undefined { + if (index === undefined) { + return undefined; + } + const bytes = new Uint8Array(8); + const dv = new DataView(bytes.buffer); + dv.setUint32(4, index); + + return Utils.bytesToHex(bytes); +} + +export function makeNumericIndex(index: Index): number { + if (index instanceof Uint8Array) { + return Binary.uint64BEToNumber(index); + } + + if (typeof index === 'string') { + const base = 16; + const ix = parseInt(index, base); + if (isNaN(ix)) { + throw new TypeError(`Invalid index: ${index}`); + } + return ix; + } + + if (typeof index === 'number') { + return index; + } + + throw new TypeError(`Unknown type of index: ${index}`); +} From dc387dbf4873a63aafe318edba70b224f234389b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Thu, 23 Jan 2025 19:48:53 +0100 Subject: [PATCH 09/18] refactor owner topic and uploadFile --- src/constants.ts | 1 - src/fileManager.ts | 281 ++++++++++++++++++++++----------------------- src/types.ts | 25 ++-- 3 files changed, 145 insertions(+), 162 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 65d53cb..5316229 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -2,7 +2,6 @@ import { FeedType } from './types'; export const DEFAULT_FEED_TYPE: FeedType = 'sequence'; export const REFERENCE_LIST_TOPIC = 'reference-list'; -export const FILEINFO_TOPIC = 'fileinfo'; export const SHARED_INBOX_TOPIC = 'shared-inbox'; export const OWNER_FEED_STAMP_LABEL = 'owner-stamp'; export const INVALID_STMAP = '0'.repeat(64); diff --git a/src/fileManager.ts b/src/fileManager.ts index 8c58758..4419e31 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -3,16 +3,14 @@ import { Bee, BeeRequestOptions, Data, - FileUploadOptions, GranteesResult, PostageBatch, PssSubscription, RedundancyLevel, Reference, REFERENCE_HEX_LENGTH, + Signer, TOPIC_HEX_LENGTH, - UploadRedundancyOptions, - UploadResultWithCid, Utils, } from '@ethersphere/bee-js'; import { initManifestNode, MantarayNode, MetadataMapping, Reference as MantarayRef } from '@solarpunkltd/mantaray-js'; @@ -27,12 +25,11 @@ import { FILEIINFO_PATH, FILEINFO_HISTORY_NAME, FILEINFO_HISTORY_PATH, - INVALID_STMAP, OWNER_FEED_STAMP_LABEL, REFERENCE_LIST_TOPIC, SHARED_INBOX_TOPIC, } from './constants'; -import { FileInfo, FileInfoHistory, OwnerFeedData, ShareItem, WrappedMantarayFeed } from './types'; +import { FileInfo, ReferenceWithHistory, ShareItem } from './types'; import { assertSharedMessage, decodeBytesToPath, @@ -45,22 +42,20 @@ import { export class FileManager { // TODO: private vars - // TODO: store shared refs and own files in the same array ? public bee: Bee; // TODO: is this.mantaray needed ? always a new mantaray instance is created when wokring on an item public mantaray: MantarayNode; + private wallet: Wallet; + private signer: Signer; private importedReferences: string[]; private stampList: PostageBatch[]; - private mantarayFeedList: WrappedMantarayFeed[]; + private mantarayFeedList: ReferenceWithHistory[]; private fileInfoList: FileInfo[]; private nextOwnerFeedIndex: number; - private wallet: Wallet; - private privateKey: string; private granteeLists: string[]; private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; - private address: string; private topic: string; constructor(beeUrl: string, privateKey: string) { @@ -73,29 +68,29 @@ export class FileManager { console.log('Initializing Bee client...'); this.bee = new Bee(beeUrl); + this.sharedSubscription = this.subscribeToSharedInbox(); + this.wallet = new Wallet(privateKey); + this.signer = { + address: Utils.hexToBytes(this.wallet.address.slice(2)), + sign: async (data: Data): Promise => { + return await this.wallet.signMessage(data); + }, + }; this.mantaray = initManifestNode(); - this.importedReferences = []; this.stampList = []; + this.importedReferences = []; this.fileInfoList = []; this.mantarayFeedList = []; this.nextOwnerFeedIndex = -1; - this.privateKey = privateKey; - this.wallet = new Wallet(privateKey); - this.address = this.wallet.address; this.topic = ''; this.granteeLists = []; this.sharedWithMe = []; - this.sharedSubscription = {} as PssSubscription; } // Start init methods // TODO: use allSettled for file fetching and only save the ones that are successful async initialize(items: any | undefined): Promise { - try { - this.sharedSubscription = this.subscribeToSharedInbox(); - } catch (error: any) { - console.log('Error during shared inbox subscription: ', error); - } + this.initOwnerTopic(); try { console.log('Importing stamps...'); @@ -136,35 +131,34 @@ export class FileManager { } } - // TODO: save owner feed topic on a separate feed protected by ACT lol async initOwnerTopic(): Promise { try { const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); - const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.address); - const feedData = await fw.download(); - if (feedData.feedIndexNext === undefined) { - const ownerStamp = await this.getOwnerFeedStamp(); - if (ownerStamp === undefined) { + const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.wallet.address); + const topicData = await fw.download({ index: numberToFeedIndex(0) }); + if (makeNumericIndex(topicData.feedIndexNext) === 0) { + const ownerFeedStamp = await this.getOwnerFeedStamp(); + if (ownerFeedStamp === undefined) { console.log('Owner stamp not found'); return; } - // TODO: handle firsrt init -> udpate feed with random data this.topic = Utils.bytesToHex(randomBytes(TOPIC_HEX_LENGTH), TOPIC_HEX_LENGTH); - const dataRes = await this.bee.uploadData(ownerStamp.batchID, this.topic, { act: true }); - await fw.upload(ownerStamp.batchID, dataRes.reference, { index: numberToFeedIndex(0) }); + const topicDataRes = await this.bee.uploadData(ownerFeedStamp.batchID, this.topic, { act: true }); + await fw.upload(ownerFeedStamp.batchID, topicDataRes.reference, { index: numberToFeedIndex(0) }); + await fw.upload(ownerFeedStamp.batchID, topicDataRes.historyAddress as Reference, { + index: numberToFeedIndex(1), + }); } else { - const feedTopic = await this.bee.downloadData(feedData.reference); - const feedTopicData = JSON.parse(JSON.stringify(feedTopic)) as OwnerFeedData; - - const options = makeBeeRequestOptions(feedTopicData.historyRef, this.address); - this.topic = (await this.bee.downloadData(feedTopicData.mantarayListFeedRef, options)).text(); + const topicHistory = await fw.download({ index: numberToFeedIndex(1) }); + const options = makeBeeRequestOptions(topicHistory.reference, this.wallet.address); + this.topic = Utils.bytesToHex(await this.bee.downloadData(topicData.reference, options)); } } catch (error: any) { - console.log('error reading owner feed topic: ', error); + console.log('error reading owner topic efeed: ', error); + throw error; } } - // TODO: import other stamps in order to topup: owner(s) ? async initStamps(): Promise { try { this.stampList = await this.getUsableStamps(); @@ -178,26 +172,24 @@ export class FileManager { // TODO: shared file feed similarly // TODO: heavy refactoring needed async initFileInfoList(): Promise { - const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); - const feedReader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topicHex, this.address); + const ownerFR = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); try { - const latestFeedData = await feedReader.download(); + const latestFeedData = await ownerFR.download(); this.nextOwnerFeedIndex = makeNumericIndex(latestFeedData.feedIndexNext); - const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); - const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as OwnerFeedData; - const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.address); - const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.mantarayListFeedRef, options)).text(); - this.mantarayFeedList = JSON.parse(mantarayFeedList) as WrappedMantarayFeed[]; + const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as ReferenceWithHistory; + const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.wallet.address); + const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.reference, options)).text(); + this.mantarayFeedList = JSON.parse(mantarayFeedList) as ReferenceWithHistory[]; for (const mantaryFeedItem of this.mantarayFeedList) { - const wrappedMantarayFr = this.bee.makeFeedReader( + const wrappedMantarayFR = this.bee.makeFeedReader( DEFAULT_FEED_TYPE, - mantaryFeedItem.mantarayFeedTopic, - this.address, + mantaryFeedItem.reference, + this.wallet.address, ); // TODO: donwload encrypted file info data - const wrappedMantarayData = await wrappedMantarayFr.download(); - let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.address); + const wrappedMantarayData = await wrappedMantarayFR.download(); + let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.wallet.address); const wrappedMantarayRef = ( await this.bee.downloadData(wrappedMantarayData.reference, options) ).hex() as Reference; @@ -219,7 +211,7 @@ export class FileManager { continue; } const historyRef = Utils.bytesToHex(historyRefBytes); - options = makeBeeRequestOptions(historyRef, this.address); + options = makeBeeRequestOptions(historyRef, this.wallet.address); const fileInfo = JSON.parse(JSON.stringify(await this.bee.downloadData(fileInfoRef, options))) as FileInfo; this.fileInfoList.push(fileInfo); } @@ -350,7 +342,6 @@ export class FileManager { } // End stamp methods - // TODO: only download metadata files for listing -> only download the whole file on demand async importReferences(referenceList: Reference[], isLocal = false): Promise { const processPromises = referenceList.map(async (item: any) => { const reference: Reference = isLocal ? item.hash : item; @@ -401,7 +392,7 @@ export class FileManager { } async importLocalReferences(items: any): Promise { - await this.importReferences(items, undefined, true); + await this.importReferences(items, undefined); } async downloadFile( @@ -506,138 +497,141 @@ export class FileManager { return validResults; // Return successful download results } - // TODO: always upload with ACT, only adding the publisher as grantee first (by defualt), then when shared, add the grantees - // TODO: store filerefs with the historyrefs - async uploadFile( - file: string, - mantaray: MantarayNode | undefined, + async upload( batchId: string | BatchId, - customMetadata = {}, + mantaray: MantarayNode | undefined, + file: string, + customMetadata: Record = {}, + refAsTopicHex: string | undefined = undefined, redundancyLevel = RedundancyLevel.MEDIUM, - refAsTopicHex?: string, - ): Promise { + ): Promise { mantaray = mantaray || this.mantaray; + + const uploadFileRes = await this.uploadFile(batchId, file, redundancyLevel); + + const fileInfo = { + fileRef: uploadFileRes.reference, + batchId: batchId, + fileName: path.basename(file), + owner: this.wallet.address, + shared: false, + historyRef: uploadFileRes.historyRef, + timestamp: new Date().getTime(), + customMetadata: customMetadata, + } as FileInfo; + + const fileInfoRes = await this.uploadFileInfo(batchId, fileInfo, redundancyLevel); + mantaray.addFork(encodePathToBytes(FILEIINFO_PATH), Utils.hexToBytes(fileInfoRes.reference), { + 'Content-Type': 'application/json', + Filename: FILEIINFO_NAME, + }); + + const uploadHistoryRef = await this.uploadFileInfoHistory(batchId, fileInfoRes.historyRef); + mantaray.addFork(encodePathToBytes(FILEINFO_HISTORY_PATH), Utils.hexToBytes(uploadHistoryRef), { + 'Content-Type': 'application/json', + Filename: FILEINFO_HISTORY_NAME, + }); + + let wrappedMantarayRef: string; + try { + wrappedMantarayRef = await this.saveMantaray(batchId, mantaray); + } catch (error: any) { + console.error(`Failed to save wrapped mantaray: ${error}`); + throw error; + } + + await this.updateWrappedMantarayFeed(batchId, wrappedMantarayRef, refAsTopicHex); + } + + private async uploadFile( + batchId: string | BatchId, + file: string, + redundancyLevel = RedundancyLevel.MEDIUM, + ): Promise { console.log(`Uploading file: ${file}`); const fileData = readFileSync(file); const fileName = path.basename(file); const contentType = getContentType(file); - const metadata = { - 'Content-Type': contentType, - 'Content-Size': fileData.length.toString(), - 'Time-Uploaded': new Date().toISOString(), - Filename: fileName, - 'Custom-Metadata': JSON.stringify(customMetadata), - } as MetadataMapping; - - const defaultOptions = { - act: true, - redundancyLevel: redundancyLevel, - } as FileUploadOptions & UploadRedundancyOptions; - - let uploadFileRes: UploadResultWithCid; try { - uploadFileRes = await this.bee.uploadFile(batchId, fileData, fileName, { - ...defaultOptions, + const uploadFileRes = await this.bee.uploadFile(batchId, fileData, fileName, { + act: true, + redundancyLevel: redundancyLevel, contentType: contentType, }); - // this.addToMantaray(mantaray, uploadFileRes.reference, metadata); - // mantaray.addFork(encodePathToBytes(ROOT_PATH), Utils.hexToBytes(uploadFileRes.reference), metadata); + + console.log(`File uploaded successfully: ${file}, Reference: ${uploadFileRes.reference}`); + return { reference: uploadFileRes.reference, historyRef: uploadFileRes.historyAddress }; } catch (error: any) { console.error(`Failed to upload file ${file}: ${error.message}`); throw error; } + } - const fileInfo = { - fileRef: uploadFileRes.reference, - batchId: batchId, - fileName: fileName, - owner: this.address, - shared: false, - historyRef: uploadFileRes.historyAddress, - timestamp: new Date().getTime(), - eGlRef: undefined, - } as FileInfo; - - let fileInfoHistoryRef: string; + private async uploadFileInfo( + batchId: string | BatchId, + fileInfo: FileInfo, + redundancyLevel: RedundancyLevel = RedundancyLevel.MEDIUM, + ): Promise { try { - const uploadInfoRes = await this.bee.uploadFile(batchId, JSON.stringify(fileInfo), FILEIINFO_NAME, { - ...defaultOptions, - contentType: 'application/json', + const uploadInfoRes = await this.bee.uploadData(batchId, JSON.stringify(fileInfo), { + act: true, + redundancyLevel: redundancyLevel, }); - - fileInfoHistoryRef = uploadInfoRes.historyAddress; console.log('Fileinfo updated: ', uploadInfoRes.reference); - // this.addToMantaray(mantaray, uploadInfoRes.reference, {}); - mantaray.addFork(encodePathToBytes(FILEIINFO_PATH), Utils.hexToBytes(uploadInfoRes.reference), { - 'Content-Type': 'application/json', - Filename: FILEIINFO_NAME, - }); + this.fileInfoList.push(fileInfo); + + return { reference: uploadInfoRes.reference, historyRef: uploadInfoRes.historyAddress }; } catch (error: any) { console.error(`Failed to save fileinfo: ${error}`); throw error; } + } - // TODO: do not even save separately fileInfoHistory just let it be the feed data - const fileInfoHistory = { - fileInfoHistoryRef: fileInfoHistoryRef, - } as FileInfoHistory; - + private async uploadFileInfoHistory( + batchId: string | BatchId, + hisoryRef: string, + redundancyLevel: RedundancyLevel = RedundancyLevel.MEDIUM, + ): Promise { try { - const uploadHistoryRes = await this.bee.uploadFile( - batchId, - JSON.stringify(fileInfoHistory), - FILEINFO_HISTORY_NAME, - { - redundancyLevel: redundancyLevel, - contentType: 'application/json', - }, - ); + const uploadHistoryRes = await this.bee.uploadData(batchId, hisoryRef, { + redundancyLevel: redundancyLevel, + }); console.log('Fileinfo history updated: ', uploadHistoryRes.reference); - // this.addToMantaray(mantaray, uploadHistoryRes.reference, {}); - // TODO: this can be simple data - mantaray.addFork(encodePathToBytes(FILEINFO_HISTORY_PATH), Utils.hexToBytes(uploadHistoryRes.reference), { - 'Content-Type': 'application/json', - Filename: FILEINFO_HISTORY_NAME, - }); - } catch (error: any) { - console.error(`Failed to save fileinfo history: ${error}`); - throw error; - } - let wrappedMantarayRef: string; - try { - wrappedMantarayRef = await this.saveMantaray(mantaray, batchId); + return uploadHistoryRes.reference; } catch (error: any) { - console.error(`Failed to save wrapped mantaray: ${error}`); + console.error(`Failed to save fileinfo history: ${error}`); throw error; } + } + private async updateWrappedMantarayFeed( + batchId: string | BatchId, + wrappedMantarayRef: string, + refAsTopicHex: string | undefined, + ): Promise { try { // TODO: test if feed ACT up/down actually works !!! - // TODO: uploadfile should accept a topic if a new version is uploaded const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); - const wrappedMantarayFw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.privateKey); + const wrappedMantarayFw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.signer); const wrappedMantarayData = await this.bee.uploadData(batchId, wrappedMantarayRef, { act: true }); const feedWriteResult = await wrappedMantarayFw.upload(batchId, wrappedMantarayData.reference, { index: undefined, // todo: keep track of the latest index ?? }); // TODO: properly handle feed data of wrapped feed addresses and histories const feedUpdate = { - mantarayFeedTopic: topicHex, + reference: topicHex, historyRef: wrappedMantarayData.historyAddress, - } as WrappedMantarayFeed; + } as ReferenceWithHistory; await this.saveFileInfoListFeed(feedUpdate, topicHex); } catch (error: any) { console.error(`Failed to save owner info feed: ${error}`); throw error; } - - console.log(`File uploaded successfully: ${file}, Reference: ${wrappedMantarayRef}`); - return wrappedMantarayRef; } addToMantaray(mantaray: MantarayNode | undefined, reference: string, metadata: MetadataMapping = {}): void { @@ -657,7 +651,7 @@ export class FileManager { } // TODO: problem: mantary impl. is old and does not return the history address - async saveMantaray(mantaray: MantarayNode | undefined, batchId: string | BatchId): Promise { + async saveMantaray(batchId: string | BatchId, mantaray: MantarayNode | undefined): Promise { mantaray = mantaray || this.mantaray; console.log('Saving Mantaray manifest...'); @@ -880,22 +874,21 @@ export class FileManager { } // Start feed handler methods - async saveFileInfoListFeed(feedUpdate: WrappedMantarayFeed, topicHex: string): Promise { + async saveFileInfoListFeed(feedUpdate: ReferenceWithHistory, topicHex: string): Promise { const ownerFeedStamp = await this.getOwnerFeedStamp(); if (!ownerFeedStamp) { console.error('Owner feed stamp is not found.'); return; } - const ix = this.mantarayFeedList.findIndex((f) => f.mantarayFeedTopic === topicHex); + const ix = this.mantarayFeedList.findIndex((f) => f.reference === topicHex); if (ix !== -1) { this.mantarayFeedList[ix] = feedUpdate; } else { this.mantarayFeedList.push(feedUpdate); } - const ownerFeedTopicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); - const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, ownerFeedTopicHex, this.privateKey); + const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, this.topic, this.signer); try { const mantarayFeedListData = await this.bee.uploadData( ownerFeedStamp.batchID, @@ -905,9 +898,9 @@ export class FileManager { }, ); const ownerFeedData = { - mantarayListFeedRef: mantarayFeedListData.reference, + reference: mantarayFeedListData.reference, historyRef: mantarayFeedListData.historyAddress, - } as OwnerFeedData; + } as ReferenceWithHistory; const ownerFeedRawData = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(ownerFeedData)); const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawData.reference, { index: numberToFeedIndex(this.nextOwnerFeedIndex), @@ -1070,7 +1063,7 @@ export class FileManager { // TODO: create a fileinfo and update a the wrapped mantaray with it if (grantResult !== undefined) { - const feedMetadatRef = await this.updateWrappedMantaray({ + const feedMetadatRef = await this.updateWrappedMantarayFeed({ ...file, eGlRef: grantResult.ref, historyRef: grantResult.historyref, @@ -1085,7 +1078,7 @@ export class FileManager { } const item = { - owner: this.address, + owner: this.wallet.address, references: historyRefs, timestamp: Date.now(), message: message, diff --git a/src/types.ts b/src/types.ts index 82cde36..de81b01 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,27 +1,18 @@ import { BatchId, Reference, ReferenceResponse } from '@ethersphere/bee-js'; export interface FileInfo { - fileRef: string | Reference; batchId: string | BatchId; - shared?: boolean; - fileName?: string; - owner?: string; - eGlRef?: string | Reference; + fileRef: string | Reference; historyRef?: string | Reference; + owner?: string; + fileName?: string; timestamp?: number; + shared?: boolean; + customMetadata?: Record; } -export interface FileInfoHistory { - fileInfoHistoryRef: string; -} - -export interface WrappedMantarayFeed { - mantarayFeedTopic: string; - historyRef: string; -} - -export interface OwnerFeedData { - mantarayListFeedRef: string; +export interface ReferenceWithHistory { + reference: string; historyRef: string; } @@ -32,6 +23,7 @@ export interface ShareItem { timestamp?: number; message?: string; } + export interface Bytes extends Uint8Array { readonly length: Length; } @@ -40,7 +32,6 @@ export interface Epoch { time: number; level: number; } - export type Index = number | Epoch | IndexBytes | string; interface FeedUpdateHeaders { feedIndex: Index; From 9dffe279612610ddfcd765164bbb8c293f50fc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Mon, 27 Jan 2025 10:22:39 +0100 Subject: [PATCH 10/18] improve grantee handling --- src/constants.ts | 4 +- src/fileManager.ts | 277 +++++++++++++++++++++++---------------------- src/types.ts | 20 ++-- src/utils.ts | 22 ++-- 4 files changed, 159 insertions(+), 164 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 5316229..06fa2f5 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,10 +3,12 @@ import { FeedType } from './types'; export const DEFAULT_FEED_TYPE: FeedType = 'sequence'; export const REFERENCE_LIST_TOPIC = 'reference-list'; export const SHARED_INBOX_TOPIC = 'shared-inbox'; +export const SHARED_WTIHME_TOPIC = 'shared-with-me'; export const OWNER_FEED_STAMP_LABEL = 'owner-stamp'; -export const INVALID_STMAP = '0'.repeat(64); export const ROOT_PATH = '/'; export const FILEIINFO_NAME = 'fileinfo.json'; export const FILEIINFO_PATH = ROOT_PATH + 'fileinfo.json'; export const FILEINFO_HISTORY_NAME = 'history.json'; export const FILEINFO_HISTORY_PATH = ROOT_PATH + 'history.json'; +export const INVALID_STMAP = '0'.repeat(64); +export const SWARM_ZERO_ADDRESS = '0'.repeat(64); diff --git a/src/fileManager.ts b/src/fileManager.ts index 4419e31..5cc1bd7 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -28,10 +28,11 @@ import { OWNER_FEED_STAMP_LABEL, REFERENCE_LIST_TOPIC, SHARED_INBOX_TOPIC, + SWARM_ZERO_ADDRESS, } from './constants'; -import { FileInfo, ReferenceWithHistory, ShareItem } from './types'; +import { FileInfo, ReferenceWithHistory, ShareItem, WrappedMantarayFeed } from './types'; import { - assertSharedMessage, + assertShareItem, decodeBytesToPath, encodePathToBytes, getContentType, @@ -50,10 +51,10 @@ export class FileManager { private signer: Signer; private importedReferences: string[]; private stampList: PostageBatch[]; - private mantarayFeedList: ReferenceWithHistory[]; + private mantarayFeedList: WrappedMantarayFeed[]; private fileInfoList: FileInfo[]; private nextOwnerFeedIndex: number; - private granteeLists: string[]; + private granteeLists: WrappedMantarayFeed[]; private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; private topic: string; @@ -154,7 +155,7 @@ export class FileManager { this.topic = Utils.bytesToHex(await this.bee.downloadData(topicData.reference, options)); } } catch (error: any) { - console.log('error reading owner topic efeed: ', error); + console.log('error reading owner topic feed: ', error); throw error; } } @@ -169,18 +170,31 @@ export class FileManager { } } + // TODO: refactor initFileInfoList + // async fetchWrappedFeedData( + // topic: string, + // address: string, + // index?: number, + // options?: BeeRequestOptions, + // ): Promise { + // const reader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topic, address); + // const refFeedData = await reader.download({ index: numberToFeedIndex(index) }); + // } + // TODO: shared file feed similarly - // TODO: heavy refactoring needed async initFileInfoList(): Promise { - const ownerFR = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); try { + const ownerFR = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); const latestFeedData = await ownerFR.download(); this.nextOwnerFeedIndex = makeNumericIndex(latestFeedData.feedIndexNext); + const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); - const ownerFeedData = JSON.parse(JSON.stringify(ownerFeedRawData)) as ReferenceWithHistory; + const ownerFeedData = JSON.parse(ownerFeedRawData.text()) as ReferenceWithHistory; + const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.wallet.address); const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.reference, options)).text(); this.mantarayFeedList = JSON.parse(mantarayFeedList) as ReferenceWithHistory[]; + for (const mantaryFeedItem of this.mantarayFeedList) { const wrappedMantarayFR = this.bee.makeFeedReader( DEFAULT_FEED_TYPE, @@ -194,16 +208,14 @@ export class FileManager { await this.bee.downloadData(wrappedMantarayData.reference, options) ).hex() as Reference; - await this.loadMantaray(wrappedMantarayRef); + await this.loadMantaray(wrappedMantarayRef, this.mantaray); const fileInfoFork = this.mantaray.getForkAtPath(encodePathToBytes(FILEIINFO_PATH)); const refBytes = fileInfoFork?.node.getEntry; if (refBytes === undefined) { console.log("object doesn't have a fileinfo entry, ref: ", wrappedMantarayRef); continue; } - const fileInfoRef = Utils.bytesToHex(refBytes); - // TODO: download history data const histsoryFork = this.mantaray.getForkAtPath(encodePathToBytes(FILEINFO_HISTORY_PATH)); const historyRefBytes = histsoryFork?.node.getEntry; if (historyRefBytes === undefined) { @@ -212,23 +224,24 @@ export class FileManager { } const historyRef = Utils.bytesToHex(historyRefBytes); options = makeBeeRequestOptions(historyRef, this.wallet.address); + const fileInfoRef = Utils.bytesToHex(refBytes); const fileInfo = JSON.parse(JSON.stringify(await this.bee.downloadData(fileInfoRef, options))) as FileInfo; this.fileInfoList.push(fileInfo); } console.log('Stamps fetched from feed.'); } catch (error: any) { console.error(`Failed to fetch stamps from feed: ${error}`); - return; } } // End init methods - async loadMantaray(manifestReference: Reference): Promise { + async loadMantaray(manifestReference: Reference, mantaray: MantarayNode | undefined): Promise { + mantaray = mantaray || this.mantaray; const loadFunction = async (address: MantarayRef): Promise => { return this.bee.downloadData(Utils.bytesToHex(address)); }; - await this.mantaray.load(loadFunction, Utils.hexToBytes(manifestReference)); + await mantaray.load(loadFunction, Utils.hexToBytes(manifestReference)); } async initializeFeed(stamp: string | BatchId): Promise { @@ -334,10 +347,8 @@ export class FileManager { this.stampList.push(newStamp); return newStamp; } - return undefined; } catch (error: any) { console.error(`Failed to get stamp with batchID ${batchId}: ${error.message}`); - return undefined; } } // End stamp methods @@ -497,6 +508,11 @@ export class FileManager { return validResults; // Return successful download results } + async download(fileRef: string): Promise { + // TODO: connect downdloadfile with fileinfolist and wrappee mantaray feed + return {}; + } + async upload( batchId: string | BatchId, mantaray: MantarayNode | undefined, @@ -540,7 +556,13 @@ export class FileManager { throw error; } - await this.updateWrappedMantarayFeed(batchId, wrappedMantarayRef, refAsTopicHex); + const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); + const wrappedMantarayHistory = await this.updateWrappedMantarayFeed(batchId, wrappedMantarayRef, topicHex); + + await this.saveFileInfoList({ + reference: topicHex, + historyRef: wrappedMantarayHistory, + }); } private async uploadFile( @@ -611,23 +633,17 @@ export class FileManager { private async updateWrappedMantarayFeed( batchId: string | BatchId, wrappedMantarayRef: string, - refAsTopicHex: string | undefined, - ): Promise { + topicHex: string, + ): Promise { try { // TODO: test if feed ACT up/down actually works !!! - const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); const wrappedMantarayFw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.signer); const wrappedMantarayData = await this.bee.uploadData(batchId, wrappedMantarayRef, { act: true }); - const feedWriteResult = await wrappedMantarayFw.upload(batchId, wrappedMantarayData.reference, { + await wrappedMantarayFw.upload(batchId, wrappedMantarayData.reference, { index: undefined, // todo: keep track of the latest index ?? }); - // TODO: properly handle feed data of wrapped feed addresses and histories - const feedUpdate = { - reference: topicHex, - historyRef: wrappedMantarayData.historyAddress, - } as ReferenceWithHistory; - await this.saveFileInfoListFeed(feedUpdate, topicHex); + return wrappedMantarayData.historyAddress; } catch (error: any) { console.error(`Failed to save owner info feed: ${error}`); throw error; @@ -650,7 +666,6 @@ export class FileManager { mantaray.addFork(bytesPath, Utils.hexToBytes(reference), metadataWithOriginalName); } - // TODO: problem: mantary impl. is old and does not return the history address async saveMantaray(batchId: string | BatchId, mantaray: MantarayNode | undefined): Promise { mantaray = mantaray || this.mantaray; console.log('Saving Mantaray manifest...'); @@ -874,14 +889,13 @@ export class FileManager { } // Start feed handler methods - async saveFileInfoListFeed(feedUpdate: ReferenceWithHistory, topicHex: string): Promise { + async saveFileInfoList(feedUpdate: WrappedMantarayFeed): Promise { const ownerFeedStamp = await this.getOwnerFeedStamp(); if (!ownerFeedStamp) { - console.error('Owner feed stamp is not found.'); - return; + throw 'Owner feed stamp is not found.'; } - const ix = this.mantarayFeedList.findIndex((f) => f.reference === topicHex); + const ix = this.mantarayFeedList.findIndex((f) => f.reference === feedUpdate.reference); if (ix !== -1) { this.mantarayFeedList[ix] = feedUpdate; } else { @@ -909,51 +923,43 @@ export class FileManager { console.log('Metdata and owner feed updated: ', writeResult.reference); } catch (error: any) { console.error(`Failed to update metdata and owner feed: ${error}`); - return; + throw error; } } // Start feed handler methods // Start grantee methods // fetches the list of grantees under the given reference - async getGrantees(eGlRef: string | Reference): Promise { + async getGrantees(eGlRef: string | Reference): Promise { if (eGlRef.length !== REFERENCE_HEX_LENGTH) { - console.error('Invalid reference: ', eGlRef); - return; + throw `Invalid reference: ${eGlRef}`; } - try { - // TODO: parse data as ref array - const grantResult = await this.bee.getGrantees(eGlRef); - const grantees = grantResult.data; - const granteeList = this.granteeLists.find((glref) => glref === eGlRef); - if (granteeList !== undefined) { - this.granteeLists.push(eGlRef); - } - console.log('Grantees fetched: ', grantees); - return grantees; - } catch (error: any) { - console.error(`Failed to get share grantee list: ${error}`); - return undefined; + const grantResult = await this.bee.getGrantees(eGlRef); + const grantees = grantResult.data; + const granteeList = this.granteeLists.find((gl) => gl.eGranteeRef === eGlRef); + if (granteeList !== undefined) { + this.granteeLists.push(granteeList); } + console.log('Grantees fetched: ', grantees); + return grantees; } - // TODO: do not store encrypted grantee ref in the wrapedd mantaray just in the owner mtdt feed // fetches the list of grantees who can access the file reference - async getGranteesOfFile(fileRef: string | Reference): Promise { + async getGranteesOfFile(fileRef: string | Reference): Promise { + const granteeList = this.granteeLists.find((f) => f.reference === fileRef); + if (granteeList === undefined) { + throw `Grantee list not found for file reference: ${fileRef}`; + } + const file = this.fileInfoList.find((f) => f.fileRef === fileRef); - if (file === undefined || file.eGlRef === undefined) { - console.error('File or grantee ref not found for reference: ', fileRef); - return undefined; + if (file === undefined || granteeList.eGranteeRef === undefined) { + throw `File or grantee ref not found for reference: ${fileRef}`; } - return await this.getGrantees(file.eGlRef); + return await this.getGrantees(granteeList.eGranteeRef); } - // TODO: separate revoke function or frontend will handle it by creating a new act ? - // TODO: create a feed just like for the stamps to store the grantee list refs - // TODO: create a feed for the share access that can be read by each grantee - // TODO: notify user if it has been granted access by someone else - // TODO: stamp of the file vs grantees stamp? + // TODO: as of not only add is supported // updates the list of grantees who can access the file reference under the history reference async handleGrantees( batchId: string | BatchId, @@ -962,58 +968,69 @@ export class FileManager { add?: string[]; revoke?: string[]; }, - historyRef: string | Reference, eGlRef?: string | Reference, - ): Promise { - console.log('Allowing grantees to share files with me'); - - try { - let grantResult: GranteesResult; - if (eGlRef !== undefined && eGlRef.length === REFERENCE_HEX_LENGTH) { - grantResult = await this.bee.patchGrantees(batchId, eGlRef, historyRef, grantees); - console.log('Access patched, grantee list reference: ', grantResult.ref); - } else { - if (grantees.add === undefined || grantees.add.length === 0) { - console.error('No grantees specified.'); - return undefined; - } + ): Promise { + console.log('Granting access to file: ', file.fileRef); + const fIx = this.fileInfoList.findIndex((f) => f.fileRef === file.fileRef); + if (fIx === -1) { + throw `Provided file reference not found: ${file.fileRef}`; + } - grantResult = await this.bee.createGrantees(batchId, grantees.add); - console.log('Access granted, new grantee list reference: ', grantResult.ref); + let grantResult: GranteesResult; + if (eGlRef !== undefined) { + // TODO: history ref should be optional in bee-js + grantResult = await this.bee.patchGrantees(batchId, eGlRef, file.historyRef || SWARM_ZERO_ADDRESS, grantees); + console.log('Access patched, grantee list reference: ', grantResult.ref); + } else { + if (grantees.add === undefined || grantees.add.length === 0) { + throw `No grantees specified.`; } - // TODO: how to handle sharing: base fileref remains but the latest & encrypted ref that is shared changes -> versioning ?? - const currentGranteesIx = this.granteeLists.findIndex((glref) => glref === file.eGlRef); - if (currentGranteesIx === -1) { - this.granteeLists.push(grantResult.ref); - } else { - this.granteeLists[currentGranteesIx] = grantResult.ref; - // TODO: maybe don't need to check if upload + patch happens at the same time -> add to import ? - const fIx = this.fileInfoList.findIndex((f) => f.fileRef === file.fileRef); - if (fIx === -1) { - console.log('Provided file reference not found in imported files: ', file.fileRef); - return undefined; - } else { - this.fileInfoList[fIx].eGlRef = grantResult.ref; - } - } + grantResult = await this.bee.createGrantees(batchId, grantees.add); + console.log('Access granted, new grantee list reference: ', grantResult.ref); + } - console.log('Grantees updated: ', grantResult); - return grantResult; - } catch (error: any) { - console.error(`Failed to grant share access: ${error}`); - return undefined; + const currentGranteesIx = this.granteeLists.findIndex((g) => g.eGranteeRef === eGlRef); + const granteeInfo = { + reference: file.fileRef, + historyRef: grantResult.historyref, + eGranteeRef: grantResult.ref, + } as WrappedMantarayFeed; + if (currentGranteesIx === -1) { + this.granteeLists.push(granteeInfo); + } else { + this.granteeLists[currentGranteesIx] = granteeInfo; } + + console.log('Grantees updated: ', grantResult); + return grantResult; } + + // private async saveGranteeList(grantee: GranteeReference) { + // const currentGranteesIx = this.granteeLists.findIndex((g) => g.eGlRef === eGlRef); + // const granteeInfo = { + // reference: file.fileRef, + // historyRef: grantResult.historyref, + // eGlRef: grantResult.ref, + // } as GranteeReference; + // if (currentGranteesIx === -1) { + // this.granteeLists.push(granteeInfo); + // } else { + // this.granteeLists[currentGranteesIx] = granteeInfo; + // } + // } + // End grantee methods // Start share methods + // TODO: do we want to save the shared items on a feed ? subscribeToSharedInbox(): PssSubscription { return this.bee.pssSubscribe(SHARED_INBOX_TOPIC, { onMessage: (message) => { console.log('Received shared inbox message: ', message); - assertSharedMessage(message); - this.sharedWithMe.push(message); + assertShareItem(message); + const msg = message as ShareItem; + this.sharedWithMe.push(msg); }, onError: (e) => { console.log('Error received in shared inbox: ', e.message); @@ -1031,55 +1048,39 @@ export class FileManager { } // TODO: allsettled - // TODO: history handling ? -> bee-js: is historyref mandatory ? patch can create a granteelist and update it in place async shareItems( batchId: string, - references: Reference[], + files: FileInfo[], targetOverlays: string[], recipients: string[], message?: string, ): Promise { + const fileRefs = files.map((f) => f.fileRef); try { - const historyRefs = new Array(references.length); - for (let i = 0; i < references.length; i++) { - const ref = references[i]; - const file = this.fileInfoList.find((f) => f.fileRef === ref); - if (file === undefined) { - console.log('File not found for reference: ', ref); - continue; + for (let i = 0; i < fileRefs.length; i++) { + const ref = fileRefs[i]; + const mf = this.mantarayFeedList.find((mf) => mf.reference === ref); + if (mf === undefined) { + throw 'File reference not found in mantaray feed list.'; } - if (file.historyRef === undefined) { - console.log('History not found for reference: ', ref); + + const fileInfo = this.fileInfoList.find((f) => f.fileRef === ref); + if (fileInfo === undefined) { + console.log('File not found for reference: ', ref); continue; } - // TODO: how to update fileinfo with new eglref, href and not separate params - const grantResult = await this.handleGrantees( - batchId, - { fileRef: ref, batchId: batchId }, - { add: recipients }, - file.historyRef, - file.eGlRef, - ); - // TODO: create a fileinfo and update a the wrapped mantaray with it - if (grantResult !== undefined) { - const feedMetadatRef = await this.updateWrappedMantarayFeed({ - ...file, - eGlRef: grantResult.ref, - historyRef: grantResult.historyref, - }); - - if (feedMetadatRef !== undefined) { - historyRefs[i] = grantResult.historyref; - } else { - console.log('Failed to update file metadata: ', ref); - } - } + const granteeRef = await this.handleGrantees(batchId, fileInfo, { add: recipients }); + + await this.saveFileInfoList({ + reference: mf.reference, + historyRef: mf.historyRef, + eGranteeRef: granteeRef.ref, + }); } const item = { - owner: this.wallet.address, - references: historyRefs, + fileInfoList: files, timestamp: Date.now(), message: message, } as ShareItem; @@ -1087,7 +1088,6 @@ export class FileManager { await this.sendShareMessage(batchId, targetOverlays, item, recipients); } catch (error: any) { console.log('Failed to share items: ', error); - return undefined; } } @@ -1101,7 +1101,7 @@ export class FileManager { // TODO: valid length check of recipient and target if (recipients.length === 0 || recipients.length !== targetOverlays.length) { console.log('Invalid recipients or targetoverlays specified for sharing.'); - return undefined; + return; } for (let i = 0; i < recipients.length; i++) { @@ -1116,9 +1116,11 @@ export class FileManager { } // TODO: maybe store only the encrypted refs for security and use async downloadSharedItem(file: FileInfo, path?: string): Promise { - if (!this.sharedWithMe.find((msg) => msg.references.includes(file.fileRef))) { + const references = this.sharedWithMe.map((si) => si.fileInfoList.map((fi) => fi.fileRef)).flat(); + + if (!references.includes(file.fileRef)) { console.log('Cannot find file reference in shared messages: ', file.fileRef); - return undefined; + return; } const options = makeBeeRequestOptions(file.historyRef, file.owner, file.timestamp); @@ -1129,7 +1131,6 @@ export class FileManager { return data.data; } catch (error: any) { console.error(`Failed to download shared file ${file.fileRef}\n: ${error}`); - return undefined; } } // End share methods diff --git a/src/types.ts b/src/types.ts index de81b01..8b3623d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,9 +1,7 @@ -import { BatchId, Reference, ReferenceResponse } from '@ethersphere/bee-js'; - export interface FileInfo { - batchId: string | BatchId; - fileRef: string | Reference; - historyRef?: string | Reference; + batchId: string; + fileRef: string; + historyRef?: string; owner?: string; fileName?: string; timestamp?: number; @@ -16,10 +14,13 @@ export interface ReferenceWithHistory { historyRef: string; } +export interface WrappedMantarayFeed extends ReferenceWithHistory { + eGranteeRef?: string; +} + // TODO: unify own files with shared and add stamp data potentially export interface ShareItem { - owner: string; - references: string[]; + fileInfoList: FileInfo[]; timestamp?: number; message?: string; } @@ -33,10 +34,5 @@ export interface Epoch { level: number; } export type Index = number | Epoch | IndexBytes | string; -interface FeedUpdateHeaders { - feedIndex: Index; - feedIndexNext: string; -} -export interface FetchFeedUpdateResponse extends ReferenceResponse, FeedUpdateHeaders {} const feedTypes = ['sequence', 'epoch'] as const; export type FeedType = (typeof feedTypes)[number]; diff --git a/src/utils.ts b/src/utils.ts index 88fa327..8ea8187 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -25,27 +25,23 @@ export function isStrictlyObject(value: unknown): value is Record Date: Mon, 27 Jan 2025 10:37:18 +0100 Subject: [PATCH 11/18] add getFileInfoList --- src/fileManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fileManager.ts b/src/fileManager.ts index 7c1f224..aff3e56 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -233,6 +233,10 @@ export class FileManager { } // End init methods + getFileInfoList(): FileInfo[] { + return this.fileInfoList; + } + async loadMantaray(manifestReference: Reference, mantaray: MantarayNode | undefined): Promise { mantaray = mantaray || this.mantaray; const loadFunction = async (address: Reference): Promise => { From 4ce3c171c4036bffd81ac44e5c14bde67e6ebfb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Mon, 27 Jan 2025 17:08:22 +0100 Subject: [PATCH 12/18] refactor init, wrapped feed save and add volume destroy --- src/fileManager.ts | 228 +++++++++++++++++++++++++-------------------- src/types.ts | 5 +- src/utils.ts | 4 - 3 files changed, 128 insertions(+), 109 deletions(-) diff --git a/src/fileManager.ts b/src/fileManager.ts index aff3e56..f57d14d 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -10,6 +10,7 @@ import { Reference, REFERENCE_HEX_LENGTH, Signer, + STAMPS_DEPTH_MAX, TOPIC_HEX_LENGTH, Utils, } from '@ethersphere/bee-js'; @@ -54,7 +55,7 @@ export class FileManager { private mantarayFeedList: WrappedMantarayFeed[]; private fileInfoList: FileInfo[]; private nextOwnerFeedIndex: number; - private granteeLists: WrappedMantarayFeed[]; + private granteeLists: WrappedMantarayFeed[]; // TODO: ? do I need this private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; private topic: string; @@ -91,29 +92,20 @@ export class FileManager { // Start init methods // TODO: use allSettled for file fetching and only save the ones that are successful async initialize(items: any | undefined): Promise { - this.initOwnerTopic(); - - try { - console.log('Importing stamps...'); - await this.initStamps(); - } catch (error: any) { - console.error(`[ERROR] Failed to initialize stamps: ${error.message}`); - throw error; - } - - try { - console.log('Importing metadata of files...'); - await this.initFileInfoList(); - } catch (error: any) { - console.error(`[ERROR] Failed to initialize file metadata: ${error.message}`); - throw error; - } - - // if stamp is not found than the file cannot be downloaded? is this necessary ?? - for (const stamp of this.stampList) { - const ix = this.fileInfoList.findIndex((f) => stamp.batchID === f.batchId); - if (ix === undefined) { - this.fileInfoList.splice(ix, 1); + console.log('Importing stamps...'); + await this.initStamps(); + + const topicSuccess = await this.initOwnerTopic(); + + if (topicSuccess) { + console.log('Importing file info list...'); + await this.initFileAndMantarayInfoList(); + // if stamp is not found than the file cannot be downloaded? is this necessary ?? + for (const stamp of this.stampList) { + const ix = this.fileInfoList.findIndex((f) => stamp.batchID === f.batchId); + if (ix === undefined) { + this.fileInfoList.splice(ix, 1); + } } } @@ -128,22 +120,24 @@ export class FileManager { console.log('References imported successfully.'); } catch (error: any) { console.error(`[ERROR] Failed to import references: ${error.message}`); - throw error; } } - async initOwnerTopic(): Promise { + async initOwnerTopic(): Promise { try { const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.wallet.address); const topicData = await fw.download({ index: numberToFeedIndex(0) }); + if (makeNumericIndex(topicData.feedIndexNext) === 0) { const ownerFeedStamp = await this.getOwnerFeedStamp(); if (ownerFeedStamp === undefined) { console.log('Owner stamp not found'); - return; + return false; } + this.topic = Utils.bytesToHex(randomBytes(TOPIC_HEX_LENGTH), TOPIC_HEX_LENGTH); + const topicDataRes = await this.bee.uploadData(ownerFeedStamp.batchID, this.topic, { act: true }); await fw.upload(ownerFeedStamp.batchID, topicDataRes.reference, { index: numberToFeedIndex(0) }); await fw.upload(ownerFeedStamp.batchID, topicDataRes.historyAddress as Reference, { @@ -152,11 +146,14 @@ export class FileManager { } else { const topicHistory = await fw.download({ index: numberToFeedIndex(1) }); const options = makeBeeRequestOptions(topicHistory.reference, this.wallet.address); + this.topic = Utils.bytesToHex(await this.bee.downloadData(topicData.reference, options)); } + + return true; } catch (error: any) { console.log('error reading owner topic feed: ', error); - throw error; + return false; } } @@ -166,23 +163,12 @@ export class FileManager { console.log('Usable stamps fetched successfully.'); } catch (error: any) { console.error(`Failed to fetch stamps: ${error}`); - throw error; } } - // TODO: refactor initFileInfoList - // async fetchWrappedFeedData( - // topic: string, - // address: string, - // index?: number, - // options?: BeeRequestOptions, - // ): Promise { - // const reader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topic, address); - // const refFeedData = await reader.download({ index: numberToFeedIndex(index) }); - // } - + // TODO: refactor -> seprate manataryfeed and fileinfo init // TODO: shared file feed similarly - async initFileInfoList(): Promise { + async initFileAndMantarayInfoList(): Promise { try { const ownerFR = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); const latestFeedData = await ownerFR.download(); @@ -193,7 +179,7 @@ export class FileManager { const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.wallet.address); const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.reference, options)).text(); - this.mantarayFeedList = JSON.parse(mantarayFeedList) as ReferenceWithHistory[]; + this.mantarayFeedList = JSON.parse(mantarayFeedList) as WrappedMantarayFeed[]; for (const mantaryFeedItem of this.mantarayFeedList) { const wrappedMantarayFR = this.bee.makeFeedReader( @@ -201,7 +187,7 @@ export class FileManager { mantaryFeedItem.reference, this.wallet.address, ); - // TODO: donwload encrypted file info data + const wrappedMantarayData = await wrappedMantarayFR.download(); let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.wallet.address); const wrappedMantarayRef = ( @@ -226,9 +212,9 @@ export class FileManager { const fileInfo = JSON.parse(JSON.stringify(await this.bee.downloadData(fileInfoRef, options))) as FileInfo; this.fileInfoList.push(fileInfo); } - console.log('Stamps fetched from feed.'); + console.log('File info list fetched successfully.'); } catch (error: any) { - console.error(`Failed to fetch stamps from feed: ${error}`); + console.error(`Failed to fetch info list: ${error}`); } } // End init methods @@ -334,7 +320,7 @@ export class FileManager { return this.stampList; } - async getOwnerFeedStamp(): Promise { + getOwnerFeedStamp(): PostageBatch | undefined { return this.stampList.find((s) => s.label === OWNER_FEED_STAMP_LABEL); } @@ -354,6 +340,39 @@ export class FileManager { console.error(`Failed to get stamp with batchID ${batchId}: ${error.message}`); } } + + async destroyVolume(batchId: string): Promise { + if (batchId === this.getOwnerFeedStamp()?.batchID) { + throw 'Cannot destroy owner stamp'; + } + + await this.bee.diluteBatch(batchId, STAMPS_DEPTH_MAX); + for (let i = 0; i < this.stampList.length; i++) { + if (this.stampList[i].batchID === batchId) { + this.stampList.splice(i, 1); + break; + } + } + + // TODO: how to identify the fileinforef itself ? + const fileInfoRefs: string[] = []; + for (let i = 0; i < this.fileInfoList.length, ++i; ) { + if (this.fileInfoList[i].batchId === batchId) { + fileInfoRefs.push(this.fileInfoList[i]); + this.fileInfoList.splice(i, 1); + } + } + + for (let i = 0; i < this.mantarayFeedList.length && i < fileInfoRefs.length, ++i; ) { + if (fileInfoRefs.includes(this.mantarayFeedList[i].fileInfoRef)) { + this.mantarayFeedList.splice(i, 1); + } + } + + await this.saveMantarayFeedList(); + + console.log(`Volume destroyed: ${batchId}`); + } // End stamp methods async importReferences(referenceList: Reference[], isLocal = false): Promise { @@ -554,17 +573,25 @@ export class FileManager { try { wrappedMantarayRef = await this.saveMantaray(batchId, mantaray); } catch (error: any) { - console.error(`Failed to save wrapped mantaray: ${error}`); - throw error; + throw `Failed to save wrapped mantaray: ${error}`; } const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); const wrappedMantarayHistory = await this.updateWrappedMantarayFeed(batchId, wrappedMantarayRef, topicHex); - await this.saveFileInfoList({ + const feedUpdate = { reference: topicHex, historyRef: wrappedMantarayHistory, - }); + fileInfoRef: fileInfoRes.reference, + } as WrappedMantarayFeed; + const ix = this.mantarayFeedList.findIndex((f) => f.reference === feedUpdate.reference); + if (ix !== -1) { + this.mantarayFeedList[ix] = feedUpdate; + } else { + this.mantarayFeedList.push(feedUpdate); + } + + await this.saveMantarayFeedList(); } private async uploadFile( @@ -587,8 +614,7 @@ export class FileManager { console.log(`File uploaded successfully: ${file}, Reference: ${uploadFileRes.reference}`); return { reference: uploadFileRes.reference, historyRef: uploadFileRes.historyAddress }; } catch (error: any) { - console.error(`Failed to upload file ${file}: ${error.message}`); - throw error; + throw `Failed to upload file ${file}: ${error}`; } } @@ -608,8 +634,7 @@ export class FileManager { return { reference: uploadInfoRes.reference, historyRef: uploadInfoRes.historyAddress }; } catch (error: any) { - console.error(`Failed to save fileinfo: ${error}`); - throw error; + throw `Failed to save fileinfo: ${error}`; } } @@ -627,8 +652,7 @@ export class FileManager { return uploadHistoryRes.reference; } catch (error: any) { - console.error(`Failed to save fileinfo history: ${error}`); - throw error; + throw `Failed to save fileinfo history: ${error}`; } } @@ -647,8 +671,7 @@ export class FileManager { return wrappedMantarayData.historyAddress; } catch (error: any) { - console.error(`Failed to save owner info feed: ${error}`); - throw error; + throw `Failed to save owner info feed: ${error}`; } } @@ -890,19 +913,12 @@ export class FileManager { } // Start feed handler methods - async saveFileInfoList(feedUpdate: WrappedMantarayFeed): Promise { - const ownerFeedStamp = await this.getOwnerFeedStamp(); + async saveMantarayFeedList(): Promise { + const ownerFeedStamp = this.getOwnerFeedStamp(); if (!ownerFeedStamp) { throw 'Owner feed stamp is not found.'; } - const ix = this.mantarayFeedList.findIndex((f) => f.reference === feedUpdate.reference); - if (ix !== -1) { - this.mantarayFeedList[ix] = feedUpdate; - } else { - this.mantarayFeedList.push(feedUpdate); - } - const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, this.topic, this.signer); try { const mantarayFeedListData = await this.bee.uploadData( @@ -912,19 +928,21 @@ export class FileManager { act: true, }, ); + const ownerFeedData = { reference: mantarayFeedListData.reference, historyRef: mantarayFeedListData.historyAddress, } as ReferenceWithHistory; + const ownerFeedRawData = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(ownerFeedData)); const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawData.reference, { index: numberToFeedIndex(this.nextOwnerFeedIndex), }); + this.nextOwnerFeedIndex += 1; - console.log('Metdata and owner feed updated: ', writeResult.reference); + console.log('File info list feed and owner feed updated: ', writeResult.reference); } catch (error: any) { - console.error(`Failed to update metdata and owner feed: ${error}`); - throw error; + throw `Failed to update File info list feed: ${error}`; } } // Start feed handler methods @@ -1025,13 +1043,16 @@ export class FileManager { // Start share methods // TODO: do we want to save the shared items on a feed ? - subscribeToSharedInbox(): PssSubscription { + subscribeToSharedInbox(callback?: (data: ShareItem) => void): PssSubscription { return this.bee.pssSubscribe(SHARED_INBOX_TOPIC, { onMessage: (message) => { console.log('Received shared inbox message: ', message); assertShareItem(message); const msg = message as ShareItem; this.sharedWithMe.push(msg); + if (callback) { + callback(msg); + } }, onError: (e) => { console.log('Error received in shared inbox: ', e.message); @@ -1057,39 +1078,41 @@ export class FileManager { message?: string, ): Promise { const fileRefs = files.map((f) => f.fileRef); - try { - for (let i = 0; i < fileRefs.length; i++) { - const ref = fileRefs[i]; - const mf = this.mantarayFeedList.find((mf) => mf.reference === ref); - if (mf === undefined) { - throw 'File reference not found in mantaray feed list.'; - } - - const fileInfo = this.fileInfoList.find((f) => f.fileRef === ref); - if (fileInfo === undefined) { - console.log('File not found for reference: ', ref); - continue; - } + for (let i = 0; i < fileRefs.length; i++) { + const ref = fileRefs[i]; + const mfIx = this.mantarayFeedList.findIndex((mf) => mf.reference === ref); + if (mfIx === -1) { + console.log('File reference not found in mantaray feed list.'); + continue; + } - const granteeRef = await this.handleGrantees(batchId, fileInfo, { add: recipients }); + const fileInfo = this.fileInfoList.find((f) => f.fileRef === ref); + if (fileInfo === undefined) { + console.log('File not found for reference: ', ref); + continue; + } - await this.saveFileInfoList({ - reference: mf.reference, - historyRef: mf.historyRef, - eGranteeRef: granteeRef.ref, - }); + try { + const grantResult = await this.handleGrantees(batchId, fileInfo, { add: recipients }); + + this.mantarayFeedList[mfIx] = { + ...this.mantarayFeedList[mfIx], + eGranteeRef: grantResult.ref, + } as WrappedMantarayFeed; + } catch (error) { + console.error(`Falied to update grantee list for reference ${ref}:\n${error}`); } + } - const item = { - fileInfoList: files, - timestamp: Date.now(), - message: message, - } as ShareItem; + this.saveMantarayFeedList(); - await this.sendShareMessage(batchId, targetOverlays, item, recipients); - } catch (error: any) { - console.log('Failed to share items: ', error); - } + const item = { + fileInfoList: files, + timestamp: Date.now(), + message: message, + } as ShareItem; + + this.sendShareMessage(batchId, targetOverlays, item, recipients); } // recipient is optional, if not provided the message will be broadcasted == pss public key @@ -1109,7 +1132,7 @@ export class FileManager { try { const target = Utils.makeMaxTarget(targetOverlays[i]); const msgData = new Uint8Array(Buffer.from(JSON.stringify(item))); - await this.bee.pssSend(batchId, SHARED_INBOX_TOPIC, target, msgData, recipients[i]); + this.bee.pssSend(batchId, SHARED_INBOX_TOPIC, target, msgData, recipients[i]); } catch (error: any) { console.log(`Failed to share item with recipient: ${recipients[i]}\n `, error); } @@ -1120,8 +1143,7 @@ export class FileManager { const references = this.sharedWithMe.map((si) => si.fileInfoList.map((fi) => fi.fileRef)).flat(); if (!references.includes(file.fileRef)) { - console.log('Cannot find file reference in shared messages: ', file.fileRef); - return; + throw `Cannot find file reference in shared messages: ${file.fileRef}`; } const options = makeBeeRequestOptions(file.historyRef, file.owner, file.timestamp); @@ -1131,7 +1153,7 @@ export class FileManager { const data = await this.bee.downloadFile(file.fileRef, path, options); return data.data; } catch (error: any) { - console.error(`Failed to download shared file ${file.fileRef}\n: ${error}`); + throw `Failed to download shared file ${file.fileRef}\n: ${error}`; } } // End share methods diff --git a/src/types.ts b/src/types.ts index 8b3623d..c3609c9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,7 +6,8 @@ export interface FileInfo { fileName?: string; timestamp?: number; shared?: boolean; - customMetadata?: Record; + preview?: string; + customMetadata?: Record; } export interface ReferenceWithHistory { @@ -15,10 +16,10 @@ export interface ReferenceWithHistory { } export interface WrappedMantarayFeed extends ReferenceWithHistory { + fileInfoRef: string; eGranteeRef?: string; } -// TODO: unify own files with shared and add stamp data potentially export interface ShareItem { fileInfoList: FileInfo[]; timestamp?: number; diff --git a/src/utils.ts b/src/utils.ts index 8ea8187..87719dd 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -32,10 +32,6 @@ export function assertShareItem(value: unknown): asserts value is ShareItem { const item = value as unknown as ShareItem; - if (!isObject(item)) { - throw new TypeError('ShareItem has to be object!'); - } - if (item.timestamp !== undefined && typeof item.timestamp !== 'number') { throw new TypeError('timestamp property of ShareItem has to be number!'); } From 1be57947c21fb7cc7dde17d041974e624fa52b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Tue, 28 Jan 2025 00:21:25 +0100 Subject: [PATCH 13/18] fix pr comments; refactor grantee handlers; fix destroyvolume --- src/fileManager.ts | 196 ++++++++++++++++----------------------------- src/types.ts | 5 +- 2 files changed, 71 insertions(+), 130 deletions(-) diff --git a/src/fileManager.ts b/src/fileManager.ts index f57d14d..0173bc2 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -1,14 +1,14 @@ import { - BatchId, Bee, BeeRequestOptions, Data, + FileData, + GetGranteesResult, GranteesResult, PostageBatch, PssSubscription, RedundancyLevel, Reference, - REFERENCE_HEX_LENGTH, Signer, STAMPS_DEPTH_MAX, TOPIC_HEX_LENGTH, @@ -55,7 +55,6 @@ export class FileManager { private mantarayFeedList: WrappedMantarayFeed[]; private fileInfoList: FileInfo[]; private nextOwnerFeedIndex: number; - private granteeLists: WrappedMantarayFeed[]; // TODO: ? do I need this private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; private topic: string; @@ -85,7 +84,6 @@ export class FileManager { this.mantarayFeedList = []; this.nextOwnerFeedIndex = -1; this.topic = ''; - this.granteeLists = []; this.sharedWithMe = []; } @@ -100,13 +98,6 @@ export class FileManager { if (topicSuccess) { console.log('Importing file info list...'); await this.initFileAndMantarayInfoList(); - // if stamp is not found than the file cannot be downloaded? is this necessary ?? - for (const stamp of this.stampList) { - const ix = this.fileInfoList.findIndex((f) => stamp.batchID === f.batchId); - if (ix === undefined) { - this.fileInfoList.splice(ix, 1); - } - } } try { @@ -229,10 +220,10 @@ export class FileManager { return this.bee.downloadData(address); }; - await mantaray.load(loadFunction, manifestReference); + mantaray.load(loadFunction, manifestReference); } - async initializeFeed(stamp: string | BatchId): Promise { + async initializeFeed(stamp: string): Promise { console.log('Initializing wallet and checking for existing feed...'); const reader = this.bee.makeFeedReader('sequence', this.topic, this.wallet.address); @@ -251,7 +242,7 @@ export class FileManager { } } - async saveFeed(stamp: string | BatchId): Promise { + async saveFeed(stamp: string): Promise { console.log('Saving Mantaray structure to feed...'); // Save the Mantaray structure and get the manifest reference (Uint8Array) @@ -324,7 +315,7 @@ export class FileManager { return this.stampList.find((s) => s.label === OWNER_FEED_STAMP_LABEL); } - async getCachedStamp(batchId: string | BatchId): Promise { + async getCachedStamp(batchId: string): Promise { return this.stampList.find((s) => s.batchID === batchId); } @@ -347,6 +338,7 @@ export class FileManager { } await this.bee.diluteBatch(batchId, STAMPS_DEPTH_MAX); + for (let i = 0; i < this.stampList.length; i++) { if (this.stampList[i].batchID === batchId) { this.stampList.splice(i, 1); @@ -354,18 +346,14 @@ export class FileManager { } } - // TODO: how to identify the fileinforef itself ? - const fileInfoRefs: string[] = []; for (let i = 0; i < this.fileInfoList.length, ++i; ) { - if (this.fileInfoList[i].batchId === batchId) { - fileInfoRefs.push(this.fileInfoList[i]); + const fileInfo = this.fileInfoList[i]; + if (fileInfo.batchId === batchId) { this.fileInfoList.splice(i, 1); - } - } - - for (let i = 0; i < this.mantarayFeedList.length && i < fileInfoRefs.length, ++i; ) { - if (fileInfoRefs.includes(this.mantarayFeedList[i].fileInfoRef)) { - this.mantarayFeedList.splice(i, 1); + const mfIx = this.mantarayFeedList.findIndex((mf) => mf.eFileRef === fileInfo.eFileRef); + if (mfIx !== -1) { + this.mantarayFeedList.splice(mfIx, 1); + } } } @@ -529,13 +517,13 @@ export class FileManager { return validResults; // Return successful download results } - async download(fileRef: string): Promise { - // TODO: connect downdloadfile with fileinfolist and wrappee mantaray feed - return {}; - } + // TODO: connect downdloadfile with fileinfolist and wrapped mantaray feed + // async download(eFileRef: string): Promise { + // return {}; + // } async upload( - batchId: string | BatchId, + batchId: string, mantaray: MantarayNode | undefined, file: string, customMetadata: Record = {}, @@ -546,8 +534,8 @@ export class FileManager { const uploadFileRes = await this.uploadFile(batchId, file, redundancyLevel); - const fileInfo = { - fileRef: uploadFileRes.reference, + const fileInfo: FileInfo = { + eFileRef: uploadFileRes.reference, batchId: batchId, fileName: path.basename(file), owner: this.wallet.address, @@ -555,7 +543,7 @@ export class FileManager { historyRef: uploadFileRes.historyRef, timestamp: new Date().getTime(), customMetadata: customMetadata, - } as FileInfo; + }; const fileInfoRes = await this.uploadFileInfo(batchId, fileInfo, redundancyLevel); mantaray.addFork(encodePathToBytes(FILEIINFO_PATH), fileInfoRes.reference as Reference, { @@ -579,14 +567,14 @@ export class FileManager { const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); const wrappedMantarayHistory = await this.updateWrappedMantarayFeed(batchId, wrappedMantarayRef, topicHex); - const feedUpdate = { + const feedUpdate: WrappedMantarayFeed = { reference: topicHex, historyRef: wrappedMantarayHistory, - fileInfoRef: fileInfoRes.reference, - } as WrappedMantarayFeed; + eFileRef: fileInfoRes.reference, + }; const ix = this.mantarayFeedList.findIndex((f) => f.reference === feedUpdate.reference); if (ix !== -1) { - this.mantarayFeedList[ix] = feedUpdate; + this.mantarayFeedList[ix] = { ...feedUpdate, eGranteeRef: this.mantarayFeedList[ix].eGranteeRef }; } else { this.mantarayFeedList.push(feedUpdate); } @@ -595,7 +583,7 @@ export class FileManager { } private async uploadFile( - batchId: string | BatchId, + batchId: string, file: string, redundancyLevel = RedundancyLevel.MEDIUM, ): Promise { @@ -619,7 +607,7 @@ export class FileManager { } private async uploadFileInfo( - batchId: string | BatchId, + batchId: string, fileInfo: FileInfo, redundancyLevel: RedundancyLevel = RedundancyLevel.MEDIUM, ): Promise { @@ -639,7 +627,7 @@ export class FileManager { } private async uploadFileInfoHistory( - batchId: string | BatchId, + batchId: string, hisoryRef: string, redundancyLevel: RedundancyLevel = RedundancyLevel.MEDIUM, ): Promise { @@ -657,7 +645,7 @@ export class FileManager { } private async updateWrappedMantarayFeed( - batchId: string | BatchId, + batchId: string, wrappedMantarayRef: string, topicHex: string, ): Promise { @@ -691,18 +679,15 @@ export class FileManager { mantaray.addFork(bytesPath, reference as Reference, metadataWithOriginalName); } - async saveMantaray(batchId: string | BatchId, mantaray: MantarayNode | undefined): Promise { + async saveMantaray(batchId: string, mantaray: MantarayNode | undefined): Promise { mantaray = mantaray || this.mantaray; - console.log('Saving Mantaray manifest...'); const saveFunction = async (data: Uint8Array): Promise => { const uploadResponse = await this.bee.uploadData(batchId, data); return uploadResponse.reference; }; - const manifestReference = await mantaray.save(saveFunction); - console.log(`Mantaray manifest saved, reference: ${manifestReference}`); - return manifestReference; + return mantaray.save(saveFunction); } searchFilesByName(fileNameQuery: string, includeMetadata = false): any { @@ -912,7 +897,7 @@ export class FileManager { return contents; } - // Start feed handler methods + // Start owner mantaray feed handler methods async saveMantarayFeedList(): Promise { const ownerFeedStamp = this.getOwnerFeedStamp(); if (!ownerFeedStamp) { @@ -929,10 +914,10 @@ export class FileManager { }, ); - const ownerFeedData = { + const ownerFeedData: ReferenceWithHistory = { reference: mantarayFeedListData.reference, historyRef: mantarayFeedListData.historyAddress, - } as ReferenceWithHistory; + }; const ownerFeedRawData = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(ownerFeedData)); const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawData.reference, { @@ -945,60 +930,40 @@ export class FileManager { throw `Failed to update File info list feed: ${error}`; } } - // Start feed handler methods - - // Start grantee methods - // fetches the list of grantees under the given reference - async getGrantees(eGlRef: string | Reference): Promise { - if (eGlRef.length !== REFERENCE_HEX_LENGTH) { - throw `Invalid reference: ${eGlRef}`; - } - - const grantResult = await this.bee.getGrantees(eGlRef); - const grantees = grantResult.data; - const granteeList = this.granteeLists.find((gl) => gl.eGranteeRef === eGlRef); - if (granteeList !== undefined) { - this.granteeLists.push(granteeList); - } - console.log('Grantees fetched: ', grantees); - return grantees; - } + // End owner mantaray feed handler methods + // Start grantee handler methods // fetches the list of grantees who can access the file reference - async getGranteesOfFile(fileRef: string | Reference): Promise { - const granteeList = this.granteeLists.find((f) => f.reference === fileRef); - if (granteeList === undefined) { - throw `Grantee list not found for file reference: ${fileRef}`; + async getGranteesOfFile(eFileRef: string): Promise { + const mf = this.mantarayFeedList.find((f) => f.eFileRef === eFileRef); + if (mf?.eGranteeRef === undefined) { + throw `Grantee list not found for file reference: ${eFileRef}`; } - const file = this.fileInfoList.find((f) => f.fileRef === fileRef); - if (file === undefined || granteeList.eGranteeRef === undefined) { - throw `File or grantee ref not found for reference: ${fileRef}`; - } - return await this.getGrantees(granteeList.eGranteeRef); + return this.bee.getGrantees(mf.eGranteeRef); } // TODO: as of not only add is supported // updates the list of grantees who can access the file reference under the history reference async handleGrantees( - batchId: string | BatchId, - file: FileInfo, + batchId: string, + fileInfo: FileInfo, grantees: { add?: string[]; revoke?: string[]; }, eGlRef?: string | Reference, ): Promise { - console.log('Granting access to file: ', file.fileRef); - const fIx = this.fileInfoList.findIndex((f) => f.fileRef === file.fileRef); + console.log('Granting access to file: ', fileInfo.eFileRef); + const fIx = this.fileInfoList.findIndex((f) => f.eFileRef === fileInfo.eFileRef); if (fIx === -1) { - throw `Provided file reference not found: ${file.fileRef}`; + throw `Provided file reference not found: ${fileInfo.eFileRef}`; } let grantResult: GranteesResult; if (eGlRef !== undefined) { // TODO: history ref should be optional in bee-js - grantResult = await this.bee.patchGrantees(batchId, eGlRef, file.historyRef || SWARM_ZERO_ADDRESS, grantees); + grantResult = await this.bee.patchGrantees(batchId, eGlRef, fileInfo.historyRef || SWARM_ZERO_ADDRESS, grantees); console.log('Access patched, grantee list reference: ', grantResult.ref); } else { if (grantees.add === undefined || grantees.add.length === 0) { @@ -1009,37 +974,10 @@ export class FileManager { console.log('Access granted, new grantee list reference: ', grantResult.ref); } - const currentGranteesIx = this.granteeLists.findIndex((g) => g.eGranteeRef === eGlRef); - const granteeInfo = { - reference: file.fileRef, - historyRef: grantResult.historyref, - eGranteeRef: grantResult.ref, - } as WrappedMantarayFeed; - if (currentGranteesIx === -1) { - this.granteeLists.push(granteeInfo); - } else { - this.granteeLists[currentGranteesIx] = granteeInfo; - } - - console.log('Grantees updated: ', grantResult); return grantResult; } - // private async saveGranteeList(grantee: GranteeReference) { - // const currentGranteesIx = this.granteeLists.findIndex((g) => g.eGlRef === eGlRef); - // const granteeInfo = { - // reference: file.fileRef, - // historyRef: grantResult.historyref, - // eGlRef: grantResult.ref, - // } as GranteeReference; - // if (currentGranteesIx === -1) { - // this.granteeLists.push(granteeInfo); - // } else { - // this.granteeLists[currentGranteesIx] = granteeInfo; - // } - // } - - // End grantee methods + // End grantee handler methods // Start share methods // TODO: do we want to save the shared items on a feed ? @@ -1077,7 +1015,7 @@ export class FileManager { recipients: string[], message?: string, ): Promise { - const fileRefs = files.map((f) => f.fileRef); + const fileRefs = files.map((f) => f.eFileRef); for (let i = 0; i < fileRefs.length; i++) { const ref = fileRefs[i]; const mfIx = this.mantarayFeedList.findIndex((mf) => mf.reference === ref); @@ -1086,7 +1024,7 @@ export class FileManager { continue; } - const fileInfo = this.fileInfoList.find((f) => f.fileRef === ref); + const fileInfo = this.fileInfoList.find((f) => f.eFileRef === ref); if (fileInfo === undefined) { console.log('File not found for reference: ', ref); continue; @@ -1098,7 +1036,7 @@ export class FileManager { this.mantarayFeedList[mfIx] = { ...this.mantarayFeedList[mfIx], eGranteeRef: grantResult.ref, - } as WrappedMantarayFeed; + }; } catch (error) { console.error(`Falied to update grantee list for reference ${ref}:\n${error}`); } @@ -1106,11 +1044,11 @@ export class FileManager { this.saveMantarayFeedList(); - const item = { + const item: ShareItem = { fileInfoList: files, timestamp: Date.now(), message: message, - } as ShareItem; + }; this.sendShareMessage(batchId, targetOverlays, item, recipients); } @@ -1138,23 +1076,25 @@ export class FileManager { } } } - // TODO: maybe store only the encrypted refs for security and use - async downloadSharedItem(file: FileInfo, path?: string): Promise { - const references = this.sharedWithMe.map((si) => si.fileInfoList.map((fi) => fi.fileRef)).flat(); - if (!references.includes(file.fileRef)) { - throw `Cannot find file reference in shared messages: ${file.fileRef}`; + // TODO: maybe use a fileinfo object instead of eFileRef + async downloadSharedItem(eFileRef: string, path?: string): Promise | undefined> { + let fileInfo: FileInfo | undefined; + for (let i = 0; i < this.sharedWithMe.length; i++) { + for (let j = 0; j < this.sharedWithMe[i].fileInfoList.length; j++) { + if (this.sharedWithMe[i].fileInfoList[j].eFileRef === eFileRef) { + fileInfo = this.sharedWithMe[i].fileInfoList[j]; + break; + } + } } - const options = makeBeeRequestOptions(file.historyRef, file.owner, file.timestamp); - - try { - // TODO: publisher and history headers - const data = await this.bee.downloadFile(file.fileRef, path, options); - return data.data; - } catch (error: any) { - throw `Failed to download shared file ${file.fileRef}\n: ${error}`; + if (!fileInfo) { + throw `Cannot find file shared item with ref: ${eFileRef}`; } + + const options = makeBeeRequestOptions(fileInfo.historyRef, fileInfo.owner, fileInfo.timestamp); + return this.bee.downloadFile(fileInfo.eFileRef, path, options); } // End share methods } diff --git a/src/types.ts b/src/types.ts index c3609c9..1598be1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,6 @@ export interface FileInfo { batchId: string; - fileRef: string; + eFileRef: string; historyRef?: string; owner?: string; fileName?: string; @@ -15,8 +15,9 @@ export interface ReferenceWithHistory { historyRef: string; } +// TODO: consider using a completely seprarate type for the manifestfeed because of topic === reference export interface WrappedMantarayFeed extends ReferenceWithHistory { - fileInfoRef: string; + eFileRef?: string; eGranteeRef?: string; } From c1402981177bc582d735db49c3873b53f001e279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Tue, 28 Jan 2025 17:14:45 +0100 Subject: [PATCH 14/18] share items only one by one --- src/fileManager.ts | 87 ++++++++++++++++------------------------------ src/types.ts | 2 +- src/utils.ts | 4 +++ 3 files changed, 35 insertions(+), 58 deletions(-) diff --git a/src/fileManager.ts b/src/fileManager.ts index 0173bc2..796086d 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -45,7 +45,6 @@ import { export class FileManager { // TODO: private vars public bee: Bee; - // TODO: is this.mantaray needed ? always a new mantaray instance is created when wokring on an item public mantaray: MantarayNode; private wallet: Wallet; @@ -263,9 +262,6 @@ export class FileManager { async fetchFeed(): Promise { console.log('Fetching the latest feed reference...'); - if (!this.wallet) { - throw new Error('Wallet not initialized. Please call initializeFeed first.'); - } const reader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); try { @@ -946,7 +942,6 @@ export class FileManager { // TODO: as of not only add is supported // updates the list of grantees who can access the file reference under the history reference async handleGrantees( - batchId: string, fileInfo: FileInfo, grantees: { add?: string[]; @@ -963,14 +958,19 @@ export class FileManager { let grantResult: GranteesResult; if (eGlRef !== undefined) { // TODO: history ref should be optional in bee-js - grantResult = await this.bee.patchGrantees(batchId, eGlRef, fileInfo.historyRef || SWARM_ZERO_ADDRESS, grantees); + grantResult = await this.bee.patchGrantees( + fileInfo.batchId, + eGlRef, + fileInfo.historyRef || SWARM_ZERO_ADDRESS, + grantees, + ); console.log('Access patched, grantee list reference: ', grantResult.ref); } else { if (grantees.add === undefined || grantees.add.length === 0) { throw `No grantees specified.`; } - grantResult = await this.bee.createGrantees(batchId, grantees.add); + grantResult = await this.bee.createGrantees(fileInfo.batchId, grantees.add); console.log('Access granted, new grantee list reference: ', grantResult.ref); } @@ -999,7 +999,6 @@ export class FileManager { }); } - // TODO: do we need to cancel sub at shutdown ? unsubscribeFromSharedInbox(): void { if (this.sharedSubscription) { console.log('Unsubscribed from shared inbox, topic: ', this.sharedSubscription.topic); @@ -1009,58 +1008,41 @@ export class FileManager { // TODO: allsettled async shareItems( - batchId: string, - files: FileInfo[], + fileInfo: FileInfo, targetOverlays: string[], recipients: string[], message?: string, ): Promise { - const fileRefs = files.map((f) => f.eFileRef); - for (let i = 0; i < fileRefs.length; i++) { - const ref = fileRefs[i]; - const mfIx = this.mantarayFeedList.findIndex((mf) => mf.reference === ref); - if (mfIx === -1) { - console.log('File reference not found in mantaray feed list.'); - continue; - } - - const fileInfo = this.fileInfoList.find((f) => f.eFileRef === ref); - if (fileInfo === undefined) { - console.log('File not found for reference: ', ref); - continue; - } + const mfIx = this.mantarayFeedList.findIndex((mf) => mf.reference === fileInfo.eFileRef); + if (mfIx === -1) { + console.log('File reference not found in mantaray feed list.'); + return; + } - try { - const grantResult = await this.handleGrantees(batchId, fileInfo, { add: recipients }); + try { + const grantResult = await this.handleGrantees(fileInfo, { add: recipients }); - this.mantarayFeedList[mfIx] = { - ...this.mantarayFeedList[mfIx], - eGranteeRef: grantResult.ref, - }; - } catch (error) { - console.error(`Falied to update grantee list for reference ${ref}:\n${error}`); - } + this.mantarayFeedList[mfIx] = { + ...this.mantarayFeedList[mfIx], + eGranteeRef: grantResult.ref, + }; + } catch (error) { + console.error(`Falied to update grantee list for reference ${fileInfo.eFileRef}:\n${error}`); } this.saveMantarayFeedList(); const item: ShareItem = { - fileInfoList: files, + fileInfo: fileInfo, timestamp: Date.now(), message: message, }; - this.sendShareMessage(batchId, targetOverlays, item, recipients); + this.sendShareMessage(targetOverlays, item, recipients); } // recipient is optional, if not provided the message will be broadcasted == pss public key - async sendShareMessage( - batchId: string, - targetOverlays: string[], - item: ShareItem, - recipients: string[], - ): Promise { - // TODO: valid length check of recipient and target + async sendShareMessage(targetOverlays: string[], item: ShareItem, recipients: string[]): Promise { if (recipients.length === 0 || recipients.length !== targetOverlays.length) { console.log('Invalid recipients or targetoverlays specified for sharing.'); return; @@ -1068,29 +1050,20 @@ export class FileManager { for (let i = 0; i < recipients.length; i++) { try { + // TODO: mining will take too long, 2 bytes are enough const target = Utils.makeMaxTarget(targetOverlays[i]); const msgData = new Uint8Array(Buffer.from(JSON.stringify(item))); - this.bee.pssSend(batchId, SHARED_INBOX_TOPIC, target, msgData, recipients[i]); + this.bee.pssSend(item.fileInfo.batchId, SHARED_INBOX_TOPIC, target, msgData, recipients[i]); } catch (error: any) { console.log(`Failed to share item with recipient: ${recipients[i]}\n `, error); } } } - // TODO: maybe use a fileinfo object instead of eFileRef - async downloadSharedItem(eFileRef: string, path?: string): Promise | undefined> { - let fileInfo: FileInfo | undefined; - for (let i = 0; i < this.sharedWithMe.length; i++) { - for (let j = 0; j < this.sharedWithMe[i].fileInfoList.length; j++) { - if (this.sharedWithMe[i].fileInfoList[j].eFileRef === eFileRef) { - fileInfo = this.sharedWithMe[i].fileInfoList[j]; - break; - } - } - } - - if (!fileInfo) { - throw `Cannot find file shared item with ref: ${eFileRef}`; + // TODO: download can be handled by the UI ? + async downloadSharedItem(fileInfo: FileInfo, path?: string): Promise | undefined> { + if (!this.sharedWithMe.find((item) => item.fileInfo.eFileRef === fileInfo.eFileRef)) { + throw `Cannot find shared item with ref: ${fileInfo.eFileRef}`; } const options = makeBeeRequestOptions(fileInfo.historyRef, fileInfo.owner, fileInfo.timestamp); diff --git a/src/types.ts b/src/types.ts index 1598be1..ebcd1de 100644 --- a/src/types.ts +++ b/src/types.ts @@ -22,7 +22,7 @@ export interface WrappedMantarayFeed extends ReferenceWithHistory { } export interface ShareItem { - fileInfoList: FileInfo[]; + fileInfo: FileInfo; timestamp?: number; message?: string; } diff --git a/src/utils.ts b/src/utils.ts index 87719dd..4b4e796 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -32,6 +32,10 @@ export function assertShareItem(value: unknown): asserts value is ShareItem { const item = value as unknown as ShareItem; + if (!isStrictlyObject(item.fileInfo)) { + throw new TypeError('ShareItem fileInfo has to be object!'); + } + if (item.timestamp !== undefined && typeof item.timestamp !== 'number') { throw new TypeError('timestamp property of ShareItem has to be number!'); } From eda43dc0bf13b6c2d1f05c2e2a9f4facd52dcad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Wed, 29 Jan 2025 13:27:35 +0100 Subject: [PATCH 15/18] refactor init feed and info list, remove this.mantaray --- src/fileManager.ts | 219 ++++++++++++++++++++------------------------- 1 file changed, 95 insertions(+), 124 deletions(-) diff --git a/src/fileManager.ts b/src/fileManager.ts index 796086d..d444390 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -43,10 +43,7 @@ import { } from './utils'; export class FileManager { - // TODO: private vars - public bee: Bee; - public mantaray: MantarayNode; - + private bee: Bee; private wallet: Wallet; private signer: Signer; private importedReferences: string[]; @@ -56,7 +53,7 @@ export class FileManager { private nextOwnerFeedIndex: number; private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; - private topic: string; + private ownerFeedTopic: string; constructor(beeUrl: string, privateKey: string) { if (!beeUrl) { @@ -76,28 +73,22 @@ export class FileManager { return await this.wallet.signMessage(data); }, }; - this.mantaray = initManifestNode(); this.stampList = []; this.importedReferences = []; this.fileInfoList = []; this.mantarayFeedList = []; this.nextOwnerFeedIndex = -1; - this.topic = ''; + this.ownerFeedTopic = ''; this.sharedWithMe = []; } // Start init methods // TODO: use allSettled for file fetching and only save the ones that are successful async initialize(items: any | undefined): Promise { - console.log('Importing stamps...'); await this.initStamps(); - - const topicSuccess = await this.initOwnerTopic(); - - if (topicSuccess) { - console.log('Importing file info list...'); - await this.initFileAndMantarayInfoList(); - } + await this.initOwnerFeedTopic(); + await this.initMantarayFeedList(); + await this.initFileInfoList(); try { if (items) { @@ -113,37 +104,29 @@ export class FileManager { } } - async initOwnerTopic(): Promise { - try { - const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); - const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.wallet.address); - const topicData = await fw.download({ index: numberToFeedIndex(0) }); - - if (makeNumericIndex(topicData.feedIndexNext) === 0) { - const ownerFeedStamp = await this.getOwnerFeedStamp(); - if (ownerFeedStamp === undefined) { - console.log('Owner stamp not found'); - return false; - } + async initOwnerFeedTopic(): Promise { + const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); + const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.wallet.address); + const topicData = await fw.download({ index: numberToFeedIndex(0) }); - this.topic = Utils.bytesToHex(randomBytes(TOPIC_HEX_LENGTH), TOPIC_HEX_LENGTH); + if (makeNumericIndex(topicData.feedIndexNext) === 0) { + const ownerFeedStamp = this.getOwnerFeedStamp(); + if (ownerFeedStamp === undefined) { + throw 'Owner stamp not found'; + } - const topicDataRes = await this.bee.uploadData(ownerFeedStamp.batchID, this.topic, { act: true }); - await fw.upload(ownerFeedStamp.batchID, topicDataRes.reference, { index: numberToFeedIndex(0) }); - await fw.upload(ownerFeedStamp.batchID, topicDataRes.historyAddress as Reference, { - index: numberToFeedIndex(1), - }); - } else { - const topicHistory = await fw.download({ index: numberToFeedIndex(1) }); - const options = makeBeeRequestOptions(topicHistory.reference, this.wallet.address); + this.ownerFeedTopic = Utils.bytesToHex(randomBytes(TOPIC_HEX_LENGTH), TOPIC_HEX_LENGTH); - this.topic = Utils.bytesToHex(await this.bee.downloadData(topicData.reference, options)); - } + const topicDataRes = await this.bee.uploadData(ownerFeedStamp.batchID, this.ownerFeedTopic, { act: true }); + await fw.upload(ownerFeedStamp.batchID, topicDataRes.reference, { index: numberToFeedIndex(0) }); + await fw.upload(ownerFeedStamp.batchID, topicDataRes.historyAddress as Reference, { + index: numberToFeedIndex(1), + }); + } else { + const topicHistory = await fw.download({ index: numberToFeedIndex(1) }); + const options = makeBeeRequestOptions(topicHistory.reference, this.wallet.address); - return true; - } catch (error: any) { - console.log('error reading owner topic feed: ', error); - return false; + this.ownerFeedTopic = Utils.bytesToHex(await this.bee.downloadData(topicData.reference, options)); } } @@ -156,56 +139,56 @@ export class FileManager { } } - // TODO: refactor -> seprate manataryfeed and fileinfo init - // TODO: shared file feed similarly - async initFileAndMantarayInfoList(): Promise { - try { - const ownerFR = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); - const latestFeedData = await ownerFR.download(); - this.nextOwnerFeedIndex = makeNumericIndex(latestFeedData.feedIndexNext); - - const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); - const ownerFeedData = JSON.parse(ownerFeedRawData.text()) as ReferenceWithHistory; - - const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.wallet.address); - const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.reference, options)).text(); - this.mantarayFeedList = JSON.parse(mantarayFeedList) as WrappedMantarayFeed[]; - - for (const mantaryFeedItem of this.mantarayFeedList) { - const wrappedMantarayFR = this.bee.makeFeedReader( - DEFAULT_FEED_TYPE, - mantaryFeedItem.reference, - this.wallet.address, - ); + async initMantarayFeedList(): Promise { + const ownerFR = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.ownerFeedTopic, this.wallet.address); + const latestFeedData = await ownerFR.download(); + this.nextOwnerFeedIndex = makeNumericIndex(latestFeedData.feedIndexNext); - const wrappedMantarayData = await wrappedMantarayFR.download(); - let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.wallet.address); - const wrappedMantarayRef = ( - await this.bee.downloadData(wrappedMantarayData.reference, options) - ).hex() as Reference; - - await this.loadMantaray(wrappedMantarayRef, this.mantaray); - const fileInfoFork = this.mantaray.getForkAtPath(encodePathToBytes(FILEIINFO_PATH)); - const fileInfoRef = fileInfoFork?.node.getEntry; - if (fileInfoRef === undefined) { - console.log("object doesn't have a fileinfo entry, ref: ", wrappedMantarayRef); - continue; - } + const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); + const ownerFeedData = JSON.parse(ownerFeedRawData.text()) as ReferenceWithHistory; - const histsoryFork = this.mantaray.getForkAtPath(encodePathToBytes(FILEINFO_HISTORY_PATH)); - const historyRef = histsoryFork?.node.getEntry; - if (historyRef === undefined) { - console.log("object doesn't have a history entry, ref: ", wrappedMantarayRef); - continue; - } - options = makeBeeRequestOptions(historyRef, this.wallet.address); - const fileInfo = JSON.parse(JSON.stringify(await this.bee.downloadData(fileInfoRef, options))) as FileInfo; - this.fileInfoList.push(fileInfo); + const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.wallet.address); + const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.reference, options)).text(); + this.mantarayFeedList = JSON.parse(mantarayFeedList) as WrappedMantarayFeed[]; + console.log('Mantaray feed list fetched successfully.'); + } + + async initFileInfoList(): Promise { + for (const mantaryFeedItem of this.mantarayFeedList) { + const wrappedMantarayFR = this.bee.makeFeedReader( + DEFAULT_FEED_TYPE, + mantaryFeedItem.reference, + this.wallet.address, + ); + + const wrappedMantarayData = await wrappedMantarayFR.download(); + let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.wallet.address); + const wrappedMantarayRef = ( + await this.bee.downloadData(wrappedMantarayData.reference, options) + ).hex() as Reference; + + const mantaray = initManifestNode(); + await this.loadMantaray(wrappedMantarayRef, mantaray); + const fileInfoFork = mantaray.getForkAtPath(encodePathToBytes(FILEIINFO_PATH)); + const fileInfoRef = fileInfoFork?.node.getEntry; + if (fileInfoRef === undefined) { + console.log("object doesn't have a fileinfo entry, ref: ", wrappedMantarayRef); + continue; } - console.log('File info list fetched successfully.'); - } catch (error: any) { - console.error(`Failed to fetch info list: ${error}`); + + const histsoryFork = mantaray.getForkAtPath(encodePathToBytes(FILEINFO_HISTORY_PATH)); + const historyRef = histsoryFork?.node.getEntry; + if (historyRef === undefined) { + console.log("object doesn't have a history entry, ref: ", wrappedMantarayRef); + continue; + } + + const history = (await this.bee.downloadData(historyRef, options)).text(); + options = makeBeeRequestOptions(history, this.wallet.address); + const fileInfo = JSON.parse(JSON.stringify(await this.bee.downloadData(fileInfoRef, options))) as FileInfo; + this.fileInfoList.push(fileInfo); } + console.log('File info list fetched successfully.'); } // End init methods @@ -213,8 +196,7 @@ export class FileManager { return this.fileInfoList; } - async loadMantaray(manifestReference: Reference, mantaray: MantarayNode | undefined): Promise { - mantaray = mantaray || this.mantaray; + async loadMantaray(manifestReference: Reference, mantaray: MantarayNode): Promise { const loadFunction = async (address: Reference): Promise => { return this.bee.downloadData(address); }; @@ -222,30 +204,30 @@ export class FileManager { mantaray.load(loadFunction, manifestReference); } - async initializeFeed(stamp: string): Promise { + async initializeFeed(stamp: string, mantaray: MantarayNode): Promise { console.log('Initializing wallet and checking for existing feed...'); - const reader = this.bee.makeFeedReader('sequence', this.topic, this.wallet.address); + const reader = this.bee.makeFeedReader('sequence', this.ownerFeedTopic, this.wallet.address); try { const { reference } = await reader.download(); console.log(`Existing feed found. Reference: ${reference}`); const manifestData = await this.bee.downloadData(reference); - this.mantaray.deserialize(Buffer.from(manifestData)); + mantaray.deserialize(Buffer.from(manifestData)); console.log('Mantaray structure initialized from feed.'); } catch (error) { console.log('No existing feed found. Initializing new Mantaray structure...'); - this.mantaray = new MantarayNode(); - await this.saveFeed(stamp); + mantaray = new MantarayNode(); + await this.saveFeed(stamp, mantaray); } } - async saveFeed(stamp: string): Promise { + async saveFeed(stamp: string, mantaray: MantarayNode): Promise { console.log('Saving Mantaray structure to feed...'); // Save the Mantaray structure and get the manifest reference (Uint8Array) - const manifestReference = await this.mantaray.save(async (data) => { + const manifestReference = await mantaray.save(async (data) => { const uploadResponse = await this.bee.uploadData(stamp, data); return uploadResponse.reference; // Ensure 64-byte reference }); @@ -254,7 +236,7 @@ export class FileManager { const hexManifestReference = manifestReference; // Create a feed writer and upload the manifest reference - const writer = this.bee.makeFeedWriter('sequence', this.topic, this.wallet.privateKey); + const writer = this.bee.makeFeedWriter('sequence', this.ownerFeedTopic, this.wallet.privateKey); await writer.upload(stamp, hexManifestReference as Reference); // Explicitly cast to Reference console.log(`Feed updated with reference: ${hexManifestReference}`); @@ -263,7 +245,7 @@ export class FileManager { async fetchFeed(): Promise { console.log('Fetching the latest feed reference...'); - const reader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.topic, this.wallet.address); + const reader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.ownerFeedTopic, this.wallet.address); try { const { reference } = await reader.download(); console.log(`Latest feed reference fetched: ${reference}`); @@ -391,7 +373,8 @@ export class FileManager { console.log(`Adding Reference: ${reference} as ${fileName}`); // Add the file to the Mantaray node with enriched metadata - this.addToMantaray(undefined, reference, metadata); + const mantaray = initManifestNode(); + this.addToMantaray(mantaray, reference, metadata); // Track imported files this.importedReferences.push(reference); @@ -418,7 +401,6 @@ export class FileManager { onlyMetadata = false, options?: BeeRequestOptions, ): Promise { - mantaray = mantaray || this.mantaray; console.log(`Downloading file: ${filePath}`); const normalizedPath = path.normalize(filePath); const segments = normalizedPath.split(path.sep); @@ -463,7 +445,6 @@ export class FileManager { } async downloadFiles(mantaray: MantarayNode): Promise { - mantaray = mantaray || this.mantaray; console.log('Downloading all files from Mantaray...'); const forks = mantaray.forks; if (!forks) { @@ -520,14 +501,12 @@ export class FileManager { async upload( batchId: string, - mantaray: MantarayNode | undefined, + mantaray: MantarayNode, file: string, customMetadata: Record = {}, refAsTopicHex: string | undefined = undefined, redundancyLevel = RedundancyLevel.MEDIUM, ): Promise { - mantaray = mantaray || this.mantaray; - const uploadFileRes = await this.uploadFile(batchId, file, redundancyLevel); const fileInfo: FileInfo = { @@ -659,9 +638,7 @@ export class FileManager { } } - addToMantaray(mantaray: MantarayNode | undefined, reference: string, metadata: MetadataMapping = {}): void { - mantaray = mantaray || this.mantaray; - + addToMantaray(mantaray: MantarayNode, reference: string, metadata: MetadataMapping = {}): void { const filePath = metadata.fullPath || metadata.Filename || 'file'; const originalFileName = metadata.originalFileName || path.basename(filePath); @@ -675,9 +652,7 @@ export class FileManager { mantaray.addFork(bytesPath, reference as Reference, metadataWithOriginalName); } - async saveMantaray(batchId: string, mantaray: MantarayNode | undefined): Promise { - mantaray = mantaray || this.mantaray; - + async saveMantaray(batchId: string, mantaray: MantarayNode): Promise { const saveFunction = async (data: Uint8Array): Promise => { const uploadResponse = await this.bee.uploadData(batchId, data); return uploadResponse.reference; @@ -686,10 +661,10 @@ export class FileManager { return mantaray.save(saveFunction); } - searchFilesByName(fileNameQuery: string, includeMetadata = false): any { + searchFilesByName(fileNameQuery: string, mantaray: MantarayNode, includeMetadata = false): any { console.log(`Searching for files by name: ${fileNameQuery}`); - const allFiles = this.listFiles(this.mantaray, includeMetadata); + const allFiles = this.listFiles(mantaray, includeMetadata); const filteredFiles = allFiles.filter((file: any) => file.path.includes(fileNameQuery)); @@ -697,6 +672,7 @@ export class FileManager { } searchFiles( + mantaray: MantarayNode, { fileName, directory, @@ -714,7 +690,7 @@ export class FileManager { }, includeMetadata = false, ): any { - let results = this.listFiles(this.mantaray, true); + let results = this.listFiles(mantaray, true); if (fileName) { results = results.filter((file: any) => path.posix.basename(file.path).includes(fileName)); @@ -753,8 +729,7 @@ export class FileManager { return results.map((file: any) => (includeMetadata ? file : { path: file.path })); } - listFiles(mantaray: MantarayNode | undefined, includeMetadata = false): any { - mantaray = mantaray || this.mantaray; + listFiles(mantaray: MantarayNode, includeMetadata = false): any { console.log('Listing files in Mantaray...'); const fileList = []; @@ -798,8 +773,7 @@ export class FileManager { return fileList; } - getDirectoryStructure(mantaray: MantarayNode | undefined, rootDirName: string): any { - mantaray = mantaray || this.mantaray; + getDirectoryStructure(mantaray: MantarayNode, rootDirName: string): any { console.log('Building directory structure from Mantaray...'); const structure = this.buildDirectoryStructure(mantaray); @@ -812,7 +786,6 @@ export class FileManager { } buildDirectoryStructure(mantaray: MantarayNode): any { - mantaray = mantaray || this.mantaray; console.log('Building raw directory structure...'); const structure: { [key: string]: any } = {}; @@ -843,9 +816,7 @@ export class FileManager { return structure; } - getContentsOfDirectory(targetPath: string, mantaray: MantarayNode | undefined, rootDirName: string): any { - mantaray = mantaray || this.mantaray; - + getContentsOfDirectory(targetPath: string, mantaray: MantarayNode, rootDirName: string): any { const directoryStructure: { [key: string]: any } = this.getDirectoryStructure(mantaray, rootDirName); if (targetPath === rootDirName || targetPath === '.') { @@ -900,7 +871,7 @@ export class FileManager { throw 'Owner feed stamp is not found.'; } - const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, this.topic, this.signer); + const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, this.ownerFeedTopic, this.signer); try { const mantarayFeedListData = await this.bee.uploadData( ownerFeedStamp.batchID, @@ -917,7 +888,7 @@ export class FileManager { const ownerFeedRawData = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(ownerFeedData)); const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawData.reference, { - index: numberToFeedIndex(this.nextOwnerFeedIndex), + index: this.nextOwnerFeedIndex, }); this.nextOwnerFeedIndex += 1; From 46c585f0e5044e1a43a2ced5919e210e114af254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Thu, 30 Jan 2025 15:32:17 +0100 Subject: [PATCH 16/18] add assertions --- src/fileManager.ts | 104 ++++++++++++++++++++++++++++++--------------- src/types.ts | 10 +++-- src/utils.ts | 102 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 176 insertions(+), 40 deletions(-) diff --git a/src/fileManager.ts b/src/fileManager.ts index d444390..bf47ea2 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -1,4 +1,5 @@ import { + BatchId, Bee, BeeRequestOptions, Data, @@ -11,6 +12,7 @@ import { Reference, Signer, STAMPS_DEPTH_MAX, + Topic, TOPIC_HEX_LENGTH, Utils, } from '@ethersphere/bee-js'; @@ -33,7 +35,11 @@ import { } from './constants'; import { FileInfo, ReferenceWithHistory, ShareItem, WrappedMantarayFeed } from './types'; import { + assertFileInfo, + assertReference, + assertReferenceWithHistory, assertShareItem, + assertWrappedMantarayFeed, decodeBytesToPath, encodePathToBytes, getContentType, @@ -53,7 +59,7 @@ export class FileManager { private nextOwnerFeedIndex: number; private sharedWithMe: ShareItem[]; private sharedSubscription: PssSubscription; - private ownerFeedTopic: string; + private ownerFeedTopic: Topic; constructor(beeUrl: string, privateKey: string) { if (!beeUrl) { @@ -78,7 +84,7 @@ export class FileManager { this.fileInfoList = []; this.mantarayFeedList = []; this.nextOwnerFeedIndex = -1; - this.ownerFeedTopic = ''; + this.ownerFeedTopic = this.bee.makeFeedTopic(SWARM_ZERO_ADDRESS); this.sharedWithMe = []; } @@ -126,7 +132,12 @@ export class FileManager { const topicHistory = await fw.download({ index: numberToFeedIndex(1) }); const options = makeBeeRequestOptions(topicHistory.reference, this.wallet.address); - this.ownerFeedTopic = Utils.bytesToHex(await this.bee.downloadData(topicData.reference, options)); + const topicHex = (await this.bee.downloadData(topicData.reference, options)).text(); + if (!Utils.isHexString(topicHex, TOPIC_HEX_LENGTH)) { + throw `Invalid owner feed topic_: ${topicHex}`; + } + + this.ownerFeedTopic = topicHex as Topic; } } @@ -145,11 +156,22 @@ export class FileManager { this.nextOwnerFeedIndex = makeNumericIndex(latestFeedData.feedIndexNext); const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); - const ownerFeedData = JSON.parse(ownerFeedRawData.text()) as ReferenceWithHistory; + const ownerFeedData = JSON.parse(ownerFeedRawData.text()); + assertReferenceWithHistory(ownerFeedData); const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.wallet.address); - const mantarayFeedList = (await this.bee.downloadData(ownerFeedData.reference, options)).text(); - this.mantarayFeedList = JSON.parse(mantarayFeedList) as WrappedMantarayFeed[]; + const mantarayFeedListRawData = await this.bee.downloadData(ownerFeedData.reference, options); + const mantarayFeedListData = JSON.parse(mantarayFeedListRawData.text()); + + for (const wmf of mantarayFeedListData) { + try { + assertWrappedMantarayFeed(wmf); + this.mantarayFeedList.push(wmf); + } catch (error: any) { + console.error(`Invalid WrappedMantarayFeed item, skipping it: ${error}`); + } + } + console.log('Mantaray feed list fetched successfully.'); } @@ -163,31 +185,45 @@ export class FileManager { const wrappedMantarayData = await wrappedMantarayFR.download(); let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.wallet.address); - const wrappedMantarayRef = ( - await this.bee.downloadData(wrappedMantarayData.reference, options) - ).hex() as Reference; + const wrappedMantarayRef = (await this.bee.downloadData(wrappedMantarayData.reference, options)).text(); + assertReference(wrappedMantarayRef); const mantaray = initManifestNode(); await this.loadMantaray(wrappedMantarayRef, mantaray); - const fileInfoFork = mantaray.getForkAtPath(encodePathToBytes(FILEIINFO_PATH)); - const fileInfoRef = fileInfoFork?.node.getEntry; - if (fileInfoRef === undefined) { - console.log("object doesn't have a fileinfo entry, ref: ", wrappedMantarayRef); + const histsoryFork = mantaray.getForkAtPath(encodePathToBytes(FILEINFO_HISTORY_PATH)); + const historyEntry = histsoryFork?.node.getEntry; + if (historyEntry === undefined) { + console.log("object doesn't have a history entry, ref: ", wrappedMantarayRef); continue; } - const histsoryFork = mantaray.getForkAtPath(encodePathToBytes(FILEINFO_HISTORY_PATH)); - const historyRef = histsoryFork?.node.getEntry; - if (historyRef === undefined) { - console.log("object doesn't have a history entry, ref: ", wrappedMantarayRef); + const historyRef = (await this.bee.downloadData(historyEntry, options)).text(); + try { + assertReference(historyRef); + } catch (error: any) { + console.error(`Invalid history reference: ${historyRef}`); continue; } - const history = (await this.bee.downloadData(historyRef, options)).text(); - options = makeBeeRequestOptions(history, this.wallet.address); - const fileInfo = JSON.parse(JSON.stringify(await this.bee.downloadData(fileInfoRef, options))) as FileInfo; - this.fileInfoList.push(fileInfo); + const fileInfoFork = mantaray.getForkAtPath(encodePathToBytes(FILEIINFO_PATH)); + const fileInfoEntry = fileInfoFork?.node.getEntry; + if (fileInfoEntry === undefined) { + console.log("object doesn't have a fileinfo entry, ref: ", wrappedMantarayRef); + continue; + } + + options = makeBeeRequestOptions(historyRef, this.wallet.address); + const fileInfoRawData = await this.bee.downloadData(fileInfoEntry, options); + const fileInfoData: FileInfo = JSON.parse(fileInfoRawData.text()); + + try { + assertFileInfo(fileInfoData); + this.fileInfoList.push(fileInfoData); + } catch (error: any) { + console.error(`Invalid FileInfo item, skipping it: ${error}`); + } } + console.log('File info list fetched successfully.'); } // End init methods @@ -204,7 +240,7 @@ export class FileManager { mantaray.load(loadFunction, manifestReference); } - async initializeFeed(stamp: string, mantaray: MantarayNode): Promise { + async initializeFeed(batchId: string | BatchId, mantaray: MantarayNode): Promise { console.log('Initializing wallet and checking for existing feed...'); const reader = this.bee.makeFeedReader('sequence', this.ownerFeedTopic, this.wallet.address); @@ -219,16 +255,16 @@ export class FileManager { } catch (error) { console.log('No existing feed found. Initializing new Mantaray structure...'); mantaray = new MantarayNode(); - await this.saveFeed(stamp, mantaray); + await this.saveFeed(batchId, mantaray); } } - async saveFeed(stamp: string, mantaray: MantarayNode): Promise { + async saveFeed(batchId: string | BatchId, mantaray: MantarayNode): Promise { console.log('Saving Mantaray structure to feed...'); // Save the Mantaray structure and get the manifest reference (Uint8Array) const manifestReference = await mantaray.save(async (data) => { - const uploadResponse = await this.bee.uploadData(stamp, data); + const uploadResponse = await this.bee.uploadData(batchId, data); return uploadResponse.reference; // Ensure 64-byte reference }); @@ -237,7 +273,7 @@ export class FileManager { // Create a feed writer and upload the manifest reference const writer = this.bee.makeFeedWriter('sequence', this.ownerFeedTopic, this.wallet.privateKey); - await writer.upload(stamp, hexManifestReference as Reference); // Explicitly cast to Reference + await writer.upload(batchId, hexManifestReference as Reference); // Explicitly cast to Reference console.log(`Feed updated with reference: ${hexManifestReference}`); } @@ -293,7 +329,7 @@ export class FileManager { return this.stampList.find((s) => s.label === OWNER_FEED_STAMP_LABEL); } - async getCachedStamp(batchId: string): Promise { + async getCachedStamp(batchId: string | BatchId): Promise { return this.stampList.find((s) => s.batchID === batchId); } @@ -310,7 +346,7 @@ export class FileManager { } } - async destroyVolume(batchId: string): Promise { + async destroyVolume(batchId: string | BatchId): Promise { if (batchId === this.getOwnerFeedStamp()?.batchID) { throw 'Cannot destroy owner stamp'; } @@ -500,7 +536,7 @@ export class FileManager { // } async upload( - batchId: string, + batchId: string | BatchId, mantaray: MantarayNode, file: string, customMetadata: Record = {}, @@ -526,6 +562,7 @@ export class FileManager { Filename: FILEIINFO_NAME, }); + // TODO: upload history as plain data, no need for file const uploadHistoryRef = await this.uploadFileInfoHistory(batchId, fileInfoRes.historyRef); mantaray.addFork(encodePathToBytes(FILEINFO_HISTORY_PATH), uploadHistoryRef as Reference, { 'Content-Type': 'application/json', @@ -602,7 +639,7 @@ export class FileManager { } private async uploadFileInfoHistory( - batchId: string, + batchId: string | BatchId, hisoryRef: string, redundancyLevel: RedundancyLevel = RedundancyLevel.MEDIUM, ): Promise { @@ -652,7 +689,7 @@ export class FileManager { mantaray.addFork(bytesPath, reference as Reference, metadataWithOriginalName); } - async saveMantaray(batchId: string, mantaray: MantarayNode): Promise { + async saveMantaray(batchId: string | BatchId, mantaray: MantarayNode): Promise { const saveFunction = async (data: Uint8Array): Promise => { const uploadResponse = await this.bee.uploadData(batchId, data); return uploadResponse.reference; @@ -957,10 +994,9 @@ export class FileManager { onMessage: (message) => { console.log('Received shared inbox message: ', message); assertShareItem(message); - const msg = message as ShareItem; - this.sharedWithMe.push(msg); + this.sharedWithMe.push(message); if (callback) { - callback(msg); + callback(message); } }, onError: (e) => { diff --git a/src/types.ts b/src/types.ts index ebcd1de..71f122a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,13 +1,15 @@ +import { BatchId, Reference } from '@ethersphere/bee-js'; + export interface FileInfo { - batchId: string; - eFileRef: string; - historyRef?: string; + batchId: string | BatchId; + eFileRef: string | Reference; + historyRef?: string | Reference; owner?: string; fileName?: string; timestamp?: number; shared?: boolean; preview?: string; - customMetadata?: Record; + customMetadata?: Record; } export interface ReferenceWithHistory { diff --git a/src/utils.ts b/src/utils.ts index 4b4e796..01c8449 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,8 +1,14 @@ -import { BeeRequestOptions, Utils } from '@ethersphere/bee-js'; +import { + BeeRequestOptions, + ENCRYPTED_REFERENCE_HEX_LENGTH, + Reference, + REFERENCE_HEX_LENGTH, + Utils, +} from '@ethersphere/bee-js'; import { Binary } from 'cafe-utility'; import path from 'path'; -import { Index, ShareItem } from './types'; +import { FileInfo, Index, ReferenceWithHistory, ShareItem, WrappedMantarayFeed } from './types'; export function getContentType(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); @@ -17,6 +23,14 @@ export function getContentType(filePath: string): string { return contentTypes.get(ext) || 'application/octet-stream'; } +export function assertReference(value: unknown): asserts value is Reference { + try { + Utils.assertHexString(value, REFERENCE_HEX_LENGTH); + } catch (e) { + Utils.assertHexString(value, ENCRYPTED_REFERENCE_HEX_LENGTH); + } +} + export function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object'; } @@ -25,6 +39,52 @@ export function isStrictlyObject(value: unknown): value is Record | string[]): value is Record { + return typeof value === 'object' && 'key' in value; +} + +export function assertFileInfo(value: unknown): asserts value is FileInfo { + if (!isStrictlyObject(value)) { + throw new TypeError('FileInfo has to be object!'); + } + + const fi = value as unknown as FileInfo; + + if (fi.customMetadata !== undefined && !isRecord(fi.customMetadata)) { + throw new TypeError('FileInfo customMetadata has to be object!'); + } + + if (fi.timestamp !== undefined && typeof fi.timestamp !== 'number') { + throw new TypeError('timestamp property of FileInfo has to be number!'); + } + + if (fi.owner !== undefined && !Utils.isHexEthAddress(fi.owner)) { + throw new TypeError('owner property of FileInfo has to be string!'); + } + + if (fi.fileName !== undefined && typeof fi.fileName !== 'string') { + throw new TypeError('fileName property of FileInfo has to be string!'); + } + + if (fi.preview !== undefined && typeof fi.preview !== 'string') { + throw new TypeError('preview property of FileInfo has to be string!'); + } + + if (fi.shared !== undefined && typeof fi.shared !== 'boolean') { + throw new TypeError('shared property of FileInfo has to be boolean!'); + } + + if (fi.historyRef !== undefined) { + assertReference(fi.historyRef); + throw new TypeError('historyRef property of FileInfo has to be a valid reference!'); + } + + if (fi.eFileRef !== undefined) { + assertReference(fi.eFileRef); + throw new TypeError('eFileRef property of FileInfo has to be a valid reference!'); + } +} + export function assertShareItem(value: unknown): asserts value is ShareItem { if (!isStrictlyObject(value)) { throw new TypeError('ShareItem has to be object!'); @@ -45,6 +105,44 @@ export function assertShareItem(value: unknown): asserts value is ShareItem { } } +export function assertReferenceWithHistory(value: unknown): asserts value is ReferenceWithHistory { + if (!isStrictlyObject(value)) { + throw new TypeError('ReferenceWithHistory has to be object!'); + } + + const rwh = value as unknown as ReferenceWithHistory; + + if (rwh.historyRef !== undefined) { + assertReference(rwh.historyRef); + throw new TypeError('historyRef property of ReferenceWithHistory has to be a valid reference!'); + } + + if (rwh.reference !== undefined) { + assertReference(rwh.reference); + throw new TypeError('reference property of ReferenceWithHistory has to be a valid reference!'); + } +} + +export function assertWrappedMantarayFeed(value: unknown): asserts value is WrappedMantarayFeed { + if (!isStrictlyObject(value)) { + throw new TypeError('WrappedMantarayFeed has to be object!'); + } + + assertReferenceWithHistory(value); + + const wmf = value as unknown as WrappedMantarayFeed; + + if (wmf.eFileRef !== undefined) { + assertReference(wmf.eFileRef); + throw new TypeError('eFileRef property of WrappedMantarayFeed has to be a valid reference!'); + } + + if (wmf.eGranteeRef !== undefined) { + assertReference(wmf.eGranteeRef); + throw new TypeError('eGranteeRef property of WrappedMantarayFeed has to be a valid reference!'); + } +} + export function decodeBytesToPath(bytes: Uint8Array): string { if (bytes.length !== 32) { const paddedBytes = new Uint8Array(32); From 801710ad2abaa4b31b0ffce9bd9c67480bdcb8c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Fri, 31 Jan 2025 12:15:09 +0100 Subject: [PATCH 17/18] =?UTF-8?q?refactor=20feedreaders;=C2=A0uploadfile?= =?UTF-8?q?=20with=20existing=20history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/constants.ts | 4 +- src/fileManager.ts | 243 +++++++++++++++++++++------------------------ src/types.ts | 16 ++- src/utils.ts | 19 ++++ 4 files changed, 145 insertions(+), 137 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index 06fa2f5..34b5494 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -8,7 +8,7 @@ export const OWNER_FEED_STAMP_LABEL = 'owner-stamp'; export const ROOT_PATH = '/'; export const FILEIINFO_NAME = 'fileinfo.json'; export const FILEIINFO_PATH = ROOT_PATH + 'fileinfo.json'; -export const FILEINFO_HISTORY_NAME = 'history.json'; -export const FILEINFO_HISTORY_PATH = ROOT_PATH + 'history.json'; +export const FILEINFO_HISTORY_NAME = 'history'; +export const FILEINFO_HISTORY_PATH = ROOT_PATH + 'history'; export const INVALID_STMAP = '0'.repeat(64); export const SWARM_ZERO_ADDRESS = '0'.repeat(64); diff --git a/src/fileManager.ts b/src/fileManager.ts index bf47ea2..07ffece 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -3,7 +3,6 @@ import { Bee, BeeRequestOptions, Data, - FileData, GetGranteesResult, GranteesResult, PostageBatch, @@ -33,16 +32,18 @@ import { SHARED_INBOX_TOPIC, SWARM_ZERO_ADDRESS, } from './constants'; -import { FileInfo, ReferenceWithHistory, ShareItem, WrappedMantarayFeed } from './types'; +import { FetchFeedUpdateResponse, FileInfo, ReferenceWithHistory, ShareItem, WrappedMantarayFeed } from './types'; import { assertFileInfo, assertReference, assertReferenceWithHistory, assertShareItem, + assertTopic, assertWrappedMantarayFeed, decodeBytesToPath, encodePathToBytes, getContentType, + isNotFoundError, makeBeeRequestOptions, makeNumericIndex, numberToFeedIndex, @@ -71,7 +72,7 @@ export class FileManager { console.log('Initializing Bee client...'); this.bee = new Bee(beeUrl); - this.sharedSubscription = this.subscribeToSharedInbox(); + this.sharedSubscription = this.subscribeToSharedInbox(SHARED_INBOX_TOPIC); this.wallet = new Wallet(privateKey); this.signer = { address: Utils.hexToBytes(this.wallet.address.slice(2)), @@ -110,12 +111,11 @@ export class FileManager { } } - async initOwnerFeedTopic(): Promise { - const topicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); - const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.wallet.address); - const topicData = await fw.download({ index: numberToFeedIndex(0) }); + private async initOwnerFeedTopic(): Promise { + const referenceListTopicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); + const feedTopicData = await this.getFeedData(referenceListTopicHex, this.wallet.address, 0); - if (makeNumericIndex(topicData.feedIndexNext) === 0) { + if (feedTopicData.feedIndex === numberToFeedIndex(-1)) { const ownerFeedStamp = this.getOwnerFeedStamp(); if (ownerFeedStamp === undefined) { throw 'Owner stamp not found'; @@ -124,24 +124,22 @@ export class FileManager { this.ownerFeedTopic = Utils.bytesToHex(randomBytes(TOPIC_HEX_LENGTH), TOPIC_HEX_LENGTH); const topicDataRes = await this.bee.uploadData(ownerFeedStamp.batchID, this.ownerFeedTopic, { act: true }); + const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, referenceListTopicHex, this.wallet.address); await fw.upload(ownerFeedStamp.batchID, topicDataRes.reference, { index: numberToFeedIndex(0) }); await fw.upload(ownerFeedStamp.batchID, topicDataRes.historyAddress as Reference, { index: numberToFeedIndex(1), }); } else { - const topicHistory = await fw.download({ index: numberToFeedIndex(1) }); + const topicHistory = await this.getFeedData(referenceListTopicHex, this.wallet.address, 1); const options = makeBeeRequestOptions(topicHistory.reference, this.wallet.address); - const topicHex = (await this.bee.downloadData(topicData.reference, options)).text(); - if (!Utils.isHexString(topicHex, TOPIC_HEX_LENGTH)) { - throw `Invalid owner feed topic_: ${topicHex}`; - } - - this.ownerFeedTopic = topicHex as Topic; + const topicHex = (await this.bee.downloadData(feedTopicData.reference, options)).text(); + assertTopic(topicHex); + this.ownerFeedTopic = topicHex; } } - async initStamps(): Promise { + private async initStamps(): Promise { try { this.stampList = await this.getUsableStamps(); console.log('Usable stamps fetched successfully.'); @@ -150,9 +148,12 @@ export class FileManager { } } - async initMantarayFeedList(): Promise { - const ownerFR = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.ownerFeedTopic, this.wallet.address); - const latestFeedData = await ownerFR.download(); + private async initMantarayFeedList(): Promise { + const latestFeedData = await this.getFeedData(this.ownerFeedTopic, this.wallet.address); + if (latestFeedData.feedIndex === numberToFeedIndex(-1)) { + console.log("Owner mantaray feed doesn't exist yet."); + return; + } this.nextOwnerFeedIndex = makeNumericIndex(latestFeedData.feedIndexNext); const ownerFeedRawData = await this.bee.downloadData(latestFeedData.reference); @@ -161,7 +162,7 @@ export class FileManager { const options = makeBeeRequestOptions(ownerFeedData.historyRef, this.wallet.address); const mantarayFeedListRawData = await this.bee.downloadData(ownerFeedData.reference, options); - const mantarayFeedListData = JSON.parse(mantarayFeedListRawData.text()); + const mantarayFeedListData: WrappedMantarayFeed[] = JSON.parse(mantarayFeedListRawData.text()); for (const wmf of mantarayFeedListData) { try { @@ -175,15 +176,14 @@ export class FileManager { console.log('Mantaray feed list fetched successfully.'); } - async initFileInfoList(): Promise { + private async initFileInfoList(): Promise { for (const mantaryFeedItem of this.mantarayFeedList) { - const wrappedMantarayFR = this.bee.makeFeedReader( - DEFAULT_FEED_TYPE, - mantaryFeedItem.reference, - this.wallet.address, - ); + const wrappedMantarayData = await this.getFeedData(mantaryFeedItem.reference, this.wallet.address); + if (wrappedMantarayData.feedIndex === numberToFeedIndex(-1)) { + console.log("mantaryFeedItem doesn't exist, skipping it."); + continue; + } - const wrappedMantarayData = await wrappedMantarayFR.download(); let options = makeBeeRequestOptions(mantaryFeedItem.historyRef, this.wallet.address); const wrappedMantarayRef = (await this.bee.downloadData(wrappedMantarayData.reference, options)).text(); assertReference(wrappedMantarayRef); @@ -232,15 +232,7 @@ export class FileManager { return this.fileInfoList; } - async loadMantaray(manifestReference: Reference, mantaray: MantarayNode): Promise { - const loadFunction = async (address: Reference): Promise => { - return this.bee.downloadData(address); - }; - - mantaray.load(loadFunction, manifestReference); - } - - async initializeFeed(batchId: string | BatchId, mantaray: MantarayNode): Promise { + private async initializeFeed(batchId: string | BatchId, mantaray: MantarayNode): Promise { console.log('Initializing wallet and checking for existing feed...'); const reader = this.bee.makeFeedReader('sequence', this.ownerFeedTopic, this.wallet.address); @@ -259,7 +251,7 @@ export class FileManager { } } - async saveFeed(batchId: string | BatchId, mantaray: MantarayNode): Promise { + private async saveFeed(batchId: string | BatchId, mantaray: MantarayNode): Promise { console.log('Saving Mantaray structure to feed...'); // Save the Mantaray structure and get the manifest reference (Uint8Array) @@ -278,7 +270,7 @@ export class FileManager { console.log(`Feed updated with reference: ${hexManifestReference}`); } - async fetchFeed(): Promise { + private async fetchFeed(): Promise { console.log('Fetching the latest feed reference...'); const reader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, this.ownerFeedTopic, this.wallet.address); @@ -371,22 +363,19 @@ export class FileManager { } } - await this.saveMantarayFeedList(); + this.saveMantarayFeedList(); console.log(`Volume destroyed: ${batchId}`); } // End stamp methods - async importReferences(referenceList: Reference[], isLocal = false): Promise { + private async importReferences(referenceList: Reference[], isLocal = false): Promise { const processPromises = referenceList.map(async (item: any) => { const reference: Reference = isLocal ? item.hash : item; try { console.log(`Processing reference: ${reference}`); // Download the file to extract its metadata - - // TODO: act headers - // TODO: maybe use path to get the rootmetadata and store it locally const path = '/rootmetadata.json'; const fileData = await this.bee.downloadFile(reference, path); const content = Buffer.from(fileData.data.toString() || ''); @@ -422,12 +411,12 @@ export class FileManager { await Promise.all(processPromises); // Wait for all references to be processed } - async importPinnedReferences(): Promise { + private async importPinnedReferences(): Promise { const allPins = await this.bee.getAllPins(); await this.importReferences(allPins); } - async importLocalReferences(items: any): Promise { + private async importLocalReferences(items: any): Promise { await this.importReferences(items, undefined); } @@ -530,20 +519,15 @@ export class FileManager { return validResults; // Return successful download results } - // TODO: connect downdloadfile with fileinfolist and wrapped mantaray feed - // async download(eFileRef: string): Promise { - // return {}; - // } - async upload( batchId: string | BatchId, mantaray: MantarayNode, file: string, customMetadata: Record = {}, - refAsTopicHex: string | undefined = undefined, - redundancyLevel = RedundancyLevel.MEDIUM, + currentFileInfo: FileInfo | undefined = undefined, ): Promise { - const uploadFileRes = await this.uploadFile(batchId, file, redundancyLevel); + const redundancyLevel = currentFileInfo?.redundancyLevel || RedundancyLevel.MEDIUM; + const uploadFileRes = await this.uploadFile(batchId, file, currentFileInfo); const fileInfo: FileInfo = { eFileRef: uploadFileRes.reference, @@ -553,35 +537,27 @@ export class FileManager { shared: false, historyRef: uploadFileRes.historyRef, timestamp: new Date().getTime(), + redundancyLevel: redundancyLevel, customMetadata: customMetadata, }; - const fileInfoRes = await this.uploadFileInfo(batchId, fileInfo, redundancyLevel); + const fileInfoRes = await this.uploadFileInfo(batchId, fileInfo); mantaray.addFork(encodePathToBytes(FILEIINFO_PATH), fileInfoRes.reference as Reference, { 'Content-Type': 'application/json', Filename: FILEIINFO_NAME, }); - // TODO: upload history as plain data, no need for file - const uploadHistoryRef = await this.uploadFileInfoHistory(batchId, fileInfoRes.historyRef); - mantaray.addFork(encodePathToBytes(FILEINFO_HISTORY_PATH), uploadHistoryRef as Reference, { - 'Content-Type': 'application/json', - Filename: FILEINFO_HISTORY_NAME, - }); + const uploadHistoryRes = await this.uploadFileInfoHistory(batchId, fileInfoRes.historyRef, redundancyLevel); + mantaray.addFork(encodePathToBytes(FILEINFO_HISTORY_PATH), uploadHistoryRes.historyRef as Reference); - let wrappedMantarayRef: string; - try { - wrappedMantarayRef = await this.saveMantaray(batchId, mantaray); - } catch (error: any) { - throw `Failed to save wrapped mantaray: ${error}`; - } - - const topicHex = refAsTopicHex || this.bee.makeFeedTopic(wrappedMantarayRef); - const wrappedMantarayHistory = await this.updateWrappedMantarayFeed(batchId, wrappedMantarayRef, topicHex); + const wrappedMantarayRef = await this.saveMantaray(batchId, mantaray); + const topicHex = currentFileInfo?.eFileRef || this.bee.makeFeedTopic(wrappedMantarayRef); + assertTopic(topicHex); + const wrappedFeedUpdateRes = await this.updateWrappedMantarayFeed(batchId, wrappedMantarayRef, topicHex); const feedUpdate: WrappedMantarayFeed = { reference: topicHex, - historyRef: wrappedMantarayHistory, + historyRef: wrappedFeedUpdateRes.historyRef, eFileRef: fileInfoRes.reference, }; const ix = this.mantarayFeedList.findIndex((f) => f.reference === feedUpdate.reference); @@ -595,9 +571,9 @@ export class FileManager { } private async uploadFile( - batchId: string, + batchId: string | BatchId, file: string, - redundancyLevel = RedundancyLevel.MEDIUM, + currentFileInfo: FileInfo | undefined = undefined, ): Promise { console.log(`Uploading file: ${file}`); const fileData = readFileSync(file); @@ -605,11 +581,18 @@ export class FileManager { const contentType = getContentType(file); try { - const uploadFileRes = await this.bee.uploadFile(batchId, fileData, fileName, { - act: true, - redundancyLevel: redundancyLevel, - contentType: contentType, - }); + const options = makeBeeRequestOptions(currentFileInfo?.historyRef); + const uploadFileRes = await this.bee.uploadFile( + batchId, + fileData, + fileName, + { + act: true, + redundancyLevel: currentFileInfo?.redundancyLevel || RedundancyLevel.MEDIUM, + contentType: contentType, + }, + options, + ); console.log(`File uploaded successfully: ${file}, Reference: ${uploadFileRes.reference}`); return { reference: uploadFileRes.reference, historyRef: uploadFileRes.historyAddress }; @@ -618,15 +601,11 @@ export class FileManager { } } - private async uploadFileInfo( - batchId: string, - fileInfo: FileInfo, - redundancyLevel: RedundancyLevel = RedundancyLevel.MEDIUM, - ): Promise { + private async uploadFileInfo(batchId: string | BatchId, fileInfo: FileInfo): Promise { try { const uploadInfoRes = await this.bee.uploadData(batchId, JSON.stringify(fileInfo), { act: true, - redundancyLevel: redundancyLevel, + redundancyLevel: fileInfo.redundancyLevel, }); console.log('Fileinfo updated: ', uploadInfoRes.reference); @@ -642,7 +621,7 @@ export class FileManager { batchId: string | BatchId, hisoryRef: string, redundancyLevel: RedundancyLevel = RedundancyLevel.MEDIUM, - ): Promise { + ): Promise { try { const uploadHistoryRes = await this.bee.uploadData(batchId, hisoryRef, { redundancyLevel: redundancyLevel, @@ -650,32 +629,32 @@ export class FileManager { console.log('Fileinfo history updated: ', uploadHistoryRes.reference); - return uploadHistoryRes.reference; + return { reference: uploadHistoryRes.reference, historyRef: uploadHistoryRes.reference }; } catch (error: any) { throw `Failed to save fileinfo history: ${error}`; } } private async updateWrappedMantarayFeed( - batchId: string, - wrappedMantarayRef: string, - topicHex: string, - ): Promise { + batchId: string | BatchId, + wrappedMantarayRef: Reference, + topicHex: Topic, + ): Promise { try { // TODO: test if feed ACT up/down actually works !!! const wrappedMantarayFw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, topicHex, this.signer); const wrappedMantarayData = await this.bee.uploadData(batchId, wrappedMantarayRef, { act: true }); - await wrappedMantarayFw.upload(batchId, wrappedMantarayData.reference, { + const { reference } = await wrappedMantarayFw.upload(batchId, wrappedMantarayData.reference, { index: undefined, // todo: keep track of the latest index ?? }); - return wrappedMantarayData.historyAddress; + return { reference: reference, historyRef: wrappedMantarayData.historyAddress }; } catch (error: any) { - throw `Failed to save owner info feed: ${error}`; + throw `Failed to wrapped mantaray feed: ${error}`; } } - addToMantaray(mantaray: MantarayNode, reference: string, metadata: MetadataMapping = {}): void { + private addToMantaray(mantaray: MantarayNode, reference: string, metadata: MetadataMapping = {}): void { const filePath = metadata.fullPath || metadata.Filename || 'file'; const originalFileName = metadata.originalFileName || path.basename(filePath); @@ -689,7 +668,7 @@ export class FileManager { mantaray.addFork(bytesPath, reference as Reference, metadataWithOriginalName); } - async saveMantaray(batchId: string | BatchId, mantaray: MantarayNode): Promise { + private async saveMantaray(batchId: string | BatchId, mantaray: MantarayNode): Promise { const saveFunction = async (data: Uint8Array): Promise => { const uploadResponse = await this.bee.uploadData(batchId, data); return uploadResponse.reference; @@ -698,6 +677,14 @@ export class FileManager { return mantaray.save(saveFunction); } + private async loadMantaray(manifestReference: Reference, mantaray: MantarayNode): Promise { + const loadFunction = async (address: Reference): Promise => { + return this.bee.downloadData(address); + }; + + mantaray.load(loadFunction, manifestReference); + } + searchFilesByName(fileNameQuery: string, mantaray: MantarayNode, includeMetadata = false): any { console.log(`Searching for files by name: ${fileNameQuery}`); @@ -810,7 +797,7 @@ export class FileManager { return fileList; } - getDirectoryStructure(mantaray: MantarayNode, rootDirName: string): any { + private getDirectoryStructure(mantaray: MantarayNode, rootDirName: string): any { console.log('Building directory structure from Mantaray...'); const structure = this.buildDirectoryStructure(mantaray); @@ -822,7 +809,7 @@ export class FileManager { return wrappedStructure; } - buildDirectoryStructure(mantaray: MantarayNode): any { + private buildDirectoryStructure(mantaray: MantarayNode): any { console.log('Building raw directory structure...'); const structure: { [key: string]: any } = {}; @@ -902,13 +889,12 @@ export class FileManager { } // Start owner mantaray feed handler methods - async saveMantarayFeedList(): Promise { + private async saveMantarayFeedList(): Promise { const ownerFeedStamp = this.getOwnerFeedStamp(); if (!ownerFeedStamp) { throw 'Owner feed stamp is not found.'; } - const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, this.ownerFeedTopic, this.signer); try { const mantarayFeedListData = await this.bee.uploadData( ownerFeedStamp.batchID, @@ -923,15 +909,16 @@ export class FileManager { historyRef: mantarayFeedListData.historyAddress, }; + const ownerFeedWriter = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, this.ownerFeedTopic, this.signer); const ownerFeedRawData = await this.bee.uploadData(ownerFeedStamp.batchID, JSON.stringify(ownerFeedData)); const writeResult = await ownerFeedWriter.upload(ownerFeedStamp.batchID, ownerFeedRawData.reference, { index: this.nextOwnerFeedIndex, }); this.nextOwnerFeedIndex += 1; - console.log('File info list feed and owner feed updated: ', writeResult.reference); + console.log('Owner feed list updated: ', writeResult.reference); } catch (error: any) { - throw `Failed to update File info list feed: ${error}`; + throw `Failed to update owner feed list: ${error}`; } } // End owner mantaray feed handler methods @@ -947,9 +934,9 @@ export class FileManager { return this.bee.getGrantees(mf.eGranteeRef); } - // TODO: as of not only add is supported + // TODO: only add is supported // updates the list of grantees who can access the file reference under the history reference - async handleGrantees( + private async handleGrantees( fileInfo: FileInfo, grantees: { add?: string[]; @@ -988,9 +975,9 @@ export class FileManager { // End grantee handler methods // Start share methods - // TODO: do we want to save the shared items on a feed ? - subscribeToSharedInbox(callback?: (data: ShareItem) => void): PssSubscription { - return this.bee.pssSubscribe(SHARED_INBOX_TOPIC, { + subscribeToSharedInbox(topic: string, callback?: (data: ShareItem) => void): PssSubscription { + console.log('Subscribing to shared inbox, topic: ', topic); + return this.bee.pssSubscribe(topic, { onMessage: (message) => { console.log('Received shared inbox message: ', message); assertShareItem(message); @@ -1013,29 +1000,23 @@ export class FileManager { } } - // TODO: allsettled - async shareItems( - fileInfo: FileInfo, - targetOverlays: string[], - recipients: string[], - message?: string, - ): Promise { + async shareItem(fileInfo: FileInfo, targetOverlays: string[], recipients: string[], message?: string): Promise { const mfIx = this.mantarayFeedList.findIndex((mf) => mf.reference === fileInfo.eFileRef); if (mfIx === -1) { console.log('File reference not found in mantaray feed list.'); return; } - try { - const grantResult = await this.handleGrantees(fileInfo, { add: recipients }); + const grantResult = await this.handleGrantees( + fileInfo, + { add: recipients }, + this.mantarayFeedList[mfIx].eGranteeRef, + ); - this.mantarayFeedList[mfIx] = { - ...this.mantarayFeedList[mfIx], - eGranteeRef: grantResult.ref, - }; - } catch (error) { - console.error(`Falied to update grantee list for reference ${fileInfo.eFileRef}:\n${error}`); - } + this.mantarayFeedList[mfIx] = { + ...this.mantarayFeedList[mfIx], + eGranteeRef: grantResult.ref, + }; this.saveMantarayFeedList(); @@ -1049,7 +1030,7 @@ export class FileManager { } // recipient is optional, if not provided the message will be broadcasted == pss public key - async sendShareMessage(targetOverlays: string[], item: ShareItem, recipients: string[]): Promise { + private async sendShareMessage(targetOverlays: string[], item: ShareItem, recipients: string[]): Promise { if (recipients.length === 0 || recipients.length !== targetOverlays.length) { console.log('Invalid recipients or targetoverlays specified for sharing.'); return; @@ -1066,15 +1047,17 @@ export class FileManager { } } } + // End share methods - // TODO: download can be handled by the UI ? - async downloadSharedItem(fileInfo: FileInfo, path?: string): Promise | undefined> { - if (!this.sharedWithMe.find((item) => item.fileInfo.eFileRef === fileInfo.eFileRef)) { - throw `Cannot find shared item with ref: ${fileInfo.eFileRef}`; + private async getFeedData(topic: string, address: string, index?: number): Promise { + try { + const feedReader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topic, address); + return await feedReader.download({ index: numberToFeedIndex(index) }); + } catch (error) { + if (isNotFoundError(error)) { + return { feedIndex: -1, feedIndexNext: (0).toString(), reference: SWARM_ZERO_ADDRESS as Reference }; + } + throw error; } - - const options = makeBeeRequestOptions(fileInfo.historyRef, fileInfo.owner, fileInfo.timestamp); - return this.bee.downloadFile(fileInfo.eFileRef, path, options); } - // End share methods } diff --git a/src/types.ts b/src/types.ts index 71f122a..ca83a1f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,4 @@ -import { BatchId, Reference } from '@ethersphere/bee-js'; +import { BatchId, RedundancyLevel, Reference, ReferenceResponse } from '@ethersphere/bee-js'; export interface FileInfo { batchId: string | BatchId; @@ -9,18 +9,19 @@ export interface FileInfo { timestamp?: number; shared?: boolean; preview?: string; + redundancyLevel?: RedundancyLevel; customMetadata?: Record; } export interface ReferenceWithHistory { - reference: string; - historyRef: string; + reference: string | Reference; + historyRef: string | Reference; } // TODO: consider using a completely seprarate type for the manifestfeed because of topic === reference export interface WrappedMantarayFeed extends ReferenceWithHistory { - eFileRef?: string; - eGranteeRef?: string; + eFileRef?: string | Reference; + eGranteeRef?: string | Reference; } export interface ShareItem { @@ -37,6 +38,11 @@ export interface Epoch { time: number; level: number; } +export interface FeedUpdateHeaders { + feedIndex: Index; + feedIndexNext: string; +} +export interface FetchFeedUpdateResponse extends ReferenceResponse, FeedUpdateHeaders {} export type Index = number | Epoch | IndexBytes | string; const feedTypes = ['sequence', 'epoch'] as const; export type FeedType = (typeof feedTypes)[number]; diff --git a/src/utils.ts b/src/utils.ts index 01c8449..73c8fb2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,8 +1,11 @@ import { + Bee, BeeRequestOptions, ENCRYPTED_REFERENCE_HEX_LENGTH, Reference, REFERENCE_HEX_LENGTH, + Topic, + TOPIC_HEX_LENGTH, Utils, } from '@ethersphere/bee-js'; import { Binary } from 'cafe-utility'; @@ -31,6 +34,12 @@ export function assertReference(value: unknown): asserts value is Reference { } } +export function assertTopic(value: unknown): asserts value is Topic { + if (!Utils.isHexString(value, TOPIC_HEX_LENGTH)) { + throw `Invalid feed topic: ${value}`; + } +} + export function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object'; } @@ -74,6 +83,10 @@ export function assertFileInfo(value: unknown): asserts value is FileInfo { throw new TypeError('shared property of FileInfo has to be boolean!'); } + if (fi.redundancyLevel !== undefined && typeof fi.redundancyLevel !== 'number') { + throw new TypeError('redundancyLevel property of FileInfo has to be number!'); + } + if (fi.historyRef !== undefined) { assertReference(fi.historyRef); throw new TypeError('historyRef property of FileInfo has to be a valid reference!'); @@ -205,3 +218,9 @@ export function makeNumericIndex(index: Index): number { throw new TypeError(`Unknown type of index: ${index}`); } + +// status is undefined in the error object +// Determines if the error is about 'Not Found' +export function isNotFoundError(error: any): boolean { + return error.stack.includes('404') || error.message.includes('Not Found') || error.message.includes('404'); +} From d33e054e146d92e017f30dc2d68992180d823be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Fri, 31 Jan 2025 16:42:12 +0100 Subject: [PATCH 18/18] write basic init integration tests --- jest.config.ts | 6 +- package.json | 2 +- src/fileManager.ts | 44 +++++++------- ...ileManager.test.ts => fileManager.spec.ts} | 6 +- tests/integration/fileManager-class.spec.ts | 60 +++++++++++++++++++ tests/utils.ts | 21 +++++++ tsconfig.json | 2 +- 7 files changed, 111 insertions(+), 30 deletions(-) rename tests/{fileManager.test.ts => fileManager.spec.ts} (99%) create mode 100644 tests/integration/fileManager-class.spec.ts create mode 100644 tests/utils.ts diff --git a/jest.config.ts b/jest.config.ts index f2420da..cbe45e1 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -4,6 +4,10 @@ module.exports = { transform: { '^.+\\.tsx?$': 'ts-jest', }, - testMatch: ['**/tests/**/*.test.ts'], + rootDir: 'tests', + testMatch: ['**/tests/**/*.spec.ts'], moduleFileExtensions: ['ts', 'js'], + globals: { + window: {}, + }, }; diff --git a/package.json b/package.json index 3351f2e..73616d3 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "dist/fileManager.js", "scripts": { "build": "tsc", - "test": "node --loader ts-node/esm node_modules/jest/bin/jest.js", + "test": "jest --config=jest.config.ts --runInBand --verbose --detectOpenHandles", "test:coverage": "jest --coverage", "start": "npm run build && node dist/index.js", "lint": "eslint . --ext ts --report-unused-disable-directives", diff --git a/src/fileManager.ts b/src/fileManager.ts index 07ffece..b9ce26b 100644 --- a/src/fileManager.ts +++ b/src/fileManager.ts @@ -12,6 +12,7 @@ import { Signer, STAMPS_DEPTH_MAX, Topic, + TOPIC_BYTES_LENGTH, TOPIC_HEX_LENGTH, Utils, } from '@ethersphere/bee-js'; @@ -59,20 +60,13 @@ export class FileManager { private fileInfoList: FileInfo[]; private nextOwnerFeedIndex: number; private sharedWithMe: ShareItem[]; - private sharedSubscription: PssSubscription; + private sharedSubscription: PssSubscription | undefined; private ownerFeedTopic: Topic; - constructor(beeUrl: string, privateKey: string) { - if (!beeUrl) { - throw new Error('Bee URL is required for initializing the FileManager.'); - } - if (!privateKey) { - throw new Error('privateKey is required for initializing the FileManager.'); - } - + constructor(bee: Bee, privateKey: string) { console.log('Initializing Bee client...'); - this.bee = new Bee(beeUrl); - this.sharedSubscription = this.subscribeToSharedInbox(SHARED_INBOX_TOPIC); + this.bee = bee; + this.sharedSubscription = undefined; this.wallet = new Wallet(privateKey); this.signer = { address: Utils.hexToBytes(this.wallet.address.slice(2)), @@ -91,7 +85,7 @@ export class FileManager { // Start init methods // TODO: use allSettled for file fetching and only save the ones that are successful - async initialize(items: any | undefined): Promise { + async initialize(items?: any): Promise { await this.initStamps(); await this.initOwnerFeedTopic(); await this.initMantarayFeedList(); @@ -115,16 +109,16 @@ export class FileManager { const referenceListTopicHex = this.bee.makeFeedTopic(REFERENCE_LIST_TOPIC); const feedTopicData = await this.getFeedData(referenceListTopicHex, this.wallet.address, 0); - if (feedTopicData.feedIndex === numberToFeedIndex(-1)) { + if (feedTopicData.reference === SWARM_ZERO_ADDRESS) { const ownerFeedStamp = this.getOwnerFeedStamp(); if (ownerFeedStamp === undefined) { throw 'Owner stamp not found'; } - this.ownerFeedTopic = Utils.bytesToHex(randomBytes(TOPIC_HEX_LENGTH), TOPIC_HEX_LENGTH); + this.ownerFeedTopic = Utils.bytesToHex(randomBytes(TOPIC_BYTES_LENGTH), TOPIC_HEX_LENGTH); const topicDataRes = await this.bee.uploadData(ownerFeedStamp.batchID, this.ownerFeedTopic, { act: true }); - const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, referenceListTopicHex, this.wallet.address); + const fw = this.bee.makeFeedWriter(DEFAULT_FEED_TYPE, referenceListTopicHex, this.signer); await fw.upload(ownerFeedStamp.batchID, topicDataRes.reference, { index: numberToFeedIndex(0) }); await fw.upload(ownerFeedStamp.batchID, topicDataRes.historyAddress as Reference, { index: numberToFeedIndex(1), @@ -149,8 +143,8 @@ export class FileManager { } private async initMantarayFeedList(): Promise { - const latestFeedData = await this.getFeedData(this.ownerFeedTopic, this.wallet.address); - if (latestFeedData.feedIndex === numberToFeedIndex(-1)) { + const latestFeedData = await this.getFeedData(this.ownerFeedTopic); + if (latestFeedData.reference === SWARM_ZERO_ADDRESS) { console.log("Owner mantaray feed doesn't exist yet."); return; } @@ -178,8 +172,8 @@ export class FileManager { private async initFileInfoList(): Promise { for (const mantaryFeedItem of this.mantarayFeedList) { - const wrappedMantarayData = await this.getFeedData(mantaryFeedItem.reference, this.wallet.address); - if (wrappedMantarayData.feedIndex === numberToFeedIndex(-1)) { + const wrappedMantarayData = await this.getFeedData(mantaryFeedItem.reference); + if (wrappedMantarayData.reference === SWARM_ZERO_ADDRESS) { console.log("mantaryFeedItem doesn't exist, skipping it."); continue; } @@ -285,7 +279,7 @@ export class FileManager { } // Start stamp methods - async getUsableStamps(): Promise { + private async getUsableStamps(): Promise { try { return (await this.bee.getAllPostageBatch()).filter((s) => s.usable); } catch (error: any) { @@ -321,7 +315,7 @@ export class FileManager { return this.stampList.find((s) => s.label === OWNER_FEED_STAMP_LABEL); } - async getCachedStamp(batchId: string | BatchId): Promise { + getCachedStamp(batchId: string | BatchId): PostageBatch | undefined { return this.stampList.find((s) => s.batchID === batchId); } @@ -977,7 +971,7 @@ export class FileManager { // Start share methods subscribeToSharedInbox(topic: string, callback?: (data: ShareItem) => void): PssSubscription { console.log('Subscribing to shared inbox, topic: ', topic); - return this.bee.pssSubscribe(topic, { + this.sharedSubscription = this.bee.pssSubscribe(topic, { onMessage: (message) => { console.log('Received shared inbox message: ', message); assertShareItem(message); @@ -991,6 +985,8 @@ export class FileManager { throw e; }, }); + + return this.sharedSubscription; } unsubscribeFromSharedInbox(): void { @@ -1049,9 +1045,9 @@ export class FileManager { } // End share methods - private async getFeedData(topic: string, address: string, index?: number): Promise { + public async getFeedData(topic: string, address?: string, index?: number): Promise { try { - const feedReader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topic, address); + const feedReader = this.bee.makeFeedReader(DEFAULT_FEED_TYPE, topic, address || this.wallet.address); return await feedReader.download({ index: numberToFeedIndex(index) }); } catch (error) { if (isNotFoundError(error)) { diff --git a/tests/fileManager.test.ts b/tests/fileManager.spec.ts similarity index 99% rename from tests/fileManager.test.ts rename to tests/fileManager.spec.ts index e61841d..13cce59 100644 --- a/tests/fileManager.test.ts +++ b/tests/fileManager.spec.ts @@ -7,6 +7,7 @@ import { FileManager } from '../src/fileManager'; import { encodePathToBytes } from '../src/utils'; import { createMockBee, createMockMantarayNode } from './mockHelpers'; +import { BEE_URL } from './utils'; jest.mock('@solarpunkltd/mantaray-js', () => { const mockMantarayNode = jest.fn(() => createMockMantarayNode()); @@ -24,10 +25,9 @@ jest.mock('fs', () => { describe('FileManager - Setup', () => { it('should initialize with a valid Bee URL', () => { + const bee = new Bee(BEE_URL); const validPrivateKey = '0x'.padEnd(66, 'a'); // 64-character hex string padded with 'a' - const fileManager = new FileManager('http://localhost:1633', validPrivateKey); - expect(fileManager.bee).toBeTruthy(); - expect(fileManager.mantaray).toBeTruthy(); + const fileManager = new FileManager(bee, validPrivateKey); }); }); diff --git a/tests/integration/fileManager-class.spec.ts b/tests/integration/fileManager-class.spec.ts new file mode 100644 index 0000000..c0a17d3 --- /dev/null +++ b/tests/integration/fileManager-class.spec.ts @@ -0,0 +1,60 @@ +import { Bee, Reference, Topic, Utils } from '@ethersphere/bee-js'; +import { beforeEach, describe, expect, it, jest } from '@jest/globals'; +import { MantarayNode } from '@solarpunkltd/mantaray-js'; + +import { OWNER_FEED_STAMP_LABEL, REFERENCE_LIST_TOPIC, SWARM_ZERO_ADDRESS } from '../../src/constants'; +import { FileManager } from '../../src/fileManager'; +import { encodePathToBytes, makeBeeRequestOptions } from '../../src/utils'; +import { BEE_URL, buyStamp, MOCK_PK, MOCK_WALLET } from '../utils'; + +jest.mock('fs', () => { + const mockBuffer = jest.fn(() => Buffer.from('Mock file content')); + return { + readFileSync: mockBuffer, + }; +}); + +describe('FileManager instantiation', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should create and initialize a new instance', async () => { + const bee = new Bee(BEE_URL); + // const f = async (): Promise => new FileManager(bee, MOCK_PK); + // const fileManager = (await f()) as FileManager; + const fileManager = new FileManager(bee, MOCK_PK); + try { + await fileManager.initialize(); + } catch (error: any) { + console.log('bagoy error: ', error); + expect(error).toEqual('Owner stamp not found'); + } + const stamps = await fileManager.getStamps(); + expect(stamps).toEqual([]); + expect(fileManager.getFileInfoList()).toEqual([]); + }); + + // TODO: test if no-one else can read the topic but the owner + it('should fetch the owner stamp and initialize the owner feed', async () => { + const bee = new Bee(BEE_URL); + const batchId = await buyStamp(bee, OWNER_FEED_STAMP_LABEL); + const fileManager = new FileManager(bee, MOCK_PK); + await fileManager.initialize(); + + const stamps = await fileManager.getStamps(); + expect(stamps[0].batchID).toEqual(batchId); + expect(stamps[0].label).toEqual(OWNER_FEED_STAMP_LABEL); + expect(fileManager.getCachedStamp(batchId)).toEqual(stamps[0]); + expect(fileManager.getFileInfoList()).toEqual([]); + + const referenceListTopicHex = bee.makeFeedTopic(REFERENCE_LIST_TOPIC); + const feedTopicData = await fileManager.getFeedData(referenceListTopicHex, MOCK_WALLET.address, 0); + const topicHistory = await fileManager.getFeedData(referenceListTopicHex, MOCK_WALLET.address, 1); + const options = makeBeeRequestOptions(topicHistory.reference, MOCK_WALLET.address); + // TODO: fails here + const topicHex = (await bee.downloadData(feedTopicData.reference, options)).text() as Topic; + expect(topicHex !== SWARM_ZERO_ADDRESS).toBeTruthy(); + console.log('owner feed topicHex: ', topicHex); + }); +}); diff --git a/tests/utils.ts b/tests/utils.ts new file mode 100644 index 0000000..28012f7 --- /dev/null +++ b/tests/utils.ts @@ -0,0 +1,21 @@ +import { BatchId, Bee, Data, Utils } from '@ethersphere/bee-js'; +import { hexlify, Wallet } from 'ethers'; + +export const BEE_URL = 'http://localhost:1633'; +export const MOCK_PK = '634fb5a872396d9693e5c9f9d7233cfa93f395c093371017ff44aa9ae6564cdd'; +export const DEFAULT_BATCH_DEPTH = 22; +export const DEFAULT_BATCH_AMOUNT = '1200000000'; +export const MOCK_WALLET = new Wallet(MOCK_PK); +export const MOCK_SIGNER = { + address: Utils.hexToBytes(MOCK_WALLET.address.slice(2)), + sign: async (data: Data): Promise => { + return await MOCK_WALLET.signMessage(data); + }, +}; + +export async function buyStamp(bee: Bee, label?: string): Promise { + return await bee.createPostageBatch(DEFAULT_BATCH_AMOUNT, DEFAULT_BATCH_DEPTH, { + waitForUsable: true, + label: label, + }); +} diff --git a/tsconfig.json b/tsconfig.json index b07cf0d..732e86c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,7 @@ "removeComments": true, "outDir": "./dist", "strict": true, - "noUnusedLocals": true, + "noUnusedLocals": false, "noUnusedParameters": true, "noFallthroughCasesInSwitch": true, "strictPropertyInitialization": false,