From cd578a86e46aab9236999ed1f011a482f14870de Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 16 Sep 2024 09:38:15 -0400 Subject: [PATCH 1/2] feat: add the ability to request chunks of data from integrity tree --- src/DataIntegrityTree/DataIntegrityTree.ts | 32 ++++++++++++++++++++-- src/utils/directoryUtils.ts | 26 ++++++++---------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/DataIntegrityTree/DataIntegrityTree.ts b/src/DataIntegrityTree/DataIntegrityTree.ts index 699de98..9429877 100644 --- a/src/DataIntegrityTree/DataIntegrityTree.ts +++ b/src/DataIntegrityTree/DataIntegrityTree.ts @@ -495,7 +495,12 @@ class DataIntegrityTree { * @param rootHash - The root hash of the tree. Defaults to the latest root hash. * @returns The readable stream for the file. */ - getValueStream(hexKey: string, rootHash: string | null = null): Readable { + getValueStream( + hexKey: string, + rootHash: string | null = null, + byteOffset: number | null = null, + length: number | null = null + ): Readable { if (!isHexString(hexKey)) { throw new Error("key must be a valid hex string"); } @@ -526,8 +531,29 @@ class DataIntegrityTree { throw new Error(`File at path ${filePath} does not exist`); } - // Create a read stream and pipe it through a decompression stream using the same algorithm (gzip) - const readStream = fs.createReadStream(filePath); + const fileSize = fs.statSync(filePath).size; + + // Validate offset and length + if (byteOffset !== null && length !== null) { + if (byteOffset + length > fileSize) { + throw new Error( + `Offset (${byteOffset}) and length (${length}) exceed the file size (${fileSize}).` + ); + } + } + + // Create the read stream with optional byte range + const options: { start?: number; end?: number } = {}; + + if (byteOffset !== null) { + options.start = byteOffset; + } + + if (length !== null && byteOffset !== null) { + options.end = byteOffset + length - 1; // `end` is inclusive, hence `byteOffset + length - 1` + } + + const readStream = fs.createReadStream(filePath, options); const decompressStream = zlib.createGunzip(); // Return the combined stream as a generic Readable stream diff --git a/src/utils/directoryUtils.ts b/src/utils/directoryUtils.ts index 3a9b7d9..69255f2 100644 --- a/src/utils/directoryUtils.ts +++ b/src/utils/directoryUtils.ts @@ -8,27 +8,28 @@ const limitConcurrency = async ( concurrencyLimit: number, tasks: (() => Promise)[] ) => { - const results = []; const executing: Promise[] = []; for (const task of tasks) { const p = task(); - results.push(p); - - // Once the limit is reached, wait for one to complete - if (executing.length >= concurrencyLimit) { - await Promise.race(executing); - } // Add the new task to the executing array executing.push(p); // When a task completes, remove it from the executing array - p.finally(() => executing.splice(executing.indexOf(p), 1)); + const cleanup = p.finally(() => { + executing.splice(executing.indexOf(cleanup), 1); + }); + + // Once the limit is reached, wait for one to complete + if (executing.length >= concurrencyLimit) { + await new Promise((resolve) => setTimeout(resolve, 100)); + await Promise.race(executing); + } } // Wait for all remaining tasks to complete - return Promise.all(results); + return Promise.all(executing); }; export const addDirectory = async ( @@ -70,16 +71,11 @@ export const addDirectory = async ( const stream = fs.createReadStream(filePath); datalayer .upsertKey(stream, Buffer.from(relativePath).toString("hex")) - .then(async () => { - await new Promise((resolve) => setTimeout(resolve, 100)); - resolve(); - }) + .then(() => resolve()) .catch(reject); }) ); } - - await new Promise((resolve) => setTimeout(resolve, 100)); } // Run tasks with limited concurrency (set the concurrency limit as needed) From 9b57b8951de044df5b2ca24e05f1054bd2b8a4d9 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Mon, 16 Sep 2024 09:39:29 -0400 Subject: [PATCH 2/2] chore(release): 0.0.1-alpha.17 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23e243a..ecd5443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.17](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.16...v0.0.1-alpha.17) (2024-09-16) + + +### Features + +* add the ability to request chunks of data from integrity tree ([cd578a8](https://github.com/DIG-Network/dig-chia-sdk/commit/cd578a86e46aab9236999ed1f011a482f14870de)) + ### [0.0.1-alpha.16](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.15...v0.0.1-alpha.16) (2024-09-10) diff --git a/package-lock.json b/package-lock.json index 695758a..a65fe25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.16", + "version": "0.0.1-alpha.17", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.16", + "version": "0.0.1-alpha.17", "license": "ISC", "dependencies": { "bip39": "^3.1.0", diff --git a/package.json b/package.json index 8607fac..270e070 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.16", + "version": "0.0.1-alpha.17", "description": "", "type": "commonjs", "main": "./dist/index.js",