diff --git a/README.md b/README.md index aeed5ab9..f2ba5a96 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ You can enable fuse by adding it to the `workspaces` value in `package.json`: ```json "workspaces": [ - "packages/!(fuse)", + "packages/!(fuse|benchmarks)", "packages/fuse" ] ``` @@ -191,3 +191,29 @@ node packages/cli/dist/src/index.js list-groups ``` node packages/cli/dist/src/index.js join-group $GROUP ``` + +## Benchmarks + +You can enable benchmarks by adding it to the `workspaces` value in `package.json`: + +```json +"workspaces": [ + "packages/!(fuse|benchmarks)", + "packages/benchmarks" +] +``` + +Then you can build the benchmarks package as usual: + +``` +npm ci +npm run build +``` + +### Transfer + +To run the transfer benchmark: + +``` +node packages/benchmarks/dist/src/transfer/index.js +``` diff --git a/package-lock.json b/package-lock.json index be1c0b9c..179cd9e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3233,11 +3233,11 @@ } }, "node_modules/@libp2p/interface": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-1.1.5.tgz", - "integrity": "sha512-BjFgv/3VwEDNRcFKL4KW6g29IcUWUjaTJhyZVGWtodFuPjZsZHJgoQU7T/FFxDcfTdI90qpFbTREycOB+VL9NQ==", + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-1.1.6.tgz", + "integrity": "sha512-CLz6TAZf+Mw1PCIU8pjMIct1uh3A1fIene2/t+E57Tw4uJLCBJE9CLed/Opxliy5RH0e32Aa6bi4QSXtkJTK7A==", "dependencies": { - "@multiformats/multiaddr": "^12.1.14", + "@multiformats/multiaddr": "^12.2.1", "it-pushable": "^3.2.3", "it-stream-types": "^2.0.1", "multiformats": "^13.1.0", @@ -3257,7 +3257,8 @@ }, "node_modules/@libp2p/interfaces": { "version": "3.3.2", - "license": "Apache-2.0 OR MIT", + "resolved": "https://registry.npmjs.org/@libp2p/interfaces/-/interfaces-3.3.2.tgz", + "integrity": "sha512-p/M7plbrxLzuQchvNwww1Was7ZeGE2NaOFulMaZBYIihU8z3fhaV+a033OqnC/0NTX/yhfdNOG7znhYq3XoR/g==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" @@ -3653,13 +3654,14 @@ } }, "node_modules/@multiformats/multiaddr": { - "version": "12.1.14", - "license": "Apache-2.0 OR MIT", + "version": "12.2.1", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-12.2.1.tgz", + "integrity": "sha512-UwjoArBbv64FlaetV4DDwh+PUMfzXUBltxQwdh+uTYnGFzVa8ZfJsn1vt1RJlJ6+Xtrm3RMekF/B+K338i2L5Q==", "dependencies": { "@chainsafe/is-ip": "^2.0.1", "@chainsafe/netmask": "^2.0.0", "@libp2p/interface": "^1.0.0", - "dns-over-http-resolver": "^3.0.2", + "@multiformats/dns": "^1.0.3", "multiformats": "^13.0.0", "uint8-varint": "^2.0.1", "uint8arrays": "^5.0.0" @@ -3909,7 +3911,12 @@ }, "node_modules/@open-draft/deferred-promise": { "version": "2.2.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==" + }, + "node_modules/@organicdesign/db-benchamrks": { + "resolved": "packages/benchmarks", + "link": true }, "node_modules/@organicdesign/db-cipher": { "resolved": "packages/cipher", @@ -14036,7 +14043,8 @@ }, "node_modules/interface-blockstore": { "version": "5.2.10", - "license": "Apache-2.0 OR MIT", + "resolved": "https://registry.npmjs.org/interface-blockstore/-/interface-blockstore-5.2.10.tgz", + "integrity": "sha512-9K48hTvBCGsKVD3pF4ILgDcf+W2P/gq0oxLcsHGB6E6W6nDutYkzR+7k7bCs9REHrBEfKzcVDEKieiuNM9WRZg==", "dependencies": { "interface-store": "^5.0.0", "multiformats": "^13.0.1" @@ -14064,7 +14072,8 @@ }, "node_modules/interface-store": { "version": "5.1.8", - "license": "Apache-2.0 OR MIT" + "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-5.1.8.tgz", + "integrity": "sha512-7na81Uxkl0vqk0CBPO5PvyTkdaJBaezwUJGsMOz7riPOq0rJt+7W31iaopaMICWea/iykUsvNlPx/Tc+MxC3/w==" }, "node_modules/internal-slot": { "version": "1.0.7", @@ -15782,7 +15791,8 @@ }, "node_modules/it-parallel": { "version": "3.0.6", - "license": "Apache-2.0 OR MIT", + "resolved": "https://registry.npmjs.org/it-parallel/-/it-parallel-3.0.6.tgz", + "integrity": "sha512-i7UM7I9LTkDJw3YIqXHFAPZX6CWYzGc+X3irdNrVExI4vPazrJdI7t5OqrSVN8CONXLAunCiqaSV/zZRbQR56A==", "dependencies": { "p-defer": "^4.0.0" } @@ -15800,7 +15810,8 @@ }, "node_modules/it-pipe": { "version": "3.0.1", - "license": "Apache-2.0 OR MIT", + "resolved": "https://registry.npmjs.org/it-pipe/-/it-pipe-3.0.1.tgz", + "integrity": "sha512-sIoNrQl1qSRg2seYSBH/3QxWhJFn9PKYvOf/bHdtCBF0bnghey44VyASsWzn5dAx0DCDDABq1hZIuzKmtBZmKA==", "dependencies": { "it-merge": "^3.0.0", "it-pushable": "^3.1.2", @@ -25093,6 +25104,17 @@ "node": ">=6" } }, + "node_modules/pretty-bytes": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz", + "integrity": "sha512-mQUvGU6aUFQ+rNvTIAcZuWGRT9a6f6Yrg9bHs4ImKF+HZCEK+plBvnAZYSIQztknZF2qnzNtr6F8s0+IuptdlQ==", + "engines": { + "node": "^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/pretty-format": { "version": "26.6.2", "license": "MIT", @@ -28461,6 +28483,11 @@ "node": ">=14.0.0" } }, + "node_modules/tinybench": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.6.0.tgz", + "integrity": "sha512-N8hW3PG/3aOoZAN5V/NSAEDz0ZixDSSt5b/a05iqtpgfLWMSVuCo7w0k2vVvEjdrIoeGqZzweX2WlyioNIHchA==" + }, "node_modules/tmp": { "version": "0.0.33", "dev": true, @@ -30477,7 +30504,8 @@ }, "node_modules/yargs": { "version": "17.7.2", - "license": "MIT", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -30593,6 +30621,26 @@ "url": "https://github.com/sponsors/wooorm" } }, + "packages/benchmarks": { + "name": "@organicdesign/db-benchamrks", + "version": "0.1.0", + "license": "GPL-3.0-or-later", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@organicdesign/db-client": "^0.1.0", + "debug": "^4.3.4", + "pretty-bytes": "^6.1.1", + "tinybench": "^2.6.0", + "uint8arrays": "^5.0.3", + "yargs": "^17.7.2" + }, + "devDependencies": { + "aegir": "^42.2.4" + }, + "peerDependencies": { + "@organicdesign/db-daemon": "^0.1.0" + } + }, "packages/cipher": { "name": "@organicdesign/db-cipher", "version": "0.1.0", @@ -30641,6 +30689,7 @@ "@chainsafe/libp2p-yamux": "^6.0.2", "@helia/block-brokers": "^2.0.3", "@helia/car": "^3.1.0", + "@helia/interface": "^4.1.0", "@helia/unixfs": "^3.0.0", "@ipld/car": "^5.3.0", "@ipld/dag-cbor": "^9.2.0", @@ -30649,6 +30698,8 @@ "@libp2p/circuit-relay-v2": "^1.0.15", "@libp2p/dcutr": "^1.0.12", "@libp2p/identify": "^1.0.14", + "@libp2p/interface": "^1.1.6", + "@libp2p/interfaces": "^3.3.2", "@libp2p/kad-dht": "^12.0.7", "@libp2p/logger": "^4.0.6", "@libp2p/pnet": "^1.0.1", @@ -30669,6 +30720,7 @@ "datastore-fs": "^9.1.8", "debug": "^4.3.4", "helia": "^4.0.1", + "interface-blockstore": "^5.2.10", "interface-datastore": "^8.2.11", "it-all": "^3.0.4", "it-parallel": "^3.0.6", @@ -30684,14 +30736,9 @@ "zod": "^3.22.4" }, "devDependencies": { - "@helia/interface": "^4.0.0", - "@ipld/dag-pb": "^4.1.0", - "@libp2p/interface": "^1.1.3", - "@libp2p/interfaces": "^3.3.2", "@organicdesign/db-test-utils": "^0.1.0", "@types/yargs": "^17.0.32", "aegir": "^42.2.4", - "interface-blockstore": "^5.2.10", "tsc-alias": "^1.8.8" } }, @@ -30823,6 +30870,7 @@ "version": "0.1.0", "license": "GPL-3.0-or-later", "dependencies": { + "@helia/interface": "^4.1.0", "@open-draft/deferred-promise": "^2.2.0", "@organicdesign/db-utils": "^0.1.0", "cborg": "^4.0.9", @@ -30835,7 +30883,6 @@ "zod": "^3.22.4" }, "devDependencies": { - "@helia/interface": "^4.0.0", "@ipld/dag-pb": "^4.1.0", "aegir": "^42.2.4", "helia": "^4.0.1", @@ -30850,6 +30897,7 @@ "dependencies": { "@ipld/dag-cbor": "^9.2.0", "@libp2p/crypto": "^4.0.2", + "@libp2p/interface": "^1.1.6", "@libp2p/peer-id": "^4.0.6", "bip32": "^4.0.0", "bip39": "^3.1.0", @@ -30860,7 +30908,6 @@ "zod": "^3.22.4" }, "devDependencies": { - "@libp2p/interface": "^1.1.3", "aegir": "^42.2.4" } }, @@ -30869,13 +30916,13 @@ "version": "0.1.0", "license": "GPL-3.0-or-later", "dependencies": { - "@open-draft/deferred-promise": "^2.2.0" - }, - "devDependencies": { - "@helia/interface": "^4.0.0", - "aegir": "^42.2.4", + "@helia/interface": "^4.1.0", + "@open-draft/deferred-promise": "^2.2.0", "interface-store": "^5.1.8", "multiformats": "^13.1.0" + }, + "devDependencies": { + "aegir": "^42.2.4" } }, "packages/rpc-interfaces": { @@ -30926,14 +30973,17 @@ "version": "0.1.0", "license": "GPL-3.0-or-later", "dependencies": { + "@helia/interface": "^4.1.0", "@helia/unixfs": "^3.0.2", + "@ipld/dag-pb": "^4.1.0", "blockstore-core": "^4.4.0", + "interface-blockstore": "^5.2.10", "it-all": "^3.0.4", + "multiformats": "^13.1.0", "uint8arrays": "^5.0.3" }, "devDependencies": { - "aegir": "^42.2.4", - "multiformats": "^13.1.0" + "aegir": "^42.2.4" } }, "packages/utils": { @@ -30941,25 +30991,27 @@ "version": "0.1.0", "license": "GPL-3.0-or-later", "dependencies": { + "@helia/interface": "^4.1.0", "@helia/unixfs": "^3.0.2", "@ipld/dag-cbor": "^9.2.0", "@ipld/dag-json": "^10.2.0", "@ipld/dag-pb": "^4.1.0", + "@libp2p/interface": "^1.1.6", "cborg": "^4.1.4", "datastore-core": "^9.2.9", + "interface-blockstore": "^5.2.10", "interface-datastore": "^8.2.11", "ipfs-unixfs-exporter": "^13.5.0", "ipfs-unixfs-importer": "^15.2.4", + "it-all": "^3.0.4", "multiformats": "^13.1.0" }, "devDependencies": { - "@helia/interface": "^4.1.0", - "@libp2p/interface": "^1.1.5", "@organicdesign/db-test-utils": "^0.1.0", "aegir": "^42.2.4", "blockstore-core": "^4.4.0", - "interface-blockstore": "^5.2.10", - "it-all": "^3.0.4" + "it-parallel": "^3.0.6", + "it-pipe": "^3.0.1" } } } diff --git a/package.json b/package.json index 408024e1..08d0bc59 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,6 @@ }, "private": true, "workspaces": [ - "packages/!(fuse)" + "packages/!(fuse|benchmarks)" ] } diff --git a/packages/benchmarks/.aegir.js b/packages/benchmarks/.aegir.js new file mode 100644 index 00000000..c3f0c16d --- /dev/null +++ b/packages/benchmarks/.aegir.js @@ -0,0 +1,8 @@ +export default { + build: { + config: { + platform: 'node', + format: 'esm' + } + } +} diff --git a/packages/benchmarks/README.md b/packages/benchmarks/README.md new file mode 100644 index 00000000..86a214fd --- /dev/null +++ b/packages/benchmarks/README.md @@ -0,0 +1,3 @@ +# Benchmarks + +Benchmarks for the distributed backup. diff --git a/packages/benchmarks/package.json b/packages/benchmarks/package.json new file mode 100644 index 00000000..1eb786cd --- /dev/null +++ b/packages/benchmarks/package.json @@ -0,0 +1,43 @@ +{ + "type": "module", + "name": "@organicdesign/db-benchamrks", + "version": "0.1.0", + "description": "Benchmarks for the distributed backup.", + "main": "dist/src/index.js", + "types": "dist/src/index.d.ts", + "files": [ + "dist/src" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "prepublishOnly": "npm run build", + "build": "aegir build" + }, + "author": "Saul Boyd", + "license": "GPL-3.0-or-later", + "devDependencies": { + "aegir": "^42.2.4" + }, + "private": true, + "sideEffects": false, + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@organicdesign/db-client": "^0.1.0", + "debug": "^4.3.4", + "pretty-bytes": "^6.1.1", + "tinybench": "^2.6.0", + "uint8arrays": "^5.0.3", + "yargs": "^17.7.2" + }, + "peerDependencies": { + "@organicdesign/db-daemon": "^0.1.0" + } +} diff --git a/packages/benchmarks/src/import/import-bench.ts b/packages/benchmarks/src/import/import-bench.ts new file mode 100644 index 00000000..41d2895f --- /dev/null +++ b/packages/benchmarks/src/import/import-bench.ts @@ -0,0 +1,36 @@ +import Path from 'path' +import { Client } from '@organicdesign/db-client' +import { packagePath } from '../utils/paths.js' +import runNode from '../utils/run-node.js' +import type { ImportBenchmark } from './interface.js' + +export const createImportBench = async (size: number, persistent: boolean): Promise => { + const dataPath = Path.join(packagePath, 'test-out') + + const name = `import-${size}` + + const proc = await runNode(name, { persistent }) + + await proc.start() + + const client = new Client(Path.join(dataPath, name, 'socket')) + + const group = await client.createGroup('test') + + const dataFile = Path.join(dataPath, `${size}.data`) + + return { + async teardown () { + client.stop() + + await proc.stop() + }, + + async run () { + const [{ cid }] = await client.import(group, dataFile, { path: '/test' }) + const [item] = await client.getStatus([cid]) + + return item + } + } +} diff --git a/packages/benchmarks/src/import/index.ts b/packages/benchmarks/src/import/index.ts new file mode 100644 index 00000000..dc716073 --- /dev/null +++ b/packages/benchmarks/src/import/index.ts @@ -0,0 +1,155 @@ +/* eslint-disable no-console,no-loop-func */ +import fs from 'fs/promises' +import Path from 'path' +import debug from 'debug' +import prettyBytes from 'pretty-bytes' +import { Bench } from 'tinybench' +import { hideBin } from 'yargs/helpers' +import yargs from 'yargs/yargs' +import generateFile from '../utils/generate-file.js' +import { packagePath } from '../utils/paths.js' +import { createImportBench } from './import-bench.js' +import type { ImportImplementation } from './interface.js' + +const argv = await yargs(hideBin(process.argv)) + .option({ + iterations: { + alias: 'i', + type: 'number', + default: 3 + } + }) + .option({ + minTime: { + type: 'number', + default: 1 + } + }) + .option({ + precision: { + type: 'number', + default: 2 + } + }) + .option({ + persistent: { + alias: 'p', + type: 'boolean', + default: false + } + }) + .parse() + +const log = debug('bench:import') + +const sizes = [ + 0, // 1b + 3, // 1kb + 6, // 1mb + 7, // 10mb + 8, // 100mb + 9 // 1gb +].map(z => 10 ** z) + +const impls: ImportImplementation[] = sizes.map(size => ({ + name: `${prettyBytes(size)}`, + create: async () => createImportBench(size, argv.persistent), + results: [], + fileSize: size, + size: 0, + blocks: 0 +})) + +const dataPath = Path.join(packagePath, 'test-out') + +async function main (): Promise { + const suite = new Bench({ + iterations: argv.iterations, + time: argv.minTime, + + async setup (task) { + const impl = impls.find(({ name }) => task.name.includes(name)) + + if (impl == null) { + return + } + + await fs.mkdir(dataPath, { recursive: true }) + const dataFile = Path.join(dataPath, `${impl.fileSize}.data`) + await generateFile(dataFile, impl.fileSize) + }, + + async teardown (task) { + const impl = impls.find(({ name }) => task.name.includes(name)) + + if (impl == null) { + return + } + + const dataFile = Path.join(dataPath, `${impl.fileSize}.data`) + await fs.rm(dataFile) + } + }) + + for (const impl of impls) { + const run = async function (): Promise { + log('Start: setup') + const subject = await impl.create() + log('End: setup') + + const start = performance.now() + const { size, blocks } = await subject.run() + + impl.size = size + impl.blocks = blocks + impl.results.push(performance.now() - start) + + log('Start: teardown') + await subject.teardown() + log('End: teardown') + } + + const hooks = { + beforeEach: async () => { + log(`Start: test ${impl.name}`) + }, + afterEach: async () => { + log(`End: test ${impl.name}`) + } + } + + suite.add(impl.name, run, hooks) + } + + await suite.run() + + console.table(suite.tasks.map(({ name, result }) => { + const impl = impls.find(impl => impl.name === name) + + if (impl == null) { + throw new Error('got result without implementation') + } + + const seconds = (result?.period ?? 0) / 1000 + const speed = impl.size / seconds + const bps = impl.blocks / seconds + + return { + 'File Size': name, + Size: prettyBytes(impl.size), + Blocks: impl.blocks, + 'Speed (Size)': `${prettyBytes(speed)}/s`, + 'Speed (Blocks)': `${bps.toFixed(argv.precision)} blocks/s`, + 'Run Time': `${result?.period.toFixed(argv.precision)}ms`, + Runs: result?.samples.length, + p99: `${result?.p99.toFixed(argv.precision)}ms` + } + })) + + await fs.rm(dataPath, { recursive: true }) +} + +main().catch(err => { + console.error(err) // eslint-disable-line no-console + process.exit(1) +}) diff --git a/packages/benchmarks/src/import/interface.ts b/packages/benchmarks/src/import/interface.ts new file mode 100644 index 00000000..ecebf3de --- /dev/null +++ b/packages/benchmarks/src/import/interface.ts @@ -0,0 +1,17 @@ +export interface ImportBenchmark { + teardown(): Promise + run(): Promise<{ + cid: string + blocks: number + size: number + }> +} + +export interface ImportImplementation { + name: string + create(): Promise + results: number[] + fileSize: number + size: number + blocks: number +} diff --git a/packages/benchmarks/src/index.ts b/packages/benchmarks/src/index.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/benchmarks/src/transfer/index.ts b/packages/benchmarks/src/transfer/index.ts new file mode 100644 index 00000000..a0cf738e --- /dev/null +++ b/packages/benchmarks/src/transfer/index.ts @@ -0,0 +1,156 @@ +/* eslint-disable no-console,no-loop-func */ +import fs from 'fs/promises' +import Path from 'path' +import debug from 'debug' +import prettyBytes from 'pretty-bytes' +import { Bench } from 'tinybench' +import { hideBin } from 'yargs/helpers' +import yargs from 'yargs/yargs' +import generateFile from '../utils/generate-file.js' +import { packagePath } from '../utils/paths.js' +import { createTransferBench } from './transfer-bench.js' +import type { TransferImplementation } from './interface.js' + +const argv = await yargs(hideBin(process.argv)) + .option({ + iterations: { + alias: 'i', + type: 'number', + default: 3 + } + }) + .option({ + minTime: { + type: 'number', + default: 1 + } + }) + .option({ + precision: { + type: 'number', + default: 2 + } + }) + .option({ + persistent: { + alias: 'p', + type: 'boolean', + default: false + } + }) + .parse() + +const log = debug('bench:transfer') + +const sizes = [ + 0, // 1b + 3, // 1kb + 6, // 1mb + 7, // 10mb + 8, // 100mb + 9 // 1gb +].map(z => 10 ** z) + +const impls: TransferImplementation[] = sizes.map(size => ({ + name: `${prettyBytes(size)}`, + create: async () => createTransferBench(size, argv.persistent), + results: [], + fileSize: size, + size: 0, + blocks: 0 +})) + +const dataPath = Path.join(packagePath, 'test-out') + +async function main (): Promise { + const suite = new Bench({ + iterations: argv.iterations, + time: argv.minTime, + + async setup (task) { + const impl = impls.find(({ name }) => task.name.includes(name)) + + if (impl == null) { + return + } + + await fs.mkdir(dataPath, { recursive: true }) + const dataFile = Path.join(dataPath, `${impl.fileSize}.data`) + await generateFile(dataFile, impl.fileSize) + }, + + async teardown (task) { + const impl = impls.find(({ name }) => task.name.includes(name)) + + if (impl == null) { + return + } + + const dataFile = Path.join(dataPath, `${impl.fileSize}.data`) + await fs.rm(dataFile) + } + }) + + for (const impl of impls) { + const run = async function (): Promise { + log('Start: setup') + const subject = await impl.create() + log('End: setup') + + await subject.warmup() + const start = performance.now() + await subject.run() + + impl.size = subject.size + impl.blocks = subject.blocks + impl.results.push(performance.now() - start) + + log('Start: teardown') + await subject.teardown() + log('End: teardown') + } + + const hooks = { + beforeEach: async () => { + log(`Start: test ${impl.name}`) + }, + afterEach: async () => { + log(`End: test ${impl.name}`) + } + } + + suite.add(impl.name, run, hooks) + } + + await suite.run() + + console.table(suite.tasks.map(({ name, result }) => { + const impl = impls.find(impl => impl.name === name) + + if (impl == null) { + throw new Error('got result without implementation') + } + + const seconds = (result?.period ?? 0) / 1000 + const speed = impl.size / seconds + const bps = impl.blocks / seconds + + return { + 'File Size': name, + Size: prettyBytes(impl.size), + Blocks: impl.blocks, + 'Speed (Size)': `${prettyBytes(speed)}/s`, + 'Speed (Blocks)': `${bps.toFixed(argv.precision)} blocks/s`, + 'Run Time': `${result?.period.toFixed(argv.precision)}ms`, + Runs: result?.samples.length, + p99: `${result?.p99.toFixed(argv.precision)}ms` + } + })) + + await fs.rm(dataPath, { recursive: true }) +} + +main().catch(err => { + console.error(err) // eslint-disable-line no-console + process.exit(1) +}) diff --git a/packages/benchmarks/src/transfer/interface.ts b/packages/benchmarks/src/transfer/interface.ts new file mode 100644 index 00000000..33493a07 --- /dev/null +++ b/packages/benchmarks/src/transfer/interface.ts @@ -0,0 +1,16 @@ +export interface TransferBenchmark { + blocks: number + size: number + teardown(): Promise + run(): Promise + warmup(): Promise +} + +export interface TransferImplementation { + name: string + create(): Promise + results: number[] + fileSize: number + size: number + blocks: number +} diff --git a/packages/benchmarks/src/transfer/transfer-bench.ts b/packages/benchmarks/src/transfer/transfer-bench.ts new file mode 100644 index 00000000..b7f47ee3 --- /dev/null +++ b/packages/benchmarks/src/transfer/transfer-bench.ts @@ -0,0 +1,57 @@ +import Path from 'path' +import { Client } from '@organicdesign/db-client' +import { packagePath } from '../utils/paths.js' +import runNode from '../utils/run-node.js' +import type { TransferBenchmark } from './interface.js' + +export const createTransferBench = async (size: number, persistent: boolean): Promise => { + const dataPath = Path.join(packagePath, 'test-out') + + const names = [...Array(2).keys()].map(i => `transfer-${size}-${i}`) + + const procs = await Promise.all(names.map(async n => runNode(n, { persistent }))) + + await Promise.all(procs.map(async p => p.start())) + + const clients = names.map(n => new Client(Path.join(dataPath, n, 'socket'))) + + const addresses = await clients[0].addresses() + + await clients[1].connect(addresses[0]) + + const group = await clients[0].createGroup('test') + + const dataFile = Path.join(dataPath, `${size}.data`) + const [{ cid }] = await clients[0].import(group, dataFile, { path: '/test' }) + const [item] = await clients[0].getStatus([cid]) + + return { + blocks: item.blocks, + size: item.size, + + async teardown () { + for (const client of clients) { + client.stop() + } + + await Promise.all(procs.map(async p => p.stop())) + }, + + async warmup () { + await clients[1].joinGroup(group) + await clients[1].sync() + }, + + async run () { + for (;;) { + const [{ state }] = await clients[1].getStatus([cid]) + + if (state === 'COMPLETED') { + break + } + + await new Promise(resolve => setTimeout(resolve, 10)) + } + } + } +} diff --git a/packages/benchmarks/src/utils/generate-file.ts b/packages/benchmarks/src/utils/generate-file.ts new file mode 100644 index 00000000..e92d75f1 --- /dev/null +++ b/packages/benchmarks/src/utils/generate-file.ts @@ -0,0 +1,24 @@ +import crypto from 'crypto' +import fs from 'fs' + +export default async (path: string, size: number, options?: { chunkSize: number }): Promise => { + const chunkSize = options?.chunkSize ?? 2 ** 14 + const stream = fs.createWriteStream(path) + + for (let i = 0; i < Math.ceil(size / chunkSize); i++) { + const thisSize = (i + 1) * chunkSize < size ? chunkSize : size - i * chunkSize + const data = crypto.randomBytes(thisSize) + + await new Promise((resolve, reject) => { + stream.write(data, (error) => { + if (error != null) { + reject(error) + } else { + resolve(error) + } + }) + }) + } + + await new Promise(resolve => stream.end(resolve)) +} diff --git a/packages/benchmarks/src/utils/paths.ts b/packages/benchmarks/src/utils/paths.ts new file mode 100644 index 00000000..7ab99848 --- /dev/null +++ b/packages/benchmarks/src/utils/paths.ts @@ -0,0 +1,6 @@ +import Path from 'path' +import { fileURLToPath } from 'url' + +export const packagePath = Path.join(Path.dirname(fileURLToPath(import.meta.url)), '../../../') + +export const modulesPath = Path.join(packagePath, '../../node_modules') diff --git a/packages/benchmarks/src/utils/run-node.ts b/packages/benchmarks/src/utils/run-node.ts new file mode 100644 index 00000000..9bad75e0 --- /dev/null +++ b/packages/benchmarks/src/utils/run-node.ts @@ -0,0 +1,89 @@ +import { spawn, type ChildProcessWithoutNullStreams } from 'child_process' +import fs from 'fs/promises' +import Path from 'path' +import { DeferredPromise } from '@open-draft/deferred-promise' +import { toString as uint8ArrayToString } from 'uint8arrays' +import { packagePath, modulesPath } from './paths.js' + +export default async (name: string, options: { persistent?: boolean } = {}): Promise<{ start(): Promise, stop(): Promise }> => { + const rootPath = Path.join(packagePath, 'test-out', name) + const socket = Path.join(rootPath, 'socket') + const config = Path.join(rootPath, 'config.json') + const isReceiver = name[name.length - 1] === '1' + const storagePath = Path.join(rootPath, 'storage') + + await fs.mkdir(rootPath, { recursive: true }) + + await fs.writeFile(config, JSON.stringify({ + storage: options.persistent === true ? storagePath : ':memory:' + })) + + const args = [ + Path.join(modulesPath, '@organicdesign/db-daemon/dist/src/index.js'), + '-s', socket, + '-c', config + ] + + if (isReceiver) { + // args.unshift('--trace-gc') + // args.unshift('--inspect') + } + + let proc: ChildProcessWithoutNullStreams + + const forceQuit = async (): Promise => { + // Ensure it is really dead. + proc.kill(9) + + await Promise.allSettled([ + fs.rm(socket) + ]) + } + + return { + async start () { + proc = spawn('node', args, { env: { ...process.env, DEBUG: 'backup:*' } }) + + const promise = new DeferredPromise() + + const listener = (chunk: Uint8Array): void => { + if (uint8ArrayToString(chunk).includes('started') === true) { + promise.resolve() + } + } + + proc.stderr.on('data', listener) + + await promise + + proc.stderr.off('data', listener) + }, + + async stop () { + const promise = new DeferredPromise() + + const listener = (chunk: Uint8Array): void => { + if (uint8ArrayToString(chunk).includes('exiting...') === true) { + promise.resolve() + } + } + + proc.stderr.on('data', listener) + proc.kill('SIGINT') + + // Kill it if it fails to do it cleanly. + setTimeout(() => { + forceQuit().finally(() => { + promise.reject(new Error('process did not exit cleanly')) + }) + }, 3000) + + await promise + + proc.stderr.off('data', listener) + + // Make sure things are cleaned up. + await forceQuit() + } + } +} diff --git a/packages/benchmarks/tsconfig.json b/packages/benchmarks/tsconfig.json new file mode 100644 index 00000000..90133985 --- /dev/null +++ b/packages/benchmarks/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "esModuleInterop": true, + "outDir": "dist", + "baseUrl": ".", + "module": "ES2022", + "target": "ES2022" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../client" + }, + { + "path": "../daemon" + } + ] +} diff --git a/packages/daemon/package.json b/packages/daemon/package.json index ade3fb50..01b921ac 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -25,14 +25,9 @@ "author": "Saul Boyd", "license": "GPL-3.0-or-later", "devDependencies": { - "@helia/interface": "^4.0.0", - "@ipld/dag-pb": "^4.1.0", - "@libp2p/interface": "^1.1.3", - "@libp2p/interfaces": "^3.3.2", "@organicdesign/db-test-utils": "^0.1.0", "@types/yargs": "^17.0.32", "aegir": "^42.2.4", - "interface-blockstore": "^5.2.10", "tsc-alias": "^1.8.8" }, "private": true, @@ -43,6 +38,7 @@ "@chainsafe/libp2p-yamux": "^6.0.2", "@helia/block-brokers": "^2.0.3", "@helia/car": "^3.1.0", + "@helia/interface": "^4.1.0", "@helia/unixfs": "^3.0.0", "@ipld/car": "^5.3.0", "@ipld/dag-cbor": "^9.2.0", @@ -51,6 +47,8 @@ "@libp2p/circuit-relay-v2": "^1.0.15", "@libp2p/dcutr": "^1.0.12", "@libp2p/identify": "^1.0.14", + "@libp2p/interface": "^1.1.6", + "@libp2p/interfaces": "^3.3.2", "@libp2p/kad-dht": "^12.0.7", "@libp2p/logger": "^4.0.6", "@libp2p/pnet": "^1.0.1", @@ -71,6 +69,7 @@ "datastore-fs": "^9.1.8", "debug": "^4.3.4", "helia": "^4.0.1", + "interface-blockstore": "^5.2.10", "interface-datastore": "^8.2.11", "it-all": "^3.0.4", "it-parallel": "^3.0.6", diff --git a/packages/daemon/src/common/downloader/index.ts b/packages/daemon/src/common/downloader/index.ts index af22abf3..ac766f30 100644 --- a/packages/daemon/src/common/downloader/index.ts +++ b/packages/daemon/src/common/downloader/index.ts @@ -59,7 +59,7 @@ export class Downloader implements Startable { } } - private async * batchDownload (itr: AsyncIterable<[CID, number]>): AsyncGenerator<() => Promise<{ cid: CID, block: Uint8Array }>, void, undefined> { + private async * batchDownload (itr: AsyncIterable<[CID, number]>): AsyncGenerator<() => Promise<{ cid: CID }>, void, undefined> { for await (const [cid, priority] of itr) { if (this.isAborted) { return diff --git a/packages/helia-pin-manager/package.json b/packages/helia-pin-manager/package.json index d9839f8d..2fcf1322 100644 --- a/packages/helia-pin-manager/package.json +++ b/packages/helia-pin-manager/package.json @@ -25,7 +25,6 @@ "author": "Saul Boyd", "license": "GPL-3.0-or-later", "devDependencies": { - "@helia/interface": "^4.0.0", "@ipld/dag-pb": "^4.1.0", "aegir": "^42.2.4", "helia": "^4.0.1", @@ -35,6 +34,7 @@ "private": true, "sideEffects": false, "dependencies": { + "@helia/interface": "^4.1.0", "@open-draft/deferred-promise": "^2.2.0", "@organicdesign/db-utils": "^0.1.0", "cborg": "^4.0.9", diff --git a/packages/helia-pin-manager/src/pin-manager.ts b/packages/helia-pin-manager/src/pin-manager.ts index b5b36859..b523887e 100644 --- a/packages/helia-pin-manager/src/pin-manager.ts +++ b/packages/helia-pin-manager/src/pin-manager.ts @@ -1,5 +1,5 @@ import { DeferredPromise } from '@open-draft/deferred-promise' -import { getWalker } from '@organicdesign/db-utils/dag' +import { walk, getWalker } from '@organicdesign/db-utils/dag' import { NamespaceDatastore } from 'datastore-core' import { Key, type Datastore } from 'interface-datastore' import all from 'it-all' @@ -17,7 +17,6 @@ export interface Components { } export interface BlockInfo { - block: Uint8Array cid: CID links: CID[] } @@ -36,7 +35,7 @@ class CIDEvent extends Event { export class PinManager { readonly events = new EventTarget<[CIDEvent]>() private readonly helia: Helia - private readonly activeDownloads = new Map>() + private readonly activeDownloads = new Map>() private readonly pins: Pins private readonly blocks: Blocks private readonly downloads: Downloads @@ -96,30 +95,20 @@ export class PinManager { throw new Error('pin find or create failed') } - const walk = async (subCid: CID, depth: number): Promise => { - const dagWalker = getWalker(subCid) + for await (const getBlock of walk(this.helia.blockstore, cid, { local: true })) { + const data = await getBlock() - if (!await this.helia.blockstore.has(subCid)) { - throw new Error('pin does not exist locally') - } - - await addBlockRef(this.helia, subCid, cid) + await addBlockRef(this.helia, data.cid, cid) - const block = await this.helia.blockstore.get(subCid) - - await this.blocks.getOrPut(cid, subCid, { - size: block.length, - depth, + await this.blocks.getOrPut(cid, data.cid, { + size: data.block.length, + depth: data.depth, timestamp: Date.now() }) - for await (const cid of dagWalker.walk(block)) { - await walk(cid, depth + 1) - } + delete (data as { block?: Uint8Array }).block } - await walk(cid, 0) - await addPinRef(this.helia, cid) await this.pins.put(cid, { @@ -285,7 +274,7 @@ export class PinManager { const enqueue = (cid: CID, depth: number): void => { queue.push(async () => { - const { links, block } = await this.download(cid) + const { links } = await this.download(cid) if (pinData.depth == null || depth < pinData.depth) { for (const cid of links) { @@ -293,7 +282,7 @@ export class PinManager { } } - return { cid, block, links } + return { cid, links } }) } @@ -303,7 +292,7 @@ export class PinManager { enqueue(head.cid, head.depth) } - const promises: Array> = [] + const promises: Array> = [] while (queue.length + promises.length !== 0) { const func = queue.shift() @@ -314,7 +303,7 @@ export class PinManager { continue } - const promise = new DeferredPromise<{ cid: CID, block: Uint8Array }>() + const promise = new DeferredPromise<{ cid: CID }>() promises.push(promise) @@ -393,7 +382,7 @@ export class PinManager { this.events.dispatchEvent(new CIDEvent('downloads:added', cid)) - return { block, cid, links } + return { cid, links } })() this.activeDownloads.set(cid.toString(), promise) diff --git a/packages/key-manager/package.json b/packages/key-manager/package.json index bce300ea..a074bee8 100644 --- a/packages/key-manager/package.json +++ b/packages/key-manager/package.json @@ -25,7 +25,6 @@ "author": "Saul Boyd", "license": "GPL-3.0-or-later", "devDependencies": { - "@libp2p/interface": "^1.1.3", "aegir": "^42.2.4" }, "private": true, @@ -33,6 +32,7 @@ "dependencies": { "@ipld/dag-cbor": "^9.2.0", "@libp2p/crypto": "^4.0.2", + "@libp2p/interface": "^1.1.6", "@libp2p/peer-id": "^4.0.6", "bip32": "^4.0.0", "bip39": "^3.1.0", diff --git a/packages/manual-block-broker/package.json b/packages/manual-block-broker/package.json index 307ba0c7..b9b4006e 100644 --- a/packages/manual-block-broker/package.json +++ b/packages/manual-block-broker/package.json @@ -25,14 +25,14 @@ "author": "Saul Boyd", "license": "GPL-3.0-or-later", "devDependencies": { - "@helia/interface": "^4.0.0", - "aegir": "^42.2.4", - "interface-store": "^5.1.8", - "multiformats": "^13.1.0" + "aegir": "^42.2.4" }, "private": true, "sideEffects": false, "dependencies": { - "@open-draft/deferred-promise": "^2.2.0" + "@helia/interface": "^4.1.0", + "@open-draft/deferred-promise": "^2.2.0", + "interface-store": "^5.1.8", + "multiformats": "^13.1.0" } } diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 3afa4227..2fafc196 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -28,15 +28,18 @@ "author": "Saul Boyd", "license": "GPL-3.0-or-later", "devDependencies": { - "aegir": "^42.2.4", - "multiformats": "^13.1.0" + "aegir": "^42.2.4" }, "private": true, "sideEffects": false, "dependencies": { + "@helia/interface": "^4.1.0", "@helia/unixfs": "^3.0.2", + "@ipld/dag-pb": "^4.1.0", "blockstore-core": "^4.4.0", + "interface-blockstore": "^5.2.10", "it-all": "^3.0.4", + "multiformats": "^13.1.0", "uint8arrays": "^5.0.3" } } diff --git a/packages/utils/package.json b/packages/utils/package.json index dc38cbe9..aed4b3d3 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -33,26 +33,28 @@ "author": "Saul Boyd", "license": "GPL-3.0-or-later", "devDependencies": { - "@helia/interface": "^4.1.0", - "@libp2p/interface": "^1.1.5", "@organicdesign/db-test-utils": "^0.1.0", "aegir": "^42.2.4", "blockstore-core": "^4.4.0", - "interface-blockstore": "^5.2.10", - "it-all": "^3.0.4" + "it-parallel": "^3.0.6", + "it-pipe": "^3.0.1" }, "private": true, "sideEffects": false, "dependencies": { + "@helia/interface": "^4.1.0", "@helia/unixfs": "^3.0.2", "@ipld/dag-cbor": "^9.2.0", "@ipld/dag-json": "^10.2.0", "@ipld/dag-pb": "^4.1.0", + "@libp2p/interface": "^1.1.6", "cborg": "^4.1.4", "datastore-core": "^9.2.9", + "interface-blockstore": "^5.2.10", "interface-datastore": "^8.2.11", "ipfs-unixfs-exporter": "^13.5.0", "ipfs-unixfs-importer": "^15.2.4", + "it-all": "^3.0.4", "multiformats": "^13.1.0" } } diff --git a/packages/utils/src/dag/index.ts b/packages/utils/src/dag/index.ts index b1b3b2fd..d518af66 100644 --- a/packages/utils/src/dag/index.ts +++ b/packages/utils/src/dag/index.ts @@ -20,7 +20,7 @@ export const getWalker = (cid: CID): DAGWalker => { return dagWalker } -export const walk = async function * (blockstore: Blockstore, cid: CID, maxDepth?: number, options?: AbortOptions): AsyncGenerator<() => Promise> { +export const walk = async function * (blockstore: Blockstore, cid: CID, options: AbortOptions & { local?: boolean, maxDepth?: number } = {}): AsyncGenerator<() => Promise> { const queue: Array<() => Promise> = [] const promises: Array> = [] @@ -33,9 +33,17 @@ export const walk = async function * (blockstore: Blockstore, cid: CID, maxDepth throw new Error(`No dag walker found for cid codec ${cid.code}`) } + if (options.local === true) { + const has = await blockstore.has(cid, options) + + if (!has) { + throw new Error(`missing block ${cid.toString()}`) + } + } + const block = await blockstore.get(cid, options) - if (maxDepth == null || depth < maxDepth) { + if (options.maxDepth == null || depth < options.maxDepth) { for await (const cid of dagWalker.walk(block)) { enqueue(cid, depth + 1) } @@ -70,10 +78,14 @@ export const getSize = async (blockstore: Blockstore, cid: CID): Promise<{ block let blocks = 0 for await (const getBlock of walk(blockstore, cid)) { - const { block } = await getBlock() + const data = await getBlock() blocks++ - size += block.length + size += data.block.length + + // This is needed to signal to NodeJS to free up the memory imediately, + // otherwise the memory can hang around for a while and can be massive. + delete (data as { block?: Uint8Array }).block } return { size, blocks } diff --git a/packages/utils/test/dag-size.spec.ts b/packages/utils/test/dag-size.spec.ts index 60373486..8c9f878e 100644 --- a/packages/utils/test/dag-size.spec.ts +++ b/packages/utils/test/dag-size.spec.ts @@ -2,7 +2,7 @@ import assert from 'assert/strict' import { createDag } from '@organicdesign/db-test-utils' import { MemoryBlockstore } from 'blockstore-core' import { type CID } from 'multiformats/cid' -import { getSize, walk } from '../src/dag/index.js' +import { getSize } from '../src/dag/index.js' describe('getDagSize', () => { let dag: CID[] @@ -28,28 +28,3 @@ describe('getDagSize', () => { assert.equal(size, totalSize) }) }) - -describe('walkDag', () => { - let dag: CID[] - let blockstore: MemoryBlockstore - - before(async () => { - blockstore = new MemoryBlockstore() - - dag = await createDag({ blockstore }, 3, 3) - }) - - it('walks over every value of the dag', async () => { - let count = 0 - - for await (const getData of walk(blockstore, dag[0])) { - const data = await getData() - - assert(dag.find(cid => cid.equals(data.cid))) - - count++ - } - - assert.equal(count, dag.length) - }) -}) diff --git a/packages/utils/test/walk.spec.ts b/packages/utils/test/walk.spec.ts new file mode 100644 index 00000000..d4cc931d --- /dev/null +++ b/packages/utils/test/walk.spec.ts @@ -0,0 +1,42 @@ +import assert from 'assert/strict' +import { createDag } from '@organicdesign/db-test-utils' +import { MemoryBlockstore } from 'blockstore-core' +import all from 'it-all' +import parallel from 'it-parallel' +import { pipe } from 'it-pipe' +import { type CID } from 'multiformats/cid' +import { walk } from '../src/dag/index.js' + +describe('walkDag', () => { + let dag: CID[] + let blockstore: MemoryBlockstore + + before(async () => { + blockstore = new MemoryBlockstore() + + dag = await createDag({ blockstore }, 3, 3) + }) + + it('walks over every value of the dag', async () => { + let count = 0 + + for await (const getData of walk(blockstore, dag[0])) { + const data = await getData() + + assert(dag.find(cid => cid.equals(data.cid))) + + count++ + } + + assert.equal(count, dag.length) + }) + + it('throws an error if the blockstore is missing an item and local is set', async () => { + const partialBlockstore = new MemoryBlockstore() + const dag = await createDag({ blockstore: partialBlockstore }, 3, 3) + + await partialBlockstore.delete(dag[1]) + + await assert.rejects(async () => all(pipe(walk(partialBlockstore, dag[0], { local: true }), parallel))) + }) +})