diff --git a/.changeset/chubby-olives-say.md b/.changeset/chubby-olives-say.md new file mode 100644 index 000000000000..0d54c117a73f --- /dev/null +++ b/.changeset/chubby-olives-say.md @@ -0,0 +1,25 @@ +--- +"fluid-framework": minor +"@fluidframework/tree": minor +--- +--- +"section": tree +--- + +New Alpha APIs for tree data import and export + +A collection of new `@alpha` APIs for importing and exporting tree content and schema from SharedTrees has been added to `TreeAlpha`. +These include import and export APIs for for `VerboseTree`, `ConciseTree` and compressed tree formats. + +`TreeAlpha.create` is also added to allow constructing trees with a more general API instead of having to use the schema constructor directly (since that doesn't handle polymorphic roots, or non-schema aware code). + +The function `independentInitializedView` has been added to provide a way to combine data from the existing `extractPersistedSchema` and new `TreeAlpha.exportCompressed` back into a `TreeView` in a way which can support safely importing data which could have been exported with a different schema. +This allows replicating the schema evolution process for Fluid documents stored in a service, but entirely locally without involving any collaboration services. +`independentView` has also been added, which is similar but handles the case of creating a new view without an existing schema or tree. + +Together these APIs address several use-cases: + +1. Using SharedTree as an in-memory non-collaborative datastore. +2. Importing and exporting data from a SharedTree to and from other services or storage locations (such as locally saved files). +3. Testing various scenarios without relying on a service. +4. Using SharedTree libraries for just the schema system and encode/decode support. diff --git a/biome.jsonc b/biome.jsonc index 154bff87904f..e59b39cb4091 100644 --- a/biome.jsonc +++ b/biome.jsonc @@ -58,6 +58,8 @@ "packages/tools/fluid-runner/src/test/localOdspSnapshots/**", "packages/tools/fluid-runner/src/test/telemetryExpectedOutputs/**", "tools/api-markdown-documenter/src/test/snapshots/**", + // TODO: why does examples/apps/tree-cli-app/*.json not work? + "**/data/*.json", // Generated type-tests "**/*.generated.ts", diff --git a/examples/apps/tree-cli-app/.eslintrc.cjs b/examples/apps/tree-cli-app/.eslintrc.cjs new file mode 100644 index 000000000000..f1aaefc67a58 --- /dev/null +++ b/examples/apps/tree-cli-app/.eslintrc.cjs @@ -0,0 +1,11 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +module.exports = { + extends: [require.resolve("@fluidframework/eslint-config-fluid/strict"), "prettier"], + parserOptions: { + project: ["./tsconfig.json", "./src/test/tsconfig.json"], + }, +}; diff --git a/examples/apps/tree-cli-app/.mocharc.cjs b/examples/apps/tree-cli-app/.mocharc.cjs new file mode 100644 index 000000000000..a2bcb0e04316 --- /dev/null +++ b/examples/apps/tree-cli-app/.mocharc.cjs @@ -0,0 +1,14 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +"use strict"; + +const getFluidTestMochaConfig = require("@fluid-internal/mocha-test-setup/mocharc-common"); + +const packageDir = __dirname; +const config = getFluidTestMochaConfig(packageDir); +config.spec = "lib/test"; + +module.exports = config; diff --git a/examples/apps/tree-cli-app/README.md b/examples/apps/tree-cli-app/README.md new file mode 100644 index 000000000000..ed6caee92a35 --- /dev/null +++ b/examples/apps/tree-cli-app/README.md @@ -0,0 +1,7 @@ +# @fluid-example/tree-cli-app + +Example application using Shared-Tree to create a non-collaborative file editing CLI application. + +Note that it's perfectly possible to write a collaborative online CLI app using tree as well: this simply is not an example of that. + +Run the app with `pnpm run app` after building. diff --git a/examples/apps/tree-cli-app/biome.jsonc b/examples/apps/tree-cli-app/biome.jsonc new file mode 100644 index 000000000000..4b65e1c0aea2 --- /dev/null +++ b/examples/apps/tree-cli-app/biome.jsonc @@ -0,0 +1,4 @@ +{ + "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", + "extends": ["../../../biome.jsonc"] +} diff --git a/examples/apps/tree-cli-app/data/concise.snapshot.json b/examples/apps/tree-cli-app/data/concise.snapshot.json new file mode 100644 index 000000000000..964c77877191 --- /dev/null +++ b/examples/apps/tree-cli-app/data/concise.snapshot.json @@ -0,0 +1 @@ +{"tree":{"version":1,"identifiers":[],"shapes":[{"c":{"type":"com.fluidframework.example.cli.List","value":false,"fields":[["",1]]}},{"a":2},{"d":0}],"data":[[0,0]]},"schema":{"version":1,"nodes":{"com.fluidframework.example.cli.Item":{"object":{"location":{"kind":"Value","types":["com.fluidframework.example.cli.Point"]},"name":{"kind":"Value","types":["com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.List":{"object":{"":{"kind":"Sequence","types":["com.fluidframework.example.cli.Item","com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.Point":{"object":{"x":{"kind":"Value","types":["com.fluidframework.leaf.number"]},"y":{"kind":"Value","types":["com.fluidframework.leaf.number"]}}},"com.fluidframework.leaf.number":{"leaf":0},"com.fluidframework.leaf.string":{"leaf":1}},"root":{"kind":"Value","types":["com.fluidframework.example.cli.List"]}},"idCompressor":"AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAAAUJLVv6TZpgqHGKmI53QQCAAAAAAAAAAAAAAAAAADwPwAAAAAAAAAA"} \ No newline at end of file diff --git a/examples/apps/tree-cli-app/data/default.compressed.json b/examples/apps/tree-cli-app/data/default.compressed.json new file mode 100644 index 000000000000..408bf9091374 --- /dev/null +++ b/examples/apps/tree-cli-app/data/default.compressed.json @@ -0,0 +1 @@ +{"version":1,"identifiers":[],"shapes":[{"c":{"type":"com.fluidframework.example.cli.List","value":false,"fields":[["",1]]}},{"a":2},{"d":0}],"data":[[0,0]]} \ No newline at end of file diff --git a/examples/apps/tree-cli-app/data/default.concise.json b/examples/apps/tree-cli-app/data/default.concise.json new file mode 100644 index 000000000000..0637a088a01e --- /dev/null +++ b/examples/apps/tree-cli-app/data/default.concise.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/examples/apps/tree-cli-app/data/default.snapshot.json b/examples/apps/tree-cli-app/data/default.snapshot.json new file mode 100644 index 000000000000..90c23add441b --- /dev/null +++ b/examples/apps/tree-cli-app/data/default.snapshot.json @@ -0,0 +1 @@ +{"tree":{"version":1,"identifiers":[],"shapes":[{"c":{"type":"com.fluidframework.example.cli.List","value":false,"fields":[["",1]]}},{"a":2},{"d":0}],"data":[[0,0]]},"schema":{"version":1,"nodes":{"com.fluidframework.example.cli.Item":{"object":{"location":{"kind":"Value","types":["com.fluidframework.example.cli.Point"]},"name":{"kind":"Value","types":["com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.List":{"object":{"":{"kind":"Sequence","types":["com.fluidframework.example.cli.Item","com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.Point":{"object":{"x":{"kind":"Value","types":["com.fluidframework.leaf.number"]},"y":{"kind":"Value","types":["com.fluidframework.leaf.number"]}}},"com.fluidframework.leaf.number":{"leaf":0},"com.fluidframework.leaf.string":{"leaf":1}},"root":{"kind":"Value","types":["com.fluidframework.example.cli.List"]}},"idCompressor":"AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAABhL9IeaYATlvd5A8Fp6aoDAAAAAAAAAAAAAAAAAADwPwAAAAAAAAAA"} \ No newline at end of file diff --git a/examples/apps/tree-cli-app/data/default.verbose-stored.json b/examples/apps/tree-cli-app/data/default.verbose-stored.json new file mode 100644 index 000000000000..a68c31892787 --- /dev/null +++ b/examples/apps/tree-cli-app/data/default.verbose-stored.json @@ -0,0 +1 @@ +{"type":"com.fluidframework.example.cli.List","fields":[]} \ No newline at end of file diff --git a/examples/apps/tree-cli-app/data/default.verbose.json b/examples/apps/tree-cli-app/data/default.verbose.json new file mode 100644 index 000000000000..a68c31892787 --- /dev/null +++ b/examples/apps/tree-cli-app/data/default.verbose.json @@ -0,0 +1 @@ +{"type":"com.fluidframework.example.cli.List","fields":[]} \ No newline at end of file diff --git a/examples/apps/tree-cli-app/package.json b/examples/apps/tree-cli-app/package.json new file mode 100644 index 000000000000..60a5796dd49d --- /dev/null +++ b/examples/apps/tree-cli-app/package.json @@ -0,0 +1,58 @@ +{ + "name": "@fluid-example/tree-cli-app", + "version": "2.5.0", + "private": true, + "description": "SharedTree CLI app demo", + "homepage": "https://fluidframework.com", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/FluidFramework.git", + "directory": "examples/apps/tree-cli-app" + }, + "license": "MIT", + "author": "Microsoft and contributors", + "type": "module", + "scripts": { + "app": "node ./lib/index.js", + "build": "fluid-build . --task build", + "build:compile": "fluid-build . --task compile", + "build:esnext": "tsc --project ./tsconfig.json", + "build:test": "npm run build:test:esm", + "build:test:esm": "tsc --project ./src/test/tsconfig.json", + "check:biome": "biome check .", + "check:format": "npm run check:biome", + "clean": "rimraf --glob dist lib \"**/*.tsbuildinfo\" \"**/*.build.log\" nyc", + "eslint": "eslint --format stylish src", + "eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout", + "format": "npm run format:biome", + "format:biome": "biome check . --write", + "lint": "fluid-build . --task lint", + "lint:fix": "fluid-build . --task eslint:fix --task format", + "test": "npm run test:mocha", + "test:mocha": "npm run test:mocha:esm", + "test:mocha:esm": "mocha", + "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha" + }, + "dependencies": { + "@fluidframework/core-interfaces": "workspace:~", + "@fluidframework/id-compressor": "workspace:~", + "@fluidframework/runtime-utils": "workspace:~", + "@fluidframework/tree": "workspace:~", + "@sinclair/typebox": "^0.32.29" + }, + "devDependencies": { + "@biomejs/biome": "~1.9.3", + "@fluid-internal/mocha-test-setup": "workspace:~", + "@fluidframework/build-tools": "^0.49.0", + "@fluidframework/eslint-config-fluid": "^5.4.0", + "@types/mocha": "^9.1.1", + "@types/node": "^18.19.0", + "cross-env": "^7.0.3", + "eslint": "~8.55.0", + "mocha": "^10.2.0", + "mocha-json-output-reporter": "^2.0.1", + "mocha-multi-reporters": "^1.5.1", + "rimraf": "^4.4.0", + "typescript": "~5.4.5" + } +} diff --git a/examples/apps/tree-cli-app/src/index.ts b/examples/apps/tree-cli-app/src/index.ts new file mode 100644 index 000000000000..28430ba62126 --- /dev/null +++ b/examples/apps/tree-cli-app/src/index.ts @@ -0,0 +1,35 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +// This is a node powered CLI application, so using node makes sense: +/* eslint-disable unicorn/no-process-exit */ + +import { applyEdit, loadDocument, saveDocument } from "./utils.js"; + +const args = process.argv.slice(2); + +console.log(`Requires arguments: [] [] []`); +console.log(`Example arguments: default data/large.concise.json string:10,item:100`); +console.log( + `File formats are specified by extension, for example ".verbose.json" uses the "verbose" format.`, +); +console.log( + `See implementation for supported formats and edit syntax: this is just a demon, not a nice app!`, +); +console.log(`Running with augments: ${args}`); + +if (args.length > 3) { + process.exit(1); +} + +const [sourceArg, destinationArg, editArg] = args; + +const node = loadDocument(sourceArg); + +if (editArg !== undefined) { + applyEdit(editArg, node); +} + +saveDocument(destinationArg, node); diff --git a/examples/apps/tree-cli-app/src/schema.ts b/examples/apps/tree-cli-app/src/schema.ts new file mode 100644 index 000000000000..c4bee3252559 --- /dev/null +++ b/examples/apps/tree-cli-app/src/schema.ts @@ -0,0 +1,38 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { SchemaFactory, TreeViewConfiguration } from "@fluidframework/tree"; + +/** + * + */ +export const schemaBuilder = new SchemaFactory("com.fluidframework.example.cli"); + +class Point extends schemaBuilder.object("Point", { + x: schemaBuilder.number, + y: schemaBuilder.number, +}) {} + +/** + * Complex list item. + */ +export class Item extends schemaBuilder.object("Item", { + position: schemaBuilder.required(Point, { key: "location" }), + name: schemaBuilder.string, +}) {} + +/** + * List node. + */ +export class List extends schemaBuilder.array("List", [schemaBuilder.string, Item]) {} + +/** + * Tree configuration. + */ +export const config = new TreeViewConfiguration({ + schema: List, + enableSchemaValidation: true, + preventAmbiguity: true, +}); diff --git a/examples/apps/tree-cli-app/src/test/schema.spec.ts b/examples/apps/tree-cli-app/src/test/schema.spec.ts new file mode 100644 index 000000000000..46147c5897e7 --- /dev/null +++ b/examples/apps/tree-cli-app/src/test/schema.spec.ts @@ -0,0 +1,144 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "node:assert"; + +import { + comparePersistedSchema, + extractPersistedSchema, + typeboxValidator, + type ForestOptions, + type ICodecOptions, + type JsonCompatible, + // eslint-disable-next-line import/no-internal-modules +} from "@fluidframework/tree/alpha"; + +import { List } from "../schema.js"; + +// This file demonstrates how applications can write tests which ensure they maintain compatibility the schema from with previously released versions. + +describe("schema", () => { + it("current schema matches latest historical schema", () => { + const current = extractPersistedSchema(List); + + // For compatibility with deep equality and simply objects, round trip via JSON to erase prototypes. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const currentRoundTripped: JsonCompatible = JSON.parse(JSON.stringify(current)); + + const previous = historicalSchema.at(-1); + assert(previous !== undefined); + // This ensures that historicalSchema's last entry is up to date with the current application code. + // This can catch: + // 1. Forgetting to update historicalSchema when intentionally making schema changes. + // 2. Accidentally changing schema in a way that impacts document compatibility. + assert.deepEqual(currentRoundTripped, previous.schema); + }); + + it("historical schema can be upgraded to current schema", () => { + const options: ForestOptions & ICodecOptions = { jsonValidator: typeboxValidator }; + + for (let documentIndex = 0; documentIndex < historicalSchema.length; documentIndex++) { + for (let viewIndex = 0; viewIndex < historicalSchema.length; viewIndex++) { + const compat = comparePersistedSchema( + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + historicalSchema[documentIndex]!.schema, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + historicalSchema[viewIndex]!.schema, + options, + false, + ); + + // We do not expect duplicates in historicalSchema. + assert.equal(compat.isEquivalent, documentIndex === viewIndex); + // Currently collaboration is only allowed between identical versions + assert.equal(compat.canView, documentIndex === viewIndex); + // Older versions should be upgradable to newer versions, but not the reverse. + assert.equal(compat.canUpgrade, documentIndex <= viewIndex); + } + } + }); +}); + +/** + * List of schema from previous versions of this application. + * Storing these as .json files in a folder may make more sense for more complex applications. + * + * The `schema` field is generated by passing the schema to `extractPersistedSchema`. + */ +const historicalSchema: { version: string; schema: JsonCompatible }[] = [ + { + version: "1.0", + schema: { + version: 1, + nodes: { + "com.fluidframework.example.cli.List": { + object: { + "": { + kind: "Sequence", + types: ["com.fluidframework.leaf.string"], + }, + }, + }, + "com.fluidframework.leaf.string": { + leaf: 1, + }, + }, + root: { + kind: "Value", + types: ["com.fluidframework.example.cli.List"], + }, + }, + }, + { + version: "2.0", + schema: { + version: 1, + nodes: { + "com.fluidframework.example.cli.Item": { + object: { + location: { + kind: "Value", + types: ["com.fluidframework.example.cli.Point"], + }, + name: { + kind: "Value", + types: ["com.fluidframework.leaf.string"], + }, + }, + }, + "com.fluidframework.example.cli.List": { + object: { + "": { + kind: "Sequence", + types: ["com.fluidframework.example.cli.Item", "com.fluidframework.leaf.string"], + }, + }, + }, + "com.fluidframework.example.cli.Point": { + object: { + x: { + kind: "Value", + types: ["com.fluidframework.leaf.number"], + }, + y: { + kind: "Value", + types: ["com.fluidframework.leaf.number"], + }, + }, + }, + "com.fluidframework.leaf.number": { + leaf: 0, + }, + "com.fluidframework.leaf.string": { + leaf: 1, + }, + }, + root: { + kind: "Value", + types: ["com.fluidframework.example.cli.List"], + }, + }, + }, +]; diff --git a/examples/apps/tree-cli-app/src/test/tsconfig.json b/examples/apps/tree-cli-app/src/test/tsconfig.json new file mode 100644 index 000000000000..acbd99a2893e --- /dev/null +++ b/examples/apps/tree-cli-app/src/test/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../../../../common/build/build-common/tsconfig.test.node16.json", + "compilerOptions": { + "rootDir": "./", + "outDir": "../../lib/test", + "types": ["mocha", "node"], + // Allows writing type checking expression without having to use the results. + "noUnusedLocals": false, + // Allow testing that declarations work properly + "declaration": true, + // Needed to ensure testExport's produce a valid d.ts + "skipLibCheck": false, + // Due to several of our own packages' exports failing to build with "exactOptionalPropertyTypes", + // disable it to prevent that from erroring when combined with "skipLibCheck". + "exactOptionalPropertyTypes": false, + }, + "include": ["./**/*"], + "references": [ + { + "path": "../..", + }, + ], +} diff --git a/examples/apps/tree-cli-app/src/utils.ts b/examples/apps/tree-cli-app/src/utils.ts new file mode 100644 index 000000000000..84ceb376c2a3 --- /dev/null +++ b/examples/apps/tree-cli-app/src/utils.ts @@ -0,0 +1,226 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +// This is a node powered CLI application, so using node makes sense: +/* eslint-disable unicorn/no-process-exit */ +/* eslint-disable import/no-nodejs-modules */ + +import { readFileSync, writeFileSync } from "node:fs"; + +import type { IFluidHandle } from "@fluidframework/core-interfaces"; +import type { SerializedIdCompressorWithOngoingSession } from "@fluidframework/id-compressor/internal"; +import { + createIdCompressor, + deserializeIdCompressor, +} from "@fluidframework/id-compressor/internal"; +import { isFluidHandle } from "@fluidframework/runtime-utils"; +import { TreeArrayNode, type InsertableTypedNode } from "@fluidframework/tree"; +import { + extractPersistedSchema, + FluidClientVersion, + independentInitializedView, + typeboxValidator, + type ForestOptions, + type ICodecOptions, + type JsonCompatible, + type VerboseTree, + type ViewContent, + type ConciseTree, + TreeAlpha, + // eslint-disable-next-line import/no-internal-modules +} from "@fluidframework/tree/alpha"; +import { type Static, Type } from "@sinclair/typebox"; + +import type { Item } from "./schema.js"; +import { config, List } from "./schema.js"; + +/** + * Load from file. + */ +export function loadDocument(source: string | undefined): List { + if (source === undefined || source === "default") { + return new List([]); + } + const parts = source.split("."); + if (parts.length < 3 || parts.at(-1) !== "json") { + console.log(`Invalid source: ${source}`); + process.exit(1); + } + + // Data parsed from JSON is safe to consider JsonCompatible. + // If file is invalid JSON, that will throw and is fine for this app. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const fileData: JsonCompatible = JSON.parse(readFileSync(source).toString()); + + switch (parts.at(-2)) { + case "concise": { + return TreeAlpha.importConcise(List, fileData as ConciseTree); + } + case "verbose": { + return TreeAlpha.importVerbose(List, fileData as VerboseTree); + } + case "verbose-stored": { + return TreeAlpha.importVerbose(List, fileData as VerboseTree, { + useStoredKeys: true, + }); + } + case "compressed": { + return TreeAlpha.importCompressed(List, fileData, { jsonValidator: typeboxValidator }); + } + case "snapshot": { + // TODO: validate + const combo: File = fileData as File; + + const content: ViewContent = { + schema: combo.schema, + tree: combo.tree, + idCompressor: deserializeIdCompressor(combo.idCompressor), + }; + const view = independentInitializedView(config, options, content); + return view.root; + } + default: { + console.log(`Invalid source format: ${parts.at(-2)}`); + process.exit(1); + } + } +} + +/** + * Save to file. + */ +export function saveDocument(destination: string | undefined, tree: List): void { + if (destination === undefined || destination === "default") { + console.log("Tree Content:"); + console.log(tree); + return; + } + const parts = destination.split("."); + if (parts.length < 3 || parts.at(-1) !== "json") { + console.log(`Invalid destination: ${destination}`); + process.exit(1); + } + + const fileData: JsonCompatible = exportContent(destination, tree); + console.log(`Writing: ${destination}`); + writeFileSync(destination, JSON.stringify(fileData, rejectHandles)); +} + +/** + * Encode to format based on file name. + */ +export function exportContent(destination: string, tree: List): JsonCompatible { + const parts = destination.split("."); + if (parts.length < 3 || parts.at(-1) !== "json") { + console.log(`Invalid destination: ${destination}`); + process.exit(1); + } + + switch (parts.at(-2)) { + case "concise": { + return TreeAlpha.exportConcise(tree) as JsonCompatible; + } + case "verbose": { + return TreeAlpha.exportVerbose(tree) as JsonCompatible; + } + case "concise-stored": { + return TreeAlpha.exportConcise(tree, { useStoredKeys: true }) as JsonCompatible; + } + case "verbose-stored": { + return TreeAlpha.exportVerbose(tree, { useStoredKeys: true }) as JsonCompatible; + } + case "compressed": { + return TreeAlpha.exportCompressed(tree, { + ...options, + oldestCompatibleClient: FluidClientVersion.v2_3, + }) as JsonCompatible; + } + case "snapshot": { + const idCompressor = createIdCompressor(); // TODO: get from tree? + const file: File = { + tree: TreeAlpha.exportCompressed(tree, { + oldestCompatibleClient: FluidClientVersion.v2_3, + idCompressor, + }), + schema: extractPersistedSchema(List), + idCompressor: idCompressor.serialize(true), + }; + return file as JsonCompatible; + } + default: { + console.log(`Invalid source format: ${parts.at(-2)}`); + process.exit(1); + } + } +} + +/** + * Encode to format based on file name. + */ +export function applyEdit(edits: string, tree: List): void { + for (const edit of edits.split(",")) { + console.log(`Applying edit ${edit}`); + const parts = edit.split(":"); + if (parts.length !== 2) { + throw new Error(`Invalid edit ${edit}`); + } + const [kind, countString] = parts; + const count = Number(countString); + if (count === 0 || !Number.isInteger(count)) { + throw new TypeError(`Invalid count in edit ${edit}`); + } + if (count > 0) { + let data: InsertableTypedNode | string; + switch (kind) { + case "string": { + data = "x"; + break; + } + case "item": { + data = { position: { x: 0, y: 0 }, name: "item" }; + break; + } + default: { + throw new TypeError(`Invalid kind in insert edit ${edit}`); + } + } + // eslint-disable-next-line unicorn/no-new-array + tree.insertAtEnd(TreeArrayNode.spread(new Array(count).fill(data))); + } else { + switch (kind) { + case "start": { + tree.removeRange(0, -count); + break; + } + case "end": { + tree.removeRange(tree.length + count, -count); + break; + } + default: { + throw new TypeError(`Invalid end in remove edit ${edit}`); + } + } + } + } +} + +/** + * Throw if handle. + */ +export function rejectHandles(key: string, value: unknown): unknown { + if (isFluidHandle(value)) { + throw new Error("Fluid handles are not supported"); + } + return value; +} + +const options: ForestOptions & ICodecOptions = { jsonValidator: typeboxValidator }; + +const File = Type.Object({ + tree: Type.Unsafe>(), + schema: Type.Unsafe(), + idCompressor: Type.Unsafe(), +}); +type File = Static; diff --git a/examples/apps/tree-cli-app/tsconfig.json b/examples/apps/tree-cli-app/tsconfig.json new file mode 100644 index 000000000000..55e7ad09857c --- /dev/null +++ b/examples/apps/tree-cli-app/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../../common/build/build-common/tsconfig.node16.json", + "include": ["src/**/*"], + "exclude": ["src/test/**/*"], + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "exactOptionalPropertyTypes": false, + "noUnusedLocals": false, + "types": ["node"], + }, +} diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index 7c0ddf84d9ff..f6f3a72b5c13 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -53,10 +53,21 @@ export interface CommitMetadata { // @alpha export function comparePersistedSchema(persisted: JsonCompatible, view: JsonCompatible, options: ICodecOptions, canInitialize: boolean): SchemaCompatibilityStatus; +// @alpha +export type ConciseTree = Exclude | THandle | ConciseTree[] | { + [key: string]: ConciseTree; +}; + // @public @sealed interface DefaultProvider extends ErasedType<"@fluidframework/tree.FieldProvider"> { } +// @alpha +export interface EncodeOptions { + readonly useStoredKeys?: boolean; + valueConverter(data: IFluidHandle): TCustom; +} + // @alpha export function enumFromStrings(factory: SchemaFactory, members: Members): ((value: TValue) => TreeNode & { readonly value: TValue; @@ -144,6 +155,18 @@ type FlexList = readonly LazyItem[]; // @public type FlexListToUnion = ExtractItemType; +// @alpha +export enum FluidClientVersion { + // (undocumented) + v2_0 = "v2_0", + // (undocumented) + v2_1 = "v2_1", + // (undocumented) + v2_2 = "v2_2", + // (undocumented) + v2_3 = "v2_3" +} + // @alpha export interface ForestOptions { readonly forest?: ForestType; @@ -176,6 +199,14 @@ export type ImplicitAllowedTypes = AllowedTypes | TreeNodeSchema; // @public export type ImplicitFieldSchema = FieldSchema | ImplicitAllowedTypes; +// @alpha +export function independentInitializedView(config: TreeViewConfiguration, options: ForestOptions & ICodecOptions, content: ViewContent): TreeView; + +// @alpha +export function independentView(config: TreeViewConfiguration, options: ForestOptions & { + idCompressor?: IIdCompressor_2 | undefined; +}): TreeView; + // @public type _InlineTrick = 0; @@ -314,11 +345,11 @@ export interface JsonArrayNodeSchema extends JsonNodeSchemaBase = string | number | boolean | null | JsonCompatible[] | JsonCompatibleObject | TExtra; // @alpha -export type JsonCompatibleObject = { - [P in string]?: JsonCompatible; +export type JsonCompatibleObject = { + [P in string]?: JsonCompatible; }; // @alpha @sealed @@ -452,6 +483,12 @@ type ObjectFromSchemaRecordUnsafe void; +// @alpha +export interface ParseOptions { + readonly useStoredKeys?: boolean; + valueConverter(data: VerboseTree): TreeLeafValue | VerboseTreeNode; +} + // @alpha export type PopUnion void : never>> = AsOverloadedFunction extends (a: infer First) => void ? First : never; @@ -604,6 +641,25 @@ export type TransactionConstraint = NodeInDocumentConstraint; // @public export const Tree: TreeApi; +// @alpha @sealed +export const TreeAlpha: { + create(schema: UnsafeUnknownSchema extends TSchema ? ImplicitFieldSchema : TSchema & ImplicitFieldSchema, data: InsertableField): Unhydrated : TreeNode | TreeLeafValue | undefined>; + importConcise(schema: UnsafeUnknownSchema extends TSchema ? ImplicitFieldSchema : TSchema & ImplicitFieldSchema, data: ConciseTree | undefined): Unhydrated : TreeNode | TreeLeafValue | undefined>; + importVerbose(schema: TSchema, data: VerboseTree | undefined, options?: Partial>): Unhydrated>; + importVerbose(schema: TSchema, data: VerboseTree | undefined, options: ParseOptions): Unhydrated>; + exportConcise(node: TreeNode | TreeLeafValue, options?: Partial>): ConciseTree; + exportConcise(node: TreeNode | TreeLeafValue, options?: EncodeOptions): ConciseTree; + exportVerbose(node: TreeNode | TreeLeafValue, options?: Partial>): VerboseTree; + exportVerbose(node: TreeNode | TreeLeafValue, options: EncodeOptions): VerboseTree; + exportCompressed(tree: TreeNode | TreeLeafValue, options: { + oldestCompatibleClient: FluidClientVersion; + idCompressor?: IIdCompressor; + }): JsonCompatible; + importCompressed(schema: TSchema, compressedData: JsonCompatible, options: { + idCompressor?: IIdCompressor; + } & ICodecOptions): Unhydrated>; +}; + // @public @sealed interface TreeApi extends TreeNodeApi { contains(node: TreeNode, other: TreeNode): boolean; @@ -862,11 +918,29 @@ export type ValidateRecursiveSchema> = true; +// @alpha +export type VerboseTree = VerboseTreeNode | Exclude | THandle; + +// @alpha +export interface VerboseTreeNode { + fields: VerboseTree[] | { + [key: string]: VerboseTree; + }; + type: string; +} + // @public @sealed export interface ViewableTree { viewWith(config: TreeViewConfiguration): TreeView; } +// @alpha +export interface ViewContent { + readonly idCompressor: IIdCompressor_2; + readonly schema: JsonCompatible; + readonly tree: JsonCompatible; +} + // @public @sealed export interface WithType { // @deprecated diff --git a/packages/dds/tree/src/codec/codec.ts b/packages/dds/tree/src/codec/codec.ts index 17d2e7db2730..b1eb3ba2c06b 100644 --- a/packages/dds/tree/src/codec/codec.ts +++ b/packages/dds/tree/src/codec/codec.ts @@ -328,3 +328,22 @@ export function withSchemaValidation< }, }; } + +/** + * Versions of FLuid Framework client packages. + * @remarks + * Used to express compatibility requirements by indicating the oldest version with which compatibility must be maintained. + * @privateRemarks + * This scheme assumes a single version will always be enough to communicate compatibility. + * For this to work, compatibility has to be strictly increasing. + * If this is violated (for example a subset of incompatible features from 3.x that are not in 3.0 are back ported to 2.x), + * a more complex scheme may be needed to allow safely opting into incompatible features in those cases: + * such a system can be added if/when its needed since it will be opt in and thus non-breaking. + * @alpha + */ +export enum FluidClientVersion { + v2_0 = "v2_0", + v2_1 = "v2_1", + v2_2 = "v2_2", + v2_3 = "v2_3", +} diff --git a/packages/dds/tree/src/codec/index.ts b/packages/dds/tree/src/codec/index.ts index 348cc6f204e4..8a9a81ca0ef6 100644 --- a/packages/dds/tree/src/codec/index.ts +++ b/packages/dds/tree/src/codec/index.ts @@ -18,6 +18,7 @@ export { unitCodec, withDefaultBinaryEncoding, withSchemaValidation, + FluidClientVersion, } from "./codec.js"; export { DiscriminatedUnionDispatcher, diff --git a/packages/dds/tree/src/feature-libraries/flex-tree/flexTreeTypes.ts b/packages/dds/tree/src/feature-libraries/flex-tree/flexTreeTypes.ts index 5a14255d545a..1d22bf5bdba6 100644 --- a/packages/dds/tree/src/feature-libraries/flex-tree/flexTreeTypes.ts +++ b/packages/dds/tree/src/feature-libraries/flex-tree/flexTreeTypes.ts @@ -9,6 +9,7 @@ import { type FieldKey, type FieldKindIdentifier, type FieldUpPath, + type ITreeCursorSynchronous, type TreeNodeSchemaIdentifier, type TreeValue, anchorSlot, @@ -184,6 +185,14 @@ export interface FlexTreeNode extends FlexTreeEntity { * If well-formed, it must follow this schema. */ readonly schema: TreeNodeSchemaIdentifier; + + /** + * Get a cursor for the underlying data. + * @remarks + * This cursor might be one the node uses in its implementation, and thus must be returned to its original location before using any other APIs to interact with the tree. + * Must not be held onto across edits or any other tree API use. + */ + borrowCursor(): ITreeCursorSynchronous; } /** diff --git a/packages/dds/tree/src/feature-libraries/flex-tree/lazyNode.ts b/packages/dds/tree/src/feature-libraries/flex-tree/lazyNode.ts index 5f54cececf3b..021c2fe305f9 100644 --- a/packages/dds/tree/src/feature-libraries/flex-tree/lazyNode.ts +++ b/packages/dds/tree/src/feature-libraries/flex-tree/lazyNode.ts @@ -10,6 +10,7 @@ import { type AnchorNode, CursorLocationType, type FieldKey, + type ITreeCursorSynchronous, type FieldKindIdentifier, type ITreeSubscriptionCursor, type TreeNavigationResult, @@ -91,6 +92,10 @@ export class LazyTreeNode extends LazyEntity implements FlexTreeNode { this.#removeDeleteCallback = anchorNode.on("afterDestroy", cleanupTree); } + public borrowCursor(): ITreeCursorSynchronous { + return this[cursorSymbol] as ITreeCursorSynchronous; + } + protected override [tryMoveCursorToAnchorSymbol]( cursor: ITreeSubscriptionCursor, ): TreeNavigationResult { diff --git a/packages/dds/tree/src/index.ts b/packages/dds/tree/src/index.ts index d67abfb2b28c..887000244552 100644 --- a/packages/dds/tree/src/index.ts +++ b/packages/dds/tree/src/index.ts @@ -60,6 +60,10 @@ export { getBranch, type TreeBranch, type TreeBranchFork, + independentInitializedView, + type ViewContent, + TreeAlpha, + independentView, } from "./shared-tree/index.js"; export { @@ -140,8 +144,13 @@ export { // Beta APIs TreeBeta, type TreeChangeEventsBeta, + type VerboseTreeNode, + type EncodeOptions, + type ParseOptions, + type VerboseTree, extractPersistedSchema, comparePersistedSchema, + type ConciseTree, // Back to normal types type JsonTreeSchema, type JsonSchemaId, @@ -168,10 +177,11 @@ export { configuredSharedTree, } from "./treeFactory.js"; -export type { - ICodecOptions, - JsonValidator, - SchemaValidationFunction, +export { + type ICodecOptions, + type JsonValidator, + type SchemaValidationFunction, + FluidClientVersion, } from "./codec/index.js"; export { noopValidator } from "./codec/index.js"; export { typeboxValidator } from "./external-utilities/index.js"; diff --git a/packages/dds/tree/src/shared-tree/independentView.ts b/packages/dds/tree/src/shared-tree/independentView.ts new file mode 100644 index 000000000000..3a590a929778 --- /dev/null +++ b/packages/dds/tree/src/shared-tree/independentView.ts @@ -0,0 +1,176 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import type { IFluidHandle } from "@fluidframework/core-interfaces"; +import { assert } from "@fluidframework/core-utils/internal"; +import { + type IIdCompressor, + createIdCompressor, +} from "@fluidframework/id-compressor/internal"; +import type { ICodecOptions } from "../codec/index.js"; +import { + type RevisionTag, + RevisionTagCodec, + TreeStoredSchemaRepository, + initializeForest, + type ITreeCursorSynchronous, + mapCursorField, +} from "../core/index.js"; +import { + createNodeKeyManager, + makeFieldBatchCodec, + makeSchemaCodec, + type FieldBatchEncodingContext, + defaultSchemaPolicy, + chunkTree, + defaultChunkPolicy, + TreeCompressionStrategy, +} from "../feature-libraries/index.js"; +// eslint-disable-next-line import/no-internal-modules +import type { Format } from "../feature-libraries/schema-index/format.js"; +import type { + TreeViewConfiguration, + TreeView, + ImplicitFieldSchema, +} from "../simple-tree/index.js"; +import type { JsonCompatibleReadOnly, JsonCompatible } from "../util/index.js"; +import { + buildConfiguredForest, + defaultSharedTreeOptions, + type ForestOptions, +} from "./sharedTree.js"; +import { createTreeCheckout } from "./treeCheckout.js"; +import { SchematizingSimpleTreeView } from "./schematizingTreeView.js"; + +/** + * Create an uninitialized {@link TreeView} that is not tied to any {@link ITree} instance. + * + * @remarks + * Such a view can never experience collaboration or be persisted to to a Fluid Container. + * + * This can be useful for testing, as well as use-cases like working on local files instead of documents stored in some Fluid service. + * @alpha + */ +export function independentView( + config: TreeViewConfiguration, + options: ForestOptions & { idCompressor?: IIdCompressor | undefined }, +): TreeView { + const idCompressor: IIdCompressor = options.idCompressor ?? createIdCompressor(); + const mintRevisionTag = (): RevisionTag => idCompressor.generateCompressedId(); + const revisionTagCodec = new RevisionTagCodec(idCompressor); + const schema = new TreeStoredSchemaRepository(); + const forest = buildConfiguredForest( + options.forest ?? defaultSharedTreeOptions.forest, + schema, + idCompressor, + ); + const checkout = createTreeCheckout(idCompressor, mintRevisionTag, revisionTagCodec, { + forest, + schema, + }); + const out: TreeView = new SchematizingSimpleTreeView( + checkout, + config, + createNodeKeyManager(idCompressor), + ); + return out; +} +/** + * Create an uninitialized {@link TreeView} that is not tied to any {@link ITree} instance. + * + * @remarks + * Such a view can never experience collaboration or be persisted to to a Fluid Container. + * + * This can be useful for testing, as well as use-cases like working on local files instead of documents stored in some Fluid service. + * @alpha + */ +export function independentInitializedView( + config: TreeViewConfiguration, + options: ForestOptions & ICodecOptions, + content: ViewContent, +): TreeView { + const idCompressor: IIdCompressor = content.idCompressor; + const mintRevisionTag = (): RevisionTag => idCompressor.generateCompressedId(); + const revisionTagCodec = new RevisionTagCodec(idCompressor); + + const fieldBatchCodec = makeFieldBatchCodec(options, 1); + const schemaCodec = makeSchemaCodec(options); + + const schema = new TreeStoredSchemaRepository(schemaCodec.decode(content.schema as Format)); + const forest = buildConfiguredForest( + options.forest ?? defaultSharedTreeOptions.forest, + schema, + idCompressor, + ); + + const context: FieldBatchEncodingContext = { + encodeType: TreeCompressionStrategy.Compressed, + idCompressor, + originatorId: idCompressor.localSessionId, // Is this right? If so, why is is needed? + schema: { schema, policy: defaultSchemaPolicy }, + }; + + const fieldCursors = fieldBatchCodec.decode(content.tree as JsonCompatibleReadOnly, context); + assert(fieldCursors.length === 1, "must have exactly 1 field in batch"); + // Checked above. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const cursors = fieldCursorToNodesCursors(fieldCursors[0]!); + + initializeForest(forest, cursors, revisionTagCodec, idCompressor, false); + + const checkout = createTreeCheckout(idCompressor, mintRevisionTag, revisionTagCodec, { + forest, + schema, + }); + const out: TreeView = new SchematizingSimpleTreeView( + checkout, + config, + createNodeKeyManager(idCompressor), + ); + return out; +} + +function fieldCursorToNodesCursors( + fieldCursor: ITreeCursorSynchronous, +): ITreeCursorSynchronous[] { + return mapCursorField(fieldCursor, copyNodeCursor); +} + +/** + * TODO: avoid needing this, or optimize it. + */ +function copyNodeCursor(cursor: ITreeCursorSynchronous): ITreeCursorSynchronous { + const copy = chunkTree(cursor, { + policy: defaultChunkPolicy, + idCompressor: undefined, + }).cursor(); + copy.enterNode(0); + return copy; +} + +/** + * The portion of SharedTree data typically persisted the container. + * Usable with {@link independentInitializedView} to create a {@link TreeView} + * without loading a container. + * @alpha + */ +export interface ViewContent { + /** + * Compressed tree from {@link TreeAlpha.exportCompressed}. + * @remarks + * This is an owning reference: + * consumers of this content might modify this data in place (for example when applying edits) to avoid copying. + */ + readonly tree: JsonCompatible; + /** + * Persisted schema from {@link extractPersistedSchema}. + */ + readonly schema: JsonCompatible; + /** + * IIdCompressor which will be used to decompress any compressed identifiers in `tree` + * as well as for any other identifiers added to the view. + */ + readonly idCompressor: IIdCompressor; +} diff --git a/packages/dds/tree/src/shared-tree/index.ts b/packages/dds/tree/src/shared-tree/index.ts index be2f40a90ab1..1645b21fa168 100644 --- a/packages/dds/tree/src/shared-tree/index.ts +++ b/packages/dds/tree/src/shared-tree/index.ts @@ -46,3 +46,11 @@ export { type RunTransaction, rollback, } from "./treeApi.js"; + +export { TreeAlpha } from "./treeApiAlpha.js"; + +export { + independentInitializedView, + type ViewContent, + independentView, +} from "./independentView.js"; diff --git a/packages/dds/tree/src/shared-tree/treeApiAlpha.ts b/packages/dds/tree/src/shared-tree/treeApiAlpha.ts new file mode 100644 index 000000000000..db2e175c14c3 --- /dev/null +++ b/packages/dds/tree/src/shared-tree/treeApiAlpha.ts @@ -0,0 +1,356 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { createIdCompressor } from "@fluidframework/id-compressor/internal"; +import { UsageError } from "@fluidframework/telemetry-utils/internal"; +import type { IFluidHandle } from "@fluidframework/core-interfaces"; +import type { IIdCompressor } from "@fluidframework/id-compressor"; + +import { + getKernel, + type TreeNode, + type Unhydrated, + TreeBeta, + tryGetSchema, + createFromCursor, + createFromInsertable, + cursorFromInsertable, + FieldKind, + normalizeFieldSchema, + type ImplicitFieldSchema, + type InsertableField, + type TreeFieldFromImplicitField, + type TreeLeafValue, + type UnsafeUnknownSchema, + conciseFromCursor, + type ConciseTree, + applySchemaToParserOptions, + cursorFromVerbose, + verboseFromCursor, + type ParseOptions, + type VerboseTree, + type VerboseTreeNode, + toStoredSchema, + type EncodeOptions, + extractPersistedSchema, + TreeViewConfiguration, +} from "../simple-tree/index.js"; +import { fail, type JsonCompatible } from "../util/index.js"; +import { noopValidator, type FluidClientVersion, type ICodecOptions } from "../codec/index.js"; +import type { ITreeCursorSynchronous } from "../core/index.js"; +import { + cursorForMapTreeField, + defaultSchemaPolicy, + isTreeValue, + makeFieldBatchCodec, + mapTreeFromCursor, + TreeCompressionStrategy, + type FieldBatch, + type FieldBatchEncodingContext, +} from "../feature-libraries/index.js"; +import { independentInitializedView, type ViewContent } from "./independentView.js"; + +/** + * Extensions to {@link Tree} and {@link TreeBeta} which are not yet stable. + * @sealed @alpha + */ +export const TreeAlpha: { + /** + * Construct tree content that is compatible with the field defined by the provided `schema`. + * @param schema - The schema for what to construct. As this is an {@link ImplicitFieldSchema}, a {@link FieldSchema}, {@link TreeNodeSchema} or {@link AllowedTypes} array can be provided. + * @param data - The data used to construct the field content. + * @remarks + * When providing a {@link TreeNodeSchemaClass}, this is the same as invoking its constructor except that an unhydrated node can also be provided. + * This function exists as a generalization that can be used in other cases as well, + * such as when `undefined` might be allowed (for an optional field), or when the type should be inferred from the data when more than one type is possible. + * + * Like with {@link TreeNodeSchemaClass}'s constructor, its an error to provide an existing node to this API. + * For that case, use {@link TreeBeta.clone}. + * @privateRemarks + * There should be a way to provide an source for defaulted identifiers, wither via this API or some way to add them to its output later. + */ + create( + schema: UnsafeUnknownSchema extends TSchema + ? ImplicitFieldSchema + : TSchema & ImplicitFieldSchema, + data: InsertableField, + ): Unhydrated< + TSchema extends ImplicitFieldSchema + ? TreeFieldFromImplicitField + : TreeNode | TreeLeafValue | undefined + >; + + /** + * Less type safe version of {@link TreeAlpha.create}, suitable for importing data. + * @remarks + * Due to {@link ConciseTree} relying on type inference from the data, its use is somewhat limited. + * This does not support {@link ConciseTree}'s with customized handle encodings or using persisted keys. + * Use "compressed" or "verbose" formats to for more flexibility. + * + * When using this function, + * it is recommend to ensure you schema is unambiguous with {@link ITreeConfigurationOptions.preventAmbiguity}. + * If the schema is ambiguous, consider using {@link TreeAlpha.create} and {@link Unhydrated} nodes where needed, + * or using {@link TreeAlpha.(importVerbose:1)} and specify all types. + * + * Documented (and thus recoverable) error handling/reporting for this is not yet implemented, + * but for now most invalid inputs will throw a recoverable error. + */ + importConcise( + schema: UnsafeUnknownSchema extends TSchema + ? ImplicitFieldSchema + : TSchema & ImplicitFieldSchema, + data: ConciseTree | undefined, + ): Unhydrated< + TSchema extends ImplicitFieldSchema + ? TreeFieldFromImplicitField + : TreeNode | TreeLeafValue | undefined + >; + + /** + * Construct tree content compatible with a field defined by the provided `schema`. + * @param schema - The schema for what to construct. As this is an {@link ImplicitFieldSchema}, a {@link FieldSchema}, {@link TreeNodeSchema} or {@link AllowedTypes} array can be provided. + * @param data - The data used to construct the field content. See `Tree.cloneToJSONVerbose`. + */ + importVerbose( + schema: TSchema, + data: VerboseTree | undefined, + options?: Partial>, + ): Unhydrated>; + + /** + * Construct tree content compatible with a field defined by the provided `schema`. + * @param schema - The schema for what to construct. As this is an {@link ImplicitFieldSchema}, a {@link FieldSchema}, {@link TreeNodeSchema} or {@link AllowedTypes} array can be provided. + * @param data - The data used to construct the field content. See `Tree.cloneToJSONVerbose`. + * @privateRemarks + * This could be exposed as a public `Tree.createFromVerbose` function. + */ + importVerbose( + schema: TSchema, + data: VerboseTree | undefined, + options: ParseOptions, + ): Unhydrated>; + + /** + * Same as generic overload, except leaves handles as is. + */ + exportConcise( + node: TreeNode | TreeLeafValue, + options?: Partial>, + ): ConciseTree; + + /** + * Copy a snapshot of the current version of a TreeNode into a {@link ConciseTree}. + */ + exportConcise( + node: TreeNode | TreeLeafValue, + options?: EncodeOptions, + ): ConciseTree; + + /** + * Same {@link TreeAlpha.(exportVerbose:2)} except leaves handles as is. + */ + exportVerbose( + node: TreeNode | TreeLeafValue, + options?: Partial>, + ): VerboseTree; + + /** + * Copy a snapshot of the current version of a TreeNode into a JSON compatible plain old JavaScript Object. + * Verbose tree format, with explicit type on every node. + * + * @remarks + * There are several cases this may be preferred to {@link TreeAlpha.(exportConcise:2)}: + * + * 1. When not using {@link ITreeConfigurationOptions.preventAmbiguity} (or when using `useStableFieldKeys`), `exportConcise` can produce ambiguous data (the type may be unclear on some nodes). + * `exportVerbose` will always be unambiguous and thus lossless. + * + * 2. When the data might be interpreted without access to the exact same view schema. In such cases, the types may be unknowable if not included. + * + * 3. When easy access to the type is desired. + */ + exportVerbose(node: TreeNode | TreeLeafValue, options: EncodeOptions): VerboseTree; + + /** + * Export the content of the provided `tree` in a compressed JSON compatible format. + * @remarks + * If an `idCompressor` is provided, it will be used to compress identifiers and thus will be needed to decompress the data. + * + * Always uses "stored" keys. + * See {@link EncodeOptions.useStoredKeys} for details. + */ + exportCompressed( + tree: TreeNode | TreeLeafValue, + options: { oldestCompatibleClient: FluidClientVersion; idCompressor?: IIdCompressor }, + ): JsonCompatible; + + /** + * Import data encoded by {@link TreeAlpha.exportCompressed}. + * + * @param schema - Schema with witch the data must be compatible. This compatibility is not verified and must be ensured by the caller. + * @param compressedData - Data compressed by {@link TreeAlpha.exportCompressed}. + * @param options - If {@link TreeAlpha.exportCompressed} was given an `idCompressor`, it must be provided here. + * + * @remarks + * If the data could have been encoded with a different schema, consider encoding the schema along side it using {@link extractPersistedSchema} and loading the data using {@link independentView}. + * + * @privateRemarks + * This API could be improved: + * + * 1. It could validate that the schema is compatible, and return or throw an error in the invalid case (maybe add a "try" version). + * 2. A "try" version of this could return an error if the data isn't in a supported format (as determined by version and/or JasonValidator). + * 3. Requiring the caller provide a JsonValidator isn't the most friendly API. It might be practical to provide a default. + */ + importCompressed( + schema: TSchema, + compressedData: JsonCompatible, + options: { idCompressor?: IIdCompressor } & ICodecOptions, + ): Unhydrated>; +} = { + create: createFromInsertable, + + importConcise( + schema: UnsafeUnknownSchema extends TSchema + ? ImplicitFieldSchema + : TSchema & ImplicitFieldSchema, + data: ConciseTree | undefined, + ): Unhydrated< + TSchema extends ImplicitFieldSchema + ? TreeFieldFromImplicitField + : TreeNode | TreeLeafValue | undefined + > { + return createFromInsertable( + schema, + data as InsertableField, + ) as Unhydrated< + TSchema extends ImplicitFieldSchema + ? TreeFieldFromImplicitField + : TreeNode | TreeLeafValue | undefined + >; + }, + + importVerbose( + schema: TSchema, + data: VerboseTree | undefined, + options?: Partial>, + ): Unhydrated> { + const config: ParseOptions = { + valueConverter: (input: VerboseTree) => { + return input as TreeLeafValue | VerboseTreeNode; + }, + ...options, + }; + const schemalessConfig = applySchemaToParserOptions(schema, config); + if (data === undefined) { + const field = normalizeFieldSchema(schema); + if (field.kind !== FieldKind.Optional) { + throw new UsageError("undefined provided for non-optional field."); + } + return undefined as Unhydrated>; + } + const cursor = cursorFromVerbose(data, schemalessConfig); + return createFromCursor(schema, cursor); + }, + + exportConcise( + node: TreeNode | TreeLeafValue, + options?: Partial>, + ): ConciseTree { + const config: EncodeOptions = { + valueConverter(handle: IFluidHandle): T { + return handle as T; + }, + ...options, + }; + + const cursor = borrowCursorFromTreeNodeOrValue(node); + return conciseFromCursor(cursor, tryGetSchema(node) ?? fail("invalid input"), config); + }, + + exportVerbose( + node: TreeNode | TreeLeafValue, + options?: Partial>, + ): VerboseTree { + const config: EncodeOptions = { + valueConverter(handle: IFluidHandle): T { + return handle as T; + }, + ...options, + }; + + const cursor = borrowCursorFromTreeNodeOrValue(node); + return verboseFromCursor(cursor, tryGetSchema(node) ?? fail("invalid input"), config); + }, + + exportCompressed( + node: TreeNode | TreeLeafValue, + options: { + oldestCompatibleClient: FluidClientVersion; + idCompressor?: IIdCompressor; + }, + ): JsonCompatible { + const schema = tryGetSchema(node) ?? fail("invalid input"); + const format = versionToFormat[options.oldestCompatibleClient]; + const codec = makeFieldBatchCodec({ jsonValidator: noopValidator }, format); + const cursor = borrowFieldCursorFromTreeNodeOrValue(node); + const batch: FieldBatch = [cursor]; + // If none provided, create a compressor which will not compress anything (TODO: is this the right way to do that?). + const idCompressor = options.idCompressor ?? createIdCompressor(); + const context: FieldBatchEncodingContext = { + encodeType: TreeCompressionStrategy.Compressed, + idCompressor, + originatorId: idCompressor.localSessionId, // Is this right? If so, why is is needed? + schema: { schema: toStoredSchema(schema), policy: defaultSchemaPolicy }, + }; + const result = codec.encode(batch, context); + return result; + }, + + importCompressed( + schema: TSchema, + compressedData: JsonCompatible, + options: { + idCompressor?: IIdCompressor; + } & ICodecOptions, + ): Unhydrated> { + const content: ViewContent = { + schema: extractPersistedSchema(schema), + tree: compressedData, + idCompressor: options.idCompressor ?? createIdCompressor(), + }; + const config = new TreeViewConfiguration({ schema }); + const view = independentInitializedView(config, options, content); + return TreeBeta.clone(view.root); + }, +}; + +function borrowCursorFromTreeNodeOrValue( + node: TreeNode | TreeLeafValue, +): ITreeCursorSynchronous { + if (isTreeValue(node)) { + return cursorFromInsertable( + tryGetSchema(node) ?? fail("missing schema"), + node, + ); + } + const kernel = getKernel(node); + const cursor = kernel.getOrCreateInnerNode().borrowCursor(); + return cursor; +} + +function borrowFieldCursorFromTreeNodeOrValue( + node: TreeNode | TreeLeafValue, +): ITreeCursorSynchronous { + const cursor = borrowCursorFromTreeNodeOrValue(node); + // TODO: avoid copy + const mapTree = mapTreeFromCursor(cursor); + return cursorForMapTreeField([mapTree]); +} + +const versionToFormat = { + v2_0: 1, + v2_1: 1, + v2_2: 1, + v2_3: 1, +}; diff --git a/packages/dds/tree/src/simple-tree/api/conciseTree.ts b/packages/dds/tree/src/simple-tree/api/conciseTree.ts index 679690bf35db..78b91643f8f3 100644 --- a/packages/dds/tree/src/simple-tree/api/conciseTree.ts +++ b/packages/dds/tree/src/simple-tree/api/conciseTree.ts @@ -23,6 +23,7 @@ import { getUnhydratedContext } from "../createContext.js"; * @privateRemarks * This can store all possible simple trees, * but it can not store all possible trees representable by our internal representations like FlexTree and JsonableTree. + * @alpha */ export type ConciseTree = | Exclude diff --git a/packages/dds/tree/src/simple-tree/api/create.ts b/packages/dds/tree/src/simple-tree/api/create.ts index 78657e20bb39..d2e962024b3f 100644 --- a/packages/dds/tree/src/simple-tree/api/create.ts +++ b/packages/dds/tree/src/simple-tree/api/create.ts @@ -3,18 +3,17 @@ * Licensed under the MIT License. */ -import type { IFluidHandle } from "@fluidframework/core-interfaces"; import { assert } from "@fluidframework/core-utils/internal"; import type { ITreeCursorSynchronous, SchemaAndPolicy } from "../../core/index.js"; import type { - TreeLeafValue, ImplicitFieldSchema, TreeFieldFromImplicitField, FieldSchema, FieldKind, UnsafeUnknownSchema, InsertableField, + TreeLeafValue, } from "../schemaTypes.js"; import { getOrCreateNodeFromInnerNode, @@ -32,13 +31,6 @@ import { import { isFieldInSchema } from "../../feature-libraries/index.js"; import { toStoredSchema } from "../toStoredSchema.js"; import { inSchemaOrThrow, mapTreeFromNodeData } from "../toMapTree.js"; -import { - applySchemaToParserOptions, - cursorFromVerbose, - type ParseOptions, - type VerboseTree, - type VerboseTreeNode, -} from "./verboseTree.js"; import { getUnhydratedContext } from "../createContext.js"; /** @@ -118,46 +110,6 @@ export function cursorFromInsertable< return cursorForMapTreeNode(mapTree); } -/** - * Construct tree content compatible with a field defined by the provided `schema`. - * @param schema - The schema for what to construct. As this is an {@link ImplicitFieldSchema}, a {@link FieldSchema}, {@link TreeNodeSchema} or {@link AllowedTypes} array can be provided. - * @param data - The data used to construct the field content. See `Tree.cloneToJSONVerbose`. - * @privateRemarks - * This could be exposed as a public `Tree.createFromVerbose` function. - */ -export function createFromVerbose( - schema: TSchema, - data: VerboseTreeNode | undefined, - options: ParseOptions, -): Unhydrated>; - -/** - * Construct tree content compatible with a field defined by the provided `schema`. - * @param schema - The schema for what to construct. As this is an {@link ImplicitFieldSchema}, a {@link FieldSchema}, {@link TreeNodeSchema} or {@link AllowedTypes} array can be provided. - * @param data - The data used to construct the field content. See `Tree.cloneToJSONVerbose`. - */ -export function createFromVerbose( - schema: TSchema, - data: VerboseTreeNode | undefined, - options?: Partial>, -): Unhydrated>; - -export function createFromVerbose( - schema: TSchema, - data: VerboseTreeNode | undefined, - options?: Partial>, -): Unhydrated> { - const config: ParseOptions = { - valueConverter: (input: VerboseTree) => { - return input as TreeLeafValue | VerboseTreeNode; - }, - ...options, - }; - const schemalessConfig = applySchemaToParserOptions(schema, config); - const cursor = cursorFromVerbose(data, schemalessConfig); - return createFromCursor(schema, cursor); -} - /** * Creates an unhydrated simple-tree field from a cursor in nodes mode. */ diff --git a/packages/dds/tree/src/simple-tree/api/customTree.ts b/packages/dds/tree/src/simple-tree/api/customTree.ts index a5a28026c90e..50c0c31f22f3 100644 --- a/packages/dds/tree/src/simple-tree/api/customTree.ts +++ b/packages/dds/tree/src/simple-tree/api/customTree.ts @@ -28,6 +28,7 @@ import { isObjectNodeSchema } from "../objectNodeTypes.js"; /** * Options for how to encode a tree. + * @alpha */ export interface EncodeOptions { /** diff --git a/packages/dds/tree/src/simple-tree/api/index.ts b/packages/dds/tree/src/simple-tree/api/index.ts index 8eb0ce304cad..dca91bfc0015 100644 --- a/packages/dds/tree/src/simple-tree/api/index.ts +++ b/packages/dds/tree/src/simple-tree/api/index.ts @@ -24,8 +24,8 @@ export { enumFromStrings, singletonSchema, } from "./schemaCreationUtilities.js"; -export { treeNodeApi, type TreeNodeApi } from "./treeNodeApi.js"; -export { createFromInsertable, cursorFromInsertable } from "./create.js"; +export { treeNodeApi, type TreeNodeApi, tryGetSchema } from "./treeNodeApi.js"; +export { createFromInsertable, cursorFromInsertable, createFromCursor } from "./create.js"; export type { SimpleTreeSchema } from "./simpleSchema.js"; export { type JsonSchemaId, @@ -69,15 +69,18 @@ export type { InsertableTreeNodeFromAllowedTypesUnsafe, } from "./typesUnsafe.js"; -export type { - VerboseTreeNode, - ParseOptions, - VerboseTree, +export { + type VerboseTreeNode, + type ParseOptions, + type VerboseTree, + applySchemaToParserOptions, + cursorFromVerbose, + verboseFromCursor, } from "./verboseTree.js"; export type { EncodeOptions } from "./customTree.js"; -export type { ConciseTree } from "./conciseTree.js"; +export { type ConciseTree, conciseFromCursor } from "./conciseTree.js"; export { TreeBeta, type NodeChangedData, type TreeChangeEventsBeta } from "./treeApiBeta.js"; diff --git a/packages/dds/tree/src/simple-tree/api/tree.ts b/packages/dds/tree/src/simple-tree/api/tree.ts index b03fa5ba14a0..ead54b721996 100644 --- a/packages/dds/tree/src/simple-tree/api/tree.ts +++ b/packages/dds/tree/src/simple-tree/api/tree.ts @@ -67,6 +67,13 @@ export interface ViewableTree { viewWith( config: TreeViewConfiguration, ): TreeView; + + // TODO: + // Add stored key versions of Tree.exportVerbose, Tree.exportConcise and Tree.exportCompressed here so tree content can be accessed without a view schema. + // Add exportSimpleSchema and exportJsonSchema methods (which should exactly match the concise format, and match the free functions for exporting view schema). + // Maybe rename "exportJsonSchema" to align on "concise" terminology. + // Ensure schema exporting APIs here aline and reference APIs for exporting view schema to the same formats (which should include stored vs property key choice). + // Make sure users of independentView can use these export APIs (maybe provide a reference back to the ViewableTree from the TreeView to accomplish that). } /** @@ -242,7 +249,7 @@ export class TreeViewConfiguration< if (ambiguityErrors.length !== 0) { // Duplicate errors are common since when two types conflict, both orders error: const deduplicated = new Set(ambiguityErrors); - throw new UsageError(`Ambigious schema found:\n${[...deduplicated].join("\n")}`); + throw new UsageError(`Ambiguous schema found:\n${[...deduplicated].join("\n")}`); } // Eagerly perform this conversion to surface errors sooner. diff --git a/packages/dds/tree/src/simple-tree/api/treeApiBeta.ts b/packages/dds/tree/src/simple-tree/api/treeApiBeta.ts index c4cec14f6f6c..34b2521b6eb2 100644 --- a/packages/dds/tree/src/simple-tree/api/treeApiBeta.ts +++ b/packages/dds/tree/src/simple-tree/api/treeApiBeta.ts @@ -12,14 +12,9 @@ import { type Unhydrated, type WithType, } from "../core/index.js"; -import type { - ImplicitFieldSchema, - TreeFieldFromImplicitField, - UnsafeUnknownSchema, -} from "../schemaTypes.js"; import { treeNodeApi } from "./treeNodeApi.js"; -import { createFromCursor, cursorFromInsertable } from "./create.js"; -import type { ITreeCursorSynchronous } from "../../core/index.js"; +import { createFromCursor } from "./create.js"; +import type { ImplicitFieldSchema, TreeFieldFromImplicitField } from "../schemaTypes.js"; /** * Data included for {@link TreeChangeEventsBeta.nodeChanged}. @@ -128,6 +123,25 @@ export const TreeBeta: { clone( node: TreeFieldFromImplicitField, ): TreeFieldFromImplicitField; + + // TODO: support more clone options + // /** + // * Like {@link TreeBeta.create}, except deeply clones existing nodes. + // * @remarks + // * This only clones the persisted data associated with a node. + // * Local state, such as properties added to customized schema classes, will not be cloned: + // * they will be initialized however they end up after running the constructor, just like if a remote client had inserted the same nodes. + // */ + // clone( + // original: TreeFieldFromImplicitField, + // options?: { + // /** + // * If set, all identifier's in the cloned tree (See {@link SchemaFactory.identifier}) will be replaced with new ones allocated using the default identifier allocation schema. + // * Otherwise any identifiers will be preserved as is. + // */ + // replaceIdentifiers?: true; + // }, + // ): TreeFieldFromImplicitField; } = { on, TNode extends TreeNode>( node: TNode, @@ -145,26 +159,9 @@ export const TreeBeta: { } const kernel = getKernel(node); - /* - * For unhydrated nodes, we can create a cursor by calling `cursorFromInsertable` because the node - * hasn't been inserted yet. We can then create a new node from the cursor. - */ - if (!kernel.isHydrated()) { - return createFromCursor( - kernel.schema, - cursorFromInsertable(kernel.schema, node), - ) as Unhydrated>; - } - - // For hydrated nodes, create a new cursor in the forest and then create a new node from the cursor. - const forest = kernel.context.flexContext.checkout.forest; - const cursor = forest.allocateCursor("tree.clone"); - forest.moveCursorToPath(kernel.anchorNode, cursor); - const clonedNode = createFromCursor( - kernel.schema, - cursor as ITreeCursorSynchronous, - ) as Unhydrated>; - cursor.free(); - return clonedNode; + const cursor = kernel.getOrCreateInnerNode().borrowCursor(); + return createFromCursor(kernel.schema, cursor) as Unhydrated< + TreeFieldFromImplicitField + >; }, }; diff --git a/packages/dds/tree/src/simple-tree/api/verboseTree.ts b/packages/dds/tree/src/simple-tree/api/verboseTree.ts index 39fb1be5a30c..e86cef6aa945 100644 --- a/packages/dds/tree/src/simple-tree/api/verboseTree.ts +++ b/packages/dds/tree/src/simple-tree/api/verboseTree.ts @@ -53,6 +53,7 @@ import { getUnhydratedContext } from "../createContext.js"; * @privateRemarks * This can store all possible simple trees, * but it can not store all possible trees representable by our internal representations like FlexTree and JsonableTree. + * @alpha */ export type VerboseTree = | VerboseTreeNode @@ -82,6 +83,7 @@ export type VerboseTree = * Unlike `JsonableTree`, leaf nodes are not boxed into node objects, and instead have their schema inferred from the value. * Additionally, sequence fields can only occur on a node that has a single sequence field (with the empty key) * replicating the behavior of simple-tree ArrayNodes. + * @alpha */ export interface VerboseTreeNode { /** @@ -109,6 +111,7 @@ export interface VerboseTreeNode { /** * Options for how to interpret a `VerboseTree` when schema information is available. + * @alpha */ export interface ParseOptions { /** diff --git a/packages/dds/tree/src/simple-tree/core/unhydratedFlexTree.ts b/packages/dds/tree/src/simple-tree/core/unhydratedFlexTree.ts index 8f2b4845a1ae..7938be81b9e6 100644 --- a/packages/dds/tree/src/simple-tree/core/unhydratedFlexTree.ts +++ b/packages/dds/tree/src/simple-tree/core/unhydratedFlexTree.ts @@ -15,6 +15,7 @@ import { type FieldKindIdentifier, type FieldUpPath, forbiddenFieldKindIdentifier, + type ITreeCursorSynchronous, type MapTree, type SchemaPolicy, type TreeNodeSchemaIdentifier, @@ -39,6 +40,7 @@ import { type FlexFieldKind, FieldKinds, type SequenceFieldEditBuilder, + cursorForMapTreeNode, } from "../../feature-libraries/index.js"; import type { Context } from "./context.js"; import { createEmitter, type Listenable } from "../../events/index.js"; @@ -168,6 +170,10 @@ export class UnhydratedFlexTreeNode implements UnhydratedFlexTreeNode { return this.location; } + public borrowCursor(): ITreeCursorSynchronous { + return cursorForMapTreeNode(this.mapTree); + } + public tryGetField(key: FieldKey): UnhydratedFlexTreeField | undefined { const field = this.mapTree.fields.get(key); // Only return the field if it is not empty, in order to fulfill the contract of `tryGetField`. diff --git a/packages/dds/tree/src/simple-tree/index.ts b/packages/dds/tree/src/simple-tree/index.ts index 864ebd4ac659..3fb617f6cce2 100644 --- a/packages/dds/tree/src/simple-tree/index.ts +++ b/packages/dds/tree/src/simple-tree/index.ts @@ -23,6 +23,7 @@ export { HydratedContext, SimpleContextSlot, getOrCreateInnerNode, + getKernel, } from "./core/index.js"; export { type ITree, @@ -97,6 +98,12 @@ export { type TreeNodeSchemaNonClassUnsafe, type InsertableTreeNodeFromAllowedTypesUnsafe, type TreeViewAlpha, + tryGetSchema, + applySchemaToParserOptions, + cursorFromVerbose, + verboseFromCursor, + conciseFromCursor, + createFromCursor, } from "./api/index.js"; export { type NodeFromSchema, diff --git a/packages/dds/tree/src/test/simple-tree/api/create.spec.ts b/packages/dds/tree/src/test/simple-tree/api/create.spec.ts index d18262c94418..4c74be3324cd 100644 --- a/packages/dds/tree/src/test/simple-tree/api/create.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/create.spec.ts @@ -6,8 +6,7 @@ import { strict as assert } from "node:assert"; import { createFromInsertable, SchemaFactory } from "../../../simple-tree/index.js"; -// eslint-disable-next-line import/no-internal-modules -import { createFromVerbose } from "../../../simple-tree/api/create.js"; +import { TreeAlpha } from "../../../shared-tree/index.js"; const schema = new SchemaFactory("com.example"); @@ -25,7 +24,7 @@ describe("simple-tree create", () => { }); it("createFromVerbose", () => { - const canvas1 = createFromVerbose(Canvas, { + const canvas1 = TreeAlpha.importVerbose(Canvas, { type: Canvas.identifier, fields: { stuff: { type: NodeList.identifier, fields: [] } }, }); diff --git a/packages/dds/tree/src/test/simple-tree/api/treeApi.spec.ts b/packages/dds/tree/src/test/simple-tree/api/treeApi.spec.ts index fac5e80a94e6..72de90a45980 100644 --- a/packages/dds/tree/src/test/simple-tree/api/treeApi.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/treeApi.spec.ts @@ -17,13 +17,16 @@ import { type StableNodeKey, } from "../../../feature-libraries/index.js"; import { + isTreeNode, type NodeFromSchema, SchemaFactory, treeNodeApi as Tree, TreeBeta, type TreeChangeEvents, + type TreeLeafValue, type TreeNode, TreeViewConfiguration, + type UnsafeUnknownSchema, } from "../../../simple-tree/index.js"; import { getView, validateUsageError } from "../../utils.js"; import { getViewForForkedBranch, hydrate } from "../utils.js"; @@ -39,6 +42,10 @@ import { } from "../../../simple-tree/leafNodeSchema.js"; // eslint-disable-next-line import/no-internal-modules import { tryGetSchema } from "../../../simple-tree/api/treeNodeApi.js"; +import { testSimpleTrees } from "../../testTrees.js"; +import { FluidClientVersion } from "../../../codec/index.js"; +import { ajvValidator } from "../../codec/index.js"; +import { TreeAlpha } from "../../../shared-tree/index.js"; const schema = new SchemaFactory("com.example"); @@ -1116,5 +1123,247 @@ describe("treeNodeApi", () => { const clonedMetadata = TreeBeta.clone(topLeftPoint.metadata); assert.equal(clonedMetadata, topLeftPoint.metadata, "String not cloned properly"); }); + + describe("test-trees", () => { + for (const testCase of testSimpleTrees) { + it(testCase.name, () => { + const tree = TreeAlpha.create(testCase.schema, testCase.root()); + const exported = TreeBeta.clone(tree); + if (isTreeNode(tree)) { + // New instance + assert.notEqual(tree, exported); + } + expectTreesEqual(tree, exported); + }); + } + }); + }); + + // create is mostly the same as node constructors which have their own tests, so just cover the new cases (optional and top level unions) here. + describe("create", () => { + it("undefined", () => { + // Valid + assert.equal(TreeAlpha.create(schema.optional([]), undefined), undefined); + // Undefined where not allowed + assert.throws( + () => TreeAlpha.create(schema.required([]), undefined as never), + validateUsageError(/undefined for non-optional field/), + ); + // Undefined required, not provided + assert.throws( + () => TreeAlpha.create(schema.optional([]), 1 as unknown as undefined), + validateUsageError(/incompatible/), + ); + }); + + it("union", () => { + // Valid + assert.equal(TreeAlpha.create([schema.null, schema.number], null), null); + // invalid + assert.throws( + () => TreeAlpha.create([schema.null, schema.number], "x" as unknown as number), + validateUsageError(/incompatible/), + ); + }); + + // Integration test object complex objects work (mainly covered by tests elsewhere) + it("object", () => { + const A = schema.object("A", { x: schema.number }); + const a = TreeAlpha.create(A, { x: 1 }); + assert.deepEqual(a, { x: 1 }); + }); + }); + + describe("concise", () => { + describe("importConcise", () => { + it("undefined", () => { + // Valid + assert.equal(TreeAlpha.importConcise(schema.optional([]), undefined), undefined); + // Undefined where not allowed + assert.throws( + () => TreeAlpha.importConcise(schema.required([]), undefined), + validateUsageError(/Got undefined for non-optional field/), + ); + // Undefined required, not provided + assert.throws( + () => TreeAlpha.importConcise(schema.optional([]), 1), + validateUsageError(/incompatible with all of the types allowed/), + ); + }); + + it("union", () => { + // Valid + assert.equal(TreeAlpha.importConcise([schema.null, schema.number], null), null); + // invalid + assert.throws( + () => TreeAlpha.importConcise([schema.null, schema.number], "x"), + validateUsageError(/The provided data is incompatible/), + ); + }); + + it("object", () => { + const A = schema.object("A", { x: schema.number }); + const a = TreeAlpha.importConcise(A, { x: 1 }); + assert.deepEqual(a, { x: 1 }); + }); + }); + + describe("roundtrip", () => { + for (const testCase of testSimpleTrees) { + if (testCase.root() !== undefined) { + it(testCase.name, () => { + const tree = TreeAlpha.create( + testCase.schema, + testCase.root(), + ); + assert(tree !== undefined); + const exported = TreeAlpha.exportConcise(tree); + if (testCase.ambiguous) { + assert.throws( + () => TreeAlpha.importConcise(testCase.schema, exported), + validateUsageError(/compatible with more than one type/), + ); + } else { + const imported = TreeAlpha.importConcise( + testCase.schema, + exported, + ); + expectTreesEqual(tree, imported); + } + }); + } + } + }); + + describe("export-stored", () => { + for (const testCase of testSimpleTrees) { + if (testCase.root() !== undefined) { + it(testCase.name, () => { + const tree = TreeAlpha.create( + testCase.schema, + testCase.root(), + ); + assert(tree !== undefined); + const _exported = TreeAlpha.exportConcise(tree, { useStoredKeys: true }); + // We have nothing that imports concise trees with stored keys, so no validation here. + }); + } + } + }); + }); + + describe("verbose", () => { + describe("importVerbose", () => { + it("undefined", () => { + // Valid + assert.equal(TreeAlpha.importVerbose(schema.optional([]), undefined), undefined); + // Undefined where not allowed + assert.throws( + () => TreeAlpha.importVerbose(schema.required([]), undefined), + validateUsageError(/non-optional/), + ); + // Undefined required, not provided + assert.throws( + () => TreeAlpha.importVerbose(schema.optional([]), 1), + validateUsageError(/does not conform to schema/), + ); + }); + + it("union", () => { + // Valid + assert.equal(TreeAlpha.importVerbose([schema.null, schema.number], null), null); + // invalid + assert.throws( + () => TreeAlpha.importVerbose([schema.null, schema.number], "x"), + validateUsageError(/does not conform to schema/), + ); + }); + + it("object", () => { + const A = schema.object("A", { x: schema.number }); + const a = TreeAlpha.importVerbose(A, { type: A.identifier, fields: { x: 1 } }); + assert.deepEqual(a, { x: 1 }); + }); + }); + + describe("roundtrip", () => { + for (const testCase of testSimpleTrees) { + if (testCase.root() !== undefined) { + it(testCase.name, () => { + const tree = TreeAlpha.create( + testCase.schema, + testCase.root(), + ); + assert(tree !== undefined); + const exported = TreeAlpha.exportVerbose(tree); + const imported = TreeAlpha.importVerbose(testCase.schema, exported); + expectTreesEqual(tree, imported); + }); + } + } + }); + + describe("roundtrip-stored", () => { + for (const testCase of testSimpleTrees) { + if (testCase.root() !== undefined) { + it(testCase.name, () => { + const tree = TreeAlpha.create( + testCase.schema, + testCase.root(), + ); + assert(tree !== undefined); + const exported = TreeAlpha.exportVerbose(tree, { useStoredKeys: true }); + const imported = TreeAlpha.importVerbose(testCase.schema, exported, { + useStoredKeys: true, + }); + expectTreesEqual(tree, imported); + }); + } + } + }); + }); + + describe("compressed", () => { + describe("roundtrip", () => { + for (const testCase of testSimpleTrees) { + if (testCase.root() !== undefined) { + it(testCase.name, () => { + const tree = TreeAlpha.create( + testCase.schema, + testCase.root(), + ); + assert(tree !== undefined); + const exported = TreeAlpha.exportCompressed(tree, { + oldestCompatibleClient: FluidClientVersion.v2_0, + }); + const imported = TreeAlpha.importCompressed(testCase.schema, exported, { + jsonValidator: ajvValidator, + }); + expectTreesEqual(tree, imported); + }); + } + } + }); }); }); + +function expectTreesEqual( + a: TreeNode | TreeLeafValue | undefined, + b: TreeNode | TreeLeafValue | undefined, +): void { + if (a === undefined || b === undefined) { + assert.equal(a === undefined, b === undefined); + return; + } + + // Validate the same schema objects are used. + assert.equal(Tree.schema(a), Tree.schema(b)); + + // This should catch all cases, assuming exportVerbose works correctly. + assert.deepEqual(TreeAlpha.exportVerbose(a), TreeAlpha.exportVerbose(b)); + + // Since this uses some of the tools to compare trees that this is testing for, perform the comparison in a few ways to reduce risk of a bug making this pass when it shouldn't: + // This case could have false negatives (two trees with ambiguous schema could export the same concise tree), + // but should have no false positives since equal trees always have the same concise tree. + assert.deepEqual(TreeAlpha.exportConcise(a), TreeAlpha.exportConcise(b)); +} diff --git a/packages/dds/tree/src/util/utils.ts b/packages/dds/tree/src/util/utils.ts index 7cb358c240a4..32d67ef33714 100644 --- a/packages/dds/tree/src/util/utils.ts +++ b/packages/dds/tree/src/util/utils.ts @@ -224,14 +224,15 @@ export function count(iterable: Iterable): number { * but instead mostly restricts access to it. * @alpha */ -export type JsonCompatible = +export type JsonCompatible = | string | number | boolean // eslint-disable-next-line @rushstack/no-new-null | null - | JsonCompatible[] - | JsonCompatibleObject; + | JsonCompatible[] + | JsonCompatibleObject + | TExtra; /** * Use for Json object compatible data. @@ -240,7 +241,7 @@ export type JsonCompatible = * but instead mostly restricts access to it. * @alpha */ -export type JsonCompatibleObject = { [P in string]?: JsonCompatible }; +export type JsonCompatibleObject = { [P in string]?: JsonCompatible }; /** * Use for readonly view of Json compatible data. diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md index 04c490b133b6..c38946175dc0 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md @@ -60,6 +60,11 @@ export interface CommitMetadata { // @alpha export function comparePersistedSchema(persisted: JsonCompatible, view: JsonCompatible, options: ICodecOptions, canInitialize: boolean): SchemaCompatibilityStatus; +// @alpha +export type ConciseTree = Exclude | THandle | ConciseTree[] | { + [key: string]: ConciseTree; +}; + // @alpha export function configuredSharedTree(options: SharedTreeOptions): SharedObjectKind; @@ -95,6 +100,12 @@ export interface ContainerSchema { interface DefaultProvider extends ErasedType<"@fluidframework/tree.FieldProvider"> { } +// @alpha +export interface EncodeOptions { + readonly useStoredKeys?: boolean; + valueConverter(data: IFluidHandle): TCustom; +} + // @alpha export function enumFromStrings(factory: SchemaFactory, members: Members): ((value: TValue) => TreeNode & { readonly value: TValue; @@ -188,6 +199,18 @@ type FlexList = readonly LazyItem[]; // @public type FlexListToUnion = ExtractItemType; +// @alpha +export enum FluidClientVersion { + // (undocumented) + v2_0 = "v2_0", + // (undocumented) + v2_1 = "v2_1", + // (undocumented) + v2_2 = "v2_2", + // (undocumented) + v2_3 = "v2_3" +} + // @public export type FluidObject = { [P in FluidObjectProviderKeys]?: T[P]; @@ -481,6 +504,14 @@ export type ImplicitAllowedTypes = AllowedTypes | TreeNodeSchema; // @public export type ImplicitFieldSchema = FieldSchema | ImplicitAllowedTypes; +// @alpha +export function independentInitializedView(config: TreeViewConfiguration, options: ForestOptions & ICodecOptions, content: ViewContent): TreeView; + +// @alpha +export function independentView(config: TreeViewConfiguration, options: ForestOptions & { + idCompressor?: IIdCompressor_2 | undefined; +}): TreeView; + // @public export type InitialObjects = { [K in keyof T["initialObjects"]]: T["initialObjects"][K] extends SharedObjectKind ? TChannel : never; @@ -654,11 +685,11 @@ export interface JsonArrayNodeSchema extends JsonNodeSchemaBase = string | number | boolean | null | JsonCompatible[] | JsonCompatibleObject | TExtra; // @alpha -export type JsonCompatibleObject = { - [P in string]?: JsonCompatible; +export type JsonCompatibleObject = { + [P in string]?: JsonCompatible; }; // @alpha @sealed @@ -800,6 +831,12 @@ type ObjectFromSchemaRecordUnsafe void; +// @alpha +export interface ParseOptions { + readonly useStoredKeys?: boolean; + valueConverter(data: VerboseTree): TreeLeafValue | VerboseTreeNode; +} + // @alpha export type PopUnion void : never>> = AsOverloadedFunction extends (a: infer First) => void ? First : never; @@ -979,6 +1016,25 @@ export type TransformedEvent = (event: E, listener: ( // @public export const Tree: TreeApi; +// @alpha @sealed +export const TreeAlpha: { + create(schema: UnsafeUnknownSchema extends TSchema ? ImplicitFieldSchema : TSchema & ImplicitFieldSchema, data: InsertableField): Unhydrated : TreeNode | TreeLeafValue | undefined>; + importConcise(schema: UnsafeUnknownSchema extends TSchema ? ImplicitFieldSchema : TSchema & ImplicitFieldSchema, data: ConciseTree | undefined): Unhydrated : TreeNode | TreeLeafValue | undefined>; + importVerbose(schema: TSchema, data: VerboseTree | undefined, options?: Partial>): Unhydrated>; + importVerbose(schema: TSchema, data: VerboseTree | undefined, options: ParseOptions): Unhydrated>; + exportConcise(node: TreeNode | TreeLeafValue, options?: Partial>): ConciseTree; + exportConcise(node: TreeNode | TreeLeafValue, options?: EncodeOptions): ConciseTree; + exportVerbose(node: TreeNode | TreeLeafValue, options?: Partial>): VerboseTree; + exportVerbose(node: TreeNode | TreeLeafValue, options: EncodeOptions): VerboseTree; + exportCompressed(tree: TreeNode | TreeLeafValue, options: { + oldestCompatibleClient: FluidClientVersion; + idCompressor?: IIdCompressor; + }): JsonCompatible; + importCompressed(schema: TSchema, compressedData: JsonCompatible, options: { + idCompressor?: IIdCompressor; + } & ICodecOptions): Unhydrated>; +}; + // @public @sealed interface TreeApi extends TreeNodeApi { contains(node: TreeNode, other: TreeNode): boolean; @@ -1237,11 +1293,29 @@ export type ValidateRecursiveSchema> = true; +// @alpha +export type VerboseTree = VerboseTreeNode | Exclude | THandle; + +// @alpha +export interface VerboseTreeNode { + fields: VerboseTree[] | { + [key: string]: VerboseTree; + }; + type: string; +} + // @public @sealed export interface ViewableTree { viewWith(config: TreeViewConfiguration): TreeView; } +// @alpha +export interface ViewContent { + readonly idCompressor: IIdCompressor_2; + readonly schema: JsonCompatible; + readonly tree: JsonCompatible; +} + // @public @sealed export interface WithType { // @deprecated diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3936bca87e84..09b70ddfdfb0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1243,6 +1243,64 @@ importers: specifier: ^6.0.1 version: 6.0.1 + examples/apps/tree-cli-app: + dependencies: + '@fluidframework/core-interfaces': + specifier: workspace:~ + version: link:../../../packages/common/core-interfaces + '@fluidframework/id-compressor': + specifier: workspace:~ + version: link:../../../packages/runtime/id-compressor + '@fluidframework/runtime-utils': + specifier: workspace:~ + version: link:../../../packages/runtime/runtime-utils + '@fluidframework/tree': + specifier: workspace:~ + version: link:../../../packages/dds/tree + '@sinclair/typebox': + specifier: ^0.32.29 + version: 0.32.35 + devDependencies: + '@biomejs/biome': + specifier: ~1.9.3 + version: 1.9.3 + '@fluid-internal/mocha-test-setup': + specifier: workspace:~ + version: link:../../../packages/test/mocha-test-setup + '@fluidframework/build-tools': + specifier: ^0.49.0 + version: 0.49.0 + '@fluidframework/eslint-config-fluid': + specifier: ^5.4.0 + version: 5.4.0(eslint@8.55.0)(typescript@5.4.5) + '@types/mocha': + specifier: ^9.1.1 + version: 9.1.1 + '@types/node': + specifier: ^18.19.0 + version: 18.19.54 + cross-env: + specifier: ^7.0.3 + version: 7.0.3 + eslint: + specifier: ~8.55.0 + version: 8.55.0 + mocha: + specifier: ^10.2.0 + version: 10.7.3 + mocha-json-output-reporter: + specifier: ^2.0.1 + version: 2.1.0(mocha@10.7.3)(moment@2.30.1) + mocha-multi-reporters: + specifier: ^1.5.1 + version: 1.5.1(mocha@10.7.3) + rimraf: + specifier: ^4.4.0 + version: 4.4.1 + typescript: + specifier: ~5.4.5 + version: 5.4.5 + examples/apps/tree-comparison: dependencies: '@fluid-example/example-utils': @@ -28383,6 +28441,7 @@ packages: /eslint@6.8.0: resolution: {integrity: sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==} engines: {node: ^8.10.0 || ^10.13.0 || >=11.10.1} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true dependencies: '@babel/code-frame': 7.25.7