tree: Import / Export APIs and demo

.changeset/chubby-olives-say.md

@@ -0,0 +1,25 @@
+---
+"fluid-framework": minor
+"@fluidframework/tree": minor
+---
+---
+"section": tree
+---
+
+New Alpha APIs for tree data import and export
+
+A collection of new `@alpha` APIs for important and exporting tree content and schema from SharedTrees has been added to `TreeAlpha`.
+These include import and export APIs for for `VerboseTree`, `ConciseTree` and compressed tree formats.
+
+`TreeAlpha.create` is also added to allow constructing trees with a more general API than having to use the schema constructor directly (since that doesn't handle polymorphic roots, or non-schema aware code).
+
+The function `independentInitializedView` has been added to provide a way to combine data from the existing `extractPersistedSchema` and new `TreeAlpha.exportCompressed` back into a `TreeView` in a way which can support safely importing data which could have been exported with a different schema.
+This allows replicating the schema evolution process for Fluid documents stored in a service, but entirely locally without involving any collaboration services.
+`independentView` has also been added, which is similar but handles the case of creating a new view without an existing schema or tree.
+
+Together these APIs address several use-cases:
+
+1. Using SharedTree as a in memory non-collaborative datastore.
+2. Importing and exporting data from a SharedTree to and from other services or storage locations (such as locally saved files).
+3. Testing various scenarios without relying on a service.
+4. Using SharedTree libraries for just the schema system and encode/decode support.

biome.jsonc

@@ -58,6 +58,8 @@
 			"packages/tools/fluid-runner/src/test/localOdspSnapshots/**",
 			"packages/tools/fluid-runner/src/test/telemetryExpectedOutputs/**",
 			"tools/api-markdown-documenter/src/test/snapshots/**",
+			// TODO: why does examples/apps/tree-cli-app/*.json not work?
+			"**/data/*.json",
 
 			// Generated type-tests
 			"**/*.generated.ts",

examples/apps/tree-cli-app/.eslintrc.cjs

@@ -0,0 +1,11 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+module.exports = {
+	extends: [require.resolve("@fluidframework/eslint-config-fluid/strict"), "prettier"],
+	parserOptions: {
+		project: ["./tsconfig.json", "./src/test/tsconfig.json"],
+	},
+};

examples/apps/tree-cli-app/.mocharc.cjs

@@ -0,0 +1,14 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+"use strict";
+
+const getFluidTestMochaConfig = require("@fluid-internal/mocha-test-setup/mocharc-common");
+
+const packageDir = __dirname;
+const config = getFluidTestMochaConfig(packageDir);
+config.spec = "lib/test";
+
+module.exports = config;

examples/apps/tree-cli-app/README.md

@@ -0,0 +1,7 @@
+# @fluid-example/tree-cli-app
+
+Example application using Shared-Tree to create a non-collaborative file editing CLI application.
+
+Note that its perfectly possible to write a collaborative online CLI app using tree as well: this simply is not an example of that.
+
+Run the app with `pnpm run app` after building.

examples/apps/tree-cli-app/biome.jsonc

@@ -0,0 +1,4 @@
+{
+	"$schema": "./node_modules/@biomejs/biome/configuration_schema.json",
+	"extends": ["../../../biome.jsonc"]
+}

examples/apps/tree-cli-app/data/concise.snapshot.json

@@ -0,0 +1 @@
+{"tree":{"version":1,"identifiers":[],"shapes":[{"c":{"type":"com.fluidframework.example.cli.List","value":false,"fields":[["",1]]}},{"a":2},{"d":0}],"data":[[0,0]]},"schema":{"version":1,"nodes":{"com.fluidframework.example.cli.Item":{"object":{"location":{"kind":"Value","types":["com.fluidframework.example.cli.Point"]},"name":{"kind":"Value","types":["com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.List":{"object":{"":{"kind":"Sequence","types":["com.fluidframework.example.cli.Item","com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.Point":{"object":{"x":{"kind":"Value","types":["com.fluidframework.leaf.number"]},"y":{"kind":"Value","types":["com.fluidframework.leaf.number"]}}},"com.fluidframework.leaf.number":{"leaf":0},"com.fluidframework.leaf.string":{"leaf":1}},"root":{"kind":"Value","types":["com.fluidframework.example.cli.List"]}},"idCompressor":"AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAAAUJLVv6TZpgqHGKmI53QQCAAAAAAAAAAAAAAAAAADwPwAAAAAAAAAA"}

examples/apps/tree-cli-app/data/default.compressed.json

@@ -0,0 +1 @@
+{"version":1,"identifiers":[],"shapes":[{"c":{"type":"com.fluidframework.example.cli.List","value":false,"fields":[["",1]]}},{"a":2},{"d":0}],"data":[[0,0]]}

examples/apps/tree-cli-app/data/default.concise.json

@@ -0,0 +1 @@
+[]

examples/apps/tree-cli-app/data/default.snapshot.json

@@ -0,0 +1 @@
+{"tree":{"version":1,"identifiers":[],"shapes":[{"c":{"type":"com.fluidframework.example.cli.List","value":false,"fields":[["",1]]}},{"a":2},{"d":0}],"data":[[0,0]]},"schema":{"version":1,"nodes":{"com.fluidframework.example.cli.Item":{"object":{"location":{"kind":"Value","types":["com.fluidframework.example.cli.Point"]},"name":{"kind":"Value","types":["com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.List":{"object":{"":{"kind":"Sequence","types":["com.fluidframework.example.cli.Item","com.fluidframework.leaf.string"]}}},"com.fluidframework.example.cli.Point":{"object":{"x":{"kind":"Value","types":["com.fluidframework.leaf.number"]},"y":{"kind":"Value","types":["com.fluidframework.leaf.number"]}}},"com.fluidframework.leaf.number":{"leaf":0},"com.fluidframework.leaf.string":{"leaf":1}},"root":{"kind":"Value","types":["com.fluidframework.example.cli.List"]}},"idCompressor":"AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAABhL9IeaYATlvd5A8Fp6aoDAAAAAAAAAAAAAAAAAADwPwAAAAAAAAAA"}

examples/apps/tree-cli-app/data/default.verbose-stored.json

@@ -0,0 +1 @@
+{"type":"com.fluidframework.example.cli.List","fields":[]}

examples/apps/tree-cli-app/data/default.verbose.json

@@ -0,0 +1 @@
+{"type":"com.fluidframework.example.cli.List","fields":[]}

examples/apps/tree-cli-app/package.json

@@ -0,0 +1,58 @@
+{
+	"name": "@fluid-example/tree-cli-app",
+	"version": "2.5.0",
+	"private": true,
+	"description": "SharedTree CLI app demo",
+	"homepage": "https://fluidframework.com",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/microsoft/FluidFramework.git",
+		"directory": "examples/apps/tree-cli-app"
+	},
+	"license": "MIT",
+	"author": "Microsoft and contributors",
+	"type": "module",
+	"scripts": {
+		"app": "node ./lib/index.js",
+		"build": "fluid-build . --task build",
+		"build:compile": "fluid-build . --task compile",
+		"build:esnext": "tsc --project ./tsconfig.json",
+		"build:test": "npm run build:test:esm",
+		"build:test:esm": "tsc --project ./src/test/tsconfig.json",
+		"check:biome": "biome check .",
+		"check:format": "npm run check:biome",
+		"clean": "rimraf --glob dist lib \"**/*.tsbuildinfo\" \"**/*.build.log\" nyc",
+		"eslint": "eslint --format stylish src",
+		"eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
+		"format": "npm run format:biome",
+		"format:biome": "biome check . --write",
+		"lint": "fluid-build . --task lint",
+		"lint:fix": "fluid-build . --task eslint:fix --task format",
+		"test": "npm run test:mocha",
+		"test:mocha": "npm run test:mocha:esm",
+		"test:mocha:esm": "mocha",
+		"test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha"
+	},
+	"dependencies": {
+		"@fluidframework/core-interfaces": "workspace:~",
+		"@fluidframework/id-compressor": "workspace:~",
+		"@fluidframework/runtime-utils": "workspace:~",
+		"@fluidframework/tree": "workspace:~",
+		"@sinclair/typebox": "^0.32.29"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "~1.9.3",
+		"@fluid-internal/mocha-test-setup": "workspace:~",
+		"@fluidframework/build-tools": "^0.49.0",
+		"@fluidframework/eslint-config-fluid": "^5.4.0",
+		"@types/mocha": "^9.1.1",
+		"@types/node": "^18.19.0",
+		"cross-env": "^7.0.3",
+		"eslint": "~8.55.0",
+		"mocha": "^10.2.0",
+		"mocha-json-output-reporter": "^2.0.1",
+		"mocha-multi-reporters": "^1.5.1",
+		"rimraf": "^4.4.0",
+		"typescript": "~5.4.5"
+	}
+}

examples/apps/tree-cli-app/src/index.ts

@@ -0,0 +1,35 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+// This is a node powered CLI application, so using node makes sense:
+/* eslint-disable unicorn/no-process-exit */
+
+import { applyEdit, loadDocument, saveDocument } from "./utils.js";
+
+const args = process.argv.slice(2);
+
+console.log(`Requires arguments: [<source>] [<destination>] [<edit>]`);
+console.log(`Example arguments: default data/large.concise.json string:10,item:100`);
+console.log(
+	`File formats are specified by extension, for example ".verbose.json" uses the "verbose" format.`,
+);
+console.log(
+	`See implementation for supported formats and edit syntax: this is just a demon, not a nice app!`,
+);
+console.log(`Running with augments: ${args}`);
+
+if (args.length > 3) {
+	process.exit(1);
+}
+
+const [sourceArg, destinationArg, editArg] = args;
+
+const node = loadDocument(sourceArg);
+
+if (editArg !== undefined) {
+	applyEdit(editArg, node);
+}
+
+saveDocument(destinationArg, node);

examples/apps/tree-cli-app/src/schema.ts

@@ -0,0 +1,38 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { SchemaFactory, TreeViewConfiguration } from "@fluidframework/tree";
+
+/**
+ *
+ */
+export const schemaBuilder = new SchemaFactory("com.fluidframework.example.cli");
+
+class Point extends schemaBuilder.object("Point", {
+	x: schemaBuilder.number,
+	y: schemaBuilder.number,
+}) {}
+
+/**
+ * Complex list item.
+ */
+export class Item extends schemaBuilder.object("Item", {
+	position: schemaBuilder.required(Point, { key: "location" }),
+	name: schemaBuilder.string,
+}) {}
+
+/**
+ * List node.
+ */
+export class List extends schemaBuilder.array("List", [schemaBuilder.string, Item]) {}
+
+/**
+ * Tree configuration.
+ */
+export const config = new TreeViewConfiguration({
+	schema: List,
+	enableSchemaValidation: true,
+	preventAmbiguity: true,
+});

examples/apps/tree-cli-app/src/test/schema.spec.ts

@@ -0,0 +1,144 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { strict as assert } from "node:assert";
+
+import {
+	comparePersistedSchema,
+	extractPersistedSchema,
+	typeboxValidator,
+	type ForestOptions,
+	type ICodecOptions,
+	type JsonCompatible,
+	// eslint-disable-next-line import/no-internal-modules
+} from "@fluidframework/tree/alpha";
+
+import { List } from "../schema.js";
+
+// This file demonstrates how applications can write tests which ensure they maintain compatibility the schema from with previously released versions.
+
+describe("schema", () => {
+	it("current schema matches latest historical schema", () => {
+		const current = extractPersistedSchema(List);
+
+		// For compatibility with deep equality and simply objects, round trip via JSON to erase prototypes.
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+		const currentRoundTripped: JsonCompatible = JSON.parse(JSON.stringify(current));
+
+		const previous = historicalSchema.at(-1);
+		assert(previous !== undefined);
+		// This ensures that historicalSchema's last entry is up to date with the current application code.
+		// This can catch:
+		// 1. Forgetting to update historicalSchema when intentionally making schema changes.
+		// 2. Accidentally changing schema in a way that impacts document compatibility.
+		assert.deepEqual(currentRoundTripped, previous.schema);
+	});
+
+	it("historical schema can be upgraded to current schema", () => {
+		const options: ForestOptions & ICodecOptions = { jsonValidator: typeboxValidator };
+
+		for (let documentIndex = 0; documentIndex < historicalSchema.length; documentIndex++) {
+			for (let viewIndex = 0; viewIndex < historicalSchema.length; viewIndex++) {
+				const compat = comparePersistedSchema(
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					historicalSchema[documentIndex]!.schema,
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					historicalSchema[viewIndex]!.schema,
+					options,
+					false,
+				);
+
+				// We do not expect duplicates in historicalSchema.
+				assert.equal(compat.isEquivalent, documentIndex === viewIndex);
+				// Currently collaboration is only allowed between identical versions
+				assert.equal(compat.canView, documentIndex === viewIndex);
+				// Older versions should be upgradable to newer versions, but not the reverse.
+				assert.equal(compat.canUpgrade, documentIndex <= viewIndex);
+			}
+		}
+	});
+});
+
+/**
+ * List of schema from previous versions of this application.
+ * Storing these as .json files in a folder may make more sense for more complex applications.
+ *
+ * The `schema` field is generated by passing the schema to `extractPersistedSchema`.
+ */
+const historicalSchema: { version: string; schema: JsonCompatible }[] = [
+	{
+		version: "1.0",
+		schema: {
+			version: 1,
+			nodes: {
+				"com.fluidframework.example.cli.List": {
+					object: {
+						"": {
+							kind: "Sequence",
+							types: ["com.fluidframework.leaf.string"],
+						},
+					},
+				},
+				"com.fluidframework.leaf.string": {
+					leaf: 1,
+				},
+			},
+			root: {
+				kind: "Value",
+				types: ["com.fluidframework.example.cli.List"],
+			},
+		},
+	},
+	{
+		version: "2.0",
+		schema: {
+			version: 1,
+			nodes: {
+				"com.fluidframework.example.cli.Item": {
+					object: {
+						location: {
+							kind: "Value",
+							types: ["com.fluidframework.example.cli.Point"],
+						},
+						name: {
+							kind: "Value",
+							types: ["com.fluidframework.leaf.string"],
+						},
+					},
+				},
+				"com.fluidframework.example.cli.List": {
+					object: {
+						"": {
+							kind: "Sequence",
+							types: ["com.fluidframework.example.cli.Item", "com.fluidframework.leaf.string"],
+						},
+					},
+				},
+				"com.fluidframework.example.cli.Point": {
+					object: {
+						x: {
+							kind: "Value",
+							types: ["com.fluidframework.leaf.number"],
+						},
+						y: {
+							kind: "Value",
+							types: ["com.fluidframework.leaf.number"],
+						},
+					},
+				},
+				"com.fluidframework.leaf.number": {
+					leaf: 0,
+				},
+				"com.fluidframework.leaf.string": {
+					leaf: 1,
+				},
+			},
+			root: {
+				kind: "Value",
+				types: ["com.fluidframework.example.cli.List"],
+			},
+		},
+	},
+];

examples/apps/tree-cli-app/src/test/tsconfig.json

@@ -0,0 +1,23 @@
+{
+	"extends": "../../../../../common/build/build-common/tsconfig.test.node16.json",
+	"compilerOptions": {
+		"rootDir": "./",
+		"outDir": "../../lib/test",
+		"types": ["mocha", "node"],
+		// Allows writing type checking expression without having to use the results.
+		"noUnusedLocals": false,
+		// Allow testing that declarations work properly
+		"declaration": true,
+		// Needed to ensure testExport's produce a valid d.ts
+		"skipLibCheck": false,
+		// Due to several of our own packages' exports failing to build with "exactOptionalPropertyTypes",
+		// disable it to prevent that from erroring when combined with "skipLibCheck".
+		"exactOptionalPropertyTypes": false,
+	},
+	"include": ["./**/*"],
+	"references": [
+		{
+			"path": "../..",
+		},
+	],
+}