From 5f912d813f311967592d35c02af7ca4062cb9d03 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 15 Oct 2024 22:02:18 +0000 Subject: [PATCH 01/28] Adds explicit strategy with a few unresovled type errors --- packages/dds/tree/src/index.ts | 11 + .../dds/tree/src/simple-tree/api/index.ts | 11 +- .../tree/src/simple-tree/api/simpleSchema.ts | 10 +- packages/dds/tree/src/simple-tree/index.ts | 8 + .../dds/tree/src/simple-tree/schemaTypes.ts | 5 + packages/framework/ai-collab/.eslintrc.cjs | 5 +- packages/framework/ai-collab/package.json | 2 + .../src/explicit-strategy/agentEditReducer.ts | 544 +++++++ .../src/explicit-strategy/agentEditTypes.ts | 164 ++ .../src/explicit-strategy/handlers.ts | 355 +++++ .../src/explicit-strategy/idGenerator.ts | 89 ++ .../explicit-strategy/json-handler/debug.ts | 13 + .../explicit-strategy/json-handler/index.ts | 11 + .../json-handler/jsonHandler.ts | 95 ++ .../json-handler/jsonHandlerImp.ts | 1418 +++++++++++++++++ .../json-handler/jsonParser.ts | 565 +++++++ .../src/explicit-strategy/promptGeneration.ts | 273 ++++ .../ai-collab/src/explicit-strategy/utils.ts | 65 + pnpm-lock.yaml | 6 + 19 files changed, 3647 insertions(+), 3 deletions(-) create mode 100644 packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/handlers.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/utils.ts diff --git a/packages/dds/tree/src/index.ts b/packages/dds/tree/src/index.ts index 26fa55fd8fd2..e875da97e530 100644 --- a/packages/dds/tree/src/index.ts +++ b/packages/dds/tree/src/index.ts @@ -151,6 +151,17 @@ export { getJsonSchema, type LazyItem, type Unenforced, + type SimpleNodeSchemaBase, + type SimpleTreeSchema, + type SimpleNodeSchema, + type SimpleFieldSchema, + type SimpleLeafNodeSchema, + type SimpleMapNodeSchema, + type SimpleArrayNodeSchema, + type SimpleObjectNodeSchema, + normalizeFieldSchema, + isTreeNodeSchemaClass, + normalizeAllowedTypes, } from "./simple-tree/index.js"; export { SharedTree, diff --git a/packages/dds/tree/src/simple-tree/api/index.ts b/packages/dds/tree/src/simple-tree/api/index.ts index 0cbde2d55957..79c78fc7d3cb 100644 --- a/packages/dds/tree/src/simple-tree/api/index.ts +++ b/packages/dds/tree/src/simple-tree/api/index.ts @@ -26,7 +26,16 @@ export { } from "./schemaCreationUtilities.js"; export { treeNodeApi, type TreeNodeApi } from "./treeNodeApi.js"; export { createFromInsertable, cursorFromInsertable } from "./create.js"; -export type { SimpleTreeSchema } from "./simpleSchema.js"; +export type { + SimpleTreeSchema, + SimpleNodeSchema, + SimpleFieldSchema, + SimpleLeafNodeSchema, + SimpleMapNodeSchema, + SimpleArrayNodeSchema, + SimpleObjectNodeSchema, + SimpleNodeSchemaBase, +} from "./simpleSchema.js"; export { type JsonSchemaId, type JsonSchemaType, diff --git a/packages/dds/tree/src/simple-tree/api/simpleSchema.ts b/packages/dds/tree/src/simple-tree/api/simpleSchema.ts index 19fe582b6742..e7c820dc8a5d 100644 --- a/packages/dds/tree/src/simple-tree/api/simpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/simpleSchema.ts @@ -10,6 +10,7 @@ import type { FieldKind } from "../schemaTypes.js"; /** * Base interface for all {@link SimpleNodeSchema} implementations. * + * @internal * @sealed */ export interface SimpleNodeSchemaBase { @@ -24,6 +25,7 @@ export interface SimpleNodeSchemaBase { /** * A {@link SimpleNodeSchema} for an object node. * + * @internal * @sealed */ export interface SimpleObjectNodeSchema extends SimpleNodeSchemaBase { @@ -36,6 +38,7 @@ export interface SimpleObjectNodeSchema extends SimpleNodeSchemaBase { @@ -51,6 +54,7 @@ export interface SimpleArrayNodeSchema extends SimpleNodeSchemaBase { @@ -66,6 +70,7 @@ export interface SimpleMapNodeSchema extends SimpleNodeSchemaBase /** * A {@link SimpleNodeSchema} for a leaf node. * + * @internal * @sealed */ export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBase { @@ -81,6 +86,8 @@ export interface SimpleLeafNodeSchema extends SimpleNodeSchemaBase, +): void { + if (typeof json === "object") { + if (json === null) { + return; + } + if (Array.isArray(json)) { + for (const element of json) { + populateDefaults(element, definitionMap); + } + } else { + assert(typeof json[typeField] === "string", "missing or invalid type field"); + const nodeSchema = definitionMap.get(json[typeField]); + assert(nodeSchema?.kind === NodeKind.Object, "Expected object schema"); + + for (const [key, fieldSchema] of Object.entries(nodeSchema.fields)) { + const defaulter = fieldSchema?.metadata?.llmDefault; + if (defaulter !== undefined) { + // TODO: Properly type. The input `json` is a JsonValue, but the output can contain nodes (from the defaulters) amidst the json. + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unnecessary-type-assertion + json[key] = defaulter() as any; + } + } + + for (const value of Object.values(json)) { + populateDefaults(value, definitionMap); + } + } + } +} + +function contentWithIds(content: TreeNode, idGenerator: IdGenerator): TreeEditObject { + return JSON.parse(toDecoratedJson(idGenerator, content)) as TreeEditObject; +} + +/** + * TBD + */ +export function applyAgentEdit( + tree: TreeView, + treeEdit: TreeEdit, + idGenerator: IdGenerator, + definitionMap: ReadonlyMap, + validator?: (edit: TreeNode) => void, +): TreeEdit { + objectIdsExist(treeEdit, idGenerator); + switch (treeEdit.type) { + case "setRoot": { + populateDefaults(treeEdit.content, definitionMap); + + const treeSchema = normalizeFieldSchema(tree.schema); + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + const schemaIdentifier = (treeEdit.content as any)[typeField]; + + let insertedObject: TreeNode | undefined; + if (treeSchema.kind === FieldKind.Optional && treeEdit.content === undefined) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (tree as any).root = treeEdit.content; + } else { + for (const allowedType of treeSchema.allowedTypeSet.values()) { + if (schemaIdentifier === allowedType.identifier) { + if (isTreeNodeSchemaClass(allowedType)) { + const simpleNodeSchema = allowedType as unknown as new ( + dummy: unknown, + ) => TreeNode; + const rootNode = new simpleNodeSchema(treeEdit.content); + validator?.(rootNode); + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (tree as any).root = rootNode; + insertedObject = rootNode; + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (tree as any).root = treeEdit.content; + } + } + } + } + + return insertedObject === undefined + ? treeEdit + : { + ...treeEdit, + content: contentWithIds(insertedObject, idGenerator), + }; + } + case "insert": { + const { array, index } = getPlaceInfo(treeEdit.destination, idGenerator); + + const parentNodeSchema = Tree.schema(array); + populateDefaults(treeEdit.content, definitionMap); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + const schemaIdentifier = (treeEdit.content as any)[typeField]; + + // We assume that the parentNode for inserts edits are guaranteed to be an arrayNode. + const allowedTypes = [ + ...normalizeAllowedTypes(parentNodeSchema.info as ImplicitAllowedTypes), + ]; + + for (const allowedType of allowedTypes.values()) { + if (allowedType.identifier === schemaIdentifier && typeof allowedType === "function") { + const simpleNodeSchema = allowedType as unknown as new (dummy: unknown) => TreeNode; + const insertNode = new simpleNodeSchema(treeEdit.content); + validator?.(insertNode); + array.insertAt(index, insertNode); + return { + ...treeEdit, + content: contentWithIds(insertNode, idGenerator), + }; + } + } + fail("inserted node must be of an allowed type"); + } + case "remove": { + const source = treeEdit.source; + if (isObjectTarget(source)) { + const node = getNodeFromTarget(source, idGenerator); + const parentNode = Tree.parent(node); + // Case for deleting rootNode + if (parentNode === undefined) { + const treeSchema = tree.schema; + if (treeSchema instanceof FieldSchema && treeSchema.kind === FieldKind.Optional) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (tree as any).root = undefined; + } else { + throw new UsageError( + "The root is required, and cannot be removed. Please use modify edit instead.", + ); + } + } else if (Tree.schema(parentNode).kind === NodeKind.Array) { + const nodeIndex = Tree.key(node) as number; + (parentNode as TreeArrayNode).removeAt(nodeIndex); + } else { + const fieldKey = Tree.key(node); + const parentSchema = Tree.schema(parentNode); + const fieldSchema = + (parentSchema.info as Record)[fieldKey] ?? + fail("Expected field schema"); + if (fieldSchema instanceof FieldSchema && fieldSchema.kind === FieldKind.Optional) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (parentNode as any)[fieldKey] = undefined; + } else { + throw new UsageError( + `${fieldKey} is required, and cannot be removed. Please use modify edit instead.`, + ); + } + } + } else if (isRange(source)) { + const { array, startIndex, endIndex } = getRangeInfo(source, idGenerator); + array.removeRange(startIndex, endIndex); + } + return treeEdit; + } + case "modify": { + const node = getNodeFromTarget(treeEdit.target, idGenerator); + const { treeNodeSchema } = getSimpleNodeSchema(node); + + const fieldSchema = + (treeNodeSchema.info as Record)[treeEdit.field] ?? + fail("Expected field schema"); + + const modification = treeEdit.modification; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + const schemaIdentifier = (modification as any)[typeField]; + + let insertedObject: TreeNode | undefined; + // if fieldSchema is a LeafnodeSchema, we can check that it's a valid type and set the field. + if (isPrimitive(modification)) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (node as any)[treeEdit.field] = modification; + } + // If the fieldSchema is a function we can grab the constructor and make an instance of that node. + else if (typeof fieldSchema === "function") { + const simpleSchema = fieldSchema as unknown as new (dummy: unknown) => TreeNode; + populateDefaults(modification, definitionMap); + const constructedModification = new simpleSchema(modification); + validator?.(constructedModification); + insertedObject = constructedModification; + + if (Array.isArray(modification)) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + const field = (node as any)[treeEdit.field] as TreeArrayNode; + assert(Array.isArray(field), "the field must be an array node"); + assert( + Array.isArray(constructedModification), + "the modification must be an array node", + ); + field.removeRange(0); + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (node as any)[treeEdit.field] = constructedModification; + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (node as any)[treeEdit.field] = constructedModification; + } + } + // If the fieldSchema is of type FieldSchema, we can check its allowed types and set the field. + else if (fieldSchema instanceof FieldSchema) { + if (fieldSchema.kind === FieldKind.Optional && modification === undefined) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (node as any)[treeEdit.field] = undefined; + } else { + for (const allowedType of fieldSchema.allowedTypeSet.values()) { + if (allowedType.identifier === schemaIdentifier) { + if (typeof allowedType === "function") { + const simpleSchema = allowedType as unknown as new ( + dummy: unknown, + ) => TreeNode; + const constructedObject = new simpleSchema(modification); + insertedObject = constructedObject; + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (node as any)[treeEdit.field] = constructedObject; + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (node as any)[treeEdit.field] = modification; + } + } + } + } + } + return insertedObject === undefined + ? treeEdit + : { + ...treeEdit, + modification: contentWithIds(insertedObject, idGenerator), + }; + } + case "move": { + // TODO: need to add schema check for valid moves + const source = treeEdit.source; + const destination = treeEdit.destination; + const { array: destinationArrayNode, index: destinationIndex } = getPlaceInfo( + destination, + idGenerator, + ); + + if (isObjectTarget(source)) { + const sourceNode = getNodeFromTarget(source, idGenerator); + const sourceIndex = Tree.key(sourceNode) as number; + + const sourceArrayNode = Tree.parent(sourceNode) as TreeArrayNode; + const sourceArraySchema = Tree.schema(sourceArrayNode); + if (sourceArraySchema.kind !== NodeKind.Array) { + throw new UsageError("the source node must be within an arrayNode"); + } + const destinationArraySchema = Tree.schema(destinationArrayNode); + const allowedTypes = [ + ...normalizeAllowedTypes(destinationArraySchema.info as ImplicitAllowedTypes), + ]; + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + const nodeToMove = sourceArrayNode[sourceIndex]; + assert(nodeToMove !== undefined, "node to move must exist"); + if (isNodeAllowedType(nodeToMove as TreeNode, allowedTypes)) { + destinationArrayNode.moveRangeToIndex( + destinationIndex, + sourceIndex, + sourceIndex + 1, + sourceArrayNode, + ); + } else { + throw new UsageError("Illegal node type in destination array"); + } + } else if (isRange(source)) { + const { + array, + startIndex: sourceStartIndex, + endIndex: sourceEndIndex, + } = getRangeInfo(source, idGenerator); + const destinationArraySchema = Tree.schema(destinationArrayNode); + const allowedTypes = [ + ...normalizeAllowedTypes(destinationArraySchema.info as ImplicitAllowedTypes), + ]; + for (let i = sourceStartIndex; i < sourceEndIndex; i++) { + const nodeToMove = array[i]; + assert(nodeToMove !== undefined, "node to move must exist"); + if (!isNodeAllowedType(nodeToMove as TreeNode, allowedTypes)) { + throw new UsageError("Illegal node type in destination array"); + } + } + destinationArrayNode.moveRangeToIndex( + destinationIndex, + sourceStartIndex, + sourceEndIndex, + array, + ); + } + return treeEdit; + } + default: + fail("invalid tree edit"); + } +} + +function isNodeAllowedType(node: TreeNode, allowedTypes: TreeNodeSchema[]): boolean { + for (const allowedType of allowedTypes) { + if (Tree.is(node, allowedType)) { + return true; + } + } + return false; +} + +function isPrimitive(content: unknown): boolean { + return ( + typeof content === "number" || + typeof content === "string" || + typeof content === "boolean" || + content === undefined || + content === null + ); +} + +function isObjectTarget(selection: Selection): selection is ObjectTarget { + return Object.keys(selection).length === 1 && "__fluid_objectId" in selection; +} + +function isRange(selection: Selection): selection is Range { + return "from" in selection && "to" in selection; +} + +interface RangeInfo { + array: TreeArrayNode; + startIndex: number; + endIndex: number; +} + +function getRangeInfo(range: Range, idGenerator: IdGenerator): RangeInfo { + const { array: arrayFrom, index: startIndex } = getPlaceInfo(range.from, idGenerator); + const { array: arrayTo, index: endIndex } = getPlaceInfo(range.to, idGenerator); + + if (arrayFrom !== arrayTo) { + throw new UsageError( + 'The "from" node and "to" nodes of the range must be in the same parent array.', + ); + } + + return { array: arrayFrom, startIndex, endIndex }; +} + +function getPlaceInfo( + place: ObjectPlace | ArrayPlace, + idGenerator: IdGenerator, +): { + array: TreeArrayNode; + index: number; +} { + if (place.type === "arrayPlace") { + const parent = idGenerator.getNode(place.parentId) ?? fail("Expected parent node"); + const child = (parent as unknown as Record)[place.field]; + if (child === undefined) { + throw new UsageError(`No child under field field`); + } + const schema = Tree.schema(child as TreeNode); + if (schema.kind !== NodeKind.Array) { + throw new UsageError("Expected child to be in an array node"); + } + return { + array: child as TreeArrayNode, + index: place.location === "start" ? 0 : (child as TreeArrayNode).length, + }; + } else { + const node = getNodeFromTarget(place, idGenerator); + const nodeIndex = Tree.key(node); + const parent = Tree.parent(node); + if (parent === undefined) { + throw new UsageError("TODO: root node target not supported"); + } + const schema = Tree.schema(parent); + if (schema.kind !== NodeKind.Array) { + throw new UsageError("Expected child to be in an array node"); + } + return { + array: parent as unknown as TreeArrayNode, + index: place.place === "before" ? (nodeIndex as number) : (nodeIndex as number) + 1, + }; + } +} + +/** + * Returns the target node with the matching internal objectId from the {@link ObjectTarget} + */ +function getNodeFromTarget(target: ObjectTarget, idGenerator: IdGenerator): TreeNode { + const node = idGenerator.getNode(target[objectIdKey]); + assert(node !== undefined, "objectId does not exist in nodeMap"); + return node; +} + +// /** +// * Returns the target node with the matching internal objectId from the {@link ObjectTarget} and the index of the nod, if the +// * parent node is an array node. +// */ +// function getTargetInfo( +// target: ObjectTarget, +// idGenerator: IdGenerator, +// ): { +// node: TreeNode; +// nodeIndex: number | undefined; +// } { +// const node = idGenerator.getNode(target[objectIdKey]); +// assert(node !== undefined, "objectId does not exist in nodeMap"); + +// Tree.key(node); + +// const nodeIndex = Tree.key(node); +// return { node, nodeIndex: typeof nodeIndex === "number" ? nodeIndex : undefined }; +// } + +function objectIdsExist(treeEdit: TreeEdit, idGenerator: IdGenerator): void { + switch (treeEdit.type) { + case "setRoot": + break; + case "insert": + if (treeEdit.destination.type === "objectPlace") { + if (idGenerator.getNode(treeEdit.destination[objectIdKey]) === undefined) { + throw new UsageError( + `objectIdKey ${treeEdit.destination[objectIdKey]} does not exist`, + ); + } + } else { + if (idGenerator.getNode(treeEdit.destination.parentId) === undefined) { + throw new UsageError(`objectIdKey ${treeEdit.destination.parentId} does not exist`); + } + } + break; + case "remove": + if (isRange(treeEdit.source)) { + const missingObjectIds = [ + treeEdit.source.from[objectIdKey], + treeEdit.source.to[objectIdKey], + ].filter((id) => !idGenerator.getNode(id)); + + if (missingObjectIds.length > 0) { + throw new UsageError(`objectIdKeys [${missingObjectIds}] does not exist`); + } + } else if ( + isObjectTarget(treeEdit.source) && + idGenerator.getNode(treeEdit.source[objectIdKey]) === undefined + ) { + throw new UsageError(`objectIdKey ${treeEdit.source[objectIdKey]} does not exist`); + } + break; + case "modify": + if (idGenerator.getNode(treeEdit.target[objectIdKey]) === undefined) { + throw new UsageError(`objectIdKey ${treeEdit.target[objectIdKey]} does not exist`); + } + break; + case "move": { + const invalidObjectIds: string[] = []; + // check the source + if (isRange(treeEdit.source)) { + const missingObjectIds = [ + treeEdit.source.from[objectIdKey], + treeEdit.source.to[objectIdKey], + ].filter((id) => !idGenerator.getNode(id)); + + if (missingObjectIds.length > 0) { + invalidObjectIds.push(...missingObjectIds); + } + } else if ( + isObjectTarget(treeEdit.source) && + idGenerator.getNode(treeEdit.source[objectIdKey]) === undefined + ) { + invalidObjectIds.push(treeEdit.source[objectIdKey]); + } + + // check the destination + if (treeEdit.destination.type === "objectPlace") { + if (idGenerator.getNode(treeEdit.destination[objectIdKey]) === undefined) { + invalidObjectIds.push(treeEdit.destination[objectIdKey]); + } + } else { + if (idGenerator.getNode(treeEdit.destination.parentId) === undefined) { + invalidObjectIds.push(treeEdit.destination.parentId); + } + } + if (invalidObjectIds.length > 0) { + throw new UsageError(`objectIdKeys [${invalidObjectIds}] does not exist`); + } + break; + } + default: + break; + } +} + +interface SchemaInfo { + treeNodeSchema: TreeNodeSchema; + simpleNodeSchema: new (dummy: unknown) => TreeNode; +} + +function getSimpleNodeSchema(node: TreeNode): SchemaInfo { + const treeNodeSchema = Tree.schema(node); + const simpleNodeSchema = treeNodeSchema as unknown as new (dummy: unknown) => TreeNode; + return { treeNodeSchema, simpleNodeSchema }; +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts new file mode 100644 index 000000000000..e86f9bd2ac8d --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts @@ -0,0 +1,164 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import type { typeField } from "./agentEditReducer.js"; +import type { JsonPrimitive } from "./json-handler/index.js"; + +/** + * TODO: The current scheme does not allow manipulation of arrays of primitive values because you cannot refer to them. + * We could accomplish this via a path (probably JSON Pointer or JSONPath) from a possibly-null objectId, or wrap arrays in an identified object. + * + * TODO: We could add a "replace" edit type to avoid tons of little modifies. + * + * TODO: only 100 object fields total are allowed by OpenAI right now, so larger schemas will fail faster if we have a bunch of schema types generated for type-specific edits. + * + * TODO: experiment using https://github.com/outlines-dev/outlines (and maybe a llama model) to avoid many of the annoyances of OpenAI's JSON Schema subset. + * + * TODO: without field count limits, we could generate a schema for valid paths from the root object to any field, but it's not clear how useful that would be. + * + * TODO: We don't supported nested arrays yet. + * + * TODO: Could omit edit contents for setRoot edits as the tree state is the result (or the other way around). + * + * TODO: Add a prompt suggestion API! + * + * TODO: Could encourage the model to output more technical explanations of the edits (e.g. "insert a new Foo after "Foo2"). + * + * TODO: Get explanation strings from o1. + * + * TODO: Tests of range edits. + * + * TODO: SetRoot might be obscure enough to make the LLM avoid it. Maybe a general replace edit would be better. + * + * TODO: Handle 429 rate limit error in streamFromLlm. + * + * TODO: Add an app-specific guidance string. + * + * TODO: Give the model a final chance to evaluate the result. + * + * TODO: Separate system prompt into [system, user, system] for security. + */ + +/** + * TBD + */ +export const objectIdKey = "__fluid_objectId"; +/** + * TBD + */ +export interface TreeEditObject { + [key: string]: TreeEditValue; + [typeField]: string; +} +/** + * TBD + */ +export type TreeEditArray = TreeEditValue[]; +/** + * TBD + */ +export type TreeEditValue = JsonPrimitive | TreeEditObject | TreeEditArray; + +/** + * TBD + * @remarks - For polymorphic edits, we need to wrap the edit in an object to avoid anyOf at the root level. + */ +export interface EditWrapper { + // eslint-disable-next-line @rushstack/no-new-null + edit: TreeEdit | null; +} + +/** + * TBD + */ +export type TreeEdit = SetRoot | Insert | Modify | Remove | Move; +/** + * TBD + */ +export interface Edit { + explanation: string; + type: "setRoot" | "insert" | "modify" | "remove" | "move"; +} +/** + * TBD + */ +export type Selection = ObjectTarget | Range; +/** + * TBD + */ +export interface ObjectTarget { + [objectIdKey]: string; +} + +/** + * @remarks - TODO: Allow support for nested arrays + */ +export interface ArrayPlace { + type: "arrayPlace"; + parentId: string; + field: string; + location: "start" | "end"; +} + +/** + * TBD + */ +export interface ObjectPlace extends ObjectTarget { + type: "objectPlace"; + // No "start" or "end" because we don't have a way to refer to arrays directly. + place: "before" | "after"; +} + +/** + * TBD + */ +export interface Range { + from: ObjectPlace; + to: ObjectPlace; +} + +/** + * TBD + */ +export interface SetRoot extends Edit { + type: "setRoot"; + content: TreeEditValue; +} + +/** + * TBD + */ +export interface Insert extends Edit { + type: "insert"; + content: TreeEditObject; + destination: ObjectPlace | ArrayPlace; +} + +/** + * TBD + */ +export interface Modify extends Edit { + type: "modify"; + target: ObjectTarget; + field: string; + modification: TreeEditValue; +} + +/** + * TBD + */ +export interface Remove extends Edit { + type: "remove"; + source: Selection; +} + +/** + * TBD + */ +export interface Move extends Edit { + type: "move"; + source: Selection; + destination: ObjectPlace | ArrayPlace; +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/handlers.ts b/packages/framework/ai-collab/src/explicit-strategy/handlers.ts new file mode 100644 index 000000000000..1b88af094919 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/handlers.ts @@ -0,0 +1,355 @@ +/* eslint-disable unicorn/no-abusive-eslint-disable */ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { FieldKind, NodeKind, ValueSchema } from "@fluidframework/tree/internal"; +import type { + SimpleFieldSchema, + SimpleNodeSchema, + SimpleTreeSchema, +} from "@fluidframework/tree/internal"; + +// import { ValueSchema } from "../core/index.js"; +import { typeField } from "./agentEditReducer.js"; +import { objectIdKey } from "./agentEditTypes.js"; +import { + type StreamedType, + type JsonObject, + JsonHandler as jh, +} from "./json-handler/index.js"; +import { fail, getOrCreate, mapIterable } from "./utils.js"; + +const objectTargetHandler = jh.object(() => ({ + description: "A pointer to an object in the tree", + properties: { + [objectIdKey]: jh.string({ description: "The id of the object that is being pointed to" }), + }, +})); + +const objectPlaceHandler = jh.object(() => ({ + description: + "A pointer to a location either just before or just after an object that is in an array", + properties: { + type: jh.enum({ values: ["objectPlace"] }), + [objectIdKey]: jh.string({ + description: `The id (${objectIdKey}) of the object that the new/moved object should be placed relative to. This must be the id of an object that already existed in the tree content that was originally supplied.`, + }), + place: jh.enum({ + values: ["before", "after"], + description: + "Where the new/moved object will be relative to the target object - either just before or just after", + }), + }, +})); + +const arrayPlaceHandler = jh.object(() => ({ + description: + "A location at either the beginning or the end of an array (useful for prepending or appending)", + properties: { + type: jh.enum({ values: ["arrayPlace"] }), + parentId: jh.string({ + description: `The id (${objectIdKey}) of the parent object of the array. This must be the id of an object that already existed in the tree content that was originally supplied.`, + }), + field: jh.string({ "description": "The key of the array to insert into" }), + location: jh.enum({ + values: ["start", "end"], + description: "Where to insert into the array - either the start or the end", + }), + }, +})); + +const rangeHandler = jh.object(() => ({ + description: + 'A span of objects that are in an array. The "to" and "from" objects MUST be in the same array.', + properties: { + from: objectPlaceHandler(), + to: objectPlaceHandler(), + }, +})); + +/** + * TBD + */ +export function generateEditHandlers( + schema: SimpleTreeSchema, + complete: (jsonObject: JsonObject) => void, +): StreamedType { + const insertSet = new Set(); + const modifyFieldSet = new Set(); + const modifyTypeSet = new Set(); + const schemaHandlers = new Map(); + + for (const name of schema.definitions.keys()) { + getOrCreateHandler( + schema.definitions, + schemaHandlers, + insertSet, + modifyFieldSet, + modifyTypeSet, + name, + ); + } + + const setRootHandler = jh.object(() => ({ + description: "A handler for setting content to the root of the tree.", + properties: { + type: jh.enum({ values: ["setRoot"] }), + explanation: jh.string({ description: editDescription }), + content: jh.anyOf( + Array.from( + schema.allowedTypes, + (nodeSchema) => schemaHandlers.get(nodeSchema) ?? fail("Unexpected schema"), + ), + ), + }, + })); + + const insertHandler = jh.object(() => ({ + description: "A handler for inserting new content into the tree.", + properties: { + type: jh.enum({ values: ["insert"] }), + explanation: jh.string({ description: editDescription }), + content: jh.anyOf( + Array.from(insertSet, (n) => schemaHandlers.get(n) ?? fail("Unexpected schema")), + ), + destination: jh.anyOf([arrayPlaceHandler(), objectPlaceHandler()]), + }, + })); + + const removeHandler = jh.object(() => ({ + description: "A handler for removing content from the tree.", + properties: { + type: jh.enum({ values: ["remove"] }), + explanation: jh.string({ description: editDescription }), + source: jh.anyOf([objectTargetHandler(), rangeHandler()]), + }, + })); + + const modifyHandler = jh.object(() => ({ + description: "A handler for modifying content in the tree.", + properties: { + type: jh.enum({ values: ["modify"] }), + explanation: jh.string({ description: editDescription }), + target: objectTargetHandler(), + field: jh.enum({ values: [...modifyFieldSet] }), + modification: jh.anyOf( + Array.from(modifyTypeSet, (n) => schemaHandlers.get(n) ?? fail("Unexpected schema")), + ), + }, + })); + + const moveHandler = jh.object(() => ({ + description: + "A handler for moving content from one location in the tree to another location in the tree.", + properties: { + type: jh.enum({ values: ["move"] }), + explanation: jh.string({ description: editDescription }), + source: jh.anyOf([objectTargetHandler(), rangeHandler()]), + destination: jh.anyOf([arrayPlaceHandler(), objectPlaceHandler()]), + }, + })); + + const editWrapper = jh.object(() => ({ + // description: + // "The next edit to apply to the tree, or null if the task is complete and no more edits are necessary.", + properties: { + edit: jh.anyOf([ + setRootHandler(), + insertHandler(), + modifyHandler(), + removeHandler(), + moveHandler(), + jh.null(), + ]), + }, + complete, + })); + + return jh.object(() => ({ + properties: { + edit: editWrapper(), + }, + }))(); +} + +const editDescription = + "A description of what this edit is meant to accomplish in human readable English"; + +function getOrCreateHandler( + definitionMap: ReadonlyMap, + handlerMap: Map, + insertSet: Set, + modifyFieldSet: Set, + modifyTypeSet: Set, + definition: string, +): StreamedType { + return getOrCreate(handlerMap, definition, () => { + const nodeSchema: SimpleNodeSchema = + definitionMap.get(definition) ?? fail("Unexpected definition"); + switch (nodeSchema.kind) { + case NodeKind.Object: { + for (const [key, field] of Object.entries(nodeSchema.fields)) { + // TODO: Remove when AI better + if ( + Array.from( + field.allowedTypes, + (n) => definitionMap.get(n) ?? fail("Unknown definition"), + ).some((n) => n.kind === NodeKind.Array) + ) { + continue; + } + modifyFieldSet.add(key); + for (const type of field.allowedTypes) { + modifyTypeSet.add(type); + } + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const properties = Object.fromEntries( + Object.entries(nodeSchema.fields) + .map(([key, field]) => { + return [ + key, + getOrCreateHandlerForField( + definitionMap, + handlerMap, + insertSet, + modifyFieldSet, + modifyFieldSet, + field, + ), + ]; + }) + .filter(([, value]) => value !== undefined), + ); + properties[typeField] = jh.enum({ values: [definition] }); + return jh.object(() => ({ + properties, + }))(); + } + case NodeKind.Array: { + for (const [name] of Array.from( + nodeSchema.allowedTypes, + (n): [string, SimpleNodeSchema] => [ + n, + definitionMap.get(n) ?? fail("Unknown definition"), + ], + ).filter(([_, schema]) => schema.kind === NodeKind.Object)) { + insertSet.add(name); + } + + return jh.array(() => ({ + items: getStreamedType( + definitionMap, + handlerMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + nodeSchema.allowedTypes, + ), + }))(); + } + case NodeKind.Leaf: + switch (nodeSchema.leafKind) { + case ValueSchema.Boolean: + return jh.boolean(); + case ValueSchema.Number: + return jh.number(); + case ValueSchema.String: + return jh.string(); + case ValueSchema.Null: + return jh.null(); + default: + throw new Error(`Unsupported leaf kind ${NodeKind[nodeSchema.leafKind]}.`); + } + default: + throw new Error(`Unsupported node kind ${NodeKind[nodeSchema.kind]}.`); + } + }); +} + +function getOrCreateHandlerForField( + definitionMap: ReadonlyMap, + handlerMap: Map, + insertSet: Set, + modifyFieldSet: Set, + modifyTypeSet: Set, + fieldSchema: SimpleFieldSchema, +): StreamedType | undefined { + if (fieldSchema.metadata?.llmDefault !== undefined) { + // Omit fields that have data which cannot be generated by an llm + return undefined; + } + + switch (fieldSchema.kind) { + case FieldKind.Required: { + return getStreamedType( + definitionMap, + handlerMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + fieldSchema.allowedTypes, + ); + } + case FieldKind.Optional: { + return jh.optional( + getStreamedType( + definitionMap, + handlerMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + fieldSchema.allowedTypes, + ), + ); + } + case FieldKind.Identifier: { + return undefined; + } + default: { + throw new Error(`Unsupported field kind ${NodeKind[fieldSchema.kind]}.`); + } + } +} + +function getStreamedType( + definitionMap: ReadonlyMap, + handlerMap: Map, + insertSet: Set, + modifyFieldSet: Set, + modifyTypeSet: Set, + allowedTypes: ReadonlySet, +): StreamedType { + const single = tryGetSingleton(allowedTypes); + return single === undefined + ? jh.anyOf([ + ...mapIterable(allowedTypes, (name) => { + return getOrCreateHandler( + definitionMap, + handlerMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + name, + ); + }), + ]) + : getOrCreateHandler( + definitionMap, + handlerMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + single, + ); +} + +function tryGetSingleton(set: ReadonlySet): T | undefined { + if (set.size === 1) { + for (const item of set) { + return item; + } + } +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts new file mode 100644 index 000000000000..bd6d7f171379 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts @@ -0,0 +1,89 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { assert, oob } from "@fluidframework/core-utils/internal"; +import { Tree, NodeKind } from "@fluidframework/tree/internal"; +import type { + TreeNode, + ImplicitFieldSchema, + TreeArrayNode, + TreeFieldFromImplicitField, +} from "@fluidframework/tree/internal"; + +import { isTreeNode } from "./utils.js"; + +/** + * TBD + */ +export class IdGenerator { + private readonly idCountMap = new Map(); + private readonly prefixMap = new Map(); + private readonly nodeToIdMap = new Map(); + private readonly idToNodeMap = new Map(); + + public constructor() {} + + public getOrCreateId(node: TreeNode): string { + const existingID = this.nodeToIdMap.get(node); + if (existingID !== undefined) { + return existingID; + } + + const schema = Tree.schema(node).identifier; + const id = this.generateID(schema); + this.nodeToIdMap.set(node, id); + this.idToNodeMap.set(id, node); + + return id; + } + + public getNode(id: string): TreeNode | undefined { + return this.idToNodeMap.get(id); + } + + public getId(node: TreeNode): string | undefined { + return this.nodeToIdMap.get(node); + } + + public assignIds(node: TreeFieldFromImplicitField): string | undefined { + if (typeof node === "object" && node !== null) { + const schema = Tree.schema(node as unknown as TreeNode); + if (schema.kind === NodeKind.Array) { + (node as unknown as TreeArrayNode).forEach((element) => { + this.assignIds(element); + }); + } else { + assert(isTreeNode(node), "Non-TreeNode value in tree."); + const objId = this.getOrCreateId(node); + Object.keys(node).forEach((key) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + this.assignIds((node as unknown as any)[key]); + }); + return objId; + } + } + return undefined; + } + + private generateID(schema: string): string { + const segments = schema.split("."); + + // If there's no period, the schema itself is the last segment + const lastSegment = segments[segments.length - 1] ?? oob(); + const prefix = segments.length > 1 ? segments.slice(0, -1).join(".") : ""; + + // Check if the last segment already exists with a different prefix + assert( + !this.prefixMap.has(lastSegment) || this.prefixMap.get(lastSegment) === prefix, + "Different scopes not supported yet.", + ); + + this.prefixMap.set(lastSegment, prefix); + const count = this.idCountMap.get(lastSegment) ?? 1; + this.idCountMap.set(lastSegment, count + 1); + + return `${lastSegment}${count}`; + } +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts new file mode 100644 index 000000000000..dc0aed7c8a77 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts @@ -0,0 +1,13 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +/** + * TBD + */ +export function assert(condition: boolean): void { + if (!condition) { + throw new Error("Assert failed"); + } +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts new file mode 100644 index 000000000000..c360694eee12 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts @@ -0,0 +1,11 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +// TODO: Only export the things we want +/* eslint-disable no-restricted-syntax */ + +export * from "./jsonHandler.js"; +export * from "./jsonHandlerImp.js"; +export * from "./jsonParser.js"; diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts new file mode 100644 index 000000000000..e08d370c28c6 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts @@ -0,0 +1,95 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { + type PartialArg, + type StreamedObjectDescriptor, + type StreamedArrayDescriptor, + type StreamedType, + getJsonHandler, + getCreateResponseHandler, +} from "./jsonHandlerImp.js"; +import type { JsonObject } from "./jsonParser.js"; + +/** + * TBD + */ +export interface ResponseHandler { + jsonSchema(): JsonObject; + processResponse(responseGenerator: { + [Symbol.asyncIterator](): AsyncGenerator; + }): Promise; + processChars(chars: string): void; + complete(): void; +} + +/** + * TBD + */ +export const createResponseHandler: ( + streamedType: StreamedType, + abortController: AbortController, +) => ResponseHandler = getCreateResponseHandler(); + +/** + * TBD + */ +export const JsonHandler: { + object: ( + getDescriptor: (input: Input) => StreamedObjectDescriptor, + ) => (getInput?: (partial: PartialArg) => Input) => StreamedType; + + array: ( + getDescriptor: (input: Input) => StreamedArrayDescriptor, + ) => (getInput?: (partial: PartialArg) => Input) => StreamedType; + + streamedStringProperty< + Parent extends Record, + Key extends keyof Parent, + >(args: { + description?: string; + target: (partial: PartialArg) => Parent; + key: Key; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType; + + streamedString(args: { + description?: string; + target: (partial: PartialArg) => Parent; + append: (chars: string, parent: Parent) => void; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType; + + string(args?: { + description?: string; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType; + + enum(args: { + description?: string; + values: string[]; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType; + + number(args?: { + description?: string; + complete?: (value: number, partial: PartialArg) => void; + }): StreamedType; + + boolean(args?: { + description?: string; + complete?: (value: boolean, partial: PartialArg) => void; + }): StreamedType; + + null(args?: { + description?: string; + // eslint-disable-next-line @rushstack/no-new-null + complete?: (value: null, partial: PartialArg) => void; + }): StreamedType; + + optional(streamedType: StreamedType): StreamedType; + + anyOf(streamedTypes: StreamedType[]): StreamedType; +} = getJsonHandler(); diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts new file mode 100644 index 000000000000..6cf4507ff044 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts @@ -0,0 +1,1418 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { assert } from "./debug.js"; +import { + type JsonArray, + type JsonBuilder, + type JsonBuilderContext, + type JsonObject, + type JsonPrimitive, + type JsonValue, + type StreamedJsonParser, + contextIsObject, + createStreamedJsonParser, +} from "./jsonParser.js"; + +type StreamedTypeGetter = ( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + getInput?: (partial: PartialArg) => any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any +) => StreamedObject | StreamedArray; +type StreamedTypeIdentity = StreamedType | StreamedTypeGetter; +type DefinitionMap = Map; + +class ResponseHandlerImpl { + public constructor( + private readonly streamedType: StreamedType, + abortController: AbortController, + ) { + if (streamedType instanceof StreamedAnyOf) { + throw new TypeError("anyOf cannot be used as root type"); + } + + if ( + streamedType instanceof StreamedStringProperty || + streamedType instanceof StreamedString + ) { + throw new TypeError( + "StreamedStringProperty and StreamedString cannot be used as root type", + ); + } + + const streamedValueHandler = ( + streamedType as InvocableStreamedType + ).invoke( + undefined, + streamedType instanceof StreamedObject + ? {} + : streamedType instanceof StreamedArray + ? [] + : undefined, + ); + const builder = new BuilderDispatcher(streamedValueHandler); + this.parser = createStreamedJsonParser(builder, abortController); + } + + public jsonSchema(): JsonObject { + const definitions = new Map(); + const visited = new Set(); + + findDefinitions(this.streamedType, visited, definitions); + + const rootIdentity = ( + this.streamedType as InvocableStreamedType + ).getIdentity(); + + // Don't call jsonSchemaFromStreamedType, as we must force the method call on the root + const schema = ( + this.streamedType as InvocableStreamedType + ).jsonSchema(rootIdentity, definitions); + + definitions.forEach((definitionName, streamedTypeOrGetter) => { + if (streamedTypeOrGetter !== rootIdentity) { + schema.$defs ??= {}; + + const streamedType = + streamedTypeOrGetter instanceof Function + ? streamedTypeOrGetter(() => guaranteedErrorObject) // No-one will call this, but this return value emphasizes this point + : streamedTypeOrGetter; + + // Again, don't call jsonSchemaFromStreamedType, as we must force the method call on each definition root + (schema.$defs as JsonObject)[definitionName] = ( + streamedType as InvocableStreamedType + ).jsonSchema(this.streamedType, definitions); + } + }); + + return schema; + } + + public async processResponse(responseGenerator: { + [Symbol.asyncIterator](): AsyncGenerator; + }): Promise { + for await (const fragment of responseGenerator) { + this.processChars(fragment); + } + this.complete(); + } + + public processChars(chars: string): void { + this.parser.addChars(chars); + } + + public complete(): void { + // Send one more whitespace token, just to ensure the parser knows we're finished + // (this is necessary for the case of a schema comprising a single number) + this.parser.addChars("\n"); + } + + private readonly parser: StreamedJsonParser; +} + +// The one createResponseHandlerImpl +const createResponseHandlerImpl = ( + streamedType: StreamedType, + abortController: AbortController, +): ResponseHandlerImpl => { + return new ResponseHandlerImpl(streamedType, abortController); +}; + +/** + * TBD + */ +export const getCreateResponseHandler: () => ( + streamedType: StreamedType, + abortController: AbortController, +) => ResponseHandlerImpl = () => createResponseHandlerImpl; + +/** + * TBD + */ +export class StreamedType { + private readonly _brand = Symbol(); +} + +class JsonHandlerImpl { + public object( + getDescriptor: (input: Input) => StreamedObjectDescriptor, + ): (getInput?: (partial: PartialArg) => Input) => StreamedType { + // The function created here serves as the identity of this type's schema, + // since the schema is independent of the input passed to the handler + return function getStreamedObject( + getInput?: (partial: PartialArg) => Input, + ): StreamedObject { + return new StreamedObject(getDescriptor, getStreamedObject, getInput); + }; + } + + public array( + getDescriptor: (input: Input) => StreamedArrayDescriptor, + ): (getInput?: (partial: PartialArg) => Input) => StreamedType { + // The function created here serves as the identity of this type's schema, + // since the schema is independent of the input passed to the handler + return function getStreamedArray( + getInput?: (partial: PartialArg) => Input, + ): StreamedArray { + return new StreamedArray(getDescriptor, getStreamedArray, getInput); + }; + } + + public streamedStringProperty< + T extends Record, + P extends keyof T, + >(args: { + description?: string; + target: (partial: PartialArg) => T; + key: P; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType { + return new StreamedStringProperty(args); + } + + public streamedString(args: { + description?: string; + target: (partial: PartialArg) => Parent; + append: (chars: string, parent: Parent) => void; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType { + return new StreamedString(args); + } + + public string(args?: { + description?: string; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType { + return new AtomicString(args); + } + + public enum(args: { + description?: string; + values: string[]; + complete?: (value: string, partial: PartialArg) => void; + }): StreamedType { + return new AtomicEnum(args); + } + + public number(args?: { + description?: string; + complete?: (value: number, partial: PartialArg) => void; + }): StreamedType { + return new AtomicNumber(args); + } + + public boolean(args?: { + description?: string; + complete?: (value: boolean, partial: PartialArg) => void; + }): StreamedType { + return new AtomicBoolean(args); + } + + public null(args?: { + description?: string; + // eslint-disable-next-line @rushstack/no-new-null + complete?: (value: null, partial: PartialArg) => void; + }): StreamedType { + return new AtomicNull(args); + } + + public optional(streamedType: StreamedType): StreamedType { + if (streamedType instanceof AtomicNull) { + throw new TypeError("Cannot have an optional null value"); + } + return new StreamedOptional(streamedType as InvocableStreamedType); + } + + public anyOf(streamedTypes: StreamedType[]): StreamedType { + return new StreamedAnyOf(streamedTypes as InvocableStreamedType[]); + } +} + +// The one JsonHandler +const jsonHandler = new JsonHandlerImpl(); + +/** + * TBD + */ +export const getJsonHandler: () => JsonHandlerImpl = () => jsonHandler; + +/** + * TBD + * @remarks - TODO: Can perhaps not export these after illustrateInteraction re-implemented (and remove all the ?s and !s) + */ +export interface StreamedObjectHandler { + addObject(key: string): StreamedObjectHandler; + addArray(key: string): StreamedArrayHandler; + addPrimitive(value: JsonPrimitive, key: string): void; + appendText(chars: string, key: string): void; + completeProperty(key: string): void; + complete(): void; +} + +/** + * TBD + */ +export interface StreamedArrayHandler { + addObject(): StreamedObjectHandler; + addArray(): StreamedArrayHandler; + addPrimitive(value: JsonPrimitive): void; + appendText(chars: string): void; + completeLast(): void; + complete(): void; +} + +type BuilderContext = JsonBuilderContext; + +class BuilderDispatcher implements JsonBuilder { + public constructor(private readonly rootHandler: StreamedValueHandler) {} + + public addObject(context?: BuilderContext): StreamedObjectHandler { + if (!context) { + // TODO: This error-handling, which really shouldn't be necessary in principle with Structured Outputs, + // is arguably "inside-out", i.e. it should report the expected type of the result, rather than + // the handler. + if (!(this.rootHandler instanceof StreamedObjectHandlerImpl)) { + throw new TypeError(`Expected object for root`); + } + return this.rootHandler; + } else if (contextIsObject(context)) { + return context.parentObject.addObject(context.key); + } else { + return context.parentArray.addObject(); + } + } + + public addArray(context?: BuilderContext): StreamedArrayHandler { + if (!context) { + if (!(this.rootHandler instanceof StreamedArrayHandlerImpl)) { + throw new TypeError(`Expected array for root`); + } + return this.rootHandler; + } else if (contextIsObject(context)) { + return context.parentObject.addArray(context.key); + } else { + return context.parentArray.addArray(); + } + } + + public addPrimitive(value: JsonPrimitive, context?: BuilderContext): void { + if (!context) { + if (value === null) { + if (!(this.rootHandler instanceof AtomicNullHandlerImpl)) { + throw new TypeError(`Expected null for root`); + } + this.rootHandler.complete(value, undefined); + } else { + switch (typeof value) { + case "string": { + if ( + !( + this.rootHandler instanceof AtomicStringHandlerImpl || + this.rootHandler instanceof AtomicEnumHandlerImpl + ) + ) { + throw new TypeError(`Expected string or enum for root`); + } + this.rootHandler.complete(value, undefined); + break; + } + case "number": { + if (!(this.rootHandler instanceof AtomicNumberHandlerImpl)) { + throw new TypeError(`Expected number for root`); + } + this.rootHandler.complete(value, undefined); + break; + } + case "boolean": { + if (!(this.rootHandler instanceof AtomicBooleanHandlerImpl)) { + throw new TypeError(`Expected boolean for root`); + } + this.rootHandler.complete(value, undefined); + break; + } + + default: { + break; + } + } + } + } else if (contextIsObject(context)) { + context.parentObject.addPrimitive(value, context.key); + } else { + context.parentArray.addPrimitive(value); + } + } + + public appendText(chars: string, context?: BuilderContext): void { + assert(context !== undefined); + if (contextIsObject(context)) { + context.parentObject.appendText(chars, context.key); + } else { + context!.parentArray.appendText(chars); + } + } + + public completeContext(context?: BuilderContext): void { + if (context !== undefined) { + if (contextIsObject(context)) { + context.parentObject.completeProperty?.(context.key); + } else { + context.parentArray.completeLast?.(); + } + } + } + + public completeContainer(container: StreamedObjectHandler | StreamedArrayHandler): void { + container.complete?.(); + } +} + +/** + * TBD + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type PartialArg = any; // Would be PartialObject | PartialArray | undefined, but doesn't work for function arguments + +type PartialObject = JsonObject; + +// TODO: May be better to distinguish between streamed object properties and array elements (because strings) +type StreamedValueHandler = + | StreamedObjectHandler + | StreamedArrayHandler + | StreamedStringPropertyHandler + | StreamedStringHandler + | AtomicStringHandler + | AtomicEnumHandler + | AtomicNumberHandler + | AtomicBooleanHandler + | AtomicNullHandler + | AtomicPrimitiveHandler; // Needed so AtomicPrimitive can implement StreamedType> + +abstract class SchemaGeneratingStreamedType extends StreamedType { + public getIdentity(): StreamedTypeIdentity { + return this; + } + public abstract findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void; + public abstract jsonSchema( + root: StreamedTypeIdentity, + definitions: DefinitionMap, + ): JsonObject; +} + +abstract class InvocableStreamedType< + T extends StreamedValueHandler | undefined, +> extends SchemaGeneratingStreamedType { + public abstract invoke(parentPartial: PartialArg, partial: PartialArg): T; +} + +// eslint-disable-next-line @rushstack/no-new-null +type FieldHandlers = Record; + +const findDefinitions = ( + streamedType: StreamedType, + visited: Set, + definitions: DefinitionMap, +): void => + (streamedType as InvocableStreamedType).findDefinitions( + visited, + definitions, + ); + +const addDefinition = ( + streamedType: StreamedTypeIdentity, + definitions: DefinitionMap, +): void => { + if (!definitions.has(streamedType)) { + definitions.set(streamedType, `d${definitions.size.toString()}`); + } +}; + +const jsonSchemaFromStreamedType = ( + streamedType: StreamedType, + root: StreamedTypeIdentity, + definitions: DefinitionMap, +): JsonObject => { + const identity = (streamedType as InvocableStreamedType).getIdentity(); + + if (root === identity) { + return { $ref: "#" }; + } else if (definitions.has(identity)) { + return { $ref: `#/$defs/${definitions.get(identity)}` }; + } else { + return (streamedType as InvocableStreamedType).jsonSchema( + root, + definitions, + ); + } +}; + +const guaranteedErrorHandler = { + get(target: T, prop: string) { + throw new Error(`Attempted to access property "${prop}" outside a handler body.`); + }, + set(target: T, prop: string, value: V) { + throw new Error( + `Attempted to set property "${prop}" to "${value}" outside a handler body.`, + ); + }, + has(target: T, prop: string) { + throw new Error( + `Attempted to check existence of property "${prop}" outside a handler body.`, + ); + }, + deleteProperty(target: T, prop: string) { + throw new Error(`Attempted to delete property "${prop}" outside a handler body.`); + }, +}; +const guaranteedErrorObject = new Proxy({}, guaranteedErrorHandler); + +type FieldTypes = Record; + +/** + * TBD + */ +export interface StreamedObjectDescriptor { + description?: string; + properties: FieldTypes; + complete?: (result: JsonObject) => void; +} + +class StreamedObject extends InvocableStreamedType { + public constructor( + private readonly getDescriptor: (input: Input) => StreamedObjectDescriptor, + private readonly identity: StreamedTypeIdentity, + private readonly getInput?: (partial: PartialArg) => Input, + ) { + super(); + } + + public override getIdentity(): StreamedTypeIdentity { + return this.identity; + } + + public findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void { + const identity = this.getIdentity(); + if (visited.has(identity)) { + addDefinition(identity, definitions); + } else { + visited.add(identity); + + // TODO: Cache descriptor here, assert it's cached in jsonSchema (ditto all other types) + const { properties } = this.getDummyDescriptor(); + Object.values(properties).forEach((streamedType) => { + findDefinitions(streamedType, visited, definitions); + }); + } + } + + public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { + const { description, properties } = this.getDummyDescriptor(); + + const propertyNames = Object.keys(properties); + const schemaProperties: Record = {}; + propertyNames.forEach((fieldName) => { + schemaProperties[fieldName] = jsonSchemaFromStreamedType( + properties[fieldName]!, + root, + definitions, + ); + }); + + const schema: JsonObject = { + type: "object", + properties: schemaProperties, + }; + + if (description !== undefined) { + schema.description = description; + } + + schema.required = Object.keys(schemaProperties); + schema.additionalProperties = false; + + return schema; + } + + public invoke(parentPartial: PartialArg, partial: PartialArg): StreamedObjectHandler { + return new StreamedObjectHandlerImpl( + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + partial, + this.getDescriptor(this.getInput?.(parentPartial) as Input), + ); + } + + public get properties(): FieldTypes { + // TODO-AnyOf: Expose this more gracefully + return this.getDummyDescriptor().properties; + } + + public delayedInvoke(parentPartial: PartialArg): StreamedObjectDescriptor { + // TODO-AnyOf: Expose this more gracefully + return this.getDescriptor(this.getInput?.(parentPartial) as Input); + } + + private getDummyDescriptor(): StreamedObjectDescriptor { + if (this.dummyDescriptor === undefined) { + this.dummyDescriptor = this.getDescriptor(guaranteedErrorObject as Input); + } + + return this.dummyDescriptor; + } + + private dummyDescriptor?: StreamedObjectDescriptor; +} + +class StreamedObjectHandlerImpl implements StreamedObjectHandler { + public constructor( + private partial: PartialObject, + private descriptor?: StreamedObjectDescriptor, + private readonly streamedAnyOf?: StreamedAnyOf, + ) {} + + public addObject(key: string): StreamedObjectHandler { + this.attemptResolution(key, StreamedObject); + + if (this.descriptor) { + let streamedType: StreamedType | undefined = this.descriptor.properties[key]; + + if (streamedType === undefined) { + throw new Error(`Unhandled key ${key}`); + } + + if (streamedType instanceof StreamedOptional) { + streamedType = streamedType.optionalType; + } + + if (streamedType instanceof StreamedAnyOf) { + const streamedAnyOf = streamedType; + if (!(streamedType = streamedType.streamedTypeIfSingleMatch(StreamedObject))) { + // The type is ambiguous, so create an "unbound" StreamedObjectHandler and wait for more input + const childPartial: PartialObject = {}; + this.partial[key] = childPartial; + this.handlers[key] = new StreamedObjectHandlerImpl( + childPartial, + undefined, + streamedAnyOf, + ); + return this.handlers[key] as StreamedObjectHandler; + } + } + + if (streamedType instanceof StreamedObject) { + const childPartial: PartialObject = {}; + this.partial[key] = childPartial; + this.handlers[key] = streamedType.invoke(this.partial, this.partial[key]); + return this.handlers[key] as StreamedObjectHandler; + } + } + + throw new Error(`Expected object for key ${key}`); + } + + public addArray(key: string): StreamedArrayHandler { + this.attemptResolution(key, StreamedArray); + + if (this.descriptor) { + let streamedType: StreamedType | undefined = this.descriptor.properties[key]; + + if (streamedType === undefined) { + throw new Error(`Unhandled key ${key}`); + } + + if (streamedType instanceof StreamedOptional) { + streamedType = streamedType.optionalType; + } + + if (streamedType instanceof StreamedAnyOf) { + const streamedAnyOf = streamedType; + if (!(streamedType = streamedType.streamedTypeIfSingleMatch(StreamedArray))) { + // The type is ambiguous, so create an "unbound" StreamedObjectHandler and wait for more input + const childPartial: PartialArray = []; + this.partial[key] = childPartial; + this.handlers[key] = new StreamedArrayHandlerImpl( + childPartial, + undefined, + streamedAnyOf, + ); + return this.handlers[key] as StreamedArrayHandler; + } + } + + if (streamedType instanceof StreamedArray) { + const childPartial = [] as PartialArray; + this.partial[key] = childPartial; + this.handlers[key] = streamedType.invoke(this.partial, childPartial); + return this.handlers[key] as StreamedArrayHandler; + } + } + + throw new Error(`Expected array for key ${key}`); + } + + // TODO: Return boolean requesting throttling if StreamedString (also in StreamedArrayHandlerImpl) + public addPrimitive(value: JsonPrimitive, key: string): void { + if (!this.descriptor) { + this.partial[key] = value; + return; + } + + let streamedType: StreamedType | undefined = this.descriptor.properties[key]; + + if (streamedType === undefined) { + throw new Error(`Unhandled key ${key}`); + } + + this.partial[key] = value; + + if (streamedType instanceof StreamedOptional) { + if (value === null) { + // Don't call the (non-null) handler, as the optional value wasn't present + return; + } + streamedType = streamedType.optionalType; + } + + if (streamedType instanceof StreamedAnyOf) { + streamedType = streamedType.streamedTypeOfFirstMatch(value); + } + + if (primitiveMatchesStreamedType(value, streamedType!)) { + this.handlers[key] = ( + streamedType as InvocableStreamedType + ).invoke(this.partial, undefined); + return; + } + + // Shouldn't happen with Structured Outputs + throw new Error(`Unexpected ${typeof value} for key ${key}`); + } + + public appendText(chars: string, key: string): void { + assert(typeof this.partial[key] === "string"); + (this.partial[key] as string) += chars; + if ( + this.handlers[key] instanceof StreamedStringPropertyHandlerImpl || + this.handlers[key] instanceof StreamedStringHandlerImpl + ) { + (this.handlers[key] as { append: (chars: string) => void }).append(chars); + } + } + + public completeProperty(key: string): void { + const value = this.partial[key]; + if (isPrimitiveValue(value!)) { + this.attemptResolution(key, value as PrimitiveType); + + // Objects and Arrays will have their complete() handler called directly + completePrimitive(this.handlers[key]!, value as PrimitiveType, this.partial); + } + } + + public complete(): void { + // TODO-AnyOf: + this.descriptor!.complete?.(this.partial); + } + + private attemptResolution( + key: string, + typeOrValue: typeof StreamedObject | typeof StreamedArray | PrimitiveType, + ): void { + if (!this.descriptor) { + assert(this.streamedAnyOf !== undefined); + for (const option of this.streamedAnyOf!.options) { + if (option instanceof StreamedObject) { + const property = option.properties[key]; + if (streamedTypeMatches(property!, typeOrValue)) { + // We now know which option in the AnyOf to use + this.descriptor = option.delayedInvoke(this.partial); + } + } + } + } + } + + private handlers: FieldHandlers = {}; // TODO: Overkill, since only one needed at a time? +} + +type PartialArray = JsonArray; + +// eslint-disable-next-line @rushstack/no-new-null +type ArrayAppendHandler = StreamedValueHandler | null; + +/** + * TBD + */ +export interface StreamedArrayDescriptor { + description?: string; + items: StreamedType; + complete?: (result: JsonArray) => void; +} + +class StreamedArray extends InvocableStreamedType { + public constructor( + private readonly getDescriptor: (input: Input) => StreamedArrayDescriptor, + private readonly identity: StreamedTypeIdentity, + private readonly getInput?: (partial: PartialArg) => Input, + ) { + super(); + } + + public override getIdentity(): StreamedTypeIdentity { + return this.identity; + } + + public findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void { + const identity = this.getIdentity(); + if (visited.has(identity)) { + addDefinition(identity, definitions); + } else { + visited.add(identity); + + const { items } = this.getDummyDescriptor(); + findDefinitions(items, visited, definitions); + } + } + + public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { + const { description, items } = this.getDummyDescriptor(); + + const schema: JsonObject = { + type: "array", + items: jsonSchemaFromStreamedType(items, root, definitions), + }; + + if (description !== undefined) { + schema.description = description; + } + + return schema; + } + + public invoke(parentPartial: PartialArg, partial: PartialArg): StreamedArrayHandler { + return new StreamedArrayHandlerImpl( + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + partial, + this.getDescriptor(this.getInput?.(parentPartial) as Input), + ); + } + + public get items(): StreamedType { + // TODO-AnyOf: Expose this more gracefully + return this.getDummyDescriptor().items; + } + + public delayedInvoke(parentPartial: PartialArg): StreamedArrayDescriptor { + // TODO-AnyOf: Expose this more gracefully + return this.getDescriptor(this.getInput?.(parentPartial) as Input); + } + + private getDummyDescriptor(): StreamedArrayDescriptor { + if (this.dummyDescriptor === undefined) { + this.dummyDescriptor = this.getDescriptor(guaranteedErrorObject as Input); + } + + return this.dummyDescriptor; + } + + private dummyDescriptor?: StreamedArrayDescriptor; +} + +class StreamedArrayHandlerImpl implements StreamedArrayHandler { + public constructor( + private readonly partial: PartialArray, + private descriptor?: StreamedArrayDescriptor, + private readonly streamedAnyOf?: StreamedAnyOf, + ) {} + + public addObject(): StreamedObjectHandler { + this.attemptResolution(StreamedObject); + + if (this.descriptor) { + let streamedType: StreamedType | undefined = this.descriptor.items; + + if (streamedType instanceof StreamedAnyOf) { + const streamedAnyOf = streamedType; + if (!(streamedType = streamedType.streamedTypeIfSingleMatch(StreamedObject))) { + const childPartial: PartialObject = {}; + this.partial.push(childPartial); + this.lastHandler = new StreamedObjectHandlerImpl( + childPartial, + undefined, + streamedAnyOf, + ); + return this.lastHandler as StreamedObjectHandler; + } + } + + if (streamedType instanceof StreamedObject) { + const childPartial: PartialObject = {}; + this.partial.push(childPartial); + this.lastHandler = streamedType.invoke(this.partial, childPartial); + return this.lastHandler as StreamedObjectHandler; + } + } + + throw new Error("Expected object for items"); + } + + public addArray(): StreamedArrayHandler { + this.attemptResolution(StreamedArray); + + if (this.descriptor) { + const streamedType = this.descriptor.items; + + if (streamedType instanceof StreamedObject) { + const childPartial = [] as PartialArray; + this.partial.push(childPartial); + this.lastHandler = streamedType.invoke(this.partial, childPartial); + return this.lastHandler as StreamedArrayHandler; + } + } + + throw new Error("Expected array for items"); + } + + public addPrimitive(value: JsonPrimitive): void { + if (!this.descriptor) { + this.partial.push(value); + return; + } + const streamedType = this.descriptor.items; + + this.partial.push(value); + + if (primitiveMatchesStreamedType(value, streamedType)) { + this.lastHandler = (streamedType as InvocableStreamedType).invoke( + this.partial, + undefined, + ); + return; + } + + // Shouldn't happen with Structured Outputs + throw new Error(`Unexpected ${typeof value}`); + } + + public appendText(chars: string): void { + assert(typeof this.partial[this.partial.length - 1] === "string"); + + (this.partial[this.partial.length - 1] as string) += chars; + if (this.lastHandler instanceof StreamedStringPropertyHandlerImpl) { + this.lastHandler.append(chars); + } + } + + public completeLast(): void { + const value = this.partial[this.partial.length - 1]; + + if (isPrimitiveValue(value!)) { + this.attemptResolution(value as PrimitiveType); + + // Objects and Arrays will have their complete() handler called directly + completePrimitive(this.lastHandler!, value as PrimitiveType, this.partial); + } + } + + public complete(): void { + // TODO-AnyOf: + this.descriptor!.complete?.(this.partial); + } + + private attemptResolution( + typeOrValue: typeof StreamedObject | typeof StreamedArray | PrimitiveType, + ): void { + if (!this.descriptor) { + assert(this.streamedAnyOf !== undefined); + for (const option of this.streamedAnyOf!.options) { + if (option instanceof StreamedArray) { + const property = option.items; + if (streamedTypeMatches(property, typeOrValue)) { + // We now know which option in the AnyOf to use + this.descriptor = option.delayedInvoke(this.partial); + } + } + } + } + } + + private lastHandler?: ArrayAppendHandler; +} + +const primitiveMatchesStreamedType = ( + value: JsonPrimitive, + streamedType: StreamedType, +): boolean => { + if (value === null) { + return streamedType instanceof AtomicNull; + } else { + switch (typeof value) { + case "string": + return ( + streamedType instanceof StreamedStringProperty || + streamedType instanceof StreamedString || + streamedType instanceof AtomicString || + streamedType instanceof AtomicEnum + ); + case "number": + return streamedType instanceof AtomicNumber; + case "boolean": + return streamedType instanceof AtomicBoolean; + default: + assert(false); + return false; + } + } +}; + +const isPrimitiveValue = (value: JsonValue): value is JsonPrimitive => { + return ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ); +}; + +const completePrimitive = ( + handler: StreamedValueHandler, + value: PrimitiveType, + partialParent: PartialArg, +): void => { + if ( + handler instanceof StreamedStringPropertyHandlerImpl || + handler instanceof StreamedStringHandlerImpl || + handler instanceof AtomicStringHandlerImpl + ) { + handler.complete(value as string, partialParent); + } else if (handler instanceof AtomicNumberHandlerImpl) { + handler.complete(value as number, partialParent); + } else if (handler instanceof AtomicBooleanHandlerImpl) { + handler.complete(value as boolean, partialParent); + } else if (handler instanceof AtomicNullHandlerImpl) { + handler.complete(value as null, partialParent); + } +}; + +const streamedTypeMatches = ( + streamedType: StreamedType, + typeOrValue: typeof StreamedObject | typeof StreamedArray | PrimitiveType, +): boolean => { + if (typeOrValue === StreamedObject || typeOrValue === StreamedArray) { + return streamedType instanceof typeOrValue; + } else { + if (typeOrValue === null) { + return streamedType instanceof AtomicNull; + } else { + switch (typeof typeOrValue) { + case "string": + return ( + streamedType instanceof AtomicString || + (streamedType instanceof AtomicEnum && streamedType.values.includes(typeOrValue)) + ); + case "number": + return streamedType instanceof AtomicNumber; + case "boolean": + return streamedType instanceof AtomicBoolean; + default: + assert(false); + return false; + } + } + } +}; + +interface SchemaArgs { + description?: string; +} + +// TODO: Also need StreamedStringElementHandler, usable only under array items:? - implementation just appends to last item in array? +interface StreamedStringPropertyHandler { + append(chars: string): void; + complete?(value: string, partial: PartialArg): void; +} + +type StreamedStringPropertyDescriptor< + T extends Record, + P extends keyof T, +> = SchemaArgs & { + target: (partial: PartialArg) => T; + key: P; + complete?: (value: string, partial: PartialArg) => void; +}; + +class StreamedStringProperty< + T extends Record, + P extends keyof T, +> extends InvocableStreamedType { + public constructor(private readonly args: StreamedStringPropertyDescriptor) { + super(); + } + + public findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void { + if (visited.has(this)) { + addDefinition(this, definitions); + } else { + visited.add(this); + } + } + + public jsonSchema(): JsonObject { + const { description } = this.args; + + const schema: { type: string; description?: string } = { + type: "string", + }; + if (description !== undefined) { + schema.description = description; + } + return schema; + } + + public invoke(parentPartial: PartialArg): StreamedStringPropertyHandler { + const { target, key, complete } = this.args; + + const item = target(parentPartial); + item[key] = "" as T[P]; + const append = (chars: string): void => { + item[key] = (item[key] + chars) as T[P]; + }; + + return new StreamedStringPropertyHandlerImpl(append, complete); + } +} + +class StreamedStringPropertyHandlerImpl< + T extends Record, + P extends keyof T, +> implements StreamedStringPropertyHandler +{ + public constructor( + private readonly onAppend: (chars: string) => void, + private readonly onComplete?: (value: string, partial: PartialArg) => void, + ) {} + + public append(chars: string): void { + return this.onAppend(chars); + } + + public complete(value: string, partial: PartialArg): void { + this.onComplete?.(value, partial); + } +} + +interface StreamedStringHandler { + append(chars: string): void; + complete?(value: string, partial: PartialArg): void; +} + +type StreamedStringDescriptor = SchemaArgs & { + target: (partial: PartialArg) => Parent; + append: (chars: string, parent: Parent) => void; + complete?: (value: string, partial: PartialArg) => void; +}; + +class StreamedString< + Parent extends object, +> extends InvocableStreamedType { + public constructor(private readonly args: StreamedStringDescriptor) { + super(); + } + + public findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void { + if (visited.has(this)) { + addDefinition(this, definitions); + } else { + visited.add(this); + } + } + + public jsonSchema(): JsonObject { + const { description } = this.args; + + const schema: { type: string; description?: string } = { + type: "string", + }; + if (description !== undefined) { + schema.description = description; + } + return schema; + } + + public invoke(parentPartial: PartialArg): StreamedStringHandler { + const { target, append, complete } = this.args; + + const parent = target?.(parentPartial); + + return new StreamedStringHandlerImpl(parent, append, complete); + } +} + +class StreamedStringHandlerImpl implements StreamedStringHandler { + public constructor( + private readonly parent: Parent, + private readonly onAppend: (chars: string, parent: Parent) => void, + private readonly onComplete?: (value: string, partial: PartialArg) => void, + ) {} + + public append(chars: string): void { + return this.onAppend(chars, this.parent); + } + + public complete(value: string, partial: PartialArg): void { + this.onComplete?.(value, partial); + } +} + +// eslint-disable-next-line @rushstack/no-new-null +type PrimitiveType = string | number | boolean | null; + +interface AtomicPrimitiveHandler { + complete(value: T, partial: PartialArg): void; +} + +type AtomicPrimitiveDescriptor = SchemaArgs & { + values?: string[]; + complete?: (value: T, partial: PartialArg) => void; +}; + +abstract class AtomicPrimitive extends InvocableStreamedType< + AtomicPrimitiveHandler +> { + public constructor(protected descriptor?: AtomicPrimitiveDescriptor) { + super(); + } + + public findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void { + if (visited.has(this)) { + addDefinition(this, definitions); + } else { + visited.add(this); + } + } + + public jsonSchema(): JsonObject { + const description = this.descriptor?.description; + + const schema: { type: string; enum?: string[]; description?: string } = { + type: this.typeName, + }; + if (this.descriptor?.values !== undefined) { + schema.enum = this.descriptor.values; + } + if (this.descriptor?.description !== undefined) { + schema.description = description; + } + return schema; + } + + public abstract override invoke(): AtomicPrimitiveHandler; + + protected abstract typeName: string; +} + +class AtomicPrimitiveHandlerImpl + implements AtomicPrimitiveHandler +{ + public constructor(private readonly onComplete?: (value: T, partial: PartialArg) => void) {} + + public complete(value: T, partial: PartialArg): void { + if (this.onComplete) { + this.onComplete(value, partial); + } + } +} + +type AtomicStringHandler = AtomicPrimitiveHandler; +class AtomicString extends AtomicPrimitive { + public override invoke(): AtomicStringHandler { + return new AtomicStringHandlerImpl(this.descriptor?.complete); + } + + public override typeName = "string"; +} +class AtomicStringHandlerImpl extends AtomicPrimitiveHandlerImpl {} + +type AtomicEnumHandler = AtomicPrimitiveHandler; +class AtomicEnum extends AtomicPrimitive { + public override invoke(): AtomicEnumHandler { + return new AtomicEnumHandlerImpl(this.descriptor?.complete); + } + + public override typeName = "string"; + + public get values(): string[] { + // TODO-AnyOf: Expose this more cleanly + return this.descriptor!.values!; + } +} +class AtomicEnumHandlerImpl extends AtomicPrimitiveHandlerImpl {} + +type AtomicNumberHandler = AtomicPrimitiveHandler; +class AtomicNumber extends AtomicPrimitive { + public override invoke(): AtomicNumberHandler { + return new AtomicNumberHandlerImpl(this.descriptor?.complete); + } + + public override typeName = "number"; +} +class AtomicNumberHandlerImpl extends AtomicPrimitiveHandlerImpl {} + +type AtomicBooleanHandler = AtomicPrimitiveHandler; +class AtomicBoolean extends AtomicPrimitive { + public override invoke(): AtomicBooleanHandler { + return new AtomicBooleanHandlerImpl(this.descriptor?.complete); + } + + public override typeName = "boolean"; +} +class AtomicBooleanHandlerImpl extends AtomicPrimitiveHandlerImpl {} + +// eslint-disable-next-line @rushstack/no-new-null +type AtomicNullHandler = AtomicPrimitiveHandler; +class AtomicNull extends AtomicPrimitive { + public override invoke(): AtomicNullHandler { + return new AtomicNullHandlerImpl(this.descriptor?.complete); + } + + public override typeName = "null"; +} +class AtomicNullHandlerImpl extends AtomicPrimitiveHandlerImpl {} + +// TODO: Only make this legal under object properties, not array items +class StreamedOptional extends SchemaGeneratingStreamedType { + public constructor(optionalType: SchemaGeneratingStreamedType) { + assert(!(optionalType instanceof AtomicNull)); + + super(); + this.optionalType = optionalType; + } + + public findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void { + if (visited.has(this)) { + addDefinition(this, definitions); + } else { + visited.add(this); + + findDefinitions(this.optionalType, visited, definitions); + } + } + + public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { + const schema = jsonSchemaFromStreamedType(this.optionalType, root, definitions); + if (root === this.optionalType || definitions.has(this.optionalType)) { + return { anyOf: [schema, { type: "null" }] }; + } else { + assert(typeof schema.type === "string"); + schema.type = [schema.type!, "null"]; + return schema; + } + } + + public optionalType: SchemaGeneratingStreamedType; +} + +class StreamedAnyOf extends SchemaGeneratingStreamedType { + public constructor(options: SchemaGeneratingStreamedType[]) { + super(); + this.options = options; + } + + public findDefinitions( + visited: Set, + definitions: DefinitionMap, + ): void { + if (visited.has(this)) { + addDefinition(this, definitions); + } else { + visited.add(this); + + for (const streamedType of this.options) { + findDefinitions(streamedType, visited, definitions); + } + } + } + + public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { + return { + anyOf: this.options.map((streamedType) => + jsonSchemaFromStreamedType(streamedType, root, definitions), + ), + }; + } + + public streamedTypeIfSingleMatch( + classType: typeof StreamedObject | typeof StreamedArray, + ): StreamedType | undefined { + // If there is exactly one child StreamedType that is of the given type, return it + let streamedType: StreamedType | undefined; + for (const option of this.options) { + // TODO-AnyOf: Must also consider Optional and AnyOf + if (option instanceof classType) { + if (streamedType) { + return undefined; + } + streamedType = option; + } + } + return streamedType; + } + + public streamedTypeOfFirstMatch( + // eslint-disable-next-line @rushstack/no-new-null + value: string | number | boolean | null, + ): StreamedType | undefined { + for (const option of this.options) { + // TODO-AnyOf: Must also consider Optional and AnyOf + if (value === null && option instanceof AtomicNull) { + return option; + } else { + switch (typeof value) { + case "string": + if (option instanceof AtomicString || option instanceof AtomicEnum) { + return option; + } + break; + case "number": + if (option instanceof AtomicNumber) { + return option; + } + break; + case "boolean": + if (option instanceof AtomicBoolean) { + return option; + } + break; + default: + break; + } + } + } + return undefined; + } + + public options: SchemaGeneratingStreamedType[]; +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts new file mode 100644 index 000000000000..ba83fa522dab --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts @@ -0,0 +1,565 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { assert } from "./debug.js"; + +/** + * TBD + */ +// eslint-disable-next-line @rushstack/no-new-null +export type JsonPrimitive = string | number | boolean | null; + +/** + * TBD + */ +// eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style +export interface JsonObject { + [key: string]: JsonValue; +} +/** + * TBD + */ +export type JsonArray = JsonValue[]; +/** + * TBD + */ +export type JsonValue = JsonPrimitive | JsonObject | JsonArray; + +/** + * TBD + */ +export type JsonBuilderContext = + | { parentObject: ObjectHandle; key: string } + | { parentArray: ArrayHandle }; + +/** + * TBD + */ +export interface JsonBuilder { + addObject(context?: JsonBuilderContext): ObjectHandle; + addArray(context?: JsonBuilderContext): ArrayHandle; + addPrimitive( + value: JsonPrimitive, + context?: JsonBuilderContext, + ): void; + appendText(chars: string, context?: JsonBuilderContext): void; + completeContext(context?: JsonBuilderContext): void; + completeContainer(container: ObjectHandle | ArrayHandle): void; +} + +/** + * TBD + */ +export function contextIsObject( + context?: JsonBuilderContext, +): context is { parentObject: ObjectHandle; key: string } { + return context !== undefined && "parentObject" in context; +} + +/** + * TBD + */ +export interface StreamedJsonParser { + addChars(text: string): void; +} + +/** + * TBD + */ +export function createStreamedJsonParser( + builder: JsonBuilder, + abortController: AbortController, +): StreamedJsonParser { + return new JsonParserImpl(builder, abortController); +} + +// Implementation + +const smoothStreaming = false; + +// prettier-ignore +enum State { + Start, + End, + + InsideObjectAtStart, + InsideObjectAfterKey, + InsideObjectAfterColon, + InsideObjectAfterProperty, + InsideObjectAfterComma, + InsideArrayAtStart, + InsideArrayAfterElement, + InsideArrayAfterComma, + InsideMarkdownAtStart, + InsideMarkdownAtEnd, + + // Special states while processing multi-character tokens (which may not arrive all at once) + InsideKeyword, + InsideNumber, + InsideKey, + InsideString, + InsideLeadingMarkdownDelimiter, + InsideTrailingMarkdownDelimiter, + + // Special momentary state + Pop, +} + +// Grammar productions - includes individual tokens +// prettier-ignore +enum Production { + Value, + Key, + Colon, + Comma, + CloseBrace, + CloseBracket, + LeadingMarkdownDelimiter, + TrailingMarkdownDelimiter, +} + +type StateTransition = [Production, State]; + +// prettier-ignore +const stateTransitionTable = new Map([ + [ + State.Start, + [ + [Production.Value, State.End], + [Production.LeadingMarkdownDelimiter, State.InsideMarkdownAtStart], + ], + ], + [ + State.InsideObjectAtStart, + [ + [Production.Key, State.InsideObjectAfterKey], + [Production.CloseBrace, State.Pop], + ], + ], + [State.InsideObjectAfterKey, [[Production.Colon, State.InsideObjectAfterColon]]], + [State.InsideObjectAfterColon, [[Production.Value, State.InsideObjectAfterProperty]]], + [ + State.InsideObjectAfterProperty, + [ + [Production.Comma, State.InsideObjectAfterComma], + [Production.CloseBrace, State.Pop], + ], + ], + [State.InsideObjectAfterComma, [[Production.Key, State.InsideObjectAfterKey]]], + [ + State.InsideArrayAtStart, + [ + [Production.Value, State.InsideArrayAfterElement], + [Production.CloseBracket, State.Pop], + ], + ], + [ + State.InsideArrayAfterElement, + [ + [Production.Comma, State.InsideArrayAfterComma], + [Production.CloseBracket, State.Pop], + ], + ], + [State.InsideArrayAfterComma, [[Production.Value, State.InsideArrayAfterElement]]], + [State.InsideMarkdownAtStart, [[Production.Value, State.InsideMarkdownAtEnd]]], + [State.InsideMarkdownAtEnd, [[Production.TrailingMarkdownDelimiter, State.End]]], +]); + +const keywords = ["true", "false", "null"]; +// eslint-disable-next-line unicorn/no-null +const keywordValues = [true, false, null]; + +interface ParserContext { + state: State; + firstToken: string; + parentObject?: ObjectHandle; + key?: string; + parentArray?: ArrayHandle; +} + +class JsonParserImpl implements StreamedJsonParser { + public constructor( + private readonly builder: JsonBuilder, + private readonly abortController: AbortController, + ) {} + + public addChars(text: string): void { + this.buffer += text; + + if (!this.throttled) { + while (this.processJsonText()) { + // Process as much of the buffer as possible + } + } + } + + // Implementation + + private buffer: string = ""; // This could be something more efficient + private throttled = false; + private readonly contexts: ParserContext[] = [ + { state: State.Start, firstToken: "" }, + ]; + + // Returns true if another token should be processed + private processJsonText(): boolean { + // Exit if there's nothing to process or the fetch has been aborted + if (this.buffer.length === 0 || this.abortController.signal.aborted) { + return false; + } + + const state = this.contexts[this.contexts.length - 1]!.state; + + // Are we in the midst of a multi-character token? + switch (state) { + case State.InsideKeyword: + return this.processJsonKeyword(); + + case State.InsideNumber: + return this.processJsonNumber(); + + case State.InsideKey: + return this.processJsonKey(); + + case State.InsideString: + return this.processJsonStringCharacters(); + + case State.InsideLeadingMarkdownDelimiter: + return this.processLeadingMarkdownDelimiter(); + + case State.InsideTrailingMarkdownDelimiter: + return this.processTrailingMarkdownDelimiter(); + + default: + break; + } + + // We're between tokens, so trim leading whitespace + // this.buffer = this.buffer.trimStart(); // REVIEW: Requires es2019 or later + this.buffer = this.buffer.replace(/^\s+/, ""); + + // Again, exit if there's nothing left to process + if (this.buffer.length === 0) { + return false; + } + + // If we're already done, there shouldn't be anything left to process + if (state === State.End) { + // REVIEW: Shouldn't be necessary with GPT4o, especially with Structured Output + this.buffer = ""; + return false; + // throw new Error("JSON already complete"); + } + + const builderContext = this.builderContextFromParserContext( + this.contexts[this.contexts.length - 1]!, + )!; + + // Start a new token + const char = this.buffer[0]!; + // eslint-disable-next-line unicorn/prefer-code-point + const charCode = char.charCodeAt(0); + + switch (charCode) { + case 123: // '{' + this.consumeCharAndPush(State.InsideObjectAtStart, { + parentObject: this.builder.addObject(builderContext), + }); + break; + case 58: // ':' + this.consumeCharAndEnterNextState(Production.Colon); + break; + case 44: // ',' + this.consumeCharAndEnterNextState(Production.Comma); + break; + case 125: // '}' + this.consumeCharAndEnterNextState(Production.CloseBrace); + break; + case 91: // '[' + this.consumeCharAndPush(State.InsideArrayAtStart, { + parentArray: this.builder.addArray(builderContext), + }); + break; + case 93: // ']' + this.consumeCharAndEnterNextState(Production.CloseBracket); + break; + case 34: // '"' + if (state === State.InsideObjectAtStart || state === State.InsideObjectAfterComma) { + // Keys shouldn't be updated incrementally, so wait until the complete key has arrived + this.pushContext(State.InsideKey, char); + } else { + this.builder.addPrimitive("", builderContext); + this.consumeCharAndPush(State.InsideString); + } + break; + case 116: // 't' + case 102: // 'f' + case 110: // 'n' + this.pushContext(State.InsideKeyword, char); + break; + case 45: // '-' + this.pushContext(State.InsideNumber, char); + break; + default: + if (charCode >= 48 && charCode <= 57) { + // '0' - '9' + this.pushContext(State.InsideNumber, char); + } else if (charCode === 96) { + // '`' + if (state === State.Start) { + this.pushContext(State.InsideLeadingMarkdownDelimiter, char); + } else if (state === State.InsideMarkdownAtEnd) { + this.pushContext(State.InsideTrailingMarkdownDelimiter, char); + } else { + this.unexpectedTokenError(char); + } + } else { + this.unexpectedTokenError(char); + } + break; + } + + return this.buffer.length > 0; + } + + private processLeadingMarkdownDelimiter(): boolean { + const leadingMarkdownDelimiter = "```json"; + if (this.buffer.startsWith(leadingMarkdownDelimiter)) { + this.buffer = this.buffer.slice(leadingMarkdownDelimiter.length); + this.popContext(Production.LeadingMarkdownDelimiter); + return this.buffer.length > 0; + } + + return false; + } + + private processTrailingMarkdownDelimiter(): boolean { + const trailingMarkdownDelimiter = "```"; + if (this.buffer.startsWith(trailingMarkdownDelimiter)) { + this.buffer = this.buffer.slice(trailingMarkdownDelimiter.length); + this.popContext(Production.TrailingMarkdownDelimiter); + return this.buffer.length > 0; + } + + return false; + } + + private processJsonKeyword(): boolean { + // Just match the keyword, let the next iteration handle the next characters + for (let i = 0; i < keywords.length; i++) { + const keyword = keywords[i]!; + if (this.buffer.startsWith(keyword)) { + this.buffer = this.buffer.slice(keyword.length); + this.setPrimitiveValueAndPop(keywordValues[i]!); + return true; + } else if (keyword.startsWith(this.buffer)) { + return false; + } + } + + this.unexpectedTokenError(this.buffer); + return false; + } + + private processJsonNumber(): boolean { + // Match the number plus a single non-number character (so we know the number is complete) + const jsonNumber = /^-?(0|([1-9]\d*))(\.\d+)?([Ee][+-]?\d+)?(?=\s|\D)/; + const match = this.buffer.match(jsonNumber); + if (match) { + const numberText = match[0]; + this.buffer = this.buffer.slice(numberText.length); + this.setPrimitiveValueAndPop(+numberText); // Unary + parses the numeric string + return true; + } + + return false; + } + + private processJsonKey(): boolean { + // Match the complete string, including start and end quotes + assert(this.buffer.startsWith('"')); + const jsonStringRegex = /^"((?:[^"\\]|\\.)*)"/; + const match = this.buffer.match(jsonStringRegex); + + if (match) { + const keyText = match[0]; + this.buffer = this.buffer.slice(keyText.length); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const key = JSON.parse(keyText); + + assert(this.contexts.length > 1); + const parentContext = this.contexts[this.contexts.length - 2]!; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + parentContext.key = key; + + this.popContext(Production.Key); + return true; + } + + return false; + } + + // String values are special because we might stream them + private processJsonStringCharacters(): boolean { + let maxCount = Number.POSITIVE_INFINITY; + + if (smoothStreaming) { + maxCount = 5; + } + + this.appendText(this.convertJsonStringCharacters(maxCount)); + + if (this.buffer.startsWith('"')) { + // The end of the string was reached + this.buffer = this.buffer.slice(1); + this.completePrimitiveAndPop(); + return this.buffer.length > 0; + } else if (this.buffer.length > 0) { + this.throttled = true; + setTimeout(() => { + this.throttled = false; + while (this.processJsonText()) { + // Process characters until it's time to pause again + } + }, 15); + } + + return false; + } + + private convertJsonStringCharacters(maxCount: number): string { + let escapeNext = false; + + let i = 0; + for (; i < Math.min(maxCount, this.buffer.length); i++) { + const char = this.buffer[i]; + + if (escapeNext) { + escapeNext = false; // JSON.parse will ensure valid escape sequence + } else if (char === "\\") { + escapeNext = true; + } else if (char === '"') { + // Unescaped " is reached + break; + } + } + + if (escapeNext) { + // Buffer ends with a single '\' character + i--; + } + + const result = this.buffer.slice(0, i); + this.buffer = this.buffer.slice(i); + return JSON.parse(`"${result}"`) as string; + } + + private appendText(text: string): void { + assert(this.contexts.length > 1); + const builderContext = this.builderContextFromParserContext( + this.contexts[this.contexts.length - 2]!, + )!; + this.builder.appendText(text, builderContext); + } + + private consumeCharAndPush( + state: State, + parent?: { parentObject?: ObjectHandle; parentArray?: ArrayHandle }, + ): void { + const firstToken = this.buffer[0]!; + this.buffer = this.buffer.slice(1); + this.pushContext(state, firstToken, parent); + } + + private pushContext( + state: State, + firstToken: string, + parent?: { parentObject?: ObjectHandle; parentArray?: ArrayHandle }, + ): void { + this.contexts.push({ + state, + firstToken, + parentObject: parent?.parentObject, + parentArray: parent?.parentArray, + }); + } + + private setPrimitiveValueAndPop(value: JsonPrimitive): void { + this.builder.addPrimitive( + value, + this.builderContextFromParserContext(this.contexts[this.contexts.length - 2]!), + ); + this.completePrimitiveAndPop(); + } + + private completePrimitiveAndPop(): void { + this.builder.completeContext( + this.builderContextFromParserContext(this.contexts[this.contexts.length - 2]!), + ); + this.popContext(Production.Value); + } + + private completeAndPopContext(): void { + const context = this.contexts[this.contexts.length - 1]!; + if (context.parentObject !== undefined) { + this.builder.completeContainer?.(context.parentObject); + } else if (context.parentArray !== undefined) { + this.builder.completeContainer?.(context.parentArray); + } + this.builder.completeContext?.( + this.builderContextFromParserContext(this.contexts[this.contexts.length - 2]!), + ); + this.popContext(); + } + + private builderContextFromParserContext( + context: ParserContext, + ): JsonBuilderContext | undefined { + if (context.parentObject !== undefined) { + return { parentObject: context.parentObject, key: context.key! }; + // eslint-disable-next-line unicorn/no-negated-condition + } else if (context.parentArray !== undefined) { + return { parentArray: context.parentArray }; + } else { + return undefined; + } + } + + private popContext(production = Production.Value): void { + assert(this.contexts.length > 1); + const poppedContext = this.contexts.pop()!; + this.nextState(poppedContext.firstToken, production); + } + + private consumeCharAndEnterNextState(production: Production): void { + const token = this.buffer[0]!; + this.buffer = this.buffer.slice(1); + this.nextState(token, production); + } + + private nextState(token: string, production: Production): void { + const context = this.contexts[this.contexts.length - 1]!; + + const stateTransitions = stateTransitionTable.get(context.state); + assert(stateTransitions !== undefined); + for (const [productionCandidate, nextState] of stateTransitions!) { + if (productionCandidate === production) { + if (nextState === State.Pop) { + this.completeAndPopContext(); + } else { + context.state = nextState; + } + return; + } + } + + this.unexpectedTokenError(token); + } + + private unexpectedTokenError(token: string): void { + throw new Error(`Unexpected token ${token}`); + } +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts new file mode 100644 index 000000000000..150559410778 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -0,0 +1,273 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { assert } from "@fluidframework/core-utils/internal"; +import { + TreeNode, + NodeKind, + normalizeFieldSchema, + type ImplicitFieldSchema, + type TreeFieldFromImplicitField, + type TreeView, + getJsonSchema, + type JsonFieldSchema, + type JsonNodeSchema, + type JsonSchemaRef, + type JsonTreeSchema, +} from "@fluidframework/tree/internal"; + +import { objectIdKey, type TreeEdit } from "./agentEditTypes.js"; +import { IdGenerator } from "./idGenerator.js"; +import { fail } from "./utils.js"; + +/** + * TBD + */ +export type EditLog = { + edit: TreeEdit; + error?: string; +}[]; + +/** + * TBD + */ +export function toDecoratedJson( + idGenerator: IdGenerator, + root: TreeFieldFromImplicitField, +): string { + idGenerator.assignIds(root); + const stringified: string = JSON.stringify(root, (_, value) => { + if (typeof value === "object" && !Array.isArray(value) && value !== null) { + assert(value instanceof TreeNode, "Non-TreeNode value in tree."); + const objId = + idGenerator.getId(value) ?? fail("ID of new node should have been assigned."); + assert( + !Object.prototype.hasOwnProperty.call(value, objectIdKey), + `Collision of object id property.`, + ); + return { + [objectIdKey]: objId, + ...value, + } as unknown; + } + return value as unknown; + }); + return stringified; +} + +/** + * TBD + */ +export function getSuggestingSystemPrompt( + view: TreeView, + suggestionCount: number, + userGuidance?: string, +): string { + const schema = normalizeFieldSchema(view.schema); + const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); + const decoratedTreeJson = toDecoratedJson(new IdGenerator(), view.root); + const guidance = + userGuidance === undefined + ? "" + : `Additionally, the user has provided some guidance to help you refine your suggestions. Here is that guidance: ${userGuidance}`; + + // TODO: security: user prompt in system prompt + return ` + You are a collaborative agent who suggests possible changes to a JSON tree that follows a specific schema. + For example, for a schema of a digital whiteboard application, you might suggest things like "Change the color of all sticky notes to blue" or "Align all the handwritten text vertically". + Or, for a schema of a calendar application, you might suggest things like "Move the meeting with Alice to 3pm" or "Add a new event called 'Lunch with Bob' on Friday". + The tree that you are suggesting for is a JSON object with the following schema: ${promptFriendlySchema} + The current state of the tree is: ${decoratedTreeJson}. + ${guidance} + Please generate exactly ${suggestionCount} suggestions for changes to the tree that you think would be useful.`; +} + +/** + * TBD + */ +export function getEditingSystemPrompt( + userPrompt: string, + idGenerator: IdGenerator, + view: TreeView, + log: EditLog, + appGuidance?: string, +): string { + const schema = normalizeFieldSchema(view.schema); + const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); + const decoratedTreeJson = toDecoratedJson(idGenerator, view.root); + + function createEditList(edits: EditLog): string { + return edits + .map((edit, index) => { + const error = + edit.error === undefined + ? "" + : ` This edit produced an error, and was discarded. The error message was: ${edit.error}`; + return `${index + 1}. ${JSON.stringify(edit.edit)}${error}`; + }) + .join("\n"); + } + + const role = `You are a collaborative agent who interacts with a JSON tree by performing edits to achieve a user-specified goal.${ + appGuidance === undefined + ? "" + : ` + The application that owns the JSON tree has the following guidance about your role: ${appGuidance}` + }`; + + // TODO: security: user prompt in system prompt + const systemPrompt = ` + ${role} + Edits are composed of the following primitives: + - ObjectTarget: a reference to an object (as specified by objectId). + - Place: either before or after a ObjectTarget (only makes sense for objects in arrays). + - ArrayPlace: either the "start" or "end" of an array, as specified by a "parent" ObjectTarget and a "field" name under which the array is stored. + - Range: a range of objects within the same array specified by a "start" and "end" Place. The range MUST be in the same array. + - Selection: a ObjectTarget or a Range. + The edits you may perform are: + - SetRoot: replaces the tree with a specific value. This is useful for initializing the tree or replacing the state entirely if appropriate. + - Insert: inserts a new object at a specific Place or ArrayPlace. + - Modify: sets a field on a specific ObjectTarget. + - Remove: deletes a Selection from the tree. + - Move: moves a Selection to a new Place or ArrayPlace. + The tree is a JSON object with the following schema: ${promptFriendlySchema} + ${ + log.length === 0 + ? "" + : `You have already performed the following edits: + ${createEditList(log)} + This means that the current state of the tree reflects these changes.` + } + The current state of the tree is: ${decoratedTreeJson}. + Before you made the above edits, the user requested you accomplish the following goal: + ${userPrompt} + If the goal is now completed, you should return null. + Otherwise, you should create an edit that makes progress towards the goal. It should have an english description ("explanation") of what edit to perform (specifying one of the allowed edit types).`; + return systemPrompt; +} + +/** + * TBD + */ +export function getReviewSystemPrompt( + userPrompt: string, + idGenerator: IdGenerator, + view: TreeView, + originalDecoratedJson: string, + appGuidance?: string, +): string { + const schema = normalizeFieldSchema(view.schema); + const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); + const decoratedTreeJson = toDecoratedJson(idGenerator, view.root); + + const role = `You are a collaborative agent who interacts with a JSON tree by performing edits to achieve a user-specified goal.${ + appGuidance === undefined + ? "" + : ` + The application that owns the JSON tree has the following guidance: ${appGuidance}` + }`; + + // TODO: security: user prompt in system prompt + const systemPrompt = ` + ${role} + You have performed a number of actions already to accomplish a user request. + You must review the resulting state to determine if the actions you performed successfully accomplished the user's goal. + The tree is a JSON object with the following schema: ${promptFriendlySchema} + The state of the tree BEFORE changes was: ${originalDecoratedJson}. + The state of the tree AFTER changes is: ${decoratedTreeJson}. + The user requested that the following goal should be accomplished: + ${userPrompt} + Was the goal accomplished?`; + return systemPrompt; +} + +/** + * TBD + */ +export function getPromptFriendlyTreeSchema(jsonSchema: JsonTreeSchema): string { + let stringifiedSchema = ""; + Object.entries(jsonSchema.$defs).forEach(([name, def]) => { + if (def.type !== "object" || def._treeNodeSchemaKind === NodeKind.Map) { + return; + } + + let stringifiedEntry = `interface ${getFriendlySchemaName(name)} {`; + + Object.entries(def.properties).forEach(([fieldName, fieldSchema]) => { + let typeString: string; + if (isJsonSchemaRef(fieldSchema)) { + const nextFieldName = fieldSchema.$ref; + const nextDef = getDef(jsonSchema.$defs, nextFieldName); + typeString = `${getTypeString(jsonSchema.$defs, [nextFieldName, nextDef])}`; + } else { + typeString = `${getAnyOfTypeString(jsonSchema.$defs, fieldSchema.anyOf, true)}`; + } + if (def.required && !def.required.includes(fieldName)) { + typeString = `${typeString} | undefined`; + } + stringifiedEntry += ` ${fieldName}: ${typeString};`; + }); + + stringifiedEntry += " }"; + + stringifiedSchema += (stringifiedSchema === "" ? "" : " ") + stringifiedEntry; + }); + return stringifiedSchema; +} + +function getTypeString( + defs: Record, + [name, currentDef]: [string, JsonNodeSchema], +): string { + const { _treeNodeSchemaKind } = currentDef; + if (_treeNodeSchemaKind === NodeKind.Leaf) { + return currentDef.type; + } + if (_treeNodeSchemaKind === NodeKind.Object) { + return getFriendlySchemaName(name); + } + if (_treeNodeSchemaKind === NodeKind.Array) { + const items = currentDef.items; + const innerType = isJsonSchemaRef(items) + ? getTypeString(defs, [items.$ref, getDef(defs, items.$ref)]) + : getAnyOfTypeString(defs, items.anyOf); + return `${innerType}[]`; + } + fail("Non-object, non-leaf, non-array schema type."); +} + +function getAnyOfTypeString( + defs: Record, + refList: JsonSchemaRef[], + topLevel = false, +): string { + const typeNames: string[] = []; + refList.forEach((ref) => { + typeNames.push(getTypeString(defs, [ref.$ref, getDef(defs, ref.$ref)])); + }); + const typeString = typeNames.join(" | "); + return topLevel ? typeString : `(${typeString})`; +} + +function isJsonSchemaRef(field: JsonFieldSchema): field is JsonSchemaRef { + return (field as JsonSchemaRef).$ref !== undefined; +} + +function getDef(defs: Record, ref: string): JsonNodeSchema { + // strip the "#/$defs/" prefix + const strippedRef = ref.slice(8); + const nextDef = defs[strippedRef]; + assert(nextDef !== undefined, "Ref not found."); + return nextDef; +} + +function getFriendlySchemaName(schemaName: string): string { + const matches = schemaName.match(/[^.]+$/); + if (matches === null) { + // empty scope + return schemaName; + } + return matches[0]; +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/utils.ts b/packages/framework/ai-collab/src/explicit-strategy/utils.ts new file mode 100644 index 000000000000..9cbbbf845a18 --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/utils.ts @@ -0,0 +1,65 @@ +import type { TreeNode } from "@fluidframework/tree"; + +/** + * Subset of Map interface. + * + * @remarks - originally from tree/src/utils.ts + */ +export interface MapGetSet { + get(key: K): V | undefined; + set(key: K, value: V): void; +} + +/** + * TBD + */ +export function fail(message: string): never { + throw new Error(message); +} + +/** + * Map one iterable to another by transforming each element one at a time + * @param iterable - the iterable to transform + * @param map - the transformation function to run on each element of the iterable + * @returns a new iterable of elements which have been transformed by the `map` function + * + * @remarks - originally from tree/src/utils.ts + */ +export function* mapIterable( + iterable: Iterable, + map: (t: T) => U, +): IterableIterator { + for (const t of iterable) { + yield map(t); + } +} + +/** + * Retrieve a value from a map with the given key, or create a new entry if the key is not in the map. + * @param map - The map to query/update + * @param key - The key to lookup in the map + * @param defaultValue - a function which returns a default value. This is called and used to set an initial value for the given key in the map if none exists + * @returns either the existing value for the given key, or the newly-created value (the result of `defaultValue`) + * + * @remarks - originally from tree/src/utils.ts + */ +export function getOrCreate( + map: MapGetSet, + key: K, + defaultValue: (key: K) => V, +): V { + let value = map.get(key); + if (value === undefined) { + value = defaultValue(key); + map.set(key, value); + } + return value; +} + +/** + * Checks if the given object is an {@link TreeNode}. + */ +export function isTreeNode(obj: unknown): obj is TreeNode { + // Check if the object is not null and has the private brand field + return obj !== null && typeof obj === "object" && "#brand" in obj; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a45541df355..71fcfb2f3840 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10440,6 +10440,12 @@ importers: packages/framework/ai-collab: dependencies: + '@fluidframework/core-utils': + specifier: workspace:~ + version: link:../../common/core-utils + '@fluidframework/telemetry-utils': + specifier: workspace:~ + version: link:../../utils/telemetry-utils '@fluidframework/tree': specifier: workspace:~ version: link:../../dds/tree From ab3b54316f60b4baf61091600f77f163bdb8de13 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:01:46 +0000 Subject: [PATCH 02/28] New SharedTree exports for aiCollab library --- packages/dds/tree/api-report/tree.alpha.api.md | 1 + packages/dds/tree/api-report/tree.beta.api.md | 1 + .../dds/tree/api-report/tree.legacy.alpha.api.md | 1 + .../dds/tree/api-report/tree.legacy.public.api.md | 1 + packages/dds/tree/api-report/tree.public.api.md | 1 + packages/dds/tree/package.json | 2 -- packages/dds/tree/src/index.ts | 1 + .../dds/tree/src/simple-tree/api/getSimpleSchema.ts | 2 ++ .../dds/tree/src/simple-tree/api/simpleSchema.ts | 6 +++--- .../src/simple-tree/api/simpleSchemaToJsonSchema.ts | 8 ++++++-- .../src/simple-tree/api/viewSchemaToSimpleSchema.ts | 10 ++++++---- packages/dds/tree/src/simple-tree/schemaTypes.ts | 6 ++++++ .../api/simpleSchemaToJsonSchema.spec.ts | 13 +++++++++++-- .../src/test/simple-tree/getSimpleSchema.spec.ts | 1 - .../api-report/fluid-framework.alpha.api.md | 1 + .../api-report/fluid-framework.beta.api.md | 1 + .../api-report/fluid-framework.legacy.alpha.api.md | 1 + .../api-report/fluid-framework.legacy.public.api.md | 1 + .../api-report/fluid-framework.public.api.md | 1 + 19 files changed, 45 insertions(+), 14 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index e924df3b12e1..e6f7bdd3b499 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -93,6 +93,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.beta.api.md b/packages/dds/tree/api-report/tree.beta.api.md index 56e084673f86..809da6e4c399 100644 --- a/packages/dds/tree/api-report/tree.beta.api.md +++ b/packages/dds/tree/api-report/tree.beta.api.md @@ -69,6 +69,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.legacy.alpha.api.md b/packages/dds/tree/api-report/tree.legacy.alpha.api.md index e319f2211630..85d599ec6de1 100644 --- a/packages/dds/tree/api-report/tree.legacy.alpha.api.md +++ b/packages/dds/tree/api-report/tree.legacy.alpha.api.md @@ -69,6 +69,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.legacy.public.api.md b/packages/dds/tree/api-report/tree.legacy.public.api.md index e4d0dde8247e..7761431dbdcb 100644 --- a/packages/dds/tree/api-report/tree.legacy.public.api.md +++ b/packages/dds/tree/api-report/tree.legacy.public.api.md @@ -69,6 +69,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.public.api.md b/packages/dds/tree/api-report/tree.public.api.md index e4d0dde8247e..7761431dbdcb 100644 --- a/packages/dds/tree/api-report/tree.public.api.md +++ b/packages/dds/tree/api-report/tree.public.api.md @@ -69,6 +69,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/package.json b/packages/dds/tree/package.json index 9b21cde0a55c..801f7d4ada3f 100644 --- a/packages/dds/tree/package.json +++ b/packages/dds/tree/package.json @@ -159,7 +159,6 @@ "@fluidframework/core-utils": "workspace:~", "@fluidframework/datastore-definitions": "workspace:~", "@fluidframework/driver-definitions": "workspace:~", - "@fluidframework/id-compressor": "workspace:~", "@fluidframework/runtime-definitions": "workspace:~", "@fluidframework/runtime-utils": "workspace:~", "@fluidframework/shared-object-base": "workspace:~", @@ -183,7 +182,6 @@ "@fluidframework/container-definitions": "workspace:~", "@fluidframework/container-loader": "workspace:~", "@fluidframework/eslint-config-fluid": "^5.4.0", - "@fluidframework/test-runtime-utils": "workspace:~", "@fluidframework/test-utils": "workspace:~", "@fluidframework/tree-previous": "npm:@fluidframework/tree@~2.3.0", "@microsoft/api-extractor": "7.47.8", diff --git a/packages/dds/tree/src/index.ts b/packages/dds/tree/src/index.ts index e875da97e530..d3e7922fe996 100644 --- a/packages/dds/tree/src/index.ts +++ b/packages/dds/tree/src/index.ts @@ -162,6 +162,7 @@ export { normalizeFieldSchema, isTreeNodeSchemaClass, normalizeAllowedTypes, + getSimpleSchema, } from "./simple-tree/index.js"; export { SharedTree, diff --git a/packages/dds/tree/src/simple-tree/api/getSimpleSchema.ts b/packages/dds/tree/src/simple-tree/api/getSimpleSchema.ts index 75bfec1ace08..ca3f8e38b758 100644 --- a/packages/dds/tree/src/simple-tree/api/getSimpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/getSimpleSchema.ts @@ -63,6 +63,8 @@ const simpleSchemaCache = new WeakMap(); * * @privateRemarks In the future, we may wish to move this to a more discoverable API location. * For now, while still an experimental API, it is surfaced as a free function. + * + * @internal */ export function getSimpleSchema(schema: ImplicitFieldSchema): SimpleTreeSchema { return getOrCreate(simpleSchemaCache, schema, () => toSimpleTreeSchema(schema)); diff --git a/packages/dds/tree/src/simple-tree/api/simpleSchema.ts b/packages/dds/tree/src/simple-tree/api/simpleSchema.ts index e7c820dc8a5d..d4b8a51a0dbb 100644 --- a/packages/dds/tree/src/simple-tree/api/simpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/simpleSchema.ts @@ -5,7 +5,7 @@ import type { ValueSchema } from "../../core/index.js"; import type { NodeKind } from "../core/index.js"; -import type { FieldKind } from "../schemaTypes.js"; +import type { FieldKind, FieldSchemaMetadata } from "../schemaTypes.js"; /** * Base interface for all {@link SimpleNodeSchema} implementations. @@ -120,9 +120,9 @@ export interface SimpleFieldSchema { readonly allowedTypes: ReadonlySet; /** - * {@inheritDoc FieldSchemaMetadata.description} + * {@inheritDoc FieldSchemaMetadata} */ - readonly description?: string | undefined; + readonly metadata?: FieldSchemaMetadata | undefined; } /** diff --git a/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts b/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts index 23005725d924..e1c5f6f480e6 100644 --- a/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts @@ -137,6 +137,10 @@ function convertObjectNodeSchema(schema: SimpleObjectNodeSchema): JsonObjectNode const properties: Record = {}; const required: string[] = []; for (const [key, value] of Object.entries(schema.fields)) { + if (value.metadata?.omitFromJson === true) { + // Don't emit JSON Schema for fields which specify they should be excluded. + continue; + } const allowedTypes: JsonSchemaRef[] = []; for (const allowedType of value.allowedTypes) { allowedTypes.push(createSchemaRef(allowedType)); @@ -150,8 +154,8 @@ function convertObjectNodeSchema(schema: SimpleObjectNodeSchema): JsonObjectNode }; // Don't include "description" property at all if it's not present in the input. - if (value.description !== undefined) { - output.description = value.description; + if (value.metadata?.description !== undefined) { + output.description = value.metadata.description; } properties[key] = output; diff --git a/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts b/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts index 891bdc0f960e..59d4b571674b 100644 --- a/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/viewSchemaToSimpleSchema.ts @@ -43,7 +43,9 @@ export function toSimpleTreeSchema(schema: ImplicitFieldSchema): SimpleTreeSchem // Include the "description" property only if it's present on the input. if (normalizedSchema.metadata?.description !== undefined) { - output.description = normalizedSchema.metadata.description; + output.metadata = { + description: normalizedSchema.metadata.description, + }; } return output; @@ -140,9 +142,9 @@ function fieldSchemaToSimpleSchema(schema: FieldSchema): SimpleFieldSchema { allowedTypes, }; - // Don't include "description" property at all if it's not present. - if (schema.metadata?.description !== undefined) { - result.description = schema.metadata.description; + // Don't include "metadata" property at all if it's not present. + if (schema.metadata !== undefined) { + result.metadata = schema.metadata; } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/dds/tree/src/simple-tree/schemaTypes.ts b/packages/dds/tree/src/simple-tree/schemaTypes.ts index bd6e5f88f447..b3435c02b634 100644 --- a/packages/dds/tree/src/simple-tree/schemaTypes.ts +++ b/packages/dds/tree/src/simple-tree/schemaTypes.ts @@ -230,6 +230,12 @@ export interface FieldSchemaMetadata { * used as the `description` field. */ readonly description?: string | undefined; + + /** + * Whether or not to include the field in JSON output generated by `getJsonSchema` (experimental). + * @defaultValue `false` + */ + readonly omitFromJson?: boolean | undefined; } /** diff --git a/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts index 9d7ffc13a123..c53eb3389453 100644 --- a/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts @@ -172,12 +172,21 @@ describe("simpleSchemaToJsonSchema", () => { "foo": { kind: FieldKind.Optional, allowedTypes: new Set(["test.number"]), - description: "A number representing the concept of Foo.", + metadata: { description: "A number representing the concept of Foo." }, }, "bar": { kind: FieldKind.Required, allowedTypes: new Set(["test.string"]), - description: "A string representing the concept of Bar.", + metadata: { description: "A string representing the concept of Bar." }, + }, + "id": { + kind: FieldKind.Identifier, + allowedTypes: new Set(["test.string"]), + metadata: { + description: "Unique identifier for the test object.", + // IDs should be generated by the system. Hide from the JSON Schema. + omitFromJson: true, + }, }, }, }, diff --git a/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts index 0587f7da0fdd..8ea48173ed23 100644 --- a/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts @@ -34,7 +34,6 @@ describe("getSimpleSchema", () => { ], ]), allowedTypes: new Set(["com.fluidframework.leaf.string"]), - description: "An optional string.", }; assert.deepEqual(actual, expected); }); diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md index a3dbff86a519..3cdb3a60eae7 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md @@ -137,6 +137,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md index 2a9f68855e1b..36111ba89a5f 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md @@ -110,6 +110,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md index 397d457a3bd9..b303e15d5788 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md @@ -113,6 +113,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md index 1f798cf2f200..6a473ab88a1f 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md @@ -110,6 +110,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md index 1192579ce6b2..4ef037ab2b98 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md @@ -110,6 +110,7 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; + readonly omitFromJson?: boolean | undefined; } // @public From 822092e204ce379b1ef0bcb9f0b89a67d1d1dc7d Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:04:10 +0000 Subject: [PATCH 03/28] Updates ai-collab to include the explicit strategy and export it under one shared api surface --- .../api-report/ai-collab.alpha.api.md | 63 +++ packages/framework/ai-collab/package.json | 3 + packages/framework/ai-collab/src/aiCollab.ts | 35 ++ .../framework/ai-collab/src/aiCollabApi.ts | 73 ++++ .../src/explicit-strategy/agentEditReducer.ts | 43 +-- .../src/explicit-strategy/handlers.ts | 8 +- .../src/explicit-strategy/idGenerator.ts | 3 +- .../ai-collab/src/explicit-strategy/index.ts | 364 ++++++++++++++++++ .../json-handler/jsonHandlerImp.ts | 2 + .../src/explicit-strategy/promptGeneration.ts | 12 +- .../ai-collab/src/explicit-strategy/utils.ts | 5 + .../README.md | 0 .../index.ts | 0 .../sharedTreeBranchManager.ts | 0 .../sharedTreeDiff.ts | 0 .../utils.ts | 0 packages/framework/ai-collab/src/index.ts | 12 +- .../sharedTreeBranchManager.spec.ts | 2 +- .../sharedTreeBranchManagerMergeDiff.spec.ts | 2 +- .../sharedTreeDiffArrays.spec.ts | 2 +- .../sharedTreeDiffMaps.spec.ts | 2 +- .../sharedTreeDiffObjects.spec.ts | 2 +- .../{ => implicit-strategy}/utils.spec.ts | 2 +- 23 files changed, 576 insertions(+), 59 deletions(-) create mode 100644 packages/framework/ai-collab/src/aiCollab.ts create mode 100644 packages/framework/ai-collab/src/aiCollabApi.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/index.ts rename packages/framework/ai-collab/src/{shared-tree-diff => implicit-strategy}/README.md (100%) rename packages/framework/ai-collab/src/{shared-tree-diff => implicit-strategy}/index.ts (100%) rename packages/framework/ai-collab/src/{shared-tree-diff => implicit-strategy}/sharedTreeBranchManager.ts (100%) rename packages/framework/ai-collab/src/{shared-tree-diff => implicit-strategy}/sharedTreeDiff.ts (100%) rename packages/framework/ai-collab/src/{shared-tree-diff => implicit-strategy}/utils.ts (100%) rename packages/framework/ai-collab/src/test/{shared-tree-diff => implicit-strategy}/sharedTreeBranchManager.spec.ts (99%) rename packages/framework/ai-collab/src/test/{shared-tree-diff => implicit-strategy}/sharedTreeBranchManagerMergeDiff.spec.ts (99%) rename packages/framework/ai-collab/src/test/{shared-tree-diff => implicit-strategy}/sharedTreeDiffArrays.spec.ts (99%) rename packages/framework/ai-collab/src/test/{shared-tree-diff => implicit-strategy}/sharedTreeDiffMaps.spec.ts (97%) rename packages/framework/ai-collab/src/test/{shared-tree-diff => implicit-strategy}/sharedTreeDiffObjects.spec.ts (99%) rename packages/framework/ai-collab/src/test/{ => implicit-strategy}/utils.spec.ts (98%) diff --git a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md index cdfbb0ae2e94..8f3e1ff3b76c 100644 --- a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md +++ b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md @@ -4,6 +4,53 @@ ```ts +// @alpha +export function aiCollab(options: AiCollabOptions): Promise; + +// @alpha +export interface AiCollabErrorResponse { + // (undocumented) + errorMessage: "tokenLimitExceeded" | "tooManyErrors" | "tooManyModelCalls" | "aborted"; + // (undocumented) + status: "failure" | "partial-failure"; + // (undocumented) + tokenUsage: TokenUsage; +} + +// @alpha +export interface AiCollabOptions { + // (undocumented) + dumpDebugLog?: boolean; + // (undocumented) + finalReviewStep?: boolean; + // (undocumented) + limiters?: { + abortController?: AbortController; + maxSequentialErrors?: number; + maxModelCalls?: number; + tokenLimits?: TokenUsage; + }; + // (undocumented) + openAI: OpenAiClientOptions; + // (undocumented) + prompt: { + systemRoleContext: string; + userAsk: string; + }; + // (undocumented) + treeView: TreeView; + // (undocumented) + validator?: (newContent: TreeNode) => void; +} + +// @alpha +export interface AiCollabSuccessResponse { + // (undocumented) + status: "success"; + // (undocumented) + tokenUsage: TokenUsage; +} + // @alpha export function createMergableDiffSeries(diffs: Difference[]): Difference[]; @@ -66,6 +113,14 @@ export interface DifferenceRemove { // @alpha export type ObjectPath = (string | number)[]; +// @alpha +export interface OpenAiClientOptions { + // (undocumented) + client: OpenAI; + // (undocumented) + modelName?: string; +} + // @alpha export interface Options { // (undocumented) @@ -107,4 +162,12 @@ export function sharedTreeDiff(obj: Record | unknown[], newObj: // @alpha export function sharedTreeTraverse(jsonObject: TreeMapNode | TreeArrayNode | Record | unknown[], path: ObjectPath): T | undefined; +// @alpha +export interface TokenUsage { + // (undocumented) + inputTokens: number; + // (undocumented) + outputTokens: number; +} + ``` diff --git a/packages/framework/ai-collab/package.json b/packages/framework/ai-collab/package.json index 3b837238528d..8ff90f07cbcf 100644 --- a/packages/framework/ai-collab/package.json +++ b/packages/framework/ai-collab/package.json @@ -92,6 +92,7 @@ "@fluidframework/core-utils": "workspace:~", "@fluidframework/telemetry-utils": "workspace:~", "@fluidframework/tree": "workspace:~", + "openai": "^4.67.3", "zod": "^3.23.8" }, "devDependencies": { @@ -102,6 +103,8 @@ "@fluidframework/build-common": "^2.0.3", "@fluidframework/build-tools": "^0.48.0", "@fluidframework/eslint-config-fluid": "^5.4.0", + "@fluidframework/id-compressor": "workspace:~", + "@fluidframework/test-runtime-utils": "workspace:^", "@microsoft/api-extractor": "7.47.8", "@types/mocha": "^9.1.1", "@types/node": "^18.19.0", diff --git a/packages/framework/ai-collab/src/aiCollab.ts b/packages/framework/ai-collab/src/aiCollab.ts new file mode 100644 index 000000000000..8c2675b58d32 --- /dev/null +++ b/packages/framework/ai-collab/src/aiCollab.ts @@ -0,0 +1,35 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import type { ImplicitFieldSchema } from "@fluidframework/tree"; + +import type { + AiCollabErrorResponse, + AiCollabOptions, + AiCollabSuccessResponse, +} from "./aiCollabApi.js"; +import { generateTreeEdits } from "./explicit-strategy/index.js"; + +/** + * Calls an LLM to modify the provided SharedTree based on the provided users input. + * @remarks This function is designed to be a controlled "all-in-one" function that handles the entire process of calling an LLM to collaborative edit a SharedTree. + * + * @alpha + */ +export async function aiCollab( + options: AiCollabOptions, +): Promise { + const response = await generateTreeEdits({ + treeView: options.treeView, + validator: options.validator, + openAI: options.openAI, + prompt: options.prompt, + limiters: options.limiters, + dumpDebugLog: options.dumpDebugLog, + finalReviewStep: options.finalReviewStep, + }); + + return response; +} diff --git a/packages/framework/ai-collab/src/aiCollabApi.ts b/packages/framework/ai-collab/src/aiCollabApi.ts new file mode 100644 index 000000000000..a63dd9193c22 --- /dev/null +++ b/packages/framework/ai-collab/src/aiCollabApi.ts @@ -0,0 +1,73 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import type { TreeNode, TreeView, ImplicitFieldSchema } from "@fluidframework/tree"; +// eslint-disable-next-line import/no-named-as-default +import type OpenAI from "openai"; + +/** + * OpenAI client options for the {@link AiCollabOptions} interface. + * + * @alpha + */ +export interface OpenAiClientOptions { + client: OpenAI; + modelName?: string; +} + +/** + * Options for the AI collaboration. + * + * @alpha + */ +export interface AiCollabOptions { + openAI: OpenAiClientOptions; + treeView: TreeView; + prompt: { + systemRoleContext: string; + userAsk: string; + }; + limiters?: { + abortController?: AbortController; + maxSequentialErrors?: number; + maxModelCalls?: number; + tokenLimits?: TokenUsage; + }; + finalReviewStep?: boolean; + validator?: (newContent: TreeNode) => void; + dumpDebugLog?: boolean; +} + +/** + * A successful response from the AI collaboration. + * + * @alpha + */ +export interface AiCollabSuccessResponse { + status: "success"; + tokenUsage: TokenUsage; +} + +/** + * An error response from the AI collaboration. + * + * @alpha + */ +export interface AiCollabErrorResponse { + status: "failure" | "partial-failure"; + errorMessage: "tokenLimitExceeded" | "tooManyErrors" | "tooManyModelCalls" | "aborted"; + tokenUsage: TokenUsage; +} + +/** + * Usage of tokens by an LLM. + * @remarks This interface is used for both tracking token usage and for setting token limits. + * + * @alpha + */ +export interface TokenUsage { + inputTokens: number; + outputTokens: number; +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts index 89cdc71325e7..e7c92729c8d9 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ /*! * Copyright (c) Microsoft Corporation and contributors. All rights reserved. * Licensed under the MIT License. @@ -8,7 +7,6 @@ import { assert } from "@fluidframework/core-utils/internal"; import { UsageError } from "@fluidframework/telemetry-utils/internal"; import { Tree, - // getOrCreateInnerNode, NodeKind, type ImplicitAllowedTypes, type TreeArrayNode, @@ -61,19 +59,6 @@ function populateDefaults( assert(typeof json[typeField] === "string", "missing or invalid type field"); const nodeSchema = definitionMap.get(json[typeField]); assert(nodeSchema?.kind === NodeKind.Object, "Expected object schema"); - - for (const [key, fieldSchema] of Object.entries(nodeSchema.fields)) { - const defaulter = fieldSchema?.metadata?.llmDefault; - if (defaulter !== undefined) { - // TODO: Properly type. The input `json` is a JsonValue, but the output can contain nodes (from the defaulters) amidst the json. - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unnecessary-type-assertion - json[key] = defaulter() as any; - } - } - - for (const value of Object.values(json)) { - populateDefaults(value, definitionMap); - } } } } @@ -98,7 +83,7 @@ export function applyAgentEdit( populateDefaults(treeEdit.content, definitionMap); const treeSchema = normalizeFieldSchema(tree.schema); - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment const schemaIdentifier = (treeEdit.content as any)[typeField]; let insertedObject: TreeNode | undefined; @@ -138,7 +123,7 @@ export function applyAgentEdit( const parentNodeSchema = Tree.schema(array); populateDefaults(treeEdit.content, definitionMap); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment const schemaIdentifier = (treeEdit.content as any)[typeField]; // We assume that the parentNode for inserts edits are guaranteed to be an arrayNode. @@ -210,7 +195,7 @@ export function applyAgentEdit( const modification = treeEdit.modification; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment const schemaIdentifier = (modification as any)[typeField]; let insertedObject: TreeNode | undefined; @@ -426,7 +411,7 @@ function getPlaceInfo( } /** - * Returns the target node with the matching internal objectId from the {@link ObjectTarget} + * Returns the target node with the matching internal objectId using the provided {@link ObjectTarget} */ function getNodeFromTarget(target: ObjectTarget, idGenerator: IdGenerator): TreeNode { const node = idGenerator.getNode(target[objectIdKey]); @@ -434,26 +419,6 @@ function getNodeFromTarget(target: ObjectTarget, idGenerator: IdGenerator): Tree return node; } -// /** -// * Returns the target node with the matching internal objectId from the {@link ObjectTarget} and the index of the nod, if the -// * parent node is an array node. -// */ -// function getTargetInfo( -// target: ObjectTarget, -// idGenerator: IdGenerator, -// ): { -// node: TreeNode; -// nodeIndex: number | undefined; -// } { -// const node = idGenerator.getNode(target[objectIdKey]); -// assert(node !== undefined, "objectId does not exist in nodeMap"); - -// Tree.key(node); - -// const nodeIndex = Tree.key(node); -// return { node, nodeIndex: typeof nodeIndex === "number" ? nodeIndex : undefined }; -// } - function objectIdsExist(treeEdit: TreeEdit, idGenerator: IdGenerator): void { switch (treeEdit.type) { case "setRoot": diff --git a/packages/framework/ai-collab/src/explicit-strategy/handlers.ts b/packages/framework/ai-collab/src/explicit-strategy/handlers.ts index 1b88af094919..5e05c04da876 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/handlers.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/handlers.ts @@ -1,4 +1,3 @@ -/* eslint-disable unicorn/no-abusive-eslint-disable */ /*! * Copyright (c) Microsoft Corporation and contributors. All rights reserved. * Licensed under the MIT License. @@ -223,8 +222,10 @@ function getOrCreateHandler( }) .filter(([, value]) => value !== undefined), ); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access properties[typeField] = jh.enum({ values: [definition] }); return jh.object(() => ({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment properties, }))(); } @@ -277,11 +278,6 @@ function getOrCreateHandlerForField( modifyTypeSet: Set, fieldSchema: SimpleFieldSchema, ): StreamedType | undefined { - if (fieldSchema.metadata?.llmDefault !== undefined) { - // Omit fields that have data which cannot be generated by an llm - return undefined; - } - switch (fieldSchema.kind) { case FieldKind.Required: { return getStreamedType( diff --git a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts index bd6d7f171379..a69b4a85829c 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts @@ -15,7 +15,8 @@ import type { import { isTreeNode } from "./utils.js"; /** - * TBD + * Given a tree node, generates a set of LLM friendly, unique ids for each node in a given Shared Tree. + * @remarks - simple id's are important for the LLM and this library to create and distinguish between different types certain TreeEdits */ export class IdGenerator { private readonly idCountMap = new Map(); diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts new file mode 100644 index 000000000000..5fd6aaabd78c --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -0,0 +1,364 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { assert } from "@fluidframework/core-utils/internal"; +import { + getSimpleSchema, + normalizeFieldSchema, + type ImplicitFieldSchema, + type SimpleTreeSchema, + type TreeNode, + type TreeView, +} from "@fluidframework/tree/internal"; +import type { + ChatCompletionCreateParams, + ResponseFormatJSONSchema, + // eslint-disable-next-line import/no-internal-modules +} from "openai/resources/index.mjs"; + +import type { OpenAiClientOptions, TokenUsage } from "../aiCollabApi.js"; + +import { applyAgentEdit } from "./agentEditReducer.js"; +import type { EditWrapper, TreeEdit } from "./agentEditTypes.js"; +import { generateEditHandlers } from "./handlers.js"; +import { IdGenerator } from "./idGenerator.js"; +import { createResponseHandler, JsonHandler, type JsonObject } from "./json-handler/index.js"; +import { + getEditingSystemPrompt, + getReviewSystemPrompt, + getSuggestingSystemPrompt, + toDecoratedJson, + type EditLog, +} from "./promptGeneration.js"; +import { fail } from "./utils.js"; + +const DEBUG_LOG: string[] = []; + +/** + * {@link generateTreeEdits} options. + * + * @internal + */ +export interface GenerateTreeEditsOptions { + openAI: OpenAiClientOptions; + treeView: TreeView; + prompt: { + systemRoleContext: string; + userAsk: string; + }; + limiters?: { + abortController?: AbortController; + maxSequentialErrors?: number; + maxModelCalls?: number; + tokenLimits?: TokenUsage; + }; + finalReviewStep?: boolean; + validator?: (newContent: TreeNode) => void; + dumpDebugLog?: boolean; +} + +interface GenerateTreeEditsSuccessResponse { + status: "success"; + tokenUsage: TokenUsage; +} + +interface GenerateTreeEditsErrorResponse { + status: "failure" | "partial-failure"; + errorMessage: "tokenLimitExceeded" | "tooManyErrors" | "tooManyModelCalls" | "aborted"; + tokenUsage: TokenUsage; +} + +/** + * Prompts the provided LLM client to generate valid tree edits. + * Applies those edits to the provided tree branch before returning. + * + * @internal + */ +export async function generateTreeEdits( + options: GenerateTreeEditsOptions, +): Promise { + const idGenerator = new IdGenerator(); + const editLog: EditLog = []; + let editCount = 0; + let sequentialErrorCount = 0; + const simpleSchema = getSimpleSchema( + normalizeFieldSchema(options.treeView.schema).allowedTypes, + ); + const tokenUsage = { inputTokens: 0, outputTokens: 0 }; + + for await (const edit of generateEdits( + options, + simpleSchema, + idGenerator, + editLog, + options.limiters?.tokenLimits, + tokenUsage, + )) { + try { + const result = applyAgentEdit( + options.treeView, + edit, + idGenerator, + simpleSchema.definitions, + options.validator, + ); + editLog.push({ edit: result }); + sequentialErrorCount = 0; + } catch (error: unknown) { + if (error instanceof Error) { + const { message } = error; + sequentialErrorCount += 1; + editLog.push({ edit, error: message }); + DEBUG_LOG?.push(`Error: ${message}`); + + if (error instanceof TokenLimitExceededError) { + return { + status: "failure", + errorMessage: "tokenLimitExceeded", + tokenUsage, + }; + } + } else { + throw error; + } + } + + if (options.limiters?.abortController?.signal.aborted === true) { + return { + status: "failure", + errorMessage: "aborted", + tokenUsage, + }; + } + + if ( + sequentialErrorCount > + (options.limiters?.maxSequentialErrors ?? Number.POSITIVE_INFINITY) + ) { + return { + status: "failure", + errorMessage: "tooManyErrors", + tokenUsage, + }; + } + + if (++editCount >= (options.limiters?.maxModelCalls ?? Number.POSITIVE_INFINITY)) { + return { + status: "failure", + errorMessage: "tooManyModelCalls", + tokenUsage, + }; + } + } + + if (options.dumpDebugLog ?? false) { + console.log(DEBUG_LOG.join("\n\n")); + DEBUG_LOG.length = 0; + } + + return { + status: "success", + tokenUsage, + }; +} + +interface ReviewResult { + goalAccomplished: "yes" | "no"; +} + +async function* generateEdits( + options: GenerateTreeEditsOptions, + simpleSchema: SimpleTreeSchema, + idGenerator: IdGenerator, + editLog: EditLog, + tokenLimits: TokenUsage | undefined, + tokenUsage: TokenUsage, +): AsyncGenerator { + const originalDecoratedJson = + options.finalReviewStep ?? false + ? toDecoratedJson(idGenerator, options.treeView.root) + : undefined; + // reviewed is implicitly true if finalReviewStep is false + let hasReviewed = options.finalReviewStep ?? false ? false : true; + + async function getNextEdit(): Promise { + const systemPrompt = getEditingSystemPrompt( + options.prompt.userAsk, + idGenerator, + options.treeView, + editLog, + options.prompt.systemRoleContext, + ); + + DEBUG_LOG?.push(systemPrompt); + + return new Promise((resolve: (value: TreeEdit | undefined) => void) => { + const editHandler = generateEditHandlers(simpleSchema, (jsonObject: JsonObject) => { + // eslint-disable-next-line unicorn/no-null + DEBUG_LOG?.push(JSON.stringify(jsonObject, null, 2)); + const wrapper = jsonObject as unknown as EditWrapper; + if (wrapper.edit === null) { + DEBUG_LOG?.push("No more edits."); + return resolve(undefined); + } else { + return resolve(wrapper.edit); + } + }); + + const responseHandler = createResponseHandler( + editHandler, + options.limiters?.abortController ?? new AbortController(), + ); + + // eslint-disable-next-line no-void + void responseHandler.processResponse( + streamFromLlm(systemPrompt, responseHandler.jsonSchema(), options.openAI, tokenUsage), + ); + }).then(async (result): Promise => { + if (result === undefined && (options.finalReviewStep ?? false) && !hasReviewed) { + const reviewResult = await reviewGoal(); + // eslint-disable-next-line require-atomic-updates + hasReviewed = true; + if (reviewResult.goalAccomplished === "yes") { + return undefined; + } else { + editLog.length = 0; + return getNextEdit(); + } + } else { + return result; + } + }); + } + + async function reviewGoal(): Promise { + const systemPrompt = getReviewSystemPrompt( + options.prompt.userAsk, + idGenerator, + options.treeView, + originalDecoratedJson ?? fail("Original decorated tree not provided."), + options.prompt.systemRoleContext, + ); + + DEBUG_LOG?.push(systemPrompt); + + return new Promise((resolve: (value: ReviewResult) => void) => { + const reviewHandler = JsonHandler.object(() => ({ + properties: { + goalAccomplished: JsonHandler.enum({ + description: + 'Whether the difference the user\'s goal was met in the "after" tree.', + values: ["yes", "no"], + }), + }, + complete: (jsonObject: JsonObject) => { + // eslint-disable-next-line unicorn/no-null + DEBUG_LOG?.push(`Review result: ${JSON.stringify(jsonObject, null, 2)}`); + resolve(jsonObject as unknown as ReviewResult); + }, + }))(); + + const responseHandler = createResponseHandler( + reviewHandler, + options.limiters?.abortController ?? new AbortController(), + ); + + // eslint-disable-next-line no-void + void responseHandler.processResponse( + streamFromLlm(systemPrompt, responseHandler.jsonSchema(), options.openAI, tokenUsage), + ); + }); + } + + let edit = await getNextEdit(); + while (edit !== undefined) { + yield edit; + if (tokenUsage.inputTokens > (tokenLimits?.inputTokens ?? Number.POSITIVE_INFINITY)) { + throw new TokenLimitExceededError("Input token limit exceeded."); + } + if (tokenUsage.outputTokens > (tokenLimits?.outputTokens ?? Number.POSITIVE_INFINITY)) { + throw new TokenLimitExceededError("Output token limit exceeded."); + } + edit = await getNextEdit(); + } +} + +class TokenLimitExceededError extends Error {} + +/** + * Prompts the provided LLM client to generate a list of suggested tree edits to perform. + * + * @internal + */ +export async function generateSuggestions( + openAIClient: OpenAiClientOptions, + view: TreeView, + suggestionCount: number, + tokenUsage: TokenUsage, + guidance?: string, + abortController = new AbortController(), +): Promise { + let suggestions: string[] | undefined; + + const suggestionsHandler = JsonHandler.object(() => ({ + properties: { + edit: JsonHandler.array(() => ({ + description: + "A list of changes that a user might want a collaborative agent to make to the tree.", + items: JsonHandler.string(), + }))(), + }, + complete: (jsonObject: JsonObject) => { + suggestions = (jsonObject as { edit: string[] }).edit; + }, + }))(); + + const responseHandler = createResponseHandler(suggestionsHandler, abortController); + const systemPrompt = getSuggestingSystemPrompt(view, suggestionCount, guidance); + await responseHandler.processResponse( + streamFromLlm(systemPrompt, responseHandler.jsonSchema(), openAIClient, tokenUsage), + ); + assert(suggestions !== undefined, "No suggestions were generated."); + return suggestions; +} + +async function* streamFromLlm( + systemPrompt: string, + jsonSchema: JsonObject, + openAI: OpenAiClientOptions, + tokenUsage: TokenUsage, +): AsyncGenerator { + const llmJsonSchema: ResponseFormatJSONSchema.JSONSchema = { + schema: jsonSchema, + name: "llm-response", + strict: true, // Opt into structured output + }; + + const body: ChatCompletionCreateParams = { + messages: [{ role: "system", content: systemPrompt }], + model: openAI.modelName ?? "gpt-4o", + response_format: { + type: "json_schema", + json_schema: llmJsonSchema, + }, + // TODO + // stream: true, // Opt in to streaming responses. + max_tokens: 4096, + }; + + const result = await openAI.client.chat.completions.create(body); + const choice = result.choices[0]; + + if (result.usage !== undefined) { + tokenUsage.inputTokens += result.usage?.prompt_tokens; + tokenUsage.outputTokens += result.usage?.completion_tokens; + } + + assert(choice !== undefined, "Response included no choices."); + assert(choice.finish_reason === "stop", "Response was unfinished."); + assert(choice.message.content !== null, "Response contained no contents."); + // TODO: There is only a single yield here because we're not actually streaming + yield choice.message.content ?? ""; +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts index 6cf4507ff044..e04e381bfda9 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts @@ -133,6 +133,8 @@ export const getCreateResponseHandler: () => ( * TBD */ export class StreamedType { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore private readonly _brand = Symbol(); } diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index 150559410778..755337a1c21d 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -5,7 +5,6 @@ import { assert } from "@fluidframework/core-utils/internal"; import { - TreeNode, NodeKind, normalizeFieldSchema, type ImplicitFieldSchema, @@ -20,10 +19,10 @@ import { import { objectIdKey, type TreeEdit } from "./agentEditTypes.js"; import { IdGenerator } from "./idGenerator.js"; -import { fail } from "./utils.js"; +import { fail, isTreeNode } from "./utils.js"; /** - * TBD + * The log of edits produced by an LLM that have been performed on the Shared tree. */ export type EditLog = { edit: TreeEdit; @@ -40,7 +39,7 @@ export function toDecoratedJson( idGenerator.assignIds(root); const stringified: string = JSON.stringify(root, (_, value) => { if (typeof value === "object" && !Array.isArray(value) && value !== null) { - assert(value instanceof TreeNode, "Non-TreeNode value in tree."); + assert(isTreeNode(value), "Non-TreeNode value in tree."); const objId = idGenerator.getId(value) ?? fail("ID of new node should have been assigned."); assert( @@ -85,7 +84,7 @@ export function getSuggestingSystemPrompt( } /** - * TBD + * Creates a prompt containing unique instructions necessary for the LLM to generate explicit edits to the Shared Tree */ export function getEditingSystemPrompt( userPrompt: string, @@ -149,7 +148,8 @@ export function getEditingSystemPrompt( } /** - * TBD + * Creates a prompt asking the LLM to confirm whether the edits it has performed has successfully accomplished the user's goal. + * @remarks This is a form of self-assessment for the LLM to evaluate its work for correctness. */ export function getReviewSystemPrompt( userPrompt: string, diff --git a/packages/framework/ai-collab/src/explicit-strategy/utils.ts b/packages/framework/ai-collab/src/explicit-strategy/utils.ts index 9cbbbf845a18..d24b88a7be83 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/utils.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/utils.ts @@ -1,3 +1,8 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + import type { TreeNode } from "@fluidframework/tree"; /** diff --git a/packages/framework/ai-collab/src/shared-tree-diff/README.md b/packages/framework/ai-collab/src/implicit-strategy/README.md similarity index 100% rename from packages/framework/ai-collab/src/shared-tree-diff/README.md rename to packages/framework/ai-collab/src/implicit-strategy/README.md diff --git a/packages/framework/ai-collab/src/shared-tree-diff/index.ts b/packages/framework/ai-collab/src/implicit-strategy/index.ts similarity index 100% rename from packages/framework/ai-collab/src/shared-tree-diff/index.ts rename to packages/framework/ai-collab/src/implicit-strategy/index.ts diff --git a/packages/framework/ai-collab/src/shared-tree-diff/sharedTreeBranchManager.ts b/packages/framework/ai-collab/src/implicit-strategy/sharedTreeBranchManager.ts similarity index 100% rename from packages/framework/ai-collab/src/shared-tree-diff/sharedTreeBranchManager.ts rename to packages/framework/ai-collab/src/implicit-strategy/sharedTreeBranchManager.ts diff --git a/packages/framework/ai-collab/src/shared-tree-diff/sharedTreeDiff.ts b/packages/framework/ai-collab/src/implicit-strategy/sharedTreeDiff.ts similarity index 100% rename from packages/framework/ai-collab/src/shared-tree-diff/sharedTreeDiff.ts rename to packages/framework/ai-collab/src/implicit-strategy/sharedTreeDiff.ts diff --git a/packages/framework/ai-collab/src/shared-tree-diff/utils.ts b/packages/framework/ai-collab/src/implicit-strategy/utils.ts similarity index 100% rename from packages/framework/ai-collab/src/shared-tree-diff/utils.ts rename to packages/framework/ai-collab/src/implicit-strategy/utils.ts diff --git a/packages/framework/ai-collab/src/index.ts b/packages/framework/ai-collab/src/index.ts index 409ac84cd355..997a92fe7d03 100644 --- a/packages/framework/ai-collab/src/index.ts +++ b/packages/framework/ai-collab/src/index.ts @@ -25,4 +25,14 @@ export { createMergableDiffSeries, SharedTreeBranchManager, sharedTreeTraverse, -} from "./shared-tree-diff/index.js"; +} from "./implicit-strategy/index.js"; + +export { + type AiCollabOptions, + type AiCollabSuccessResponse, + type AiCollabErrorResponse, + type TokenUsage, + type OpenAiClientOptions, +} from "./aiCollabApi.js"; + +export { aiCollab } from "./aiCollab.js"; diff --git a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeBranchManager.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeBranchManager.spec.ts similarity index 99% rename from packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeBranchManager.spec.ts rename to packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeBranchManager.spec.ts index d2053b06fce3..021f7e22150a 100644 --- a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeBranchManager.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeBranchManager.spec.ts @@ -8,7 +8,7 @@ import { strict as assert } from "node:assert"; import { SchemaFactory } from "@fluidframework/tree"; import * as z from "zod"; -import { SharedTreeBranchManager } from "../../shared-tree-diff/index.js"; +import { SharedTreeBranchManager } from "../../implicit-strategy/index.js"; const schemaFactory = new SchemaFactory("TreeNodeTest"); diff --git a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeBranchManagerMergeDiff.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeBranchManagerMergeDiff.spec.ts similarity index 99% rename from packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeBranchManagerMergeDiff.spec.ts rename to packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeBranchManagerMergeDiff.spec.ts index bdbe54288819..16d17f17d19f 100644 --- a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeBranchManagerMergeDiff.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeBranchManagerMergeDiff.spec.ts @@ -7,7 +7,7 @@ import { strict as assert } from "node:assert"; import { SchemaFactory } from "@fluidframework/tree"; -import { SharedTreeBranchManager } from "../../shared-tree-diff/index.js"; +import { SharedTreeBranchManager } from "../../implicit-strategy/index.js"; const schemaFactory = new SchemaFactory("TreeNodeTest"); diff --git a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffArrays.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffArrays.spec.ts similarity index 99% rename from packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffArrays.spec.ts rename to packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffArrays.spec.ts index b070f844f444..f2261b8ff6bf 100644 --- a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffArrays.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffArrays.spec.ts @@ -7,7 +7,7 @@ import { strict as assert } from "node:assert"; import { SchemaFactory } from "@fluidframework/tree"; -import { createMergableIdDiffSeries, sharedTreeDiff } from "../../shared-tree-diff/index.js"; +import { createMergableIdDiffSeries, sharedTreeDiff } from "../../implicit-strategy/index.js"; const schemaFactory = new SchemaFactory("TreeNodeTest"); diff --git a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffMaps.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffMaps.spec.ts similarity index 97% rename from packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffMaps.spec.ts rename to packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffMaps.spec.ts index 3fc6792f2998..ecd8a4dcde30 100644 --- a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffMaps.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffMaps.spec.ts @@ -7,7 +7,7 @@ import { strict as assert } from "node:assert"; import { SchemaFactory } from "@fluidframework/tree"; -import { sharedTreeDiff } from "../../shared-tree-diff/index.js"; +import { sharedTreeDiff } from "../../implicit-strategy/index.js"; const schemaFactory = new SchemaFactory("TreeNodeTest"); diff --git a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffObjects.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffObjects.spec.ts similarity index 99% rename from packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffObjects.spec.ts rename to packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffObjects.spec.ts index fb7242bd3ee0..6df7334aaa42 100644 --- a/packages/framework/ai-collab/src/test/shared-tree-diff/sharedTreeDiffObjects.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffObjects.spec.ts @@ -7,7 +7,7 @@ import { strict as assert } from "node:assert"; import { SchemaFactory } from "@fluidframework/tree"; -import { sharedTreeDiff } from "../../shared-tree-diff/index.js"; +import { sharedTreeDiff } from "../../implicit-strategy/index.js"; const schemaFactory = new SchemaFactory("TreeNodeTest"); diff --git a/packages/framework/ai-collab/src/test/utils.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/utils.spec.ts similarity index 98% rename from packages/framework/ai-collab/src/test/utils.spec.ts rename to packages/framework/ai-collab/src/test/implicit-strategy/utils.spec.ts index a581f36eccdc..3fc1b04c8fd5 100644 --- a/packages/framework/ai-collab/src/test/utils.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/utils.spec.ts @@ -7,7 +7,7 @@ import { strict as assert } from "node:assert"; import { SchemaFactory } from "@fluidframework/tree"; -import { sharedTreeTraverse } from "../shared-tree-diff/index.js"; +import { sharedTreeTraverse } from "../../implicit-strategy/index.js"; const schemaFactory = new SchemaFactory("TreeTraversalTest"); From 4b5bc9137b3f9c939f1ea1384cd7e721b3a3b181 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 18 Oct 2024 19:30:20 +0000 Subject: [PATCH 04/28] Adds back accidentlaly removed dependencies from tree --- packages/dds/tree/package.json | 2 + packages/framework/ai-collab/package.json | 4 +- pnpm-lock.yaml | 53 ++++++++++++++++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/dds/tree/package.json b/packages/dds/tree/package.json index da122ede2385..93fef320898b 100644 --- a/packages/dds/tree/package.json +++ b/packages/dds/tree/package.json @@ -159,6 +159,7 @@ "@fluidframework/core-utils": "workspace:~", "@fluidframework/datastore-definitions": "workspace:~", "@fluidframework/driver-definitions": "workspace:~", + "@fluidframework/id-compressor": "workspace:~", "@fluidframework/runtime-definitions": "workspace:~", "@fluidframework/runtime-utils": "workspace:~", "@fluidframework/shared-object-base": "workspace:~", @@ -182,6 +183,7 @@ "@fluidframework/container-definitions": "workspace:~", "@fluidframework/container-loader": "workspace:~", "@fluidframework/eslint-config-fluid": "^5.4.0", + "@fluidframework/test-runtime-utils": "workspace:~", "@fluidframework/test-utils": "workspace:~", "@fluidframework/tree-previous": "npm:@fluidframework/tree@~2.4.0", "@microsoft/api-extractor": "7.47.8", diff --git a/packages/framework/ai-collab/package.json b/packages/framework/ai-collab/package.json index 5d9b13f0e41f..11dc5d642a68 100644 --- a/packages/framework/ai-collab/package.json +++ b/packages/framework/ai-collab/package.json @@ -61,8 +61,8 @@ "test": "npm run test:mocha", "test:coverage": "c8 npm test", "test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs", - "test:mocha:cjs": "echo \"TESTS ARE CURRENTLY FAILING. SKIPPING RUN.;\" # mocha --recursive \"dist/test/**/*.spec.js\"", - "test:mocha:esm": "echo \"TESTS ARE CURRENTLY FAILING. SKIPPING RUN.;\" # mocha --recursive \"lib/test/**/*.spec.js\"", + "test:mocha:cjs": "mocha --recursive \"dist/test/**/*.spec.js\"", + "test:mocha:esm": "mocha --recursive \"lib/test/**/*.spec.js\"", "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha", "tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && copyfiles -f ../../../common/build/build-common/src/cjs/package.json ./dist" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c2d3221daa96..a1a7e1355706 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10446,6 +10446,9 @@ importers: '@fluidframework/tree': specifier: workspace:~ version: link:../../dds/tree + openai: + specifier: ^4.67.3 + version: 4.68.0(zod@3.23.8) zod: specifier: ^3.23.8 version: 3.23.8 @@ -10471,6 +10474,12 @@ importers: '@fluidframework/eslint-config-fluid': specifier: ^5.4.0 version: 5.4.0(eslint@8.55.0)(typescript@5.4.5) + '@fluidframework/id-compressor': + specifier: workspace:~ + version: link:../../runtime/id-compressor + '@fluidframework/test-runtime-utils': + specifier: workspace:^ + version: link:../../runtime/test-runtime-utils '@microsoft/api-extractor': specifier: 7.47.8 version: 7.47.8(patch_hash=ldzfpsbo3oeejrejk775zxplmi)(@types/node@18.19.54) @@ -24072,7 +24081,6 @@ packages: dependencies: '@types/node': 18.19.54 form-data: 4.0.0 - dev: true /@types/node-forge@1.3.11: resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==} @@ -29147,6 +29155,10 @@ packages: dev: true optional: true + /form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + dev: false + /form-data-encoder@2.1.4: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} @@ -29193,6 +29205,14 @@ packages: engines: {node: '>=0.4.x'} dev: false + /formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + dev: false + /formidable@1.2.6: resolution: {integrity: sha512-KcpbcpuLNOwrEjnbpMC0gS+X8ciDoZE1kkqzat4a8vrprf+s9pKNQ/QIwWfbfs4ltgmFl3MD177SNTkve3BwGQ==} deprecated: 'Please upgrade to latest, formidable@v2 or formidable@v3! Check these notes: https://bit.ly/2ZEqIau' @@ -33695,6 +33715,11 @@ packages: resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==} dev: true + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + /node-emoji@2.1.3: resolution: {integrity: sha512-E2WEOVsgs7O16zsURJ/eH8BqhF029wGpEOnv7Urwdo2wmQanOACwJQh0devF9D9RhoZru0+9JXIS0dBXIAz+lA==} engines: {node: '>=18'} @@ -34155,6 +34180,27 @@ packages: is-docker: 2.2.1 is-wsl: 2.2.0 + /openai@4.68.0(zod@3.23.8): + resolution: {integrity: sha512-cVH0WMKd4cColyorwqo+Gn08lN8LQ8uKLMfWXFfvnedrLq3lCH6lRd0Rd0XJRunyfgNve/L9E7uZLAii39NBkw==} + hasBin: true + peerDependencies: + zod: ^3.23.8 + peerDependenciesMeta: + zod: + optional: true + dependencies: + '@types/node': 18.19.54 + '@types/node-fetch': 2.6.11 + abort-controller: 3.0.0 + agentkeepalive: 4.5.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + zod: 3.23.8 + transitivePeerDependencies: + - encoding + dev: false + /opener@1.5.2: resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} hasBin: true @@ -39078,6 +39124,11 @@ packages: dependencies: defaults: 1.0.4 + /web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + dev: false + /webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} From c8b6684027c943e32a0e3eaec3cf7f9fe6bf9309 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 18 Oct 2024 20:17:49 +0000 Subject: [PATCH 05/28] fixes broken tests for implicit strategy not accounting for new objectId parameter in diffs --- .../src/implicit-strategy/sharedTreeDiff.ts | 3 + .../sharedTreeDiffArrays.spec.ts | 107 ++++++++++-------- .../sharedTreeDiffMaps.spec.ts | 5 + .../sharedTreeDiffObjects.spec.ts | 14 +++ 4 files changed, 81 insertions(+), 48 deletions(-) diff --git a/packages/framework/ai-collab/src/implicit-strategy/sharedTreeDiff.ts b/packages/framework/ai-collab/src/implicit-strategy/sharedTreeDiff.ts index 6184da2aef79..b79b8d6fea21 100644 --- a/packages/framework/ai-collab/src/implicit-strategy/sharedTreeDiff.ts +++ b/packages/framework/ai-collab/src/implicit-strategy/sharedTreeDiff.ts @@ -138,6 +138,7 @@ export function sharedTreeDiff( diffs.push({ type: "REMOVE", path: [path], + objectId: undefined, oldValue: objValue, }); continue; @@ -170,6 +171,7 @@ export function sharedTreeDiff( diffs.push({ type: "REMOVE", path: [path], + objectId, oldValue: objValue, }); continue; @@ -181,6 +183,7 @@ export function sharedTreeDiff( diffs.push({ type: "REMOVE", path: [path], + objectId: undefined, oldValue: objValue, }); continue; diff --git a/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffArrays.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffArrays.spec.ts index f2261b8ff6bf..1bf1e7089fcb 100644 --- a/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffArrays.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffArrays.spec.ts @@ -24,6 +24,7 @@ describe("sharedTreeDiff() - arrays", () => { { type: "REMOVE", path: [1], + objectId: undefined, oldValue: "testing", }, ]); @@ -68,6 +69,7 @@ describe("sharedTreeDiff() - arrays", () => { { type: "CHANGE", path: ["state", 1, "test"], + objectId: undefined, value: false, oldValue: true, }, @@ -76,7 +78,13 @@ describe("sharedTreeDiff() - arrays", () => { it("array to object", () => { assert.deepStrictEqual(sharedTreeDiff({ data: [] }, { data: { val: "test" } }), [ - { type: "CHANGE", path: ["data"], value: { val: "test" }, oldValue: [] }, + { + type: "CHANGE", + path: ["data"], + objectId: undefined, + value: { val: "test" }, + oldValue: [], + }, ]); }); }); @@ -110,12 +118,14 @@ describe("sharedTreeDiff() - arrays with object ID strategy", () => { { type: "CHANGE", path: ["state", 0], + objectId: undefined, value: { id: "1", test: true }, oldValue: "test", }, { type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, newIndex: 0, value: treeNode.state[1], }, @@ -148,12 +158,14 @@ describe("sharedTreeDiff() - arrays with object ID strategy", () => { { type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, value: treeNode.state[0], newIndex: 1, }, { type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, value: treeNode.state[1], newIndex: 0, }, @@ -187,12 +199,14 @@ describe("sharedTreeDiff() - arrays with object ID strategy", () => { { type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, value: treeNode.state[0], newIndex: 1, }, { type: "REMOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, oldValue: treeNode.state[1], }, { @@ -230,18 +244,21 @@ describe("sharedTreeDiff() - arrays with object ID strategy", () => { { type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, value: treeNode.state[0], newIndex: 1, }, { type: "CHANGE", path: ["state", 0, "test"], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, oldValue: true, value: false, }, { type: "REMOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, oldValue: treeNode.state[1], }, { @@ -279,18 +296,21 @@ describe("sharedTreeDiff() - arrays with object ID strategy", () => { { type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, value: treeNode.state[0], newIndex: 1, }, { type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, value: treeNode.state[1], newIndex: 0, }, { type: "CHANGE", path: ["state", 1, "test"], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, oldValue: true, value: false, }, @@ -337,6 +357,7 @@ describe("createMergableIdDiffSeries()", () => { // good [1, 2] -> [2, 1] type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, value: treeNode.state[0], newIndex: 1, }, @@ -344,6 +365,7 @@ describe("createMergableIdDiffSeries()", () => { // should be removed, as it is redundant due to a swap. type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, value: treeNode.state[1], newIndex: 0, }, @@ -354,6 +376,7 @@ describe("createMergableIdDiffSeries()", () => { { type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, value: treeNode.state[0], newIndex: 1, }, @@ -377,12 +400,14 @@ describe("createMergableIdDiffSeries()", () => { // good [1, 2,] -> [2] type: "REMOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, oldValue: treeNode.state[0], }, { // Should be removed, 2 will be in the right position after the removal of 1 type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, newIndex: 0, value: treeNode.state[1], }, @@ -404,6 +429,7 @@ describe("createMergableIdDiffSeries()", () => { // [1, 2] -> [2] type: "REMOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, oldValue: treeNode.state[0], }, ]); @@ -440,12 +466,14 @@ describe("createMergableIdDiffSeries()", () => { { type: "REMOVE", path: ["state", 2], + objectId: (treeNode.state[2] as SimpleObjectTreeNode).id, oldValue: treeNode.state[2], }, { // expected to have the path index shifted back due to prior remove. type: "MOVE", path: ["state", 3], + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, newIndex: 4, value: treeNode.state[3], }, @@ -463,6 +491,7 @@ describe("createMergableIdDiffSeries()", () => { // expected to be removed TODO: Potential bug - Why does this diff even get created? type: "MOVE", path: ["state", 4], + objectId: "4", newIndex: 4, value: { id: "4", test: true }, }, @@ -475,6 +504,7 @@ describe("createMergableIdDiffSeries()", () => { // [1, 2, 3, 4] -> [1, 2, 4] type: "REMOVE", path: ["state", 2], + objectId: (treeNode.state[2] as SimpleObjectTreeNode).id, oldValue: treeNode.state[2], }, { @@ -493,6 +523,7 @@ describe("createMergableIdDiffSeries()", () => { // [1, 2, 4, 6, 5] -> [1, 2, 4, 6, 5, 4] type: "MOVE", path: ["state", 2], // Note the index was shifted back because of the prior remove + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, newIndex: 4, value: treeNode.state[3], }, @@ -529,24 +560,28 @@ describe("createMergableIdDiffSeries()", () => { { type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, newIndex: 2, value: treeNode.state[0], }, { type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, newIndex: 0, value: treeNode.state[1], }, { type: "MOVE", path: ["state", 2], + objectId: (treeNode.state[2] as SimpleObjectTreeNode).id, newIndex: 3, value: treeNode.state[2], }, { type: "MOVE", path: ["state", 3], + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, newIndex: 1, value: treeNode.state[3], }, @@ -561,6 +596,7 @@ describe("createMergableIdDiffSeries()", () => { // obj at index 0 moves to index 2 so move everything it jumped over, back type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, newIndex: 2, value: treeNode.state[0], }, @@ -570,6 +606,7 @@ describe("createMergableIdDiffSeries()", () => { // obj at index 1 moves to index 3, so move everything < index 3 back. (only applies to index moved over) type: "MOVE", path: ["state", 1], // source index shifted backwards + objectId: (treeNode.state[2] as SimpleObjectTreeNode).id, newIndex: 3, value: treeNode.state[2], }, @@ -577,6 +614,7 @@ describe("createMergableIdDiffSeries()", () => { // [2, 1, 4, 3] -> [2, 4, 1, 3] type: "MOVE", path: ["state", 2], // source index shifted backwards + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, newIndex: 1, value: treeNode.state[3], // keep in mind we are referencing node locations for eqaulity prior to the moves }, @@ -617,12 +655,14 @@ describe("createMergableIdDiffSeries()", () => { { type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, newIndex: 2, value: treeNode.state[0], }, { // expected to be reordered to the beginning. path: ["state", 0, "test"], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, type: "CHANGE", value: false, oldValue: true, @@ -631,24 +671,28 @@ describe("createMergableIdDiffSeries()", () => { // expected to be removed due to other moves placing this in the correct pos. type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, newIndex: 0, value: treeNode.state[1], }, { type: "MOVE", path: ["state", 2], + objectId: (treeNode.state[2] as SimpleObjectTreeNode).id, newIndex: 3, value: treeNode.state[2], }, { type: "MOVE", path: ["state", 3], + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, newIndex: 1, value: treeNode.state[3], }, { // expected to be reordered to the beginning. path: ["state", 3, "test"], + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, type: "CHANGE", value: false, oldValue: true, @@ -661,6 +705,7 @@ describe("createMergableIdDiffSeries()", () => { { // reordered to the beginning path: ["state", 0, "test"], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, type: "CHANGE", value: false, oldValue: true, @@ -668,6 +713,7 @@ describe("createMergableIdDiffSeries()", () => { { // reordered to the beginning path: ["state", 3, "test"], + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, type: "CHANGE", value: false, oldValue: true, @@ -676,6 +722,7 @@ describe("createMergableIdDiffSeries()", () => { // [1, 2, 3, 4] -> [2, 3, 1, 4] type: "MOVE", path: ["state", 0], + objectId: (treeNode.state[0] as SimpleObjectTreeNode).id, newIndex: 2, value: treeNode.state[0], }, @@ -683,6 +730,7 @@ describe("createMergableIdDiffSeries()", () => { // [2, 3, 1, 4] -> [2, 1, 4, 3] type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[2] as SimpleObjectTreeNode).id, newIndex: 3, value: treeNode.state[2], }, @@ -690,6 +738,7 @@ describe("createMergableIdDiffSeries()", () => { // [2, 1, 4, 3] -> [2, 4, 1, 3] type: "MOVE", path: ["state", 2], + objectId: (treeNode.state[3] as SimpleObjectTreeNode).id, newIndex: 1, value: treeNode.state[3], }, @@ -725,6 +774,7 @@ describe("createMergableIdDiffSeries()", () => { // expected to be reordered to to the end of the array. type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, newIndex: 3, value: treeNode.state[1], }, @@ -742,57 +792,11 @@ describe("createMergableIdDiffSeries()", () => { // Expected to be removed TODO: Potential BUG - Why does this diff even get created? type: "MOVE", path: ["state", 3], + objectId: "4", newIndex: 3, value: { id: "4", test: true }, // also records this value as pojo instead of the tree node? }, ]); - // { - // // good [1, 2, 3, 4] -> [2, 1, 3, 4] - // type: "MOVE", - // path: ["state", 0], - // newIndex: 1, - // value: treeNode.state[0], - // }, - // { - // // expected to be removed, unecessary due to swap - // type: "MOVE", - // path: ["state", 1], - // newIndex: 0, - // value: treeNode.state[1], - // }, - // { - // // good [2, 1, 3, 4] -> [2, 1, 4] - // type: "REMOVE", - // path: ["state", 2], - // oldValue: treeNode.state[2], - // }, - // { - // // expected to be reordered to the end [2, 1, 4, 6, 5] -> [2, 1, 4, 6, 5, 4] - // type: "MOVE", - // path: ["state", 3], - // newIndex: 4, - // value: treeNode.state[3], - // }, - // { - // // good [2, 1, 4] -> [2, 1, 4, 6] - // type: "CREATE", - // path: ["state", 2], - // value: { id: "6", test: true }, - // }, - // { - // // good [2, 1, 4, 6] -> [2, 1, 4, 6, 5] - // type: "CREATE", - // path: ["state", 3], - // value: { id: "5", test: true }, - // }, - // { - // // expected to be removed TODO: Potential bug - Why does this diff even get created? - // type: "MOVE", - // path: ["state", 4], - // newIndex: 4, - // value: { id: "4", test: true }, - // }, - // ]); const minimalDiffs = createMergableIdDiffSeries(treeNode, diffs, "id"); @@ -813,6 +817,7 @@ describe("createMergableIdDiffSeries()", () => { // [1, 4, 6, 5] -> [1, 6, 5, 4] type: "MOVE", path: ["state", 1], + objectId: (treeNode.state[1] as SimpleObjectTreeNode).id, newIndex: 3, value: treeNode.state[1], }, @@ -916,12 +921,14 @@ describe("createMergableIdDiffSeries()", () => { { type: "MOVE", path: ["state", 0], + objectId: treeNode.state[0]?.id, newIndex: 1, value: treeNode.state[0], }, { type: "MOVE", path: ["state", 0, "innerArray", 0], + objectId: treeNode.state[0]?.innerArray[0]?.id, newIndex: 1, value: treeNode.state[0]?.innerArray[0], }, @@ -933,11 +940,13 @@ describe("createMergableIdDiffSeries()", () => { { type: "REMOVE", path: ["state", 2], + objectId: (treeNode.state[2] as SimpleObjectTreeNodeWithObjectArray).id, oldValue: treeNode.state[2], }, { type: "MOVE", path: ["stateArrayTwo", 0, "innerArray", 0], + objectId: treeNode.stateArrayTwo[0]?.innerArray[0]?.id, newIndex: 1, value: treeNode.stateArrayTwo[0]?.innerArray[0], }, @@ -1023,6 +1032,7 @@ describe("createMergableIdDiffSeries()", () => { { type: "MOVE", path: ["state", 0, "innerArray", 0], + objectId: treeNode.state[0]?.innerArray[0]?.id, newIndex: 1, value: treeNode.state[0]?.innerArray[0], }, @@ -1037,6 +1047,7 @@ describe("createMergableIdDiffSeries()", () => { { type: "MOVE", path: ["stateArrayTwo", 0, "innerArray", 0], + objectId: treeNode.stateArrayTwo[0]?.innerArray[0]?.id, newIndex: 1, value: treeNode.stateArrayTwo[0]?.innerArray[0], }, diff --git a/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffMaps.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffMaps.spec.ts index ecd8a4dcde30..351a5c7c485b 100644 --- a/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffMaps.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffMaps.spec.ts @@ -43,6 +43,7 @@ describe("sharedTreeDiff() - Maps - Change Diffs", () => { { type: "CHANGE", path: ["stringKey"], + objectId: undefined, oldValue: "test", value: "true", }, @@ -61,6 +62,7 @@ describe("sharedTreeDiff() - Maps - Change Diffs", () => { { type: "CHANGE", path: ["booleanKey"], + objectId: undefined, oldValue: true, value: false, }, @@ -79,6 +81,7 @@ describe("sharedTreeDiff() - Maps - Change Diffs", () => { { type: "CHANGE", path: ["numberKey"], + objectId: undefined, oldValue: 0, value: 1, }, @@ -105,6 +108,7 @@ describe("sharedTreeDiff() - Maps - Change Diffs", () => { { type: "CHANGE", path: ["objectKey", "stringKey"], + objectId: undefined, oldValue: "test", value: "SomethingDifferent", }, @@ -122,6 +126,7 @@ describe("sharedTreeDiff() - Maps - Change Diffs", () => { { type: "CHANGE", path: ["arrayKey"], + objectId: undefined, oldValue: arrayNode, value: undefined, }, diff --git a/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffObjects.spec.ts b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffObjects.spec.ts index 6df7334aaa42..f42366520640 100644 --- a/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffObjects.spec.ts +++ b/packages/framework/ai-collab/src/test/implicit-strategy/sharedTreeDiffObjects.spec.ts @@ -57,6 +57,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["requiredString"], + objectId: undefined, oldValue: "test", value: "true", }, @@ -75,6 +76,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["requiredBoolean"], + objectId: undefined, oldValue: true, value: false, }, @@ -93,6 +95,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["requiredNumber"], + objectId: undefined, oldValue: 0, value: 1, }, @@ -119,6 +122,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["requiredObject", "requiredString"], + objectId: undefined, oldValue: "test", value: "SomethingDifferent", }, @@ -135,6 +139,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["optionalBoolean"], + objectId: undefined, oldValue: true, value: undefined, }, @@ -152,6 +157,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["optionalString"], + objectId: undefined, oldValue: "true", value: undefined, }, @@ -169,6 +175,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["optionalNumber"], + objectId: undefined, oldValue: 1, value: undefined, }, @@ -189,6 +196,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["optionalObject"], + objectId: undefined, oldValue: { requiredString: "test" }, value: undefined, }, @@ -211,6 +219,7 @@ describe("sharedTreeDiff() - Object - Change Diffs", () => { { type: "CHANGE", path: ["optionalArray"], + objectId: undefined, oldValue: arrayNode, value: undefined, }, @@ -298,6 +307,7 @@ describe("sharedTreeDiff() - Object - Remove Diffs", () => { assert.deepStrictEqual(diffs, [ { type: "REMOVE", + objectId: undefined, path: ["optionalBoolean"], oldValue: true, }, @@ -311,6 +321,7 @@ describe("sharedTreeDiff() - Object - Remove Diffs", () => { { type: "REMOVE", path: ["optionalString"], + objectId: undefined, oldValue: "true", }, ]); @@ -323,6 +334,7 @@ describe("sharedTreeDiff() - Object - Remove Diffs", () => { { type: "REMOVE", path: ["optionalNumber"], + objectId: undefined, oldValue: 1, }, ]); @@ -340,6 +352,7 @@ describe("sharedTreeDiff() - Object - Remove Diffs", () => { { type: "REMOVE", path: ["optionalArray"], + objectId: undefined, oldValue: arrayNode, }, ]); @@ -354,6 +367,7 @@ describe("sharedTreeDiff() - Object - Remove Diffs", () => { { type: "REMOVE", path: ["optionalObject"], + objectId: undefined, oldValue: { requiredString: "test" }, }, ]); From 34664025ee9c616b1471e2f78cdb51dbe39d4657 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 18 Oct 2024 20:22:43 +0000 Subject: [PATCH 06/28] WIP adding test for explicit-strategy --- .../ai-collab/src/explicit-strategy/index.ts | 8 +- .../agentEditingReducer.spec.ts | 1251 +++++++++++++++++ .../src/test/explicit-strategy/utils.ts | 28 + 3 files changed, 1285 insertions(+), 2 deletions(-) create mode 100644 packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts create mode 100644 packages/framework/ai-collab/src/test/explicit-strategy/utils.ts diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index 5fd6aaabd78c..6921bf821ee0 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -7,6 +7,7 @@ import { assert } from "@fluidframework/core-utils/internal"; import { getSimpleSchema, normalizeFieldSchema, + // Tree, type ImplicitFieldSchema, type SimpleTreeSchema, type TreeNode, @@ -86,6 +87,9 @@ export async function generateTreeEdits( const simpleSchema = getSimpleSchema( normalizeFieldSchema(options.treeView.schema).allowedTypes, ); + + // const simpleSchema = getSimpleSchema(Tree.schema(options.treeNode)); + const tokenUsage = { inputTokens: 0, outputTokens: 0 }; for await (const edit of generateEdits( @@ -177,11 +181,11 @@ async function* generateEdits( tokenUsage: TokenUsage, ): AsyncGenerator { const originalDecoratedJson = - options.finalReviewStep ?? false + (options.finalReviewStep ?? false) ? toDecoratedJson(idGenerator, options.treeView.root) : undefined; // reviewed is implicitly true if finalReviewStep is false - let hasReviewed = options.finalReviewStep ?? false ? false : true; + let hasReviewed = (options.finalReviewStep ?? false) ? false : true; async function getNextEdit(): Promise { const systemPrompt = getEditingSystemPrompt( diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts new file mode 100644 index 000000000000..fdfbf63a2e80 --- /dev/null +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts @@ -0,0 +1,1251 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { + strict as assert, + // fail +} from "node:assert"; + +// eslint-disable-next-line import/no-internal-modules +import { createIdCompressor } from "@fluidframework/id-compressor/internal"; +// eslint-disable-next-line import/no-internal-modules +import { MockFluidDataStoreRuntime } from "@fluidframework/test-runtime-utils/internal"; +import { + getSimpleSchema, + normalizeFieldSchema, + SchemaFactory, + TreeViewConfiguration, + // type TreeNode, + // jsonableTreeFromForest, + SharedTree, + // eslint-disable-next-line import/no-internal-modules +} from "@fluidframework/tree/internal"; + +import { + applyAgentEdit, + typeField, + // eslint-disable-next-line import/no-internal-modules +} from "../../explicit-strategy/agentEditReducer.js"; +// eslint-disable-next-line import/order +import type { + // objectIdKey, + TreeEdit, + // eslint-disable-next-line import/no-internal-modules +} from "../../explicit-strategy/agentEditTypes.js"; + +// eslint-disable-next-line import/no-internal-modules +import { IdGenerator } from "../../explicit-strategy/idGenerator.js"; +// import { validateUsageError } from "./utils.js"; + +const sf = new SchemaFactory("agentSchema"); + +class Vector extends sf.object("Vector", { + id: sf.identifier, // will be omitted from the generated JSON schema + x: sf.number, + y: sf.number, + z: sf.optional(sf.number), +}) {} + +class Vector2 extends sf.object("Vector2", { + id: sf.identifier, // will be omitted from the generated JSON schema + x2: sf.number, + y2: sf.number, + z2: sf.optional(sf.number), +}) {} + +class RootObjectPolymorphic extends sf.object("RootObject", { + str: sf.string, + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector, Vector2]), + bools: sf.array(sf.boolean), +}) {} + +class RootObject extends sf.object("RootObject", { + str: sf.string, + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector]), + bools: sf.array(sf.boolean), +}) {} + +class RootObjectWithMultipleVectorArrays extends sf.object("RootObject", { + str: sf.string, + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector]), + vectors2: sf.array([Vector]), + bools: sf.array(sf.boolean), +}) {} + +class RootObjectWithDifferentVectorArrayTypes extends sf.object("RootObject", { + str: sf.string, + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector]), + vectors2: sf.array([Vector2]), + bools: sf.array(sf.boolean), +}) {} + +class RootObjectWithNonArrayVectorField extends sf.object("RootObject", { + singleVector: sf.optional(Vector), + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector]), + bools: sf.array(sf.boolean), +}) {} + +const config = new TreeViewConfiguration({ schema: [sf.number, RootObjectPolymorphic] }); + +const factory = SharedTree.getFactory(); + +describe("applyAgentEdit", () => { + let idGenerator: IdGenerator; + beforeEach(() => { + idGenerator = new IdGenerator(); + }); + describe("setRoot edits", () => { + it("polymorphic root", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(config); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], + }); + + const setRootEdit: TreeEdit = { + explanation: "Set root to object", + type: "setRoot", + content: { + [typeField]: RootObjectPolymorphic.identifier, + str: "rootStr", + vectors: [], + bools: [], + }, + }; + + applyAgentEdit(view, setRootEdit, idGenerator, simpleSchema.definitions); + + const expectedTreeView = factory + .create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "expectedTree", + ) + .viewWith(config); + expectedTreeView.initialize({ + str: "rootStr", + vectors: [], + bools: [], + }); + + // const expected = [ + // { + // type: "agentSchema.RootObject", + // fields: { + // bools: [ + // { + // type: 'agentSchema.Array<["com.fluidframework.leaf.boolean"]>', + // }, + // ], + // str: [ + // { + // type: "com.fluidframework.leaf.string", + // value: "rootStr", + // }, + // ], + // vectors: [ + // { + // type: 'agentSchema.Array<["agentSchema.Vector","agentSchema.Vector2"]>', + // }, + // ], + // }, + // }, + // ]; + + assert.deepEqual(view.root, expectedTreeView.root); + }); + + // it("optional root", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configOptionalRoot = new TreeViewConfiguration({ schema: sf.optional(sf.number) }); + // const view = tree.viewWith(configOptionalRoot); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + // view.initialize(1); + + // const setRootEdit: TreeEdit = { + // explanation: "Set root to 2", + // type: "setRoot", + // content: 2, + // }; + + // applyAgentEdit(view, setRootEdit, idGenerator, simpleSchema.definitions); + + // const expected = [ + // { + // type: "com.fluidframework.leaf.number", + // value: 2, + // }, + // ]; + + // assert.deepEqual(jsonableTreeFromForest(view.checkout.forest), expected); + // }); + }); + + // describe("insert edits", () => { + // it("polymorphic insert edits", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const view = tree.viewWith(config); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // bools: [true], + // }); + // idGenerator.assignIds(view.root); + // const vectorId = + // idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0]) ?? + // fail("ID expected."); + + // const insertEdit: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + // destination: { + // type: "objectPlace", + // [objectIdKey]: vectorId, + // place: "after", + // }, + // }; + // applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + + // const insertEdit2: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, + // destination: { + // type: "objectPlace", + // [objectIdKey]: vectorId, + // place: "after", + // }, + // }; + // applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions); + + // const identifier1 = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; + // const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector).id; + // const identifier3 = ((view.root as RootObjectPolymorphic).vectors[2] as Vector).id; + + // const expected = { + // "str": "testStr", + // "vectors": [ + // { + // "id": identifier1, + // "x": 1, + // "y": 2, + // "z": 3, + // }, + // { + // "id": identifier2, + // "x2": 3, + // "y2": 4, + // "z2": 5, + // }, + // { + // "id": identifier3, + // "x": 2, + // "y": 3, + // "z": 4, + // }, + // ], + // "bools": [true], + // }; + + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("non polymorphic insert edits", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + // const view = tree.viewWith(config2); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId = + // idGenerator.getId((view.root as RootObject).vectors[0]) ?? fail("ID expected."); + + // const insertEdit: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + // destination: { + // type: "objectPlace", + // [objectIdKey]: vectorId, + // place: "after", + // }, + // }; + // applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + + // const identifier1 = (view.root as RootObject).vectors[0].id; + // const identifier2 = (view.root as RootObject).vectors[1].id; + + // const expected = { + // "str": "testStr", + // "vectors": [ + // { + // "id": identifier1, + // "x": 1, + // "y": 2, + // "z": 3, + // }, + // { + // "id": identifier2, + // "x": 2, + // "y": 3, + // "z": 4, + // }, + // ], + // "bools": [true], + // }; + + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("insert edit into an empty array", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + // const view = tree.viewWith(config2); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId = idGenerator.getId(view.root as RootObject) ?? fail("ID expected."); + + // const insertEdit: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + // destination: { + // type: "arrayPlace", + // parentId: vectorId, + // field: "vectors", + // location: "start", + // }, + // }; + // applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + + // const identifier1 = (view.root as RootObject).vectors[0].id; + + // const expected = { + // "str": "testStr", + // "vectors": [ + // { + // "id": identifier1, + // "x": 2, + // "y": 3, + // "z": 4, + // }, + // ], + // "bools": [true], + // }; + + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("fails for invalid content for schema type", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + // const view = tree.viewWith(config2); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId = + // idGenerator.getId((view.root as RootObject).vectors[0]) ?? fail("ID expected."); + + // const insertEdit: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, + // destination: { + // type: "objectPlace", + // [objectIdKey]: vectorId, + // place: "after", + // }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(/invalid data provided for schema/), + // ); + // }); + + // it("inserting node into an non array node fails", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const config2 = new TreeViewConfiguration({ schema: RootObjectWithNonArrayVectorField }); + // const view = tree.viewWith(config2); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // singleVector: new Vector({ x: 1, y: 2, z: 3 }), + // vectors: [new Vector({ x: 2, y: 3, z: 4 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // assert(view.root.singleVector !== undefined); + // const vectorId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + + // const insertEdit: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector.identifier, x: 3, y: 4, z: 5 }, + // destination: { + // type: "objectPlace", + // [objectIdKey]: vectorId, + // place: "before", + // }, + // }; + // assert.throws( + // () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(/Expected child to be in an array node/), + // ); + // }); + // }); + + // it("modify edits", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const view = tree.viewWith(config); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId = idGenerator.getId(view.root as TreeNode) ?? fail("ID expected."); + + // const modifyEdit: TreeEdit = { + // explanation: "Modify a vector", + // type: "modify", + // target: { __fluid_objectId: vectorId }, + // field: "vectors", + // modification: [ + // { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + // { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, + // ], + // }; + // applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions); + + // const modifyEdit2: TreeEdit = { + // explanation: "Modify a vector", + // type: "modify", + // target: { __fluid_objectId: vectorId }, + // field: "bools", + // modification: [false], + // }; + // applyAgentEdit(view, modifyEdit2, idGenerator, simpleSchema.definitions); + + // idGenerator.assignIds(view.root); + // const vectorId2 = + // idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0] as Vector) ?? + // fail("ID expected."); + + // const modifyEdit3: TreeEdit = { + // explanation: "Modify a vector", + // type: "modify", + // target: { __fluid_objectId: vectorId2 }, + // field: "x", + // modification: 111, + // }; + // applyAgentEdit(view, modifyEdit3, idGenerator, simpleSchema.definitions); + + // const identifier = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; + // const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector2).id; + + // const expected = { + // "str": "testStr", + // "vectors": [ + // { + // "id": identifier, + // "x": 111, + // "y": 3, + // "z": 4, + // }, + // { + // "id": identifier2, + // "x2": 3, + // "y2": 4, + // "z2": 5, + // }, + // ], + // "bools": [false], + // }; + + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // describe("remove edits", () => { + // it("removes a single item in an array", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObject], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); + + // const removeEdit: TreeEdit = { + // explanation: "remove a vector", + // type: "remove", + // source: { [objectIdKey]: vectorId1 }, + // }; + // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + + // const expected = { + // "str": "testStr", + // "vectors": [], + // "bools": [true], + // }; + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("removes an item in a non array field", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithNonArrayVectorField], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // singleVector: new Vector({ x: 1, y: 2, z: 3 }), + // vectors: [], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // assert(view.root.singleVector !== undefined); + + // const singleVectorId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + + // const removeEdit: TreeEdit = { + // explanation: "remove a vector", + // type: "remove", + // source: { [objectIdKey]: singleVectorId }, + // }; + // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + + // const expected = { + // "vectors": [], + // "bools": [true], + // }; + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("can remove an optional root", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: sf.optional(RootObject), + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // assert(view.root !== undefined); + // const rootId = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const removeEdit: TreeEdit = { + // explanation: "remove the root", + // type: "remove", + // source: { [objectIdKey]: rootId }, + // }; + // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + // assert.equal(view.root, undefined); + // }); + + // it("removing a required root fails", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObject], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const rootId = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const removeEdit: TreeEdit = { + // explanation: "remove the root", + // type: "remove", + // source: { [objectIdKey]: rootId }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), + + // validateUsageError( + // /The root is required, and cannot be removed. Please use modify edit instead./, + // ), + // ); + // }); + + // it("removes a range of items", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObject], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); + // const vectorId2 = idGenerator.getId(view.root.vectors[1]) ?? fail("ID expected."); + + // const removeEdit: TreeEdit = { + // explanation: "remove a vector", + // type: "remove", + // source: { + // from: { + // [objectIdKey]: vectorId1, + // type: "objectPlace", + // place: "before", + // }, + // to: { + // [objectIdKey]: vectorId2, + // type: "objectPlace", + // place: "after", + // }, + // }, + // }; + // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + + // const expected = { + // "str": "testStr", + // "vectors": [], + // "bools": [true], + // }; + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("invalid range of items fails", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithMultipleVectorArrays], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + // vectors2: [new Vector({ x: 3, y: 4, z: 5 }), new Vector({ x: 4, y: 5, z: 6 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); + // const vectorId2 = idGenerator.getId(view.root.vectors2[0]) ?? fail("ID expected."); + + // const removeEdit: TreeEdit = { + // explanation: "remove a vector", + // type: "remove", + // source: { + // from: { + // [objectIdKey]: vectorId1, + // type: "objectPlace", + // place: "before", + // }, + // to: { + // [objectIdKey]: vectorId2, + // type: "objectPlace", + // place: "after", + // }, + // }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), + // validateUsageError( + // /The "from" node and "to" nodes of the range must be in the same parent array./, + // ), + // ); + // }); + // }); + + // describe("Move Edits", () => { + // it("move a single item", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithMultipleVectorArrays], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 })], + // vectors2: [new Vector({ x: 2, y: 3, z: 4 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); + // const vectorId2 = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const moveEdit: TreeEdit = { + // explanation: "Move a vector", + // type: "move", + // source: { [objectIdKey]: vectorId1 }, + // destination: { + // type: "arrayPlace", + // parentId: vectorId2, + // field: "vectors2", + // location: "start", + // }, + // }; + // applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); + // const identifier = view.root.vectors2[0].id; + // const identifier2 = view.root.vectors2[1].id; + + // const expected = { + // "str": "testStr", + // "vectors": [], + // "vectors2": [ + // { + // "id": identifier, + // "x": 1, + // "y": 2, + // "z": 3, + // }, + // { + // "id": identifier2, + // "x": 2, + // "y": 3, + // "z": 4, + // }, + // ], + // "bools": [true], + // }; + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("move range of items", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithMultipleVectorArrays], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + // vectors2: [new Vector({ x: 3, y: 4, z: 5 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); + // const vectorId2 = idGenerator.getId(view.root.vectors[1]) ?? fail("ID expected."); + // const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const moveEdit: TreeEdit = { + // explanation: "Move a vector", + // type: "move", + // source: { + // from: { + // [objectIdKey]: vectorId1, + // type: "objectPlace", + // place: "before", + // }, + // to: { + // [objectIdKey]: vectorId2, + // type: "objectPlace", + // place: "after", + // }, + // }, + // destination: { + // type: "arrayPlace", + // parentId: vectorId3, + // field: "vectors2", + // location: "start", + // }, + // }; + // applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); + // const identifier = view.root.vectors2[0].id; + // const identifier2 = view.root.vectors2[1].id; + // const identifier3 = view.root.vectors2[2].id; + + // const expected = { + // "str": "testStr", + // "vectors": [], + // "vectors2": [ + // { + // "id": identifier, + // "x": 1, + // "y": 2, + // "z": 3, + // }, + // { + // "id": identifier2, + // "x": 2, + // "y": 3, + // "z": 4, + // }, + // { + // "id": identifier3, + // "x": 3, + // "y": 4, + // "z": 5, + // }, + // ], + // "bools": [true], + // }; + // assert.deepEqual( + // JSON.stringify(view.root, undefined, 2), + // JSON.stringify(expected, undefined, 2), + // ); + // }); + + // it("moving invalid types fails", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithDifferentVectorArrayTypes], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + // vectors2: [new Vector2({ x2: 3, y2: 4, z2: 5 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); + // const vectorId2 = idGenerator.getId(view.root.vectors[1]) ?? fail("ID expected."); + // const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const moveEdit: TreeEdit = { + // type: "move", + // explanation: "Move a vector", + // source: { + // from: { + // [objectIdKey]: vectorId1, + // type: "objectPlace", + // place: "before", + // }, + // to: { + // [objectIdKey]: vectorId2, + // type: "objectPlace", + // place: "after", + // }, + // }, + // destination: { + // type: "arrayPlace", + // parentId: vectorId3, + // field: "vectors2", + // location: "start", + // }, + // }; + // assert.throws( + // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(/Illegal node type in destination array/), + // ); + // }); + + // it("moving invalid range fails", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithMultipleVectorArrays], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + // vectors2: [new Vector({ x: 3, y: 4, z: 5 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); + // const vectorId2 = idGenerator.getId(view.root.vectors2[0]) ?? fail("ID expected."); + // const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const moveEdit: TreeEdit = { + // type: "move", + // explanation: "Move a vector", + // source: { + // from: { + // [objectIdKey]: vectorId1, + // type: "objectPlace", + // place: "before", + // }, + // to: { + // [objectIdKey]: vectorId2, + // type: "objectPlace", + // place: "after", + // }, + // }, + // destination: { + // type: "arrayPlace", + // parentId: vectorId3, + // field: "vectors2", + // location: "start", + // }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + // validateUsageError( + // /The "from" node and "to" nodes of the range must be in the same parent array./, + // ), + // ); + // }); + + // it("moving elements which aren't under an array fails", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithNonArrayVectorField], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // singleVector: new Vector({ x: 1, y: 2, z: 3 }), + // vectors: [new Vector({ x: 2, y: 3, z: 4 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // assert(view.root.singleVector !== undefined); + + // const strId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + // const vectorId = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const moveEdit: TreeEdit = { + // type: "move", + // explanation: "Move a vector", + // source: { + // [objectIdKey]: strId, + // }, + // destination: { + // type: "arrayPlace", + // parentId: vectorId, + // field: "vectors", + // location: "start", + // }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(/the source node must be within an arrayNode/), + // ); + // }); + + // it("providing arrayPlace with non-existant field fails", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithNonArrayVectorField], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // singleVector: new Vector({ x: 1, y: 2, z: 3 }), + // vectors: [new Vector({ x: 2, y: 3, z: 4 })], + // bools: [true], + // }); + + // idGenerator.assignIds(view.root); + // assert(view.root.singleVector !== undefined); + + // const strId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + // const vectorId = idGenerator.getId(view.root) ?? fail("ID expected."); + + // const moveEdit: TreeEdit = { + // type: "move", + // explanation: "Move a vector", + // source: { + // [objectIdKey]: strId, + // }, + // destination: { + // type: "arrayPlace", + // parentId: vectorId, + // field: "nonExistantField", + // location: "start", + // }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(/No child under field field/), + // ); + // }); + // }); + + // it("treeEdits with object ids that don't exist", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configWithMultipleVectors = new TreeViewConfiguration({ + // schema: [RootObjectWithMultipleVectorArrays], + // }); + // const view = tree.viewWith(configWithMultipleVectors); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + + // view.initialize({ + // str: "testStr", + // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + // vectors2: [new Vector({ x: 3, y: 4, z: 5 })], + // bools: [true], + // }); + + // const insertEdit: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, + // destination: { + // type: "objectPlace", + // [objectIdKey]: "testObjectId", + // place: "after", + // }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(/objectIdKey testObjectId does not exist/), + // ); + + // const insertEdit2: TreeEdit = { + // explanation: "Insert a vector", + // type: "insert", + // content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, + // destination: { + // type: "arrayPlace", + // parentId: "testObjectId", + // field: "vectors", + // location: "start", + // }, + // }; + + // assert.throws( + // () => applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions), + // validateUsageError(/objectIdKey testObjectId does not exist/), + // ); + + // const moveEdit: TreeEdit = { + // type: "move", + // explanation: "Move a vector", + // source: { + // from: { + // [objectIdKey]: "testObjectId1", + // type: "objectPlace", + // place: "before", + // }, + // to: { + // [objectIdKey]: "testObjectId2", + // type: "objectPlace", + // place: "after", + // }, + // }, + // destination: { + // type: "arrayPlace", + // parentId: "testObjectId3", + // field: "vectors2", + // location: "start", + // }, + // }; + // const objectIdKeys = ["testObjectId1", "testObjectId2", "testObjectId3"]; + // const errorMessage = `objectIdKeys [${objectIdKeys.join(",")}] does not exist`; + // assert.throws( + // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(errorMessage), + // ); + + // const moveEdit2: TreeEdit = { + // type: "move", + // explanation: "Move a vector", + // source: { + // [objectIdKey]: "testObjectId1", + // }, + // destination: { + // type: "objectPlace", + // [objectIdKey]: "testObjectId2", + // place: "before", + // }, + // }; + + // const objectIdKeys2 = ["testObjectId1", "testObjectId2"]; + // const errorMessage2 = `objectIdKeys [${objectIdKeys2.join(",")}] does not exist`; + // assert.throws( + // () => applyAgentEdit(view, moveEdit2, idGenerator, simpleSchema.definitions), + // validateUsageError(errorMessage2), + // ); + + // const modifyEdit: TreeEdit = { + // explanation: "Modify a vector", + // type: "modify", + // target: { __fluid_objectId: "testObjectId" }, + // field: "x", + // modification: 111, + // }; + + // assert.throws( + // () => applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions), + // validateUsageError(/objectIdKey testObjectId does not exist/), + // ); + // }); +}); diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts b/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts new file mode 100644 index 000000000000..a00f55ca87dd --- /dev/null +++ b/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts @@ -0,0 +1,28 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "node:assert"; + +// eslint-disable-next-line import/no-internal-modules +import { UsageError } from "@fluidframework/telemetry-utils/internal"; + +/** + * Validates that the error is a UsageError with the expected error message. + */ +export function validateUsageError(expectedErrorMsg: string | RegExp): (error: Error) => true { + return (error: Error) => { + assert(error instanceof UsageError); + if ( + typeof expectedErrorMsg === "string" + ? error.message !== expectedErrorMsg + : !expectedErrorMsg.test(error.message) + ) { + throw new Error( + `Unexpected assertion thrown\nActual: ${error.message}\nExpected: ${expectedErrorMsg}`, + ); + } + return true; + }; +} From 62b3783812bca7b7125fe5e764490226c9389f22 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:36:14 +0000 Subject: [PATCH 07/28] comments out TreeNode typeguarding as we don't have a working solution exported from tree --- .../src/explicit-strategy/agentEditReducer.ts | 19 +++++++------------ .../src/explicit-strategy/idGenerator.ts | 8 ++++---- .../src/explicit-strategy/promptGeneration.ts | 10 +++++++--- .../ai-collab/src/explicit-strategy/utils.ts | 10 ---------- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts index e7c92729c8d9..bb71e48d678c 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts @@ -38,9 +38,7 @@ import type { JsonValue } from "./json-handler/jsonParser.js"; import { toDecoratedJson } from "./promptGeneration.js"; import { fail } from "./utils.js"; -/** - * TBD - */ +// eslint-disable-next-line jsdoc/require-jsdoc export const typeField = "__fluid_type"; function populateDefaults( @@ -83,13 +81,12 @@ export function applyAgentEdit( populateDefaults(treeEdit.content, definitionMap); const treeSchema = normalizeFieldSchema(tree.schema); - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access const schemaIdentifier = (treeEdit.content as any)[typeField]; let insertedObject: TreeNode | undefined; if (treeSchema.kind === FieldKind.Optional && treeEdit.content === undefined) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access - (tree as any).root = treeEdit.content; + tree.root = treeEdit.content; } else { for (const allowedType of treeSchema.allowedTypeSet.values()) { if (schemaIdentifier === allowedType.identifier) { @@ -123,7 +120,7 @@ export function applyAgentEdit( const parentNodeSchema = Tree.schema(array); populateDefaults(treeEdit.content, definitionMap); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access const schemaIdentifier = (treeEdit.content as any)[typeField]; // We assume that the parentNode for inserts edits are guaranteed to be an arrayNode. @@ -195,7 +192,7 @@ export function applyAgentEdit( const modification = treeEdit.modification; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access const schemaIdentifier = (modification as any)[typeField]; let insertedObject: TreeNode | undefined; @@ -271,7 +268,6 @@ export function applyAgentEdit( if (isObjectTarget(source)) { const sourceNode = getNodeFromTarget(source, idGenerator); const sourceIndex = Tree.key(sourceNode) as number; - const sourceArrayNode = Tree.parent(sourceNode) as TreeArrayNode; const sourceArraySchema = Tree.schema(sourceArrayNode); if (sourceArraySchema.kind !== NodeKind.Array) { @@ -281,8 +277,7 @@ export function applyAgentEdit( const allowedTypes = [ ...normalizeAllowedTypes(destinationArraySchema.info as ImplicitAllowedTypes), ]; - // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const nodeToMove = sourceArrayNode[sourceIndex]; + const nodeToMove = sourceArrayNode.at(sourceIndex); assert(nodeToMove !== undefined, "node to move must exist"); if (isNodeAllowedType(nodeToMove as TreeNode, allowedTypes)) { destinationArrayNode.moveRangeToIndex( @@ -305,7 +300,7 @@ export function applyAgentEdit( ...normalizeAllowedTypes(destinationArraySchema.info as ImplicitAllowedTypes), ]; for (let i = sourceStartIndex; i < sourceEndIndex; i++) { - const nodeToMove = array[i]; + const nodeToMove = array.at(i); assert(nodeToMove !== undefined, "node to move must exist"); if (!isNodeAllowedType(nodeToMove as TreeNode, allowedTypes)) { throw new UsageError("Illegal node type in destination array"); diff --git a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts index a69b4a85829c..e0afb3245589 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts @@ -12,8 +12,6 @@ import type { TreeFieldFromImplicitField, } from "@fluidframework/tree/internal"; -import { isTreeNode } from "./utils.js"; - /** * Given a tree node, generates a set of LLM friendly, unique ids for each node in a given Shared Tree. * @remarks - simple id's are important for the LLM and this library to create and distinguish between different types certain TreeEdits @@ -56,8 +54,10 @@ export class IdGenerator { this.assignIds(element); }); } else { - assert(isTreeNode(node), "Non-TreeNode value in tree."); - const objId = this.getOrCreateId(node); + // TODO: SharedTree Team needs to either publish TreeNode as a class to use .instanceof() or a typeguard. + // Uncomment this assertion back once we have a typeguard ready. + // assert(isTreeNode(node), "Non-TreeNode value in tree."); + const objId = this.getOrCreateId(node as TreeNode); Object.keys(node).forEach((key) => { // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access this.assignIds((node as unknown as any)[key]); diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index 755337a1c21d..c192345c473b 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -15,11 +15,12 @@ import { type JsonNodeSchema, type JsonSchemaRef, type JsonTreeSchema, + type TreeNode, } from "@fluidframework/tree/internal"; import { objectIdKey, type TreeEdit } from "./agentEditTypes.js"; import { IdGenerator } from "./idGenerator.js"; -import { fail, isTreeNode } from "./utils.js"; +import { fail } from "./utils.js"; /** * The log of edits produced by an LLM that have been performed on the Shared tree. @@ -39,9 +40,12 @@ export function toDecoratedJson( idGenerator.assignIds(root); const stringified: string = JSON.stringify(root, (_, value) => { if (typeof value === "object" && !Array.isArray(value) && value !== null) { - assert(isTreeNode(value), "Non-TreeNode value in tree."); + // TODO: SharedTree Team needs to either publish TreeNode as a class to use .instanceof() or a typeguard. + // Uncomment this assertion back once we have a typeguard ready. + // assert(isTreeNode(node), "Non-TreeNode value in tree."); const objId = - idGenerator.getId(value) ?? fail("ID of new node should have been assigned."); + idGenerator.getId(value as TreeNode) ?? + fail("ID of new node should have been assigned."); assert( !Object.prototype.hasOwnProperty.call(value, objectIdKey), `Collision of object id property.`, diff --git a/packages/framework/ai-collab/src/explicit-strategy/utils.ts b/packages/framework/ai-collab/src/explicit-strategy/utils.ts index d24b88a7be83..cbd716f9d630 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/utils.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/utils.ts @@ -3,8 +3,6 @@ * Licensed under the MIT License. */ -import type { TreeNode } from "@fluidframework/tree"; - /** * Subset of Map interface. * @@ -60,11 +58,3 @@ export function getOrCreate( } return value; } - -/** - * Checks if the given object is an {@link TreeNode}. - */ -export function isTreeNode(obj: unknown): obj is TreeNode { - // Check if the object is not null and has the private brand field - return obj !== null && typeof obj === "object" && "#brand" in obj; -} From ecd974614f10df5d01f536edcd9e120d9e70ccaa Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:36:36 +0000 Subject: [PATCH 08/28] Adds agentEditingReducer tests, 2 failing currently --- .../agentEditingReducer.spec.ts | 2222 ++++++++--------- 1 file changed, 1111 insertions(+), 1111 deletions(-) diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts index fdfbf63a2e80..169c1754c1d0 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts @@ -3,10 +3,7 @@ * Licensed under the MIT License. */ -import { - strict as assert, - // fail -} from "node:assert"; +import { strict as assert, fail } from "node:assert"; // eslint-disable-next-line import/no-internal-modules import { createIdCompressor } from "@fluidframework/id-compressor/internal"; @@ -20,6 +17,7 @@ import { // type TreeNode, // jsonableTreeFromForest, SharedTree, + type TreeNode, // eslint-disable-next-line import/no-internal-modules } from "@fluidframework/tree/internal"; @@ -28,15 +26,20 @@ import { typeField, // eslint-disable-next-line import/no-internal-modules } from "../../explicit-strategy/agentEditReducer.js"; +import { + objectIdKey, + // eslint-disable-next-line import/no-internal-modules +} from "../../explicit-strategy/agentEditTypes.js"; // eslint-disable-next-line import/order import type { - // objectIdKey, TreeEdit, // eslint-disable-next-line import/no-internal-modules } from "../../explicit-strategy/agentEditTypes.js"; // eslint-disable-next-line import/no-internal-modules import { IdGenerator } from "../../explicit-strategy/idGenerator.js"; + +import { validateUsageError } from "./utils.js"; // import { validateUsageError } from "./utils.js"; const sf = new SchemaFactory("agentSchema"); @@ -129,1123 +132,1120 @@ describe("applyAgentEdit", () => { applyAgentEdit(view, setRootEdit, idGenerator, simpleSchema.definitions); + const expected = { + str: "rootStr", + vectors: [], + bools: [], + }; + + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("optional root", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configOptionalRoot = new TreeViewConfiguration({ schema: sf.optional(sf.number) }); + const view = tree.viewWith(configOptionalRoot); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + view.initialize(1); + + const setRootEdit: TreeEdit = { + explanation: "Set root to 2", + type: "setRoot", + content: 2, + }; + + applyAgentEdit(view, setRootEdit, idGenerator, simpleSchema.definitions); + const expectedTreeView = factory .create( new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "expectedTree", ) - .viewWith(config); - expectedTreeView.initialize({ - str: "rootStr", + .viewWith(configOptionalRoot); + expectedTreeView.initialize(2); + + assert.deepEqual(view.root, expectedTreeView.root); + }); + }); + + describe("insert edits", () => { + it("polymorphic insert edits", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(config); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], + }); + idGenerator.assignIds(view.root); + const vectorId = + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0]!) ?? + fail("ID expected."); + + const insertEdit: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + destination: { + type: "objectPlace", + [objectIdKey]: vectorId, + place: "after", + }, + }; + applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + + const insertEdit2: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, + destination: { + type: "objectPlace", + [objectIdKey]: vectorId, + place: "after", + }, + }; + applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions); + + const identifier1 = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; + const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector).id; + const identifier3 = ((view.root as RootObjectPolymorphic).vectors[2] as Vector).id; + + const expected = { + "str": "testStr", + "vectors": [ + { + "id": identifier1, + "x": 1, + "y": 2, + "z": 3, + }, + { + "id": identifier2, + "x2": 3, + "y2": 4, + "z2": 5, + }, + { + "id": identifier3, + "x": 2, + "y": 3, + "z": 4, + }, + ], + "bools": [true], + }; + + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("non polymorphic insert edits", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + const view = tree.viewWith(config2); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + const vectorId = + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + idGenerator.getId((view.root as RootObject).vectors[0]!) ?? fail("ID expected."); + + const insertEdit: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + destination: { + type: "objectPlace", + [objectIdKey]: vectorId, + place: "after", + }, + }; + applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier1 = (view.root as RootObject).vectors[0]!.id; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier2 = (view.root as RootObject).vectors[1]!.id; + + const expected = { + "str": "testStr", + "vectors": [ + { + "id": identifier1, + "x": 1, + "y": 2, + "z": 3, + }, + { + "id": identifier2, + "x": 2, + "y": 3, + "z": 4, + }, + ], + "bools": [true], + }; + + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("insert edit into an empty array", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + const view = tree.viewWith(config2); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", vectors: [], - bools: [], + bools: [true], }); - // const expected = [ - // { - // type: "agentSchema.RootObject", - // fields: { - // bools: [ - // { - // type: 'agentSchema.Array<["com.fluidframework.leaf.boolean"]>', - // }, - // ], - // str: [ - // { - // type: "com.fluidframework.leaf.string", - // value: "rootStr", - // }, - // ], - // vectors: [ - // { - // type: 'agentSchema.Array<["agentSchema.Vector","agentSchema.Vector2"]>', - // }, - // ], - // }, - // }, - // ]; + idGenerator.assignIds(view.root); + const vectorId = idGenerator.getId(view.root as RootObject) ?? fail("ID expected."); + + const insertEdit: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + destination: { + type: "arrayPlace", + parentId: vectorId, + field: "vectors", + location: "start", + }, + }; + applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier1 = (view.root as RootObject).vectors[0]!.id; + + const expected = { + "str": "testStr", + "vectors": [ + { + "id": identifier1, + "x": 2, + "y": 3, + "z": 4, + }, + ], + "bools": [true], + }; - assert.deepEqual(view.root, expectedTreeView.root); + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("fails for invalid content for schema type", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + const view = tree.viewWith(config2); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + const vectorId = + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + idGenerator.getId((view.root as RootObject).vectors[0]!) ?? fail("ID expected."); + + const insertEdit: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, + destination: { + type: "objectPlace", + [objectIdKey]: vectorId, + place: "after", + }, + }; + + assert.throws( + () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + validateUsageError(/invalid data provided for schema/), + ); + }); + + it("inserting node into an non array node fails", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const config2 = new TreeViewConfiguration({ schema: RootObjectWithNonArrayVectorField }); + const view = tree.viewWith(config2); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + singleVector: new Vector({ x: 1, y: 2, z: 3 }), + vectors: [new Vector({ x: 2, y: 3, z: 4 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + assert(view.root.singleVector !== undefined); + const vectorId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + + const insertEdit: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector.identifier, x: 3, y: 4, z: 5 }, + destination: { + type: "objectPlace", + [objectIdKey]: vectorId, + place: "before", + }, + }; + assert.throws( + () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + validateUsageError(/Expected child to be in an array node/), + ); + }); + }); + + it("modify edits", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(config); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], }); - // it("optional root", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configOptionalRoot = new TreeViewConfiguration({ schema: sf.optional(sf.number) }); - // const view = tree.viewWith(configOptionalRoot); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - // view.initialize(1); - - // const setRootEdit: TreeEdit = { - // explanation: "Set root to 2", - // type: "setRoot", - // content: 2, - // }; - - // applyAgentEdit(view, setRootEdit, idGenerator, simpleSchema.definitions); - - // const expected = [ - // { - // type: "com.fluidframework.leaf.number", - // value: 2, - // }, - // ]; - - // assert.deepEqual(jsonableTreeFromForest(view.checkout.forest), expected); - // }); + idGenerator.assignIds(view.root); + const vectorId = idGenerator.getId(view.root as TreeNode) ?? fail("ID expected."); + + const modifyEdit: TreeEdit = { + explanation: "Modify a vector", + type: "modify", + target: { __fluid_objectId: vectorId }, + field: "vectors", + modification: [ + { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, + { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, + ], + }; + applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions); + + const modifyEdit2: TreeEdit = { + explanation: "Modify a vector", + type: "modify", + target: { __fluid_objectId: vectorId }, + field: "bools", + modification: [false], + }; + applyAgentEdit(view, modifyEdit2, idGenerator, simpleSchema.definitions); + + idGenerator.assignIds(view.root); + const vectorId2 = + idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0] as Vector) ?? + fail("ID expected."); + + const modifyEdit3: TreeEdit = { + explanation: "Modify a vector", + type: "modify", + target: { __fluid_objectId: vectorId2 }, + field: "x", + modification: 111, + }; + applyAgentEdit(view, modifyEdit3, idGenerator, simpleSchema.definitions); + + const identifier = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; + const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector2).id; + + const expected = { + "str": "testStr", + "vectors": [ + { + "id": identifier, + "x": 111, + "y": 3, + "z": 4, + }, + { + "id": identifier2, + "x2": 3, + "y2": 4, + "z2": 5, + }, + ], + "bools": [false], + }; + + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); }); - // describe("insert edits", () => { - // it("polymorphic insert edits", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const view = tree.viewWith(config); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // bools: [true], - // }); - // idGenerator.assignIds(view.root); - // const vectorId = - // idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0]) ?? - // fail("ID expected."); - - // const insertEdit: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, - // destination: { - // type: "objectPlace", - // [objectIdKey]: vectorId, - // place: "after", - // }, - // }; - // applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); - - // const insertEdit2: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, - // destination: { - // type: "objectPlace", - // [objectIdKey]: vectorId, - // place: "after", - // }, - // }; - // applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions); - - // const identifier1 = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; - // const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector).id; - // const identifier3 = ((view.root as RootObjectPolymorphic).vectors[2] as Vector).id; - - // const expected = { - // "str": "testStr", - // "vectors": [ - // { - // "id": identifier1, - // "x": 1, - // "y": 2, - // "z": 3, - // }, - // { - // "id": identifier2, - // "x2": 3, - // "y2": 4, - // "z2": 5, - // }, - // { - // "id": identifier3, - // "x": 2, - // "y": 3, - // "z": 4, - // }, - // ], - // "bools": [true], - // }; - - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("non polymorphic insert edits", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); - // const view = tree.viewWith(config2); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId = - // idGenerator.getId((view.root as RootObject).vectors[0]) ?? fail("ID expected."); - - // const insertEdit: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, - // destination: { - // type: "objectPlace", - // [objectIdKey]: vectorId, - // place: "after", - // }, - // }; - // applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); - - // const identifier1 = (view.root as RootObject).vectors[0].id; - // const identifier2 = (view.root as RootObject).vectors[1].id; - - // const expected = { - // "str": "testStr", - // "vectors": [ - // { - // "id": identifier1, - // "x": 1, - // "y": 2, - // "z": 3, - // }, - // { - // "id": identifier2, - // "x": 2, - // "y": 3, - // "z": 4, - // }, - // ], - // "bools": [true], - // }; - - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("insert edit into an empty array", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); - // const view = tree.viewWith(config2); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId = idGenerator.getId(view.root as RootObject) ?? fail("ID expected."); - - // const insertEdit: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, - // destination: { - // type: "arrayPlace", - // parentId: vectorId, - // field: "vectors", - // location: "start", - // }, - // }; - // applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); - - // const identifier1 = (view.root as RootObject).vectors[0].id; - - // const expected = { - // "str": "testStr", - // "vectors": [ - // { - // "id": identifier1, - // "x": 2, - // "y": 3, - // "z": 4, - // }, - // ], - // "bools": [true], - // }; - - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("fails for invalid content for schema type", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); - // const view = tree.viewWith(config2); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId = - // idGenerator.getId((view.root as RootObject).vectors[0]) ?? fail("ID expected."); - - // const insertEdit: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, - // destination: { - // type: "objectPlace", - // [objectIdKey]: vectorId, - // place: "after", - // }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(/invalid data provided for schema/), - // ); - // }); - - // it("inserting node into an non array node fails", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const config2 = new TreeViewConfiguration({ schema: RootObjectWithNonArrayVectorField }); - // const view = tree.viewWith(config2); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // singleVector: new Vector({ x: 1, y: 2, z: 3 }), - // vectors: [new Vector({ x: 2, y: 3, z: 4 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // assert(view.root.singleVector !== undefined); - // const vectorId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); - - // const insertEdit: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector.identifier, x: 3, y: 4, z: 5 }, - // destination: { - // type: "objectPlace", - // [objectIdKey]: vectorId, - // place: "before", - // }, - // }; - // assert.throws( - // () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(/Expected child to be in an array node/), - // ); - // }); - // }); - - // it("modify edits", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const view = tree.viewWith(config); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId = idGenerator.getId(view.root as TreeNode) ?? fail("ID expected."); - - // const modifyEdit: TreeEdit = { - // explanation: "Modify a vector", - // type: "modify", - // target: { __fluid_objectId: vectorId }, - // field: "vectors", - // modification: [ - // { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, - // { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, - // ], - // }; - // applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions); - - // const modifyEdit2: TreeEdit = { - // explanation: "Modify a vector", - // type: "modify", - // target: { __fluid_objectId: vectorId }, - // field: "bools", - // modification: [false], - // }; - // applyAgentEdit(view, modifyEdit2, idGenerator, simpleSchema.definitions); - - // idGenerator.assignIds(view.root); - // const vectorId2 = - // idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0] as Vector) ?? - // fail("ID expected."); - - // const modifyEdit3: TreeEdit = { - // explanation: "Modify a vector", - // type: "modify", - // target: { __fluid_objectId: vectorId2 }, - // field: "x", - // modification: 111, - // }; - // applyAgentEdit(view, modifyEdit3, idGenerator, simpleSchema.definitions); - - // const identifier = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; - // const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector2).id; - - // const expected = { - // "str": "testStr", - // "vectors": [ - // { - // "id": identifier, - // "x": 111, - // "y": 3, - // "z": 4, - // }, - // { - // "id": identifier2, - // "x2": 3, - // "y2": 4, - // "z2": 5, - // }, - // ], - // "bools": [false], - // }; - - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // describe("remove edits", () => { - // it("removes a single item in an array", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObject], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); - - // const removeEdit: TreeEdit = { - // explanation: "remove a vector", - // type: "remove", - // source: { [objectIdKey]: vectorId1 }, - // }; - // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); - - // const expected = { - // "str": "testStr", - // "vectors": [], - // "bools": [true], - // }; - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("removes an item in a non array field", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithNonArrayVectorField], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // singleVector: new Vector({ x: 1, y: 2, z: 3 }), - // vectors: [], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // assert(view.root.singleVector !== undefined); - - // const singleVectorId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); - - // const removeEdit: TreeEdit = { - // explanation: "remove a vector", - // type: "remove", - // source: { [objectIdKey]: singleVectorId }, - // }; - // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); - - // const expected = { - // "vectors": [], - // "bools": [true], - // }; - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("can remove an optional root", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: sf.optional(RootObject), - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // assert(view.root !== undefined); - // const rootId = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const removeEdit: TreeEdit = { - // explanation: "remove the root", - // type: "remove", - // source: { [objectIdKey]: rootId }, - // }; - // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); - // assert.equal(view.root, undefined); - // }); - - // it("removing a required root fails", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObject], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const rootId = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const removeEdit: TreeEdit = { - // explanation: "remove the root", - // type: "remove", - // source: { [objectIdKey]: rootId }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), - - // validateUsageError( - // /The root is required, and cannot be removed. Please use modify edit instead./, - // ), - // ); - // }); - - // it("removes a range of items", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObject], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); - // const vectorId2 = idGenerator.getId(view.root.vectors[1]) ?? fail("ID expected."); - - // const removeEdit: TreeEdit = { - // explanation: "remove a vector", - // type: "remove", - // source: { - // from: { - // [objectIdKey]: vectorId1, - // type: "objectPlace", - // place: "before", - // }, - // to: { - // [objectIdKey]: vectorId2, - // type: "objectPlace", - // place: "after", - // }, - // }, - // }; - // applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); - - // const expected = { - // "str": "testStr", - // "vectors": [], - // "bools": [true], - // }; - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("invalid range of items fails", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithMultipleVectorArrays], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], - // vectors2: [new Vector({ x: 3, y: 4, z: 5 }), new Vector({ x: 4, y: 5, z: 6 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); - // const vectorId2 = idGenerator.getId(view.root.vectors2[0]) ?? fail("ID expected."); - - // const removeEdit: TreeEdit = { - // explanation: "remove a vector", - // type: "remove", - // source: { - // from: { - // [objectIdKey]: vectorId1, - // type: "objectPlace", - // place: "before", - // }, - // to: { - // [objectIdKey]: vectorId2, - // type: "objectPlace", - // place: "after", - // }, - // }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), - // validateUsageError( - // /The "from" node and "to" nodes of the range must be in the same parent array./, - // ), - // ); - // }); - // }); - - // describe("Move Edits", () => { - // it("move a single item", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithMultipleVectorArrays], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 })], - // vectors2: [new Vector({ x: 2, y: 3, z: 4 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); - // const vectorId2 = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const moveEdit: TreeEdit = { - // explanation: "Move a vector", - // type: "move", - // source: { [objectIdKey]: vectorId1 }, - // destination: { - // type: "arrayPlace", - // parentId: vectorId2, - // field: "vectors2", - // location: "start", - // }, - // }; - // applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); - // const identifier = view.root.vectors2[0].id; - // const identifier2 = view.root.vectors2[1].id; - - // const expected = { - // "str": "testStr", - // "vectors": [], - // "vectors2": [ - // { - // "id": identifier, - // "x": 1, - // "y": 2, - // "z": 3, - // }, - // { - // "id": identifier2, - // "x": 2, - // "y": 3, - // "z": 4, - // }, - // ], - // "bools": [true], - // }; - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("move range of items", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithMultipleVectorArrays], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], - // vectors2: [new Vector({ x: 3, y: 4, z: 5 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); - // const vectorId2 = idGenerator.getId(view.root.vectors[1]) ?? fail("ID expected."); - // const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const moveEdit: TreeEdit = { - // explanation: "Move a vector", - // type: "move", - // source: { - // from: { - // [objectIdKey]: vectorId1, - // type: "objectPlace", - // place: "before", - // }, - // to: { - // [objectIdKey]: vectorId2, - // type: "objectPlace", - // place: "after", - // }, - // }, - // destination: { - // type: "arrayPlace", - // parentId: vectorId3, - // field: "vectors2", - // location: "start", - // }, - // }; - // applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); - // const identifier = view.root.vectors2[0].id; - // const identifier2 = view.root.vectors2[1].id; - // const identifier3 = view.root.vectors2[2].id; - - // const expected = { - // "str": "testStr", - // "vectors": [], - // "vectors2": [ - // { - // "id": identifier, - // "x": 1, - // "y": 2, - // "z": 3, - // }, - // { - // "id": identifier2, - // "x": 2, - // "y": 3, - // "z": 4, - // }, - // { - // "id": identifier3, - // "x": 3, - // "y": 4, - // "z": 5, - // }, - // ], - // "bools": [true], - // }; - // assert.deepEqual( - // JSON.stringify(view.root, undefined, 2), - // JSON.stringify(expected, undefined, 2), - // ); - // }); - - // it("moving invalid types fails", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithDifferentVectorArrayTypes], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], - // vectors2: [new Vector2({ x2: 3, y2: 4, z2: 5 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); - // const vectorId2 = idGenerator.getId(view.root.vectors[1]) ?? fail("ID expected."); - // const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const moveEdit: TreeEdit = { - // type: "move", - // explanation: "Move a vector", - // source: { - // from: { - // [objectIdKey]: vectorId1, - // type: "objectPlace", - // place: "before", - // }, - // to: { - // [objectIdKey]: vectorId2, - // type: "objectPlace", - // place: "after", - // }, - // }, - // destination: { - // type: "arrayPlace", - // parentId: vectorId3, - // field: "vectors2", - // location: "start", - // }, - // }; - // assert.throws( - // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(/Illegal node type in destination array/), - // ); - // }); - - // it("moving invalid range fails", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithMultipleVectorArrays], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], - // vectors2: [new Vector({ x: 3, y: 4, z: 5 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // const vectorId1 = idGenerator.getId(view.root.vectors[0]) ?? fail("ID expected."); - // const vectorId2 = idGenerator.getId(view.root.vectors2[0]) ?? fail("ID expected."); - // const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const moveEdit: TreeEdit = { - // type: "move", - // explanation: "Move a vector", - // source: { - // from: { - // [objectIdKey]: vectorId1, - // type: "objectPlace", - // place: "before", - // }, - // to: { - // [objectIdKey]: vectorId2, - // type: "objectPlace", - // place: "after", - // }, - // }, - // destination: { - // type: "arrayPlace", - // parentId: vectorId3, - // field: "vectors2", - // location: "start", - // }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), - // validateUsageError( - // /The "from" node and "to" nodes of the range must be in the same parent array./, - // ), - // ); - // }); - - // it("moving elements which aren't under an array fails", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithNonArrayVectorField], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // singleVector: new Vector({ x: 1, y: 2, z: 3 }), - // vectors: [new Vector({ x: 2, y: 3, z: 4 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // assert(view.root.singleVector !== undefined); - - // const strId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); - // const vectorId = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const moveEdit: TreeEdit = { - // type: "move", - // explanation: "Move a vector", - // source: { - // [objectIdKey]: strId, - // }, - // destination: { - // type: "arrayPlace", - // parentId: vectorId, - // field: "vectors", - // location: "start", - // }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(/the source node must be within an arrayNode/), - // ); - // }); - - // it("providing arrayPlace with non-existant field fails", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithNonArrayVectorField], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // singleVector: new Vector({ x: 1, y: 2, z: 3 }), - // vectors: [new Vector({ x: 2, y: 3, z: 4 })], - // bools: [true], - // }); - - // idGenerator.assignIds(view.root); - // assert(view.root.singleVector !== undefined); - - // const strId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); - // const vectorId = idGenerator.getId(view.root) ?? fail("ID expected."); - - // const moveEdit: TreeEdit = { - // type: "move", - // explanation: "Move a vector", - // source: { - // [objectIdKey]: strId, - // }, - // destination: { - // type: "arrayPlace", - // parentId: vectorId, - // field: "nonExistantField", - // location: "start", - // }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(/No child under field field/), - // ); - // }); - // }); - - // it("treeEdits with object ids that don't exist", () => { - // const tree = factory.create( - // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - // "tree", - // ); - // const configWithMultipleVectors = new TreeViewConfiguration({ - // schema: [RootObjectWithMultipleVectorArrays], - // }); - // const view = tree.viewWith(configWithMultipleVectors); - // const schema = normalizeFieldSchema(view.schema); - // const simpleSchema = getSimpleSchema(schema.allowedTypes); - - // view.initialize({ - // str: "testStr", - // vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], - // vectors2: [new Vector({ x: 3, y: 4, z: 5 })], - // bools: [true], - // }); - - // const insertEdit: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, - // destination: { - // type: "objectPlace", - // [objectIdKey]: "testObjectId", - // place: "after", - // }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(/objectIdKey testObjectId does not exist/), - // ); - - // const insertEdit2: TreeEdit = { - // explanation: "Insert a vector", - // type: "insert", - // content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, - // destination: { - // type: "arrayPlace", - // parentId: "testObjectId", - // field: "vectors", - // location: "start", - // }, - // }; - - // assert.throws( - // () => applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions), - // validateUsageError(/objectIdKey testObjectId does not exist/), - // ); - - // const moveEdit: TreeEdit = { - // type: "move", - // explanation: "Move a vector", - // source: { - // from: { - // [objectIdKey]: "testObjectId1", - // type: "objectPlace", - // place: "before", - // }, - // to: { - // [objectIdKey]: "testObjectId2", - // type: "objectPlace", - // place: "after", - // }, - // }, - // destination: { - // type: "arrayPlace", - // parentId: "testObjectId3", - // field: "vectors2", - // location: "start", - // }, - // }; - // const objectIdKeys = ["testObjectId1", "testObjectId2", "testObjectId3"]; - // const errorMessage = `objectIdKeys [${objectIdKeys.join(",")}] does not exist`; - // assert.throws( - // () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(errorMessage), - // ); - - // const moveEdit2: TreeEdit = { - // type: "move", - // explanation: "Move a vector", - // source: { - // [objectIdKey]: "testObjectId1", - // }, - // destination: { - // type: "objectPlace", - // [objectIdKey]: "testObjectId2", - // place: "before", - // }, - // }; - - // const objectIdKeys2 = ["testObjectId1", "testObjectId2"]; - // const errorMessage2 = `objectIdKeys [${objectIdKeys2.join(",")}] does not exist`; - // assert.throws( - // () => applyAgentEdit(view, moveEdit2, idGenerator, simpleSchema.definitions), - // validateUsageError(errorMessage2), - // ); - - // const modifyEdit: TreeEdit = { - // explanation: "Modify a vector", - // type: "modify", - // target: { __fluid_objectId: "testObjectId" }, - // field: "x", - // modification: 111, - // }; - - // assert.throws( - // () => applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions), - // validateUsageError(/objectIdKey testObjectId does not exist/), - // ); - // }); + describe("remove edits", () => { + it("removes a single item in an array", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObject], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId1 = idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); + + const removeEdit: TreeEdit = { + explanation: "remove a vector", + type: "remove", + source: { [objectIdKey]: vectorId1 }, + }; + applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + + const expected = { + "str": "testStr", + "vectors": [], + "bools": [true], + }; + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("removes an item in a non array field", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithNonArrayVectorField], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + singleVector: new Vector({ x: 1, y: 2, z: 3 }), + vectors: [], + bools: [true], + }); + + idGenerator.assignIds(view.root); + assert(view.root.singleVector !== undefined); + + const singleVectorId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + + const removeEdit: TreeEdit = { + explanation: "remove a vector", + type: "remove", + source: { [objectIdKey]: singleVectorId }, + }; + applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + + const expected = { + "vectors": [], + "bools": [true], + }; + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("can remove an optional root", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: sf.optional(RootObject), + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + assert(view.root !== undefined); + const rootId = idGenerator.getId(view.root) ?? fail("ID expected."); + + const removeEdit: TreeEdit = { + explanation: "remove the root", + type: "remove", + source: { [objectIdKey]: rootId }, + }; + applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + assert.equal(view.root, undefined); + }); + + it("removing a required root fails", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObject], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + const rootId = idGenerator.getId(view.root) ?? fail("ID expected."); + + const removeEdit: TreeEdit = { + explanation: "remove the root", + type: "remove", + source: { [objectIdKey]: rootId }, + }; + + assert.throws( + () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), + + validateUsageError( + /The root is required, and cannot be removed. Please use modify edit instead./, + ), + ); + }); + + it("removes a range of items", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObject], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId1 = idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId2 = idGenerator.getId(view.root.vectors[1]!) ?? fail("ID expected."); + + const removeEdit: TreeEdit = { + explanation: "remove a vector", + type: "remove", + source: { + from: { + [objectIdKey]: vectorId1, + type: "objectPlace", + place: "before", + }, + to: { + [objectIdKey]: vectorId2, + type: "objectPlace", + place: "after", + }, + }, + }; + applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + + const expected = { + "str": "testStr", + "vectors": [], + "bools": [true], + }; + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("invalid range of items fails", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithMultipleVectorArrays], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + vectors2: [new Vector({ x: 3, y: 4, z: 5 }), new Vector({ x: 4, y: 5, z: 6 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId1 = idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId2 = idGenerator.getId(view.root.vectors2[0]!) ?? fail("ID expected."); + + const removeEdit: TreeEdit = { + explanation: "remove a vector", + type: "remove", + source: { + from: { + [objectIdKey]: vectorId1, + type: "objectPlace", + place: "before", + }, + to: { + [objectIdKey]: vectorId2, + type: "objectPlace", + place: "after", + }, + }, + }; + + assert.throws( + () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), + validateUsageError( + /The "from" node and "to" nodes of the range must be in the same parent array./, + ), + ); + }); + }); + + describe("Move Edits", () => { + it("move a single item", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithMultipleVectorArrays], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 })], + vectors2: [new Vector({ x: 2, y: 3, z: 4 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId1 = idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); + const vectorId2 = idGenerator.getId(view.root) ?? fail("ID expected."); + + const moveEdit: TreeEdit = { + explanation: "Move a vector", + type: "move", + source: { [objectIdKey]: vectorId1 }, + destination: { + type: "arrayPlace", + parentId: vectorId2, + field: "vectors2", + location: "start", + }, + }; + applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier = view.root.vectors2[0]!.id; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier2 = view.root.vectors2[1]!.id; + + const expected = { + "str": "testStr", + "vectors": [], + "vectors2": [ + { + "id": identifier, + "x": 1, + "y": 2, + "z": 3, + }, + { + "id": identifier2, + "x": 2, + "y": 3, + "z": 4, + }, + ], + "bools": [true], + }; + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("move range of items", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithMultipleVectorArrays], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + vectors2: [new Vector({ x: 3, y: 4, z: 5 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId1 = idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId2 = idGenerator.getId(view.root.vectors[1]!) ?? fail("ID expected."); + const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); + + const moveEdit: TreeEdit = { + explanation: "Move a vector", + type: "move", + source: { + from: { + [objectIdKey]: vectorId1, + type: "objectPlace", + place: "before", + }, + to: { + [objectIdKey]: vectorId2, + type: "objectPlace", + place: "after", + }, + }, + destination: { + type: "arrayPlace", + parentId: vectorId3, + field: "vectors2", + location: "start", + }, + }; + applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier = view.root.vectors2[0]!.id; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier2 = view.root.vectors2[1]!.id; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const identifier3 = view.root.vectors2[2]!.id; + + const expected = { + "str": "testStr", + "vectors": [], + "vectors2": [ + { + "id": identifier, + "x": 1, + "y": 2, + "z": 3, + }, + { + "id": identifier2, + "x": 2, + "y": 3, + "z": 4, + }, + { + "id": identifier3, + "x": 3, + "y": 4, + "z": 5, + }, + ], + "bools": [true], + }; + assert.deepEqual( + JSON.stringify(view.root, undefined, 2), + JSON.stringify(expected, undefined, 2), + ); + }); + + it("moving invalid types fails", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithDifferentVectorArrayTypes], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + vectors2: [new Vector2({ x2: 3, y2: 4, z2: 5 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId1 = idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId2 = idGenerator.getId(view.root.vectors[1]!) ?? fail("ID expected."); + const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); + + const moveEdit: TreeEdit = { + type: "move", + explanation: "Move a vector", + source: { + from: { + [objectIdKey]: vectorId1, + type: "objectPlace", + place: "before", + }, + to: { + [objectIdKey]: vectorId2, + type: "objectPlace", + place: "after", + }, + }, + destination: { + type: "arrayPlace", + parentId: vectorId3, + field: "vectors2", + location: "start", + }, + }; + assert.throws( + () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + validateUsageError(/Illegal node type in destination array/), + ); + }); + + it("moving invalid range fails", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithMultipleVectorArrays], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + vectors2: [new Vector({ x: 3, y: 4, z: 5 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId1 = idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const vectorId2 = idGenerator.getId(view.root.vectors2[0]!) ?? fail("ID expected."); + const vectorId3 = idGenerator.getId(view.root) ?? fail("ID expected."); + + const moveEdit: TreeEdit = { + type: "move", + explanation: "Move a vector", + source: { + from: { + [objectIdKey]: vectorId1, + type: "objectPlace", + place: "before", + }, + to: { + [objectIdKey]: vectorId2, + type: "objectPlace", + place: "after", + }, + }, + destination: { + type: "arrayPlace", + parentId: vectorId3, + field: "vectors2", + location: "start", + }, + }; + + assert.throws( + () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + validateUsageError( + /The "from" node and "to" nodes of the range must be in the same parent array./, + ), + ); + }); + + it("moving elements which aren't under an array fails", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithNonArrayVectorField], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + singleVector: new Vector({ x: 1, y: 2, z: 3 }), + vectors: [new Vector({ x: 2, y: 3, z: 4 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + assert(view.root.singleVector !== undefined); + + const strId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + const vectorId = idGenerator.getId(view.root) ?? fail("ID expected."); + + const moveEdit: TreeEdit = { + type: "move", + explanation: "Move a vector", + source: { + [objectIdKey]: strId, + }, + destination: { + type: "arrayPlace", + parentId: vectorId, + field: "vectors", + location: "start", + }, + }; + + assert.throws( + () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + validateUsageError(/the source node must be within an arrayNode/), + ); + }); + + it("providing arrayPlace with non-existant field fails", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithNonArrayVectorField], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + singleVector: new Vector({ x: 1, y: 2, z: 3 }), + vectors: [new Vector({ x: 2, y: 3, z: 4 })], + bools: [true], + }); + + idGenerator.assignIds(view.root); + assert(view.root.singleVector !== undefined); + + const strId = idGenerator.getId(view.root.singleVector) ?? fail("ID expected."); + const vectorId = idGenerator.getId(view.root) ?? fail("ID expected."); + + const moveEdit: TreeEdit = { + type: "move", + explanation: "Move a vector", + source: { + [objectIdKey]: strId, + }, + destination: { + type: "arrayPlace", + parentId: vectorId, + field: "nonExistantField", + location: "start", + }, + }; + + assert.throws( + () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + validateUsageError(/No child under field field/), + ); + }); + }); + + it("treeEdits with object ids that don't exist", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const configWithMultipleVectors = new TreeViewConfiguration({ + schema: [RootObjectWithMultipleVectorArrays], + }); + const view = tree.viewWith(configWithMultipleVectors); + const schema = normalizeFieldSchema(view.schema); + const simpleSchema = getSimpleSchema(schema.allowedTypes); + + view.initialize({ + str: "testStr", + vectors: [new Vector({ x: 1, y: 2, z: 3 }), new Vector({ x: 2, y: 3, z: 4 })], + vectors2: [new Vector({ x: 3, y: 4, z: 5 })], + bools: [true], + }); + + const insertEdit: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, + destination: { + type: "objectPlace", + [objectIdKey]: "testObjectId", + place: "after", + }, + }; + + assert.throws( + () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + validateUsageError(/objectIdKey testObjectId does not exist/), + ); + + const insertEdit2: TreeEdit = { + explanation: "Insert a vector", + type: "insert", + content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, + destination: { + type: "arrayPlace", + parentId: "testObjectId", + field: "vectors", + location: "start", + }, + }; + + assert.throws( + () => applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions), + validateUsageError(/objectIdKey testObjectId does not exist/), + ); + + const moveEdit: TreeEdit = { + type: "move", + explanation: "Move a vector", + source: { + from: { + [objectIdKey]: "testObjectId1", + type: "objectPlace", + place: "before", + }, + to: { + [objectIdKey]: "testObjectId2", + type: "objectPlace", + place: "after", + }, + }, + destination: { + type: "arrayPlace", + parentId: "testObjectId3", + field: "vectors2", + location: "start", + }, + }; + const objectIdKeys = ["testObjectId1", "testObjectId2", "testObjectId3"]; + const errorMessage = `objectIdKeys [${objectIdKeys.join(",")}] does not exist`; + assert.throws( + () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + validateUsageError(errorMessage), + ); + + const moveEdit2: TreeEdit = { + type: "move", + explanation: "Move a vector", + source: { + [objectIdKey]: "testObjectId1", + }, + destination: { + type: "objectPlace", + [objectIdKey]: "testObjectId2", + place: "before", + }, + }; + + const objectIdKeys2 = ["testObjectId1", "testObjectId2"]; + const errorMessage2 = `objectIdKeys [${objectIdKeys2.join(",")}] does not exist`; + assert.throws( + () => applyAgentEdit(view, moveEdit2, idGenerator, simpleSchema.definitions), + validateUsageError(errorMessage2), + ); + + const modifyEdit: TreeEdit = { + explanation: "Modify a vector", + type: "modify", + target: { __fluid_objectId: "testObjectId" }, + field: "x", + modification: 111, + }; + + assert.throws( + () => applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions), + validateUsageError(/objectIdKey testObjectId does not exist/), + ); + }); }); From ccfe5b61ba9ea36f4753eb089770cbdc02bb45de Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:30:44 +0000 Subject: [PATCH 09/28] Adds remaining tests with two failing --- .../ai-collab/src/explicit-strategy/index.ts | 6 +- .../explicit-strategy/agentEditing.spec.ts | 283 ++++++++++++++++++ .../explicit-strategy/integration.spec.ts | 220 ++++++++++++++ .../src/test/explicit-strategy/utils.ts | 51 ++++ 4 files changed, 557 insertions(+), 3 deletions(-) create mode 100644 packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts create mode 100644 packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index 6921bf821ee0..70807027c44d 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -300,7 +300,7 @@ export async function generateSuggestions( openAIClient: OpenAiClientOptions, view: TreeView, suggestionCount: number, - tokenUsage: TokenUsage, + tokenUsage?: TokenUsage, guidance?: string, abortController = new AbortController(), ): Promise { @@ -332,7 +332,7 @@ async function* streamFromLlm( systemPrompt: string, jsonSchema: JsonObject, openAI: OpenAiClientOptions, - tokenUsage: TokenUsage, + tokenUsage?: TokenUsage, ): AsyncGenerator { const llmJsonSchema: ResponseFormatJSONSchema.JSONSchema = { schema: jsonSchema, @@ -355,7 +355,7 @@ async function* streamFromLlm( const result = await openAI.client.chat.completions.create(body); const choice = result.choices[0]; - if (result.usage !== undefined) { + if (result.usage !== undefined && tokenUsage !== undefined) { tokenUsage.inputTokens += result.usage?.prompt_tokens; tokenUsage.outputTokens += result.usage?.completion_tokens; } diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts new file mode 100644 index 000000000000..796a78517289 --- /dev/null +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts @@ -0,0 +1,283 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "node:assert"; + +// eslint-disable-next-line import/no-internal-modules +import { createIdCompressor } from "@fluidframework/id-compressor/internal"; +// eslint-disable-next-line import/no-internal-modules +import { MockFluidDataStoreRuntime } from "@fluidframework/test-runtime-utils/internal"; +import { + SchemaFactory, + getJsonSchema, + SharedTree, + TreeViewConfiguration, + // eslint-disable-next-line import/no-internal-modules +} from "@fluidframework/tree/internal"; +// eslint-disable-next-line import/no-internal-modules +import type { ResponseFormatJSONSchema } from "openai/resources/shared.mjs"; + +// eslint-disable-next-line import/no-internal-modules +import { objectIdKey } from "../../explicit-strategy/agentEditTypes.js"; +// eslint-disable-next-line import/no-internal-modules +import { IdGenerator } from "../../explicit-strategy/idGenerator.js"; +// import { getResponse } from "../../explicit-strategy/llmClient.js"; +import { + getPromptFriendlyTreeSchema, + toDecoratedJson, + // eslint-disable-next-line import/no-internal-modules +} from "../../explicit-strategy/promptGeneration.js"; + +const demoSf = new SchemaFactory("agentSchema"); + +class Vector extends demoSf.object("Vector", { + x: demoSf.number, + y: demoSf.number, + z: demoSf.optional(demoSf.number), +}) {} + +class RootObject extends demoSf.object("RootObject", { + str: demoSf.string, + vectors: demoSf.array(Vector), + bools: demoSf.array(demoSf.boolean), +}) {} + +const factory = SharedTree.getFactory(); + +describe("toDecoratedJson", () => { + let idGenerator: IdGenerator; + beforeEach(() => { + idGenerator = new IdGenerator(); + }); + it("adds ID fields", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: Vector })); + view.initialize({ x: 1, y: 2 }); + + assert.equal( + toDecoratedJson(idGenerator, view.root), + JSON.stringify({ + [objectIdKey]: "Vector1", + x: 1, + y: 2, + }), + ); + + // const vector = new Vector({ x: 1, y: 2 }); + // // const hydratedObject = hydrate(Vector, vector); + + // assert.equal( + // toDecoratedJson(idGenerator, hydratedObject), + // JSON.stringify({ + // [objectIdKey]: "Vector0", + // x: 1, + // y: 2, + // }), + // ); + }); + + it("handles nested objects", () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: RootObject })); + view.initialize({ str: "hello", vectors: [{ x: 1, y: 2, z: 3 }], bools: [true] }); + + assert.equal( + toDecoratedJson(idGenerator, view.root), + JSON.stringify({ + [objectIdKey]: "RootObject1", + str: "hello", + vectors: [ + { + [objectIdKey]: "Vector1", + x: 1, + y: 2, + z: 3, + }, + ], + bools: [true], + }), + ); + + assert.equal(idGenerator.getNode("RootObject1"), view.root); + assert.equal(idGenerator.getNode("Vector1"), view.root.vectors.at(0)); + + // const hydratedObject = hydrate( + // RootObject, + // new RootObject({ str: "hello", vectors: [{ x: 1, y: 2, z: 3 }], bools: [true] }), + // ); + // assert.equal( + // toDecoratedJson(idGenerator, hydratedObject), + // JSON.stringify({ + // [objectIdKey]: "RootObject0", + // str: "hello", + // vectors: [ + // { + // [objectIdKey]: "Vector0", + // x: 1, + // y: 2, + // z: 3, + // }, + // ], + // bools: [true], + // }), + // ); + // assert.equal(idGenerator.getNode("RootObject0"), hydratedObject); + // assert.equal(idGenerator.getNode("Vector0"), hydratedObject.vectors.at(0)); + }); + + it("handles non-POJO mode arrays", () => { + const sf = new SchemaFactory("testSchema"); + class NamedArray extends sf.array("Vector", sf.number) {} + class Root extends sf.object("Root", { + arr: NamedArray, + }) {} + + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: Root })); + view.initialize({ arr: [1, 2, 3] }); + + assert.equal( + toDecoratedJson(idGenerator, view.root), + JSON.stringify({ __fluid_objectId: "Root1", arr: [1, 2, 3] }), + ); + + // const hydratedObject = hydrate(Root, new Root({ arr: [1, 2, 3] })); + // assert.equal( + // toDecoratedJson(idGenerator, hydratedObject), + // JSON.stringify({ __fluid_objectId: "Root0", arr: [1, 2, 3] }), + // ); + }); +}); + +describe("Makes TS type strings from schema", () => { + it("for objects with primitive fields", () => { + const testSf = new SchemaFactory("test"); + class Foo extends testSf.object("Foo", { + x: testSf.number, + y: testSf.string, + z: testSf.optional(testSf.null), + }) {} + assert.equal( + getPromptFriendlyTreeSchema(getJsonSchema(Foo)), + "interface Foo { x: number; y: string; z: null | undefined; }", + ); + }); + + // This test fails due to the fact that identifier fields are incorrectly set as optional in the JSON Schema + it.skip("for objects with identifier fields", () => { + const testSf = new SchemaFactory("test"); + class Foo extends testSf.object("Foo", { + y: testSf.identifier, + }) {} + assert.equal( + getPromptFriendlyTreeSchema(getJsonSchema(Foo)), + "interface Foo { y: string; }", + ); + }); + + it("for objects with polymorphic fields", () => { + const testSf = new SchemaFactory("test"); + class Bar extends testSf.object("Bar", { + z: testSf.number, + }) {} + class Foo extends testSf.object("Foo", { + y: demoSf.required([demoSf.number, demoSf.string, Bar]), + }) {} + assert.equal( + getPromptFriendlyTreeSchema(getJsonSchema(Foo)), + "interface Foo { y: number | string | Bar; } interface Bar { z: number; }", + ); + }); + + it("for objects with array fields", () => { + const testSf = new SchemaFactory("test"); + class Foo extends testSf.object("Foo", { + y: demoSf.array(demoSf.number), + }) {} + const stringified = JSON.stringify(getJsonSchema(Foo)); + assert.equal( + getPromptFriendlyTreeSchema(getJsonSchema(Foo)), + "interface Foo { y: number[]; }", + ); + }); + + it("for objects with nested array fields", () => { + const testSf = new SchemaFactory("test"); + class Foo extends testSf.object("Foo", { + y: demoSf.array([ + demoSf.number, + demoSf.array([demoSf.number, demoSf.array(demoSf.string)]), + ]), + }) {} + assert.equal( + getPromptFriendlyTreeSchema(getJsonSchema(Foo)), + "interface Foo { y: (number | (number | string[])[])[]; }", + ); + }); + + it("for objects in the demo schema", () => { + assert.equal( + getPromptFriendlyTreeSchema(getJsonSchema(RootObject)), + "interface RootObject { str: string; vectors: Vector[]; bools: boolean[]; } interface Vector { x: number; y: number; z: number | undefined; }", + ); + }); +}); + +describe.skip("llmClient", () => { + it("can accept a structured schema prompt", async () => { + const userPrompt = + "I need a catalog listing for a product. Please extract this info into the required schema. The product is a Red Ryder bicycle, which is a particularly fast bicycle, and which should be listed for one hundred dollars."; + + // const testSf = new SchemaFactory("test"); + // class CatalogEntry extends testSf.object("CatalogEntry", { + // itemTitle: testSf.string, + // itemDescription: testSf.string, + // itemPrice: testSf.number, + // }) {} + + // const jsonSchema = getJsonSchema(CatalogEntry); + + const responseSchema: ResponseFormatJSONSchema = { + type: "json_schema", + json_schema: { + name: "Catalog_Entry", + description: "An entry for an item in a product catalog", + strict: true, + schema: { + "type": "object", + "properties": { + "title": { + "type": "string", + "description": "a title which must be in all caps", + }, + "description": { + "type": "string", + "description": "the description of the item, which must be in CaMeLcAsE", + }, + "price": { + "type": "number", + "description": "the price, which must be expressed with one decimal place.", + }, + }, + "required": ["title", "description", "price"], + "additionalProperties": false, + }, + }, + }; + + // const response = await getResponse(userPrompt, responseSchema); + + // console.log(response); + }); +}); diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts new file mode 100644 index 000000000000..4eb524cccfb5 --- /dev/null +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts @@ -0,0 +1,220 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { strict as assert } from "node:assert"; + +// eslint-disable-next-line import/no-internal-modules +import { createIdCompressor } from "@fluidframework/id-compressor/internal"; +// eslint-disable-next-line import/no-internal-modules +import { MockFluidDataStoreRuntime } from "@fluidframework/test-runtime-utils/internal"; +import { + SchemaFactory, + SharedTree, + TreeViewConfiguration, + // eslint-disable-next-line import/no-internal-modules +} from "@fluidframework/tree/internal"; + +import { generateSuggestions, generateTreeEdits } from "../../explicit-strategy/index.js"; + +import { initializeOpenAIClient } from "./utils.js"; + +const sf = new SchemaFactory("Planner"); + +// eslint-disable-next-line jsdoc/require-jsdoc +export class Session extends sf.object("Session", { + id: sf.identifier, + title: sf.string, + abstract: sf.string, + sessionType: sf.required(sf.string, { + metadata: { + description: + "This is one of four possible strings: 'Session', 'Workshop', 'Panel', or 'Keynote'", + }, + }), + created: sf.required(sf.number, { + metadata: { + // llmDefault: () => Date.now(), TODO: Add this back when we have a defaulting value solution + }, + }), + lastChanged: sf.required(sf.number, { + metadata: { + // llmDefault: () => Date.now(), TODO: Add this back when we have a defaulting value solution + }, + }), +}) {} + +const SessionType = { + session: "Session", + workshop: "Workshop", + panel: "Panel", + keynote: "Keynote", +}; + +// eslint-disable-next-line jsdoc/require-jsdoc +export class Sessions extends sf.array("Sessions", Session) {} + +// eslint-disable-next-line jsdoc/require-jsdoc +export class Day extends sf.object("Day", { + sessions: sf.required(Sessions, { + metadata: { + description: "The sessions scheduled on this day.", + }, + }), +}) {} +// eslint-disable-next-line jsdoc/require-jsdoc +export class Days extends sf.array("Days", Day) {} + +// eslint-disable-next-line jsdoc/require-jsdoc +export class Conference extends sf.object("Conference", { + name: sf.string, + sessions: sf.required(Sessions, { + metadata: { + description: + "These sessions are not scheduled yet. The user (or AI agent) can move them to a specific day.", + }, + }), + days: Days, + sessionsPerDay: sf.number, +}) {} + +const factory = SharedTree.getFactory(); + +const TEST_MODEL_NAME = "gpt-4o"; + +describe.skip("Agent Editing Integration", () => { + process.env.OPENAI_API_KEY = "TODO "; // DON'T COMMIT THIS + process.env.AZURE_OPENAI_API_KEY = "TODO "; // DON'T COMMIT THIS + process.env.AZURE_OPENAI_ENDPOINT = "TODO "; + process.env.AZURE_OPENAI_DEPLOYMENT = "gpt-4o"; + + it("Suggestion Test", async () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: Conference })); + view.initialize({ name: "Plucky Penguins", sessions: [], days: [], sessionsPerDay: 3 }); + + const openAIClient = initializeOpenAIClient("azure"); + const abortController = new AbortController(); + const suggestions = await generateSuggestions( + { client: openAIClient, modelName: TEST_MODEL_NAME }, + view, + 3, + ); + for (const prompt of suggestions) { + const result = await generateTreeEdits({ + openAI: { client: openAIClient, modelName: TEST_MODEL_NAME }, + treeView: view, + prompt: { + userAsk: prompt, + systemRoleContext: "", + }, + limiters: { + abortController, + maxModelCalls: 15, + }, + }); + assert.equal(result, "success"); + } + }); + + it("Roblox Test", async () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: Conference })); + + view.initialize({ + name: "Roblox Creator x Investor Conference", + sessions: [], + days: [ + { + sessions: [ + { + title: "Can Roblox achieve 120 hz?", + abstract: + "With the latest advancements in the G transformation, we may achieve up to 120 hz and still have time for lunch.", + sessionType: SessionType.session, + created: Date.now(), + lastChanged: Date.now(), + }, + { + title: "Roblox in VR", + abstract: + "Grab your VR headset and discover the latest in Roblox VR technology. Attendees of this lecture will receive a free VR headset.", + sessionType: SessionType.workshop, + created: Date.now(), + lastChanged: Date.now(), + }, + { + title: "What about fun?", + abstract: "Can profit and the delightful smiles of the children coexist?", + sessionType: SessionType.keynote, + created: Date.now(), + lastChanged: Date.now(), + }, + { + title: "Combat in Roblox", + abstract: + "Get the latest tips and tricks for fighting your friends in roblox. Bonus: learn how to make your own sword!", + sessionType: SessionType.panel, + created: Date.now(), + lastChanged: Date.now(), + }, + ], + }, + { + sessions: [ + { + title: "Monetizing Children", + abstract: "Maximize those Robux.", + sessionType: SessionType.session, + created: Date.now(), + lastChanged: Date.now(), + }, + { + title: "Racecars!", + abstract: + "Find out how to build the fastest racecar in Roblox. Then, challenge your friends!", + sessionType: SessionType.workshop, + created: Date.now(), + lastChanged: Date.now(), + }, + { + title: + "The Gentrification of Roblox City's Downtown (and why that's a good thing)", + abstract: + "Real estate prices in Robloxia are skyrocketing, moving cash into the hands of those who can use it most wisely.", + sessionType: SessionType.session, + created: Date.now(), + lastChanged: Date.now(), + }, + ], + }, + ], + sessionsPerDay: 2, + }); + const openAIClient = initializeOpenAIClient("azure"); + const abortController = new AbortController(); + await generateTreeEdits({ + openAI: { client: openAIClient, modelName: TEST_MODEL_NAME }, + treeView: view, + prompt: { + userAsk: "Please alphabetize the sessions.", + systemRoleContext: "", + }, + limiters: { + abortController, + maxModelCalls: 15, + }, + finalReviewStep: true, + }); + + const stringified = JSON.stringify(view.root, undefined, 2); + console.log(stringified); + }); +}); diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts b/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts index a00f55ca87dd..476f17767bb1 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts @@ -7,6 +7,8 @@ import { strict as assert } from "node:assert"; // eslint-disable-next-line import/no-internal-modules import { UsageError } from "@fluidframework/telemetry-utils/internal"; +import OpenAI from "openai"; +import { AzureOpenAI } from "openai"; /** * Validates that the error is a UsageError with the expected error message. @@ -26,3 +28,52 @@ export function validateUsageError(expectedErrorMsg: string | RegExp): (error: E return true; }; } + +/** + * Creates an OpenAI Client session. + * Depends on the following environment variables: + * + * If using the OpenAI API: + * - OPENAI_API_KEY + * + * If using the Azure OpenAI API: + * - AZURE_OPENAI_API_KEY + * - AZURE_OPENAI_ENDPOINT + * - AZURE_OPENAI_DEPLOYMENT + * + */ +export function initializeOpenAIClient(service: "openai" | "azure"): OpenAI { + if (service === "azure") { + const apiKey = process.env.AZURE_OPENAI_API_KEY; + if (apiKey === null || apiKey === undefined) { + throw new Error("AZURE_OPENAI_API_KEY environment variable not set"); + } + + const endpoint = process.env.AZURE_OPENAI_ENDPOINT; + if (endpoint === null || endpoint === undefined) { + throw new Error("AZURE_OPENAI_ENDPOINT environment variable not set"); + } + + const deployment = process.env.AZURE_OPENAI_DEPLOYMENT; + if (deployment === null || deployment === undefined) { + throw new Error("AZURE_OPENAI_DEPLOYMENT environment variable not set"); + } + + const client = new AzureOpenAI({ + endpoint, + deployment, + apiKey, + apiVersion: "2024-08-01-preview", + timeout: 2500000, + }); + return client; + } else { + const apiKey = process.env.OPENAI_API_KEY; + if (apiKey === null || apiKey === undefined) { + throw new Error("OPENAI_API_KEY environment variable not set"); + } + + const client = new OpenAI({ apiKey }); + return client; + } +} From 2be83092a4331d15392e7a78d55e88f28f3e51c2 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:32:19 +0000 Subject: [PATCH 10/28] small import warning fix --- .../framework/ai-collab/src/test/explicit-strategy/utils.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts b/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts index 476f17767bb1..f880589da3b0 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/utils.ts @@ -7,8 +7,7 @@ import { strict as assert } from "node:assert"; // eslint-disable-next-line import/no-internal-modules import { UsageError } from "@fluidframework/telemetry-utils/internal"; -import OpenAI from "openai"; -import { AzureOpenAI } from "openai"; +import { OpenAI, AzureOpenAI } from "openai"; /** * Validates that the error is a UsageError with the expected error message. From a01a52ac142b50aa93980311f33c3fc430662f13 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:58:33 +0000 Subject: [PATCH 11/28] In progress adding planning step and other new updates from taylors fork --- packages/dds/tree/src/index.ts | 5 + .../tree/src/simple-tree/leafNodeSchema.ts | 19 + packages/framework/ai-collab/package.json | 3 + .../src/explicit-strategy/agentEditReducer.ts | 98 +++-- .../src/explicit-strategy/agentEditTypes.ts | 25 +- .../ai-collab/src/explicit-strategy/index.ts | 152 +++++--- .../src/explicit-strategy/promptGeneration.ts | 182 ++++++--- .../src/explicit-strategy/typeGeneration.ts | 347 ++++++++++++++++++ pnpm-lock.yaml | 11 +- 9 files changed, 704 insertions(+), 138 deletions(-) create mode 100644 packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts diff --git a/packages/dds/tree/src/index.ts b/packages/dds/tree/src/index.ts index d3e7922fe996..4b21173b18fe 100644 --- a/packages/dds/tree/src/index.ts +++ b/packages/dds/tree/src/index.ts @@ -163,6 +163,11 @@ export { isTreeNodeSchemaClass, normalizeAllowedTypes, getSimpleSchema, + numberSchema, + stringSchema, + booleanSchema, + handleSchema, + nullSchema, } from "./simple-tree/index.js"; export { SharedTree, diff --git a/packages/dds/tree/src/simple-tree/leafNodeSchema.ts b/packages/dds/tree/src/simple-tree/leafNodeSchema.ts index ab96b1ee7809..5b7221b4a269 100644 --- a/packages/dds/tree/src/simple-tree/leafNodeSchema.ts +++ b/packages/dds/tree/src/simple-tree/leafNodeSchema.ts @@ -63,8 +63,27 @@ function makeLeaf( } // Leaf schema shared between all SchemaFactory instances. +/** + * @internal + */ export const stringSchema = makeLeaf("string", ValueSchema.String); + +/** + * @internal + */ export const numberSchema = makeLeaf("number", ValueSchema.Number); + +/** + * @internal + */ export const booleanSchema = makeLeaf("boolean", ValueSchema.Boolean); + +/** + * @internal + */ export const nullSchema = makeLeaf("null", ValueSchema.Null); + +/** + * @internal + */ export const handleSchema = makeLeaf("handle", ValueSchema.FluidHandle); diff --git a/packages/framework/ai-collab/package.json b/packages/framework/ai-collab/package.json index 11dc5d642a68..b6251c1325f3 100644 --- a/packages/framework/ai-collab/package.json +++ b/packages/framework/ai-collab/package.json @@ -90,6 +90,7 @@ }, "dependencies": { "@fluidframework/core-utils": "workspace:~", + "@fluidframework/runtime-utils": "workspace:~", "@fluidframework/telemetry-utils": "workspace:~", "@fluidframework/tree": "workspace:~", "openai": "^4.67.3", @@ -104,6 +105,7 @@ "@fluidframework/build-tools": "^0.49.0", "@fluidframework/eslint-config-fluid": "^5.4.0", "@fluidframework/id-compressor": "workspace:~", + "@fluidframework/runtime-utils": "workspace:~", "@fluidframework/test-runtime-utils": "workspace:^", "@microsoft/api-extractor": "7.47.8", "@types/mocha": "^9.1.1", @@ -119,6 +121,7 @@ "mocha-multi-reporters": "^1.5.1", "prettier": "~3.0.3", "rimraf": "^4.4.0", + "typechat": "^0.1.1", "typescript": "~5.4.5" }, "typeValidation": { diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts index bb71e48d678c..38c13c7f40ec 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts @@ -4,6 +4,7 @@ */ import { assert } from "@fluidframework/core-utils/internal"; +import { isFluidHandle } from "@fluidframework/runtime-utils"; import { UsageError } from "@fluidframework/telemetry-utils/internal"; import { Tree, @@ -20,6 +21,11 @@ import { normalizeAllowedTypes, normalizeFieldSchema, type ImplicitFieldSchema, + booleanSchema, + handleSchema, + nullSchema, + numberSchema, + stringSchema, } from "@fluidframework/tree/internal"; import { @@ -28,9 +34,10 @@ import { type Selection, type Range, type ObjectPlace, - objectIdKey, type ArrayPlace, type TreeEditObject, + type TreeEditValue, + typeField, } from "./agentEditTypes.js"; import type { IdGenerator } from "./idGenerator.js"; // eslint-disable-next-line import/no-internal-modules @@ -38,9 +45,6 @@ import type { JsonValue } from "./json-handler/jsonParser.js"; import { toDecoratedJson } from "./promptGeneration.js"; import { fail } from "./utils.js"; -// eslint-disable-next-line jsdoc/require-jsdoc -export const typeField = "__fluid_type"; - function populateDefaults( json: JsonValue, definitionMap: ReadonlyMap, @@ -54,13 +58,55 @@ function populateDefaults( populateDefaults(element, definitionMap); } } else { - assert(typeof json[typeField] === "string", "missing or invalid type field"); + assert( + typeof json[typeField] === "string", + `${typeField} must be present in new JSON content`, + ); const nodeSchema = definitionMap.get(json[typeField]); assert(nodeSchema?.kind === NodeKind.Object, "Expected object schema"); } } } +function getSchemaIdentifier(content: TreeEditValue): string | undefined { + switch (typeof content) { + case "boolean": + return booleanSchema.identifier; + case "number": + return numberSchema.identifier; + case "string": + return stringSchema.identifier; + case "object": { + if (content === null) { + return nullSchema.identifier; + } + if (Array.isArray(content)) { + // TODO: Support arrays for setRoot + throw new UsageError("Arrays are not currently supported in this context"); + } + if (isFluidHandle(content)) { + return handleSchema.identifier; + } + return content[typeField]; + } + default: + throw new UsageError("Unsupported content type"); + } +} + +function isConstructable(schema: TreeNodeSchema): boolean { + switch (schema.identifier) { + case booleanSchema.identifier: + case numberSchema.identifier: + case stringSchema.identifier: + case nullSchema.identifier: + case handleSchema.identifier: + return false; + default: + return isTreeNodeSchemaClass(schema); + } +} + function contentWithIds(content: TreeNode, idGenerator: IdGenerator): TreeEditObject { return JSON.parse(toDecoratedJson(idGenerator, content)) as TreeEditObject; } @@ -81,8 +127,7 @@ export function applyAgentEdit( populateDefaults(treeEdit.content, definitionMap); const treeSchema = normalizeFieldSchema(tree.schema); - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const schemaIdentifier = (treeEdit.content as any)[typeField]; + const schemaIdentifier = getSchemaIdentifier(treeEdit.content); let insertedObject: TreeNode | undefined; if (treeSchema.kind === FieldKind.Optional && treeEdit.content === undefined) { @@ -90,7 +135,7 @@ export function applyAgentEdit( } else { for (const allowedType of treeSchema.allowedTypeSet.values()) { if (schemaIdentifier === allowedType.identifier) { - if (isTreeNodeSchemaClass(allowedType)) { + if (isConstructable(allowedType)) { const simpleNodeSchema = allowedType as unknown as new ( dummy: unknown, ) => TreeNode; @@ -120,8 +165,7 @@ export function applyAgentEdit( const parentNodeSchema = Tree.schema(array); populateDefaults(treeEdit.content, definitionMap); - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const schemaIdentifier = (treeEdit.content as any)[typeField]; + const schemaIdentifier = getSchemaIdentifier(treeEdit.content); // We assume that the parentNode for inserts edits are guaranteed to be an arrayNode. const allowedTypes = [ @@ -340,7 +384,7 @@ function isPrimitive(content: unknown): boolean { } function isObjectTarget(selection: Selection): selection is ObjectTarget { - return Object.keys(selection).length === 1 && "__fluid_objectId" in selection; + return Object.keys(selection).length === 1 && "target" in selection; } function isRange(selection: Selection): selection is Range { @@ -409,7 +453,7 @@ function getPlaceInfo( * Returns the target node with the matching internal objectId using the provided {@link ObjectTarget} */ function getNodeFromTarget(target: ObjectTarget, idGenerator: IdGenerator): TreeNode { - const node = idGenerator.getNode(target[objectIdKey]); + const node = idGenerator.getNode(target.target); assert(node !== undefined, "objectId does not exist in nodeMap"); return node; } @@ -420,10 +464,8 @@ function objectIdsExist(treeEdit: TreeEdit, idGenerator: IdGenerator): void { break; case "insert": if (treeEdit.destination.type === "objectPlace") { - if (idGenerator.getNode(treeEdit.destination[objectIdKey]) === undefined) { - throw new UsageError( - `objectIdKey ${treeEdit.destination[objectIdKey]} does not exist`, - ); + if (idGenerator.getNode(treeEdit.destination.target) === undefined) { + throw new UsageError(`objectIdKey ${treeEdit.destination.target} does not exist`); } } else { if (idGenerator.getNode(treeEdit.destination.parentId) === undefined) { @@ -434,8 +476,8 @@ function objectIdsExist(treeEdit: TreeEdit, idGenerator: IdGenerator): void { case "remove": if (isRange(treeEdit.source)) { const missingObjectIds = [ - treeEdit.source.from[objectIdKey], - treeEdit.source.to[objectIdKey], + treeEdit.source.from.target, + treeEdit.source.to.target, ].filter((id) => !idGenerator.getNode(id)); if (missingObjectIds.length > 0) { @@ -443,14 +485,14 @@ function objectIdsExist(treeEdit: TreeEdit, idGenerator: IdGenerator): void { } } else if ( isObjectTarget(treeEdit.source) && - idGenerator.getNode(treeEdit.source[objectIdKey]) === undefined + idGenerator.getNode(treeEdit.source.target) === undefined ) { - throw new UsageError(`objectIdKey ${treeEdit.source[objectIdKey]} does not exist`); + throw new UsageError(`objectIdKey ${treeEdit.source.target} does not exist`); } break; case "modify": - if (idGenerator.getNode(treeEdit.target[objectIdKey]) === undefined) { - throw new UsageError(`objectIdKey ${treeEdit.target[objectIdKey]} does not exist`); + if (idGenerator.getNode(treeEdit.target.target) === undefined) { + throw new UsageError(`objectIdKey ${treeEdit.target.target} does not exist`); } break; case "move": { @@ -458,8 +500,8 @@ function objectIdsExist(treeEdit: TreeEdit, idGenerator: IdGenerator): void { // check the source if (isRange(treeEdit.source)) { const missingObjectIds = [ - treeEdit.source.from[objectIdKey], - treeEdit.source.to[objectIdKey], + treeEdit.source.from.target, + treeEdit.source.to.target, ].filter((id) => !idGenerator.getNode(id)); if (missingObjectIds.length > 0) { @@ -467,15 +509,15 @@ function objectIdsExist(treeEdit: TreeEdit, idGenerator: IdGenerator): void { } } else if ( isObjectTarget(treeEdit.source) && - idGenerator.getNode(treeEdit.source[objectIdKey]) === undefined + idGenerator.getNode(treeEdit.source.target) === undefined ) { - invalidObjectIds.push(treeEdit.source[objectIdKey]); + invalidObjectIds.push(treeEdit.source.target); } // check the destination if (treeEdit.destination.type === "objectPlace") { - if (idGenerator.getNode(treeEdit.destination[objectIdKey]) === undefined) { - invalidObjectIds.push(treeEdit.destination[objectIdKey]); + if (idGenerator.getNode(treeEdit.destination.target) === undefined) { + invalidObjectIds.push(treeEdit.destination.target); } } else { if (idGenerator.getNode(treeEdit.destination.parentId) === undefined) { diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts index e86f9bd2ac8d..fa8787c41d18 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts @@ -3,15 +3,12 @@ * Licensed under the MIT License. */ -import type { typeField } from "./agentEditReducer.js"; import type { JsonPrimitive } from "./json-handler/index.js"; /** * TODO: The current scheme does not allow manipulation of arrays of primitive values because you cannot refer to them. * We could accomplish this via a path (probably JSON Pointer or JSONPath) from a possibly-null objectId, or wrap arrays in an identified object. * - * TODO: We could add a "replace" edit type to avoid tons of little modifies. - * * TODO: only 100 object fields total are allowed by OpenAI right now, so larger schemas will fail faster if we have a bunch of schema types generated for type-specific edits. * * TODO: experiment using https://github.com/outlines-dev/outlines (and maybe a llama model) to avoid many of the annoyances of OpenAI's JSON Schema subset. @@ -39,12 +36,23 @@ import type { JsonPrimitive } from "./json-handler/index.js"; * TODO: Give the model a final chance to evaluate the result. * * TODO: Separate system prompt into [system, user, system] for security. + * + * TODO: Top level arrays are not supported with current DSL. + * + * TODO: Structured Output fails when multiple schema types have the same first field name (e.g. id: sf.identifier on multiple types). + * + * TODO: Pass descriptions from schema metadata to the generated TS types that we put in the prompt */ +/** + * TBD + */ +export const typeField = "__fluid_type"; /** * TBD */ export const objectIdKey = "__fluid_objectId"; + /** * TBD */ @@ -63,8 +71,8 @@ export type TreeEditValue = JsonPrimitive | TreeEditObject | TreeEditArray; /** * TBD - * @remarks - For polymorphic edits, we need to wrap the edit in an object to avoid anyOf at the root level. */ +// For polymorphic edits, we need to wrap the edit in an object to avoid anyOf at the root level. export interface EditWrapper { // eslint-disable-next-line @rushstack/no-new-null edit: TreeEdit | null; @@ -74,6 +82,7 @@ export interface EditWrapper { * TBD */ export type TreeEdit = SetRoot | Insert | Modify | Remove | Move; + /** * TBD */ @@ -85,16 +94,18 @@ export interface Edit { * TBD */ export type Selection = ObjectTarget | Range; + /** * TBD */ export interface ObjectTarget { - [objectIdKey]: string; + target: string; } /** - * @remarks - TODO: Allow support for nested arrays + * TBD */ +// TODO: Allow support for nested arrays export interface ArrayPlace { type: "arrayPlace"; parentId: string; @@ -132,7 +143,7 @@ export interface SetRoot extends Edit { */ export interface Insert extends Edit { type: "insert"; - content: TreeEditObject; + content: TreeEditObject | JsonPrimitive; destination: ObjectPlace | ArrayPlace; } diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index 70807027c44d..9fb67dfe3d39 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -13,9 +13,13 @@ import { type TreeNode, type TreeView, } from "@fluidframework/tree/internal"; +import type OpenAI from "openai"; +// eslint-disable-next-line import/no-internal-modules +import { zodResponseFormat } from "openai/helpers/zod"; import type { ChatCompletionCreateParams, ResponseFormatJSONSchema, + // eslint-disable-next-line import/no-internal-modules } from "openai/resources/index.mjs"; @@ -34,6 +38,8 @@ import { type EditLog, } from "./promptGeneration.js"; import { fail } from "./utils.js"; +import { generateGenericEditTypes } from "./typeGeneration.js"; +import { z } from "zod"; const DEBUG_LOG: string[] = []; @@ -108,7 +114,8 @@ export async function generateTreeEdits( simpleSchema.definitions, options.validator, ); - editLog.push({ edit: result }); + const explanation = result.explanation; // TODO: describeEdit(result, idGenerator); + editLog.push({ edit: { ...result, explanation } }); sequentialErrorCount = 0; } catch (error: unknown) { if (error instanceof Error) { @@ -180,50 +187,56 @@ async function* generateEdits( tokenLimits: TokenUsage | undefined, tokenUsage: TokenUsage, ): AsyncGenerator { + const [types, rootTypeName] = generateGenericEditTypes(simpleSchema, true); + + let plan: string | undefined; + if (options.plan !== undefined) { + plan = await getStringFromLlm( + getPlanningSystemPrompt(options.prompt, options.treeView, options.appGuidance), + options.openAIClient, + ); + } + const originalDecoratedJson = (options.finalReviewStep ?? false) ? toDecoratedJson(idGenerator, options.treeView.root) : undefined; // reviewed is implicitly true if finalReviewStep is false let hasReviewed = (options.finalReviewStep ?? false) ? false : true; - async function getNextEdit(): Promise { const systemPrompt = getEditingSystemPrompt( - options.prompt.userAsk, + options.prompt, idGenerator, options.treeView, editLog, - options.prompt.systemRoleContext, + options.appGuidance, + plan, ); DEBUG_LOG?.push(systemPrompt); - return new Promise((resolve: (value: TreeEdit | undefined) => void) => { - const editHandler = generateEditHandlers(simpleSchema, (jsonObject: JsonObject) => { - // eslint-disable-next-line unicorn/no-null - DEBUG_LOG?.push(JSON.stringify(jsonObject, null, 2)); - const wrapper = jsonObject as unknown as EditWrapper; - if (wrapper.edit === null) { - DEBUG_LOG?.push("No more edits."); - return resolve(undefined); - } else { - return resolve(wrapper.edit); - } - }); + const schema = types[rootTypeName] ?? fail("Root type not found."); + const wrapper = await getFromLlm( + systemPrompt, + options.openAIClient, + schema, + "A JSON object that represents an edit to a JSON tree.", + ); - const responseHandler = createResponseHandler( - editHandler, - options.limiters?.abortController ?? new AbortController(), - ); + DEBUG_LOG?.push(JSON.stringify(wrapper, null, 2)); + if (wrapper === undefined) { + DEBUG_LOG?.push("Failed to get response"); + return undefined; + } - // eslint-disable-next-line no-void - void responseHandler.processResponse( - streamFromLlm(systemPrompt, responseHandler.jsonSchema(), options.openAI, tokenUsage), - ); - }).then(async (result): Promise => { - if (result === undefined && (options.finalReviewStep ?? false) && !hasReviewed) { + if (wrapper.edit === null) { + DEBUG_LOG?.push("No more edits."); + if ((options.finalReviewStep ?? false) && !hasReviewed) { const reviewResult = await reviewGoal(); - // eslint-disable-next-line require-atomic-updates + if (reviewResult === undefined) { + DEBUG_LOG?.push("Failed to get review response"); + return undefined; + } hasReviewed = true; if (reviewResult.goalAccomplished === "yes") { return undefined; @@ -231,51 +244,37 @@ async function* generateEdits( editLog.length = 0; return getNextEdit(); } - } else { - return result; } - }); + } else { + return wrapper.edit; + } } - async function reviewGoal(): Promise { + async function reviewGoal(): Promise { const systemPrompt = getReviewSystemPrompt( - options.prompt.userAsk, + options.prompt, idGenerator, options.treeView, originalDecoratedJson ?? fail("Original decorated tree not provided."), - options.prompt.systemRoleContext, + options.appGuidance, ); DEBUG_LOG?.push(systemPrompt); - return new Promise((resolve: (value: ReviewResult) => void) => { - const reviewHandler = JsonHandler.object(() => ({ - properties: { - goalAccomplished: JsonHandler.enum({ - description: - 'Whether the difference the user\'s goal was met in the "after" tree.', - values: ["yes", "no"], - }), - }, - complete: (jsonObject: JsonObject) => { - // eslint-disable-next-line unicorn/no-null - DEBUG_LOG?.push(`Review result: ${JSON.stringify(jsonObject, null, 2)}`); - resolve(jsonObject as unknown as ReviewResult); - }, - }))(); - - const responseHandler = createResponseHandler( - reviewHandler, - options.limiters?.abortController ?? new AbortController(), - ); - - // eslint-disable-next-line no-void - void responseHandler.processResponse( - streamFromLlm(systemPrompt, responseHandler.jsonSchema(), options.openAI, tokenUsage), - ); + const schema = z.object({ + goalAccomplished: z + .enum(["yes", "no"]) + .describe('Whether the user\'s goal was met in the "after" tree.'), }); + return getFromLlm(systemPrompt, options.openAIClient, schema); } + // let edit = await getNextEdit(); + // while (edit !== undefined) { + // yield edit; + // edit = await getNextEdit(); + // } + let edit = await getNextEdit(); while (edit !== undefined) { yield edit; @@ -289,6 +288,43 @@ async function* generateEdits( } } +async function getFromLlm( + prompt: string, + openAIClient: OpenAI, + structuredOutputSchema: Zod.ZodTypeAny, + description?: string, +): Promise { + const response_format = zodResponseFormat(structuredOutputSchema, "SharedTreeAI", { + description, + }); + + const body: ChatCompletionCreateParams = { + messages: [{ role: "system", content: prompt }], + model: clientModel.get(openAIClient) ?? "gpt-4o", + response_format, + max_tokens: 4096, + }; + + const result = await openAIClient.beta.chat.completions.parse(body); + // TODO: fix types so this isn't null and doesn't need a cast + // The type should be derived from the zod schema + return result.choices[0]?.message.parsed as T | undefined; +} + +async function getStringFromLlm( + prompt: string, + openAIClient: OpenAI, +): Promise { + const body: ChatCompletionCreateParams = { + messages: [{ role: "system", content: prompt }], + model: clientModel.get(openAIClient) ?? "gpt-4o", + max_tokens: 4096, + }; + + const result = await openAIClient.chat.completions.create(body); + return result.choices[0]?.message.content ?? undefined; +} + class TokenLimitExceededError extends Error {} /** diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index c192345c473b..4212969d0cab 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -15,15 +15,25 @@ import { type JsonNodeSchema, type JsonSchemaRef, type JsonTreeSchema, - type TreeNode, + getSimpleSchema, + Tree, } from "@fluidframework/tree/internal"; +// eslint-disable-next-line import/no-extraneous-dependencies, import/no-internal-modules +import { createZodJsonValidator } from "typechat/zod"; -import { objectIdKey, type TreeEdit } from "./agentEditTypes.js"; -import { IdGenerator } from "./idGenerator.js"; +import { + objectIdKey, + type ObjectTarget, + type TreeEdit, + type TreeEditValue, + type Range, +} from "./agentEditTypes.js"; +import type { IdGenerator } from "./idGenerator.js"; +import { generateGenericEditTypes } from "./typeGeneration.js"; import { fail } from "./utils.js"; /** - * The log of edits produced by an LLM that have been performed on the Shared tree. + * */ export type EditLog = { edit: TreeEdit; @@ -44,8 +54,8 @@ export function toDecoratedJson( // Uncomment this assertion back once we have a typeguard ready. // assert(isTreeNode(node), "Non-TreeNode value in tree."); const objId = - idGenerator.getId(value as TreeNode) ?? - fail("ID of new node should have been assigned."); + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + idGenerator.getId(value) ?? fail("ID of new node should have been assigned."); assert( !Object.prototype.hasOwnProperty.call(value, objectIdKey), `Collision of object id property.`, @@ -63,32 +73,35 @@ export function toDecoratedJson( /** * TBD */ -export function getSuggestingSystemPrompt( +export function getPlanningSystemPrompt( + userPrompt: string, view: TreeView, - suggestionCount: number, - userGuidance?: string, + appGuidance?: string, ): string { const schema = normalizeFieldSchema(view.schema); const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); - const decoratedTreeJson = toDecoratedJson(new IdGenerator(), view.root); - const guidance = - userGuidance === undefined + const role = `I'm an agent who makes plans for another agent to achieve a user-specified goal to update the state of an application.${ + appGuidance === undefined ? "" - : `Additionally, the user has provided some guidance to help you refine your suggestions. Here is that guidance: ${userGuidance}`; + : ` + The other agent follows this guidance: ${appGuidance}` + }`; - // TODO: security: user prompt in system prompt - return ` - You are a collaborative agent who suggests possible changes to a JSON tree that follows a specific schema. - For example, for a schema of a digital whiteboard application, you might suggest things like "Change the color of all sticky notes to blue" or "Align all the handwritten text vertically". - Or, for a schema of a calendar application, you might suggest things like "Move the meeting with Alice to 3pm" or "Add a new event called 'Lunch with Bob' on Friday". - The tree that you are suggesting for is a JSON object with the following schema: ${promptFriendlySchema} - The current state of the tree is: ${decoratedTreeJson}. - ${guidance} - Please generate exactly ${suggestionCount} suggestions for changes to the tree that you think would be useful.`; + const systemPrompt = ` + ${role} + The application state tree is a JSON object with the following schema: ${promptFriendlySchema} + The current state is: ${JSON.stringify(view.root)}. + The user requested that I accomplish the following goal: + "${userPrompt}" + I've made a plan to accomplish this goal by doing a sequence of edits to the tree. + Edits can include setting the root, inserting, modifying, removing, or moving elements in the tree. + Here is my plan:`; + + return systemPrompt; } /** - * Creates a prompt containing unique instructions necessary for the LLM to generate explicit edits to the Shared Tree + * TBD */ export function getEditingSystemPrompt( userPrompt: string, @@ -96,6 +109,7 @@ export function getEditingSystemPrompt( view: TreeView, log: EditLog, appGuidance?: string, + plan?: string, ): string { const schema = normalizeFieldSchema(view.schema); const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); @@ -107,7 +121,7 @@ export function getEditingSystemPrompt( const error = edit.error === undefined ? "" - : ` This edit produced an error, and was discarded. The error message was: ${edit.error}`; + : ` This edit produced an error, and was discarded. The error message was: "${edit.error}"`; return `${index + 1}. ${JSON.stringify(edit.edit)}${error}`; }) .join("\n"); @@ -120,22 +134,19 @@ export function getEditingSystemPrompt( The application that owns the JSON tree has the following guidance about your role: ${appGuidance}` }`; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call + const treeSchemaString = createZodJsonValidator( + ...generateGenericEditTypes(getSimpleSchema(schema), false), + ).getSchemaText(); + // TODO: security: user prompt in system prompt const systemPrompt = ` ${role} - Edits are composed of the following primitives: - - ObjectTarget: a reference to an object (as specified by objectId). - - Place: either before or after a ObjectTarget (only makes sense for objects in arrays). - - ArrayPlace: either the "start" or "end" of an array, as specified by a "parent" ObjectTarget and a "field" name under which the array is stored. - - Range: a range of objects within the same array specified by a "start" and "end" Place. The range MUST be in the same array. - - Selection: a ObjectTarget or a Range. - The edits you may perform are: - - SetRoot: replaces the tree with a specific value. This is useful for initializing the tree or replacing the state entirely if appropriate. - - Insert: inserts a new object at a specific Place or ArrayPlace. - - Modify: sets a field on a specific ObjectTarget. - - Remove: deletes a Selection from the tree. - - Move: moves a Selection to a new Place or ArrayPlace. + Edits are JSON objects that conform to the following schema. + The top level object you produce is an "EditWrapper" object which contains one of "SetRoot", "Insert", "Modify", "Remove", "Move", or null. + ${treeSchemaString} The tree is a JSON object with the following schema: ${promptFriendlySchema} + ${plan === undefined ? "" : `You have made a plan to accomplish the user's goal. The plan is: "${plan}". You will perform one or more edits that correspond to that plan to accomplish the goal.`} ${ log.length === 0 ? "" @@ -144,16 +155,15 @@ export function getEditingSystemPrompt( This means that the current state of the tree reflects these changes.` } The current state of the tree is: ${decoratedTreeJson}. - Before you made the above edits, the user requested you accomplish the following goal: - ${userPrompt} - If the goal is now completed, you should return null. - Otherwise, you should create an edit that makes progress towards the goal. It should have an english description ("explanation") of what edit to perform (specifying one of the allowed edit types).`; + ${log.length > 0 ? "Before you made the above edits t" : "T"}he user requested you accomplish the following goal: + "${userPrompt}" + If the goal is now completed or is impossible, you should return null. + Otherwise, you should create an edit that makes progress towards the goal. It should have an English description ("explanation") of which edit to perform (specifying one of the allowed edit types).`; return systemPrompt; } /** - * Creates a prompt asking the LLM to confirm whether the edits it has performed has successfully accomplished the user's goal. - * @remarks This is a form of self-assessment for the LLM to evaluate its work for correctness. + * TBD */ export function getReviewSystemPrompt( userPrompt: string, @@ -221,6 +231,81 @@ export function getPromptFriendlyTreeSchema(jsonSchema: JsonTreeSchema): string return stringifiedSchema; } +function printContent(content: TreeEditValue, idGenerator: IdGenerator): string { + switch (typeof content) { + case "boolean": + return content ? "true" : "false"; + case "number": + return content.toString(); + case "string": + return `"${truncateString(content, 32)}"`; + case "object": { + if (Array.isArray(content)) { + // TODO: Describe the types of the array contents + return "a new array"; + } + if (content === null) { + return "null"; + } + const id = content[objectIdKey]; + assert(typeof id === "string", "Object content has no id."); + const node = idGenerator.getNode(id) ?? fail("Node not found."); + const schema = Tree.schema(node); + return `a new ${getFriendlySchemaName(schema.identifier)}`; + } + default: + fail("Unexpected content type."); + } +} + +/** + * TBD + */ +export function describeEdit(edit: TreeEdit, idGenerator: IdGenerator): string { + switch (edit.type) { + case "setRoot": + return `Set the root of the tree to ${printContent(edit.content, idGenerator)}.`; + case "insert": { + if (edit.destination.type === "arrayPlace") { + return `Insert ${printContent(edit.content, idGenerator)} at the ${edit.destination.location} of the array that is under the "${edit.destination.field}" property of ${edit.destination.parentId}.`; + } else { + const target = + idGenerator.getNode(edit.destination.target) ?? fail("Target node not found."); + const array = Tree.parent(target) ?? fail("Target node has no parent."); + const container = Tree.parent(array); + if (container === undefined) { + return `Insert ${printContent(edit.content, idGenerator)} into the array at the root of the tree. Insert it ${edit.destination.place} ${edit.destination.target}.`; + } + return `Insert ${printContent(edit.content, idGenerator)} into the array that is under the "${Tree.key(array)}" property of ${idGenerator.getId(container)}. Insert it ${edit.destination.place} ${edit.destination.target}.`; + } + } + case "modify": + return `Set the "${edit.field}" field of ${edit.target.target} to ${printContent(edit.modification, idGenerator)}.`; + case "remove": + return isObjectTarget(edit.source) + ? `Remove "${edit.source.target}" from the containing array.` + : `Remove all elements from ${edit.source.from.place} ${edit.source.from.target} to ${edit.source.to.place} ${edit.source.to.target} in their containing array.`; + case "move": + if (edit.destination.type === "arrayPlace") { + const suffix = `to the ${edit.destination.location} of the array that is under the "${edit.destination.field}" property of ${edit.destination.parentId}`; + return isObjectTarget(edit.source) + ? `Move ${edit.source.target} ${suffix}.` + : `Move all elements from ${edit.source.from.place} ${edit.source.from.target} to ${edit.source.to.place} ${edit.source.to.target} ${suffix}.`; + } else { + const suffix = `to ${edit.destination.place} ${edit.destination.target}`; + return isObjectTarget(edit.source) + ? `Move ${edit.source.target} ${suffix}.` + : `Move all elements from ${edit.source.from.place} ${edit.source.from.target} to ${edit.source.to.place} ${edit.source.to.target} ${suffix}.`; + } + default: + return "Unknown edit type."; + } +} + +function isObjectTarget(value: ObjectTarget | Range): value is ObjectTarget { + return (value as Partial).target !== undefined; +} + function getTypeString( defs: Record, [name, currentDef]: [string, JsonNodeSchema], @@ -267,7 +352,10 @@ function getDef(defs: Record, ref: string): JsonNodeSche return nextDef; } -function getFriendlySchemaName(schemaName: string): string { +/** + * TBD + */ +export function getFriendlySchemaName(schemaName: string): string { const matches = schemaName.match(/[^.]+$/); if (matches === null) { // empty scope @@ -275,3 +363,11 @@ function getFriendlySchemaName(schemaName: string): string { } return matches[0]; } + +function truncateString(str: string, maxLength: number): string { + if (str.length > maxLength) { + // eslint-disable-next-line unicorn/prefer-string-slice + return `${str.substring(0, maxLength - 3)}...`; + } + return str; +} diff --git a/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts new file mode 100644 index 000000000000..600ee9fa947d --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts @@ -0,0 +1,347 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ +import { assert } from "@fluidframework/core-utils/internal"; +import { FieldKind, NodeKind, ValueSchema } from "@fluidframework/tree/internal"; +import type { + SimpleFieldSchema, + SimpleNodeSchema, + SimpleTreeSchema, +} from "@fluidframework/tree/internal"; +import { z } from "zod"; + +import { objectIdKey, typeField } from "./agentEditTypes.js"; +import { fail, getOrCreate, mapIterable } from "./utils.js"; + +const objectTarget = z.object({ + target: z + .string() + .describe( + `The id of the object (as specified by the object's ${objectIdKey} property) that is being referenced`, + ), +}); +const objectPlace = z + .object({ + type: z.enum(["objectPlace"]), + target: z + .string() + .describe( + `The id (${objectIdKey}) of the object that the new/moved object should be placed relative to. This must be the id of an object that already existed in the tree content that was originally supplied.`, + ), + place: z + .enum(["before", "after"]) + .describe( + "Where the new/moved object will be relative to the target object - either just before or just after", + ), + }) + .describe( + "A pointer to a location either just before or just after an object that is in an array", + ); +const arrayPlace = z + .object({ + type: z.enum(["arrayPlace"]), + parentId: z + .string() + .describe( + `The id (${objectIdKey}) of the parent object of the array. This must be the id of an object that already existed in the tree content that was originally supplied.`, + ), + field: z.string().describe("The key of the array to insert into"), + location: z + .enum(["start", "end"]) + .describe("Where to insert into the array - either the start or the end"), + }) + .describe( + `either the "start" or "end" of an array, as specified by a "parent" ObjectTarget and a "field" name under which the array is stored (useful for prepending or appending)`, + ); +const range = z + .object({ + from: objectPlace, + to: objectPlace, + }) + .describe( + 'A range of objects in the same array specified by a "from" and "to" Place. The "to" and "from" objects MUST be in the same array.', + ); +const cache = new WeakMap>(); + +/** + * TBD + */ +export function generateGenericEditTypes( + schema: SimpleTreeSchema, + generateDomainTypes: boolean, +): [Record, root: string] { + return getOrCreate(cache, schema, () => { + const insertSet = new Set(); + const modifyFieldSet = new Set(); + const modifyTypeSet = new Set(); + const typeMap = new Map(); + for (const name of schema.definitions.keys()) { + getOrCreateType( + schema.definitions, + typeMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + name, + ); + } + function getType(allowedTypes: ReadonlySet): Zod.ZodTypeAny { + switch (allowedTypes.size) { + case 0: + return z.never(); + case 1: + return ( + typeMap.get(tryGetSingleton(allowedTypes) ?? fail("Expected singleton")) ?? + fail("Unknown type") + ); + default: { + const types = Array.from( + allowedTypes, + (name) => typeMap.get(name) ?? fail("Unknown type"), + ); + assert(hasAtLeastTwo(types), "Expected at least two types"); + return z.union(types); + } + } + } + const setRoot = z + .object({ + type: z.literal("setRoot"), + explanation: z.string().describe(editDescription), + content: generateDomainTypes + ? getType(schema.allowedTypes) + : z.any().describe("Domain-specific content here"), + }) + .describe( + "Replaces the tree with a specific value. This is useful for initializing the tree or replacing the state entirely if appropriate.", + ); + const insert = z + .object({ + type: z.literal("insert"), + explanation: z.string().describe(editDescription), + content: generateDomainTypes + ? getType(insertSet) + : z.any().describe("Domain-specific content here"), + destination: z.union([arrayPlace, objectPlace]), + }) + .describe("Inserts a new object at a specific Place or ArrayPlace."); + const remove = z + .object({ + type: z.literal("remove"), + explanation: z.string().describe(editDescription), + source: z.union([objectTarget, range]), + }) + .describe("Deletes an object or Range of objects from the tree."); + const move = z + .object({ + type: z.literal("move"), + explanation: z.string().describe(editDescription), + source: z.union([objectTarget, range]), + destination: z.union([arrayPlace, objectPlace]), + }) + .describe("Moves an object or Range of objects to a new Place or ArrayPlace."); + const modify = z + .object({ + type: z.enum(["modify"]), + explanation: z.string().describe(editDescription), + target: objectTarget, + field: z.enum([...modifyFieldSet] as [string, ...string[]]), // Modify with appropriate fields + modification: generateDomainTypes + ? getType(modifyTypeSet) + : z.any().describe("Domain-specific content here"), + }) + .describe("Sets a field on a specific ObjectTarget."); + const editTypes = [setRoot, insert, remove, move, modify, z.null()] as const; + const editWrapper = z.object({ + edit: z + .union(editTypes) + .describe("The next edit to apply to the tree, or null if the task is complete."), + }); + const typeRecord: Record = { + ObjectTarget: objectTarget, + ObjectPlace: objectPlace, + ArrayPlace: arrayPlace, + Range: range, + SetRoot: setRoot, + Insert: insert, + Remove: remove, + Move: move, + Modify: modify, + EditWrapper: editWrapper, + }; + return [typeRecord, "EditWrapper"]; + }); +} +const editDescription = + "A description of what this edit is meant to accomplish in human readable English"; +function getOrCreateType( + definitionMap: ReadonlyMap, + typeMap: Map, + insertSet: Set, + modifyFieldSet: Set, + modifyTypeSet: Set, + definition: string, +): Zod.ZodTypeAny { + return getOrCreate(typeMap, definition, () => { + const nodeSchema = definitionMap.get(definition) ?? fail("Unexpected definition"); + switch (nodeSchema.kind) { + case NodeKind.Object: { + for (const [key, field] of Object.entries(nodeSchema.fields)) { + // TODO: Remove when AI better + if ( + Array.from( + field.allowedTypes, + (n) => definitionMap.get(n) ?? fail("Unknown definition"), + ).some((n) => n.kind === NodeKind.Array) + ) { + continue; + } + modifyFieldSet.add(key); + for (const type of field.allowedTypes) { + modifyTypeSet.add(type); + } + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const properties = Object.fromEntries( + Object.entries(nodeSchema.fields) + .map(([key, field]) => { + return [ + key, + getOrCreateTypeForField( + definitionMap, + typeMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + field, + ), + ]; + }) + .filter(([, value]) => value !== undefined), + ); + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + properties[typeField] = z.enum([definition]); + return z.object(properties); + } + case NodeKind.Array: { + for (const [name] of Array.from( + nodeSchema.allowedTypes, + (n): [string, SimpleNodeSchema] => [ + n, + definitionMap.get(n) ?? fail("Unknown definition"), + ], + ).filter( + ([_, schema]) => schema.kind === NodeKind.Object || schema.kind === NodeKind.Leaf, + )) { + insertSet.add(name); + } + return z.array( + getTypeForAllowedTypes( + definitionMap, + typeMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + nodeSchema.allowedTypes, + ), + ); + } + case NodeKind.Leaf: + switch (nodeSchema.leafKind) { + case ValueSchema.Boolean: + return z.boolean(); + case ValueSchema.Number: + return z.number(); + case ValueSchema.String: + return z.string(); + case ValueSchema.Null: + return z.null(); + default: + throw new Error(`Unsupported leaf kind ${NodeKind[nodeSchema.leafKind]}.`); + } + default: + throw new Error(`Unsupported node kind ${NodeKind[nodeSchema.kind]}.`); + } + }); +} +function getOrCreateTypeForField( + definitionMap: ReadonlyMap, + typeMap: Map, + insertSet: Set, + modifyFieldSet: Set, + modifyTypeSet: Set, + fieldSchema: SimpleFieldSchema, +): Zod.ZodTypeAny | undefined { + switch (fieldSchema.kind) { + case FieldKind.Required: + return getTypeForAllowedTypes( + definitionMap, + typeMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + fieldSchema.allowedTypes, + ); + case FieldKind.Optional: + return z.optional( + getTypeForAllowedTypes( + definitionMap, + typeMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + fieldSchema.allowedTypes, + ), + ); + case FieldKind.Identifier: + return undefined; + default: + throw new Error(`Unsupported field kind ${NodeKind[fieldSchema.kind]}.`); + } +} +function getTypeForAllowedTypes( + definitionMap: ReadonlyMap, + typeMap: Map, + insertSet: Set, + modifyFieldSet: Set, + modifyTypeSet: Set, + allowedTypes: ReadonlySet, +): Zod.ZodTypeAny { + const single = tryGetSingleton(allowedTypes); + if (single === undefined) { + const types = [ + ...mapIterable(allowedTypes, (name) => { + return getOrCreateType( + definitionMap, + typeMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + name, + ); + }), + ]; + assert(hasAtLeastTwo(types), "Expected at least two types"); + return z.union(types); + } else { + return getOrCreateType( + definitionMap, + typeMap, + insertSet, + modifyFieldSet, + modifyTypeSet, + single, + ); + } +} +function tryGetSingleton(set: ReadonlySet): T | undefined { + if (set.size === 1) { + for (const item of set) { + return item; + } + } +} +function hasAtLeastTwo(array: T[]): array is [T, T, ...T[]] { + return array.length >= 2; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a1a7e1355706..ccb65c7305a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -455,7 +455,7 @@ importers: version: 5.0.0 typechat: specifier: ^0.1.1 - version: 0.1.1(typescript@5.4.5) + version: 0.1.1(typescript@5.4.5)(zod@3.23.8) typescript: specifier: ~5.4.5 version: 5.4.5 @@ -10440,6 +10440,9 @@ importers: '@fluidframework/core-utils': specifier: workspace:~ version: link:../../common/core-utils + '@fluidframework/runtime-utils': + specifier: workspace:~ + version: link:../../runtime/runtime-utils '@fluidframework/telemetry-utils': specifier: workspace:~ version: link:../../utils/telemetry-utils @@ -10522,6 +10525,9 @@ importers: rimraf: specifier: ^4.4.0 version: 4.4.1 + typechat: + specifier: ^0.1.1 + version: 0.1.1(typescript@5.4.5)(zod@3.23.8) typescript: specifier: ~5.4.5 version: 5.4.5 @@ -38383,7 +38389,7 @@ packages: media-typer: 0.3.0 mime-types: 2.1.35 - /typechat@0.1.1(typescript@5.4.5): + /typechat@0.1.1(typescript@5.4.5)(zod@3.23.8): resolution: {integrity: sha512-Sw96vmkYqbAahqam7vCp8P/MjIGsR26Odz17UHpVGniYN5ir2B37nRRkoDuRpA5djwNQB+W5TB7w2xoF6kwbHQ==} engines: {node: '>=18'} peerDependencies: @@ -38396,6 +38402,7 @@ packages: optional: true dependencies: typescript: 5.4.5 + zod: 3.23.8 dev: true /typed-array-buffer@1.0.2: From a29d48a2731c86c476bd830731799d8521b27bbb Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:32:25 +0000 Subject: [PATCH 12/28] Updates package with new planning step and removal of json streaming logic because its not in use. --- .../api-report/ai-collab.alpha.api.md | 2 + packages/framework/ai-collab/package.json | 2 +- packages/framework/ai-collab/src/aiCollab.ts | 1 + .../framework/ai-collab/src/aiCollabApi.ts | 1 + .../src/explicit-strategy/agentEditReducer.ts | 3 +- .../src/explicit-strategy/agentEditTypes.ts | 2 +- .../src/explicit-strategy/handlers.ts | 351 ---- .../ai-collab/src/explicit-strategy/index.ts | 255 ++- .../explicit-strategy/json-handler/debug.ts | 13 - .../explicit-strategy/json-handler/index.ts | 11 - .../json-handler/jsonHandler.ts | 95 -- .../json-handler/jsonHandlerImp.ts | 1420 ----------------- .../json-handler/jsonParser.ts | 565 ------- .../src/explicit-strategy/jsonTypes.ts | 26 + .../src/explicit-strategy/promptGeneration.ts | 8 +- .../explicit-strategy/agentEditing.spec.ts | 12 - .../agentEditingReducer.spec.ts | 69 +- .../explicit-strategy/integration.spec.ts | 36 +- pnpm-lock.yaml | 8 +- 19 files changed, 171 insertions(+), 2709 deletions(-) delete mode 100644 packages/framework/ai-collab/src/explicit-strategy/handlers.ts delete mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts delete mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts delete mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts delete mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts delete mode 100644 packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts create mode 100644 packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts diff --git a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md index 8f3e1ff3b76c..7747cdf21408 100644 --- a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md +++ b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md @@ -33,6 +33,8 @@ export interface AiCollabOptions { // (undocumented) openAI: OpenAiClientOptions; // (undocumented) + planningStep?: boolean; + // (undocumented) prompt: { systemRoleContext: string; userAsk: string; diff --git a/packages/framework/ai-collab/package.json b/packages/framework/ai-collab/package.json index b6251c1325f3..f186f960958a 100644 --- a/packages/framework/ai-collab/package.json +++ b/packages/framework/ai-collab/package.json @@ -94,6 +94,7 @@ "@fluidframework/telemetry-utils": "workspace:~", "@fluidframework/tree": "workspace:~", "openai": "^4.67.3", + "typechat": "^0.1.1", "zod": "^3.23.8" }, "devDependencies": { @@ -121,7 +122,6 @@ "mocha-multi-reporters": "^1.5.1", "prettier": "~3.0.3", "rimraf": "^4.4.0", - "typechat": "^0.1.1", "typescript": "~5.4.5" }, "typeValidation": { diff --git a/packages/framework/ai-collab/src/aiCollab.ts b/packages/framework/ai-collab/src/aiCollab.ts index 8c2675b58d32..850c675155d9 100644 --- a/packages/framework/ai-collab/src/aiCollab.ts +++ b/packages/framework/ai-collab/src/aiCollab.ts @@ -28,6 +28,7 @@ export async function aiCollab( prompt: options.prompt, limiters: options.limiters, dumpDebugLog: options.dumpDebugLog, + planningStep: options.planningStep, finalReviewStep: options.finalReviewStep, }); diff --git a/packages/framework/ai-collab/src/aiCollabApi.ts b/packages/framework/ai-collab/src/aiCollabApi.ts index a63dd9193c22..ca18c47d0e46 100644 --- a/packages/framework/ai-collab/src/aiCollabApi.ts +++ b/packages/framework/ai-collab/src/aiCollabApi.ts @@ -35,6 +35,7 @@ export interface AiCollabOptions { maxModelCalls?: number; tokenLimits?: TokenUsage; }; + planningStep?: boolean; finalReviewStep?: boolean; validator?: (newContent: TreeNode) => void; dumpDebugLog?: boolean; diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts index 38c13c7f40ec..01303d492b1b 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts @@ -40,8 +40,7 @@ import { typeField, } from "./agentEditTypes.js"; import type { IdGenerator } from "./idGenerator.js"; -// eslint-disable-next-line import/no-internal-modules -import type { JsonValue } from "./json-handler/jsonParser.js"; +import type { JsonValue } from "./jsonTypes.js"; import { toDecoratedJson } from "./promptGeneration.js"; import { fail } from "./utils.js"; diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts index fa8787c41d18..1ebed48a9818 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. */ -import type { JsonPrimitive } from "./json-handler/index.js"; +import type { JsonPrimitive } from "./jsonTypes.js"; /** * TODO: The current scheme does not allow manipulation of arrays of primitive values because you cannot refer to them. diff --git a/packages/framework/ai-collab/src/explicit-strategy/handlers.ts b/packages/framework/ai-collab/src/explicit-strategy/handlers.ts deleted file mode 100644 index 5e05c04da876..000000000000 --- a/packages/framework/ai-collab/src/explicit-strategy/handlers.ts +++ /dev/null @@ -1,351 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -import { FieldKind, NodeKind, ValueSchema } from "@fluidframework/tree/internal"; -import type { - SimpleFieldSchema, - SimpleNodeSchema, - SimpleTreeSchema, -} from "@fluidframework/tree/internal"; - -// import { ValueSchema } from "../core/index.js"; -import { typeField } from "./agentEditReducer.js"; -import { objectIdKey } from "./agentEditTypes.js"; -import { - type StreamedType, - type JsonObject, - JsonHandler as jh, -} from "./json-handler/index.js"; -import { fail, getOrCreate, mapIterable } from "./utils.js"; - -const objectTargetHandler = jh.object(() => ({ - description: "A pointer to an object in the tree", - properties: { - [objectIdKey]: jh.string({ description: "The id of the object that is being pointed to" }), - }, -})); - -const objectPlaceHandler = jh.object(() => ({ - description: - "A pointer to a location either just before or just after an object that is in an array", - properties: { - type: jh.enum({ values: ["objectPlace"] }), - [objectIdKey]: jh.string({ - description: `The id (${objectIdKey}) of the object that the new/moved object should be placed relative to. This must be the id of an object that already existed in the tree content that was originally supplied.`, - }), - place: jh.enum({ - values: ["before", "after"], - description: - "Where the new/moved object will be relative to the target object - either just before or just after", - }), - }, -})); - -const arrayPlaceHandler = jh.object(() => ({ - description: - "A location at either the beginning or the end of an array (useful for prepending or appending)", - properties: { - type: jh.enum({ values: ["arrayPlace"] }), - parentId: jh.string({ - description: `The id (${objectIdKey}) of the parent object of the array. This must be the id of an object that already existed in the tree content that was originally supplied.`, - }), - field: jh.string({ "description": "The key of the array to insert into" }), - location: jh.enum({ - values: ["start", "end"], - description: "Where to insert into the array - either the start or the end", - }), - }, -})); - -const rangeHandler = jh.object(() => ({ - description: - 'A span of objects that are in an array. The "to" and "from" objects MUST be in the same array.', - properties: { - from: objectPlaceHandler(), - to: objectPlaceHandler(), - }, -})); - -/** - * TBD - */ -export function generateEditHandlers( - schema: SimpleTreeSchema, - complete: (jsonObject: JsonObject) => void, -): StreamedType { - const insertSet = new Set(); - const modifyFieldSet = new Set(); - const modifyTypeSet = new Set(); - const schemaHandlers = new Map(); - - for (const name of schema.definitions.keys()) { - getOrCreateHandler( - schema.definitions, - schemaHandlers, - insertSet, - modifyFieldSet, - modifyTypeSet, - name, - ); - } - - const setRootHandler = jh.object(() => ({ - description: "A handler for setting content to the root of the tree.", - properties: { - type: jh.enum({ values: ["setRoot"] }), - explanation: jh.string({ description: editDescription }), - content: jh.anyOf( - Array.from( - schema.allowedTypes, - (nodeSchema) => schemaHandlers.get(nodeSchema) ?? fail("Unexpected schema"), - ), - ), - }, - })); - - const insertHandler = jh.object(() => ({ - description: "A handler for inserting new content into the tree.", - properties: { - type: jh.enum({ values: ["insert"] }), - explanation: jh.string({ description: editDescription }), - content: jh.anyOf( - Array.from(insertSet, (n) => schemaHandlers.get(n) ?? fail("Unexpected schema")), - ), - destination: jh.anyOf([arrayPlaceHandler(), objectPlaceHandler()]), - }, - })); - - const removeHandler = jh.object(() => ({ - description: "A handler for removing content from the tree.", - properties: { - type: jh.enum({ values: ["remove"] }), - explanation: jh.string({ description: editDescription }), - source: jh.anyOf([objectTargetHandler(), rangeHandler()]), - }, - })); - - const modifyHandler = jh.object(() => ({ - description: "A handler for modifying content in the tree.", - properties: { - type: jh.enum({ values: ["modify"] }), - explanation: jh.string({ description: editDescription }), - target: objectTargetHandler(), - field: jh.enum({ values: [...modifyFieldSet] }), - modification: jh.anyOf( - Array.from(modifyTypeSet, (n) => schemaHandlers.get(n) ?? fail("Unexpected schema")), - ), - }, - })); - - const moveHandler = jh.object(() => ({ - description: - "A handler for moving content from one location in the tree to another location in the tree.", - properties: { - type: jh.enum({ values: ["move"] }), - explanation: jh.string({ description: editDescription }), - source: jh.anyOf([objectTargetHandler(), rangeHandler()]), - destination: jh.anyOf([arrayPlaceHandler(), objectPlaceHandler()]), - }, - })); - - const editWrapper = jh.object(() => ({ - // description: - // "The next edit to apply to the tree, or null if the task is complete and no more edits are necessary.", - properties: { - edit: jh.anyOf([ - setRootHandler(), - insertHandler(), - modifyHandler(), - removeHandler(), - moveHandler(), - jh.null(), - ]), - }, - complete, - })); - - return jh.object(() => ({ - properties: { - edit: editWrapper(), - }, - }))(); -} - -const editDescription = - "A description of what this edit is meant to accomplish in human readable English"; - -function getOrCreateHandler( - definitionMap: ReadonlyMap, - handlerMap: Map, - insertSet: Set, - modifyFieldSet: Set, - modifyTypeSet: Set, - definition: string, -): StreamedType { - return getOrCreate(handlerMap, definition, () => { - const nodeSchema: SimpleNodeSchema = - definitionMap.get(definition) ?? fail("Unexpected definition"); - switch (nodeSchema.kind) { - case NodeKind.Object: { - for (const [key, field] of Object.entries(nodeSchema.fields)) { - // TODO: Remove when AI better - if ( - Array.from( - field.allowedTypes, - (n) => definitionMap.get(n) ?? fail("Unknown definition"), - ).some((n) => n.kind === NodeKind.Array) - ) { - continue; - } - modifyFieldSet.add(key); - for (const type of field.allowedTypes) { - modifyTypeSet.add(type); - } - } - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const properties = Object.fromEntries( - Object.entries(nodeSchema.fields) - .map(([key, field]) => { - return [ - key, - getOrCreateHandlerForField( - definitionMap, - handlerMap, - insertSet, - modifyFieldSet, - modifyFieldSet, - field, - ), - ]; - }) - .filter(([, value]) => value !== undefined), - ); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - properties[typeField] = jh.enum({ values: [definition] }); - return jh.object(() => ({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - properties, - }))(); - } - case NodeKind.Array: { - for (const [name] of Array.from( - nodeSchema.allowedTypes, - (n): [string, SimpleNodeSchema] => [ - n, - definitionMap.get(n) ?? fail("Unknown definition"), - ], - ).filter(([_, schema]) => schema.kind === NodeKind.Object)) { - insertSet.add(name); - } - - return jh.array(() => ({ - items: getStreamedType( - definitionMap, - handlerMap, - insertSet, - modifyFieldSet, - modifyTypeSet, - nodeSchema.allowedTypes, - ), - }))(); - } - case NodeKind.Leaf: - switch (nodeSchema.leafKind) { - case ValueSchema.Boolean: - return jh.boolean(); - case ValueSchema.Number: - return jh.number(); - case ValueSchema.String: - return jh.string(); - case ValueSchema.Null: - return jh.null(); - default: - throw new Error(`Unsupported leaf kind ${NodeKind[nodeSchema.leafKind]}.`); - } - default: - throw new Error(`Unsupported node kind ${NodeKind[nodeSchema.kind]}.`); - } - }); -} - -function getOrCreateHandlerForField( - definitionMap: ReadonlyMap, - handlerMap: Map, - insertSet: Set, - modifyFieldSet: Set, - modifyTypeSet: Set, - fieldSchema: SimpleFieldSchema, -): StreamedType | undefined { - switch (fieldSchema.kind) { - case FieldKind.Required: { - return getStreamedType( - definitionMap, - handlerMap, - insertSet, - modifyFieldSet, - modifyTypeSet, - fieldSchema.allowedTypes, - ); - } - case FieldKind.Optional: { - return jh.optional( - getStreamedType( - definitionMap, - handlerMap, - insertSet, - modifyFieldSet, - modifyTypeSet, - fieldSchema.allowedTypes, - ), - ); - } - case FieldKind.Identifier: { - return undefined; - } - default: { - throw new Error(`Unsupported field kind ${NodeKind[fieldSchema.kind]}.`); - } - } -} - -function getStreamedType( - definitionMap: ReadonlyMap, - handlerMap: Map, - insertSet: Set, - modifyFieldSet: Set, - modifyTypeSet: Set, - allowedTypes: ReadonlySet, -): StreamedType { - const single = tryGetSingleton(allowedTypes); - return single === undefined - ? jh.anyOf([ - ...mapIterable(allowedTypes, (name) => { - return getOrCreateHandler( - definitionMap, - handlerMap, - insertSet, - modifyFieldSet, - modifyTypeSet, - name, - ); - }), - ]) - : getOrCreateHandler( - definitionMap, - handlerMap, - insertSet, - modifyFieldSet, - modifyTypeSet, - single, - ); -} - -function tryGetSingleton(set: ReadonlySet): T | undefined { - if (set.size === 1) { - for (const item of set) { - return item; - } - } -} diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index 9fb67dfe3d39..2c1a6545cd87 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. */ -import { assert } from "@fluidframework/core-utils/internal"; import { getSimpleSchema, normalizeFieldSchema, @@ -13,33 +12,28 @@ import { type TreeNode, type TreeView, } from "@fluidframework/tree/internal"; -import type OpenAI from "openai"; // eslint-disable-next-line import/no-internal-modules import { zodResponseFormat } from "openai/helpers/zod"; import type { ChatCompletionCreateParams, - ResponseFormatJSONSchema, - // eslint-disable-next-line import/no-internal-modules } from "openai/resources/index.mjs"; +import { z } from "zod"; import type { OpenAiClientOptions, TokenUsage } from "../aiCollabApi.js"; import { applyAgentEdit } from "./agentEditReducer.js"; import type { EditWrapper, TreeEdit } from "./agentEditTypes.js"; -import { generateEditHandlers } from "./handlers.js"; import { IdGenerator } from "./idGenerator.js"; -import { createResponseHandler, JsonHandler, type JsonObject } from "./json-handler/index.js"; import { getEditingSystemPrompt, + getPlanningSystemPrompt, getReviewSystemPrompt, - getSuggestingSystemPrompt, toDecoratedJson, type EditLog, } from "./promptGeneration.js"; -import { fail } from "./utils.js"; import { generateGenericEditTypes } from "./typeGeneration.js"; -import { z } from "zod"; +import { fail } from "./utils.js"; const DEBUG_LOG: string[] = []; @@ -64,6 +58,7 @@ export interface GenerateTreeEditsOptions { finalReviewStep?: boolean; validator?: (newContent: TreeNode) => void; dumpDebugLog?: boolean; + planningStep?: boolean; } interface GenerateTreeEditsSuccessResponse { @@ -98,70 +93,75 @@ export async function generateTreeEdits( const tokenUsage = { inputTokens: 0, outputTokens: 0 }; - for await (const edit of generateEdits( - options, - simpleSchema, - idGenerator, - editLog, - options.limiters?.tokenLimits, - tokenUsage, - )) { - try { - const result = applyAgentEdit( - options.treeView, - edit, - idGenerator, - simpleSchema.definitions, - options.validator, - ); - const explanation = result.explanation; // TODO: describeEdit(result, idGenerator); - editLog.push({ edit: { ...result, explanation } }); - sequentialErrorCount = 0; - } catch (error: unknown) { - if (error instanceof Error) { - const { message } = error; - sequentialErrorCount += 1; - editLog.push({ edit, error: message }); - DEBUG_LOG?.push(`Error: ${message}`); - - if (error instanceof TokenLimitExceededError) { - return { - status: "failure", - errorMessage: "tokenLimitExceeded", - tokenUsage, - }; + try { + for await (const edit of generateEdits( + options, + simpleSchema, + idGenerator, + editLog, + options.limiters?.tokenLimits, + tokenUsage, + )) { + try { + const result = applyAgentEdit( + options.treeView, + edit, + idGenerator, + simpleSchema.definitions, + options.validator, + ); + const explanation = result.explanation; // TODO: describeEdit(result, idGenerator); + editLog.push({ edit: { ...result, explanation } }); + sequentialErrorCount = 0; + } catch (error: unknown) { + if (error instanceof Error) { + sequentialErrorCount += 1; + editLog.push({ edit, error: error.message }); + DEBUG_LOG?.push(`Error: ${error.message}`); + } else { + throw error; } - } else { - throw error; } - } - if (options.limiters?.abortController?.signal.aborted === true) { - return { - status: "failure", - errorMessage: "aborted", - tokenUsage, - }; - } + if (options.limiters?.abortController?.signal.aborted === true) { + return { + status: "failure", + errorMessage: "aborted", + tokenUsage, + }; + } - if ( - sequentialErrorCount > - (options.limiters?.maxSequentialErrors ?? Number.POSITIVE_INFINITY) - ) { - return { - status: "failure", - errorMessage: "tooManyErrors", - tokenUsage, - }; - } + if ( + sequentialErrorCount > + (options.limiters?.maxSequentialErrors ?? Number.POSITIVE_INFINITY) + ) { + return { + status: "failure", + errorMessage: "tooManyErrors", + tokenUsage, + }; + } - if (++editCount >= (options.limiters?.maxModelCalls ?? Number.POSITIVE_INFINITY)) { + if (++editCount >= (options.limiters?.maxModelCalls ?? Number.POSITIVE_INFINITY)) { + return { + status: "failure", + errorMessage: "tooManyModelCalls", + tokenUsage, + }; + } + } + } catch (error: unknown) { + if (error instanceof Error) { + DEBUG_LOG?.push(`Error: ${error.message}`); + } + if (error instanceof TokenLimitExceededError) { return { status: "failure", - errorMessage: "tooManyModelCalls", + errorMessage: "tokenLimitExceeded", tokenUsage, }; } + throw error; } if (options.dumpDebugLog ?? false) { @@ -190,10 +190,14 @@ async function* generateEdits( const [types, rootTypeName] = generateGenericEditTypes(simpleSchema, true); let plan: string | undefined; - if (options.plan !== undefined) { + if (options.planningStep !== undefined) { plan = await getStringFromLlm( - getPlanningSystemPrompt(options.prompt, options.treeView, options.appGuidance), - options.openAIClient, + getPlanningSystemPrompt( + options.treeView, + options.prompt.userAsk, + options.prompt.systemRoleContext, + ), + options.openAI, ); } @@ -205,11 +209,11 @@ async function* generateEdits( let hasReviewed = (options.finalReviewStep ?? false) ? false : true; async function getNextEdit(): Promise { const systemPrompt = getEditingSystemPrompt( - options.prompt, + options.prompt.userAsk, idGenerator, options.treeView, editLog, - options.appGuidance, + options.prompt.systemRoleContext, plan, ); @@ -218,11 +222,12 @@ async function* generateEdits( const schema = types[rootTypeName] ?? fail("Root type not found."); const wrapper = await getFromLlm( systemPrompt, - options.openAIClient, + options.openAI, schema, "A JSON object that represents an edit to a JSON tree.", ); + // eslint-disable-next-line unicorn/no-null DEBUG_LOG?.push(JSON.stringify(wrapper, null, 2)); if (wrapper === undefined) { DEBUG_LOG?.push("Failed to get response"); @@ -237,10 +242,12 @@ async function* generateEdits( DEBUG_LOG?.push("Failed to get review response"); return undefined; } + // eslint-disable-next-line require-atomic-updates hasReviewed = true; if (reviewResult.goalAccomplished === "yes") { return undefined; } else { + // eslint-disable-next-line require-atomic-updates editLog.length = 0; return getNextEdit(); } @@ -252,11 +259,11 @@ async function* generateEdits( async function reviewGoal(): Promise { const systemPrompt = getReviewSystemPrompt( - options.prompt, + options.prompt.userAsk, idGenerator, options.treeView, originalDecoratedJson ?? fail("Original decorated tree not provided."), - options.appGuidance, + options.prompt.systemRoleContext, ); DEBUG_LOG?.push(systemPrompt); @@ -266,15 +273,9 @@ async function* generateEdits( .enum(["yes", "no"]) .describe('Whether the user\'s goal was met in the "after" tree.'), }); - return getFromLlm(systemPrompt, options.openAIClient, schema); + return getFromLlm(systemPrompt, options.openAI, schema); } - // let edit = await getNextEdit(); - // while (edit !== undefined) { - // yield edit; - // edit = await getNextEdit(); - // } - let edit = await getNextEdit(); while (edit !== undefined) { yield edit; @@ -290,9 +291,10 @@ async function* generateEdits( async function getFromLlm( prompt: string, - openAIClient: OpenAI, + openAi: OpenAiClientOptions, structuredOutputSchema: Zod.ZodTypeAny, description?: string, + tokenUsage?: TokenUsage, ): Promise { const response_format = zodResponseFormat(structuredOutputSchema, "SharedTreeAI", { description, @@ -300,12 +302,17 @@ async function getFromLlm( const body: ChatCompletionCreateParams = { messages: [{ role: "system", content: prompt }], - model: clientModel.get(openAIClient) ?? "gpt-4o", + model: openAi.modelName ?? "gpt-4o", response_format, - max_tokens: 4096, }; - const result = await openAIClient.beta.chat.completions.parse(body); + const result = await openAi.client.beta.chat.completions.parse(body); + + if (result.usage !== undefined && tokenUsage !== undefined) { + tokenUsage.inputTokens += result.usage?.prompt_tokens; + tokenUsage.outputTokens += result.usage?.completion_tokens; + } + // TODO: fix types so this isn't null and doesn't need a cast // The type should be derived from the zod schema return result.choices[0]?.message.parsed as T | undefined; @@ -313,92 +320,22 @@ async function getFromLlm( async function getStringFromLlm( prompt: string, - openAIClient: OpenAI, + openAi: OpenAiClientOptions, + tokenUsage?: TokenUsage, ): Promise { const body: ChatCompletionCreateParams = { messages: [{ role: "system", content: prompt }], - model: clientModel.get(openAIClient) ?? "gpt-4o", - max_tokens: 4096, + model: openAi.modelName ?? "gpt-4o", }; - const result = await openAIClient.chat.completions.create(body); - return result.choices[0]?.message.content ?? undefined; -} - -class TokenLimitExceededError extends Error {} - -/** - * Prompts the provided LLM client to generate a list of suggested tree edits to perform. - * - * @internal - */ -export async function generateSuggestions( - openAIClient: OpenAiClientOptions, - view: TreeView, - suggestionCount: number, - tokenUsage?: TokenUsage, - guidance?: string, - abortController = new AbortController(), -): Promise { - let suggestions: string[] | undefined; - - const suggestionsHandler = JsonHandler.object(() => ({ - properties: { - edit: JsonHandler.array(() => ({ - description: - "A list of changes that a user might want a collaborative agent to make to the tree.", - items: JsonHandler.string(), - }))(), - }, - complete: (jsonObject: JsonObject) => { - suggestions = (jsonObject as { edit: string[] }).edit; - }, - }))(); - - const responseHandler = createResponseHandler(suggestionsHandler, abortController); - const systemPrompt = getSuggestingSystemPrompt(view, suggestionCount, guidance); - await responseHandler.processResponse( - streamFromLlm(systemPrompt, responseHandler.jsonSchema(), openAIClient, tokenUsage), - ); - assert(suggestions !== undefined, "No suggestions were generated."); - return suggestions; -} - -async function* streamFromLlm( - systemPrompt: string, - jsonSchema: JsonObject, - openAI: OpenAiClientOptions, - tokenUsage?: TokenUsage, -): AsyncGenerator { - const llmJsonSchema: ResponseFormatJSONSchema.JSONSchema = { - schema: jsonSchema, - name: "llm-response", - strict: true, // Opt into structured output - }; - - const body: ChatCompletionCreateParams = { - messages: [{ role: "system", content: systemPrompt }], - model: openAI.modelName ?? "gpt-4o", - response_format: { - type: "json_schema", - json_schema: llmJsonSchema, - }, - // TODO - // stream: true, // Opt in to streaming responses. - max_tokens: 4096, - }; - - const result = await openAI.client.chat.completions.create(body); - const choice = result.choices[0]; + const result = await openAi.client.chat.completions.create(body); if (result.usage !== undefined && tokenUsage !== undefined) { tokenUsage.inputTokens += result.usage?.prompt_tokens; tokenUsage.outputTokens += result.usage?.completion_tokens; } - assert(choice !== undefined, "Response included no choices."); - assert(choice.finish_reason === "stop", "Response was unfinished."); - assert(choice.message.content !== null, "Response contained no contents."); - // TODO: There is only a single yield here because we're not actually streaming - yield choice.message.content ?? ""; + return result.choices[0]?.message.content ?? undefined; } + +class TokenLimitExceededError extends Error {} diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts deleted file mode 100644 index dc0aed7c8a77..000000000000 --- a/packages/framework/ai-collab/src/explicit-strategy/json-handler/debug.ts +++ /dev/null @@ -1,13 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -/** - * TBD - */ -export function assert(condition: boolean): void { - if (!condition) { - throw new Error("Assert failed"); - } -} diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts deleted file mode 100644 index c360694eee12..000000000000 --- a/packages/framework/ai-collab/src/explicit-strategy/json-handler/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -// TODO: Only export the things we want -/* eslint-disable no-restricted-syntax */ - -export * from "./jsonHandler.js"; -export * from "./jsonHandlerImp.js"; -export * from "./jsonParser.js"; diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts deleted file mode 100644 index e08d370c28c6..000000000000 --- a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandler.ts +++ /dev/null @@ -1,95 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -import { - type PartialArg, - type StreamedObjectDescriptor, - type StreamedArrayDescriptor, - type StreamedType, - getJsonHandler, - getCreateResponseHandler, -} from "./jsonHandlerImp.js"; -import type { JsonObject } from "./jsonParser.js"; - -/** - * TBD - */ -export interface ResponseHandler { - jsonSchema(): JsonObject; - processResponse(responseGenerator: { - [Symbol.asyncIterator](): AsyncGenerator; - }): Promise; - processChars(chars: string): void; - complete(): void; -} - -/** - * TBD - */ -export const createResponseHandler: ( - streamedType: StreamedType, - abortController: AbortController, -) => ResponseHandler = getCreateResponseHandler(); - -/** - * TBD - */ -export const JsonHandler: { - object: ( - getDescriptor: (input: Input) => StreamedObjectDescriptor, - ) => (getInput?: (partial: PartialArg) => Input) => StreamedType; - - array: ( - getDescriptor: (input: Input) => StreamedArrayDescriptor, - ) => (getInput?: (partial: PartialArg) => Input) => StreamedType; - - streamedStringProperty< - Parent extends Record, - Key extends keyof Parent, - >(args: { - description?: string; - target: (partial: PartialArg) => Parent; - key: Key; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType; - - streamedString(args: { - description?: string; - target: (partial: PartialArg) => Parent; - append: (chars: string, parent: Parent) => void; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType; - - string(args?: { - description?: string; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType; - - enum(args: { - description?: string; - values: string[]; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType; - - number(args?: { - description?: string; - complete?: (value: number, partial: PartialArg) => void; - }): StreamedType; - - boolean(args?: { - description?: string; - complete?: (value: boolean, partial: PartialArg) => void; - }): StreamedType; - - null(args?: { - description?: string; - // eslint-disable-next-line @rushstack/no-new-null - complete?: (value: null, partial: PartialArg) => void; - }): StreamedType; - - optional(streamedType: StreamedType): StreamedType; - - anyOf(streamedTypes: StreamedType[]): StreamedType; -} = getJsonHandler(); diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts deleted file mode 100644 index e04e381bfda9..000000000000 --- a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonHandlerImp.ts +++ /dev/null @@ -1,1420 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -import { assert } from "./debug.js"; -import { - type JsonArray, - type JsonBuilder, - type JsonBuilderContext, - type JsonObject, - type JsonPrimitive, - type JsonValue, - type StreamedJsonParser, - contextIsObject, - createStreamedJsonParser, -} from "./jsonParser.js"; - -type StreamedTypeGetter = ( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - getInput?: (partial: PartialArg) => any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any -) => StreamedObject | StreamedArray; -type StreamedTypeIdentity = StreamedType | StreamedTypeGetter; -type DefinitionMap = Map; - -class ResponseHandlerImpl { - public constructor( - private readonly streamedType: StreamedType, - abortController: AbortController, - ) { - if (streamedType instanceof StreamedAnyOf) { - throw new TypeError("anyOf cannot be used as root type"); - } - - if ( - streamedType instanceof StreamedStringProperty || - streamedType instanceof StreamedString - ) { - throw new TypeError( - "StreamedStringProperty and StreamedString cannot be used as root type", - ); - } - - const streamedValueHandler = ( - streamedType as InvocableStreamedType - ).invoke( - undefined, - streamedType instanceof StreamedObject - ? {} - : streamedType instanceof StreamedArray - ? [] - : undefined, - ); - const builder = new BuilderDispatcher(streamedValueHandler); - this.parser = createStreamedJsonParser(builder, abortController); - } - - public jsonSchema(): JsonObject { - const definitions = new Map(); - const visited = new Set(); - - findDefinitions(this.streamedType, visited, definitions); - - const rootIdentity = ( - this.streamedType as InvocableStreamedType - ).getIdentity(); - - // Don't call jsonSchemaFromStreamedType, as we must force the method call on the root - const schema = ( - this.streamedType as InvocableStreamedType - ).jsonSchema(rootIdentity, definitions); - - definitions.forEach((definitionName, streamedTypeOrGetter) => { - if (streamedTypeOrGetter !== rootIdentity) { - schema.$defs ??= {}; - - const streamedType = - streamedTypeOrGetter instanceof Function - ? streamedTypeOrGetter(() => guaranteedErrorObject) // No-one will call this, but this return value emphasizes this point - : streamedTypeOrGetter; - - // Again, don't call jsonSchemaFromStreamedType, as we must force the method call on each definition root - (schema.$defs as JsonObject)[definitionName] = ( - streamedType as InvocableStreamedType - ).jsonSchema(this.streamedType, definitions); - } - }); - - return schema; - } - - public async processResponse(responseGenerator: { - [Symbol.asyncIterator](): AsyncGenerator; - }): Promise { - for await (const fragment of responseGenerator) { - this.processChars(fragment); - } - this.complete(); - } - - public processChars(chars: string): void { - this.parser.addChars(chars); - } - - public complete(): void { - // Send one more whitespace token, just to ensure the parser knows we're finished - // (this is necessary for the case of a schema comprising a single number) - this.parser.addChars("\n"); - } - - private readonly parser: StreamedJsonParser; -} - -// The one createResponseHandlerImpl -const createResponseHandlerImpl = ( - streamedType: StreamedType, - abortController: AbortController, -): ResponseHandlerImpl => { - return new ResponseHandlerImpl(streamedType, abortController); -}; - -/** - * TBD - */ -export const getCreateResponseHandler: () => ( - streamedType: StreamedType, - abortController: AbortController, -) => ResponseHandlerImpl = () => createResponseHandlerImpl; - -/** - * TBD - */ -export class StreamedType { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - private readonly _brand = Symbol(); -} - -class JsonHandlerImpl { - public object( - getDescriptor: (input: Input) => StreamedObjectDescriptor, - ): (getInput?: (partial: PartialArg) => Input) => StreamedType { - // The function created here serves as the identity of this type's schema, - // since the schema is independent of the input passed to the handler - return function getStreamedObject( - getInput?: (partial: PartialArg) => Input, - ): StreamedObject { - return new StreamedObject(getDescriptor, getStreamedObject, getInput); - }; - } - - public array( - getDescriptor: (input: Input) => StreamedArrayDescriptor, - ): (getInput?: (partial: PartialArg) => Input) => StreamedType { - // The function created here serves as the identity of this type's schema, - // since the schema is independent of the input passed to the handler - return function getStreamedArray( - getInput?: (partial: PartialArg) => Input, - ): StreamedArray { - return new StreamedArray(getDescriptor, getStreamedArray, getInput); - }; - } - - public streamedStringProperty< - T extends Record, - P extends keyof T, - >(args: { - description?: string; - target: (partial: PartialArg) => T; - key: P; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType { - return new StreamedStringProperty(args); - } - - public streamedString(args: { - description?: string; - target: (partial: PartialArg) => Parent; - append: (chars: string, parent: Parent) => void; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType { - return new StreamedString(args); - } - - public string(args?: { - description?: string; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType { - return new AtomicString(args); - } - - public enum(args: { - description?: string; - values: string[]; - complete?: (value: string, partial: PartialArg) => void; - }): StreamedType { - return new AtomicEnum(args); - } - - public number(args?: { - description?: string; - complete?: (value: number, partial: PartialArg) => void; - }): StreamedType { - return new AtomicNumber(args); - } - - public boolean(args?: { - description?: string; - complete?: (value: boolean, partial: PartialArg) => void; - }): StreamedType { - return new AtomicBoolean(args); - } - - public null(args?: { - description?: string; - // eslint-disable-next-line @rushstack/no-new-null - complete?: (value: null, partial: PartialArg) => void; - }): StreamedType { - return new AtomicNull(args); - } - - public optional(streamedType: StreamedType): StreamedType { - if (streamedType instanceof AtomicNull) { - throw new TypeError("Cannot have an optional null value"); - } - return new StreamedOptional(streamedType as InvocableStreamedType); - } - - public anyOf(streamedTypes: StreamedType[]): StreamedType { - return new StreamedAnyOf(streamedTypes as InvocableStreamedType[]); - } -} - -// The one JsonHandler -const jsonHandler = new JsonHandlerImpl(); - -/** - * TBD - */ -export const getJsonHandler: () => JsonHandlerImpl = () => jsonHandler; - -/** - * TBD - * @remarks - TODO: Can perhaps not export these after illustrateInteraction re-implemented (and remove all the ?s and !s) - */ -export interface StreamedObjectHandler { - addObject(key: string): StreamedObjectHandler; - addArray(key: string): StreamedArrayHandler; - addPrimitive(value: JsonPrimitive, key: string): void; - appendText(chars: string, key: string): void; - completeProperty(key: string): void; - complete(): void; -} - -/** - * TBD - */ -export interface StreamedArrayHandler { - addObject(): StreamedObjectHandler; - addArray(): StreamedArrayHandler; - addPrimitive(value: JsonPrimitive): void; - appendText(chars: string): void; - completeLast(): void; - complete(): void; -} - -type BuilderContext = JsonBuilderContext; - -class BuilderDispatcher implements JsonBuilder { - public constructor(private readonly rootHandler: StreamedValueHandler) {} - - public addObject(context?: BuilderContext): StreamedObjectHandler { - if (!context) { - // TODO: This error-handling, which really shouldn't be necessary in principle with Structured Outputs, - // is arguably "inside-out", i.e. it should report the expected type of the result, rather than - // the handler. - if (!(this.rootHandler instanceof StreamedObjectHandlerImpl)) { - throw new TypeError(`Expected object for root`); - } - return this.rootHandler; - } else if (contextIsObject(context)) { - return context.parentObject.addObject(context.key); - } else { - return context.parentArray.addObject(); - } - } - - public addArray(context?: BuilderContext): StreamedArrayHandler { - if (!context) { - if (!(this.rootHandler instanceof StreamedArrayHandlerImpl)) { - throw new TypeError(`Expected array for root`); - } - return this.rootHandler; - } else if (contextIsObject(context)) { - return context.parentObject.addArray(context.key); - } else { - return context.parentArray.addArray(); - } - } - - public addPrimitive(value: JsonPrimitive, context?: BuilderContext): void { - if (!context) { - if (value === null) { - if (!(this.rootHandler instanceof AtomicNullHandlerImpl)) { - throw new TypeError(`Expected null for root`); - } - this.rootHandler.complete(value, undefined); - } else { - switch (typeof value) { - case "string": { - if ( - !( - this.rootHandler instanceof AtomicStringHandlerImpl || - this.rootHandler instanceof AtomicEnumHandlerImpl - ) - ) { - throw new TypeError(`Expected string or enum for root`); - } - this.rootHandler.complete(value, undefined); - break; - } - case "number": { - if (!(this.rootHandler instanceof AtomicNumberHandlerImpl)) { - throw new TypeError(`Expected number for root`); - } - this.rootHandler.complete(value, undefined); - break; - } - case "boolean": { - if (!(this.rootHandler instanceof AtomicBooleanHandlerImpl)) { - throw new TypeError(`Expected boolean for root`); - } - this.rootHandler.complete(value, undefined); - break; - } - - default: { - break; - } - } - } - } else if (contextIsObject(context)) { - context.parentObject.addPrimitive(value, context.key); - } else { - context.parentArray.addPrimitive(value); - } - } - - public appendText(chars: string, context?: BuilderContext): void { - assert(context !== undefined); - if (contextIsObject(context)) { - context.parentObject.appendText(chars, context.key); - } else { - context!.parentArray.appendText(chars); - } - } - - public completeContext(context?: BuilderContext): void { - if (context !== undefined) { - if (contextIsObject(context)) { - context.parentObject.completeProperty?.(context.key); - } else { - context.parentArray.completeLast?.(); - } - } - } - - public completeContainer(container: StreamedObjectHandler | StreamedArrayHandler): void { - container.complete?.(); - } -} - -/** - * TBD - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export type PartialArg = any; // Would be PartialObject | PartialArray | undefined, but doesn't work for function arguments - -type PartialObject = JsonObject; - -// TODO: May be better to distinguish between streamed object properties and array elements (because strings) -type StreamedValueHandler = - | StreamedObjectHandler - | StreamedArrayHandler - | StreamedStringPropertyHandler - | StreamedStringHandler - | AtomicStringHandler - | AtomicEnumHandler - | AtomicNumberHandler - | AtomicBooleanHandler - | AtomicNullHandler - | AtomicPrimitiveHandler; // Needed so AtomicPrimitive can implement StreamedType> - -abstract class SchemaGeneratingStreamedType extends StreamedType { - public getIdentity(): StreamedTypeIdentity { - return this; - } - public abstract findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void; - public abstract jsonSchema( - root: StreamedTypeIdentity, - definitions: DefinitionMap, - ): JsonObject; -} - -abstract class InvocableStreamedType< - T extends StreamedValueHandler | undefined, -> extends SchemaGeneratingStreamedType { - public abstract invoke(parentPartial: PartialArg, partial: PartialArg): T; -} - -// eslint-disable-next-line @rushstack/no-new-null -type FieldHandlers = Record; - -const findDefinitions = ( - streamedType: StreamedType, - visited: Set, - definitions: DefinitionMap, -): void => - (streamedType as InvocableStreamedType).findDefinitions( - visited, - definitions, - ); - -const addDefinition = ( - streamedType: StreamedTypeIdentity, - definitions: DefinitionMap, -): void => { - if (!definitions.has(streamedType)) { - definitions.set(streamedType, `d${definitions.size.toString()}`); - } -}; - -const jsonSchemaFromStreamedType = ( - streamedType: StreamedType, - root: StreamedTypeIdentity, - definitions: DefinitionMap, -): JsonObject => { - const identity = (streamedType as InvocableStreamedType).getIdentity(); - - if (root === identity) { - return { $ref: "#" }; - } else if (definitions.has(identity)) { - return { $ref: `#/$defs/${definitions.get(identity)}` }; - } else { - return (streamedType as InvocableStreamedType).jsonSchema( - root, - definitions, - ); - } -}; - -const guaranteedErrorHandler = { - get(target: T, prop: string) { - throw new Error(`Attempted to access property "${prop}" outside a handler body.`); - }, - set(target: T, prop: string, value: V) { - throw new Error( - `Attempted to set property "${prop}" to "${value}" outside a handler body.`, - ); - }, - has(target: T, prop: string) { - throw new Error( - `Attempted to check existence of property "${prop}" outside a handler body.`, - ); - }, - deleteProperty(target: T, prop: string) { - throw new Error(`Attempted to delete property "${prop}" outside a handler body.`); - }, -}; -const guaranteedErrorObject = new Proxy({}, guaranteedErrorHandler); - -type FieldTypes = Record; - -/** - * TBD - */ -export interface StreamedObjectDescriptor { - description?: string; - properties: FieldTypes; - complete?: (result: JsonObject) => void; -} - -class StreamedObject extends InvocableStreamedType { - public constructor( - private readonly getDescriptor: (input: Input) => StreamedObjectDescriptor, - private readonly identity: StreamedTypeIdentity, - private readonly getInput?: (partial: PartialArg) => Input, - ) { - super(); - } - - public override getIdentity(): StreamedTypeIdentity { - return this.identity; - } - - public findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void { - const identity = this.getIdentity(); - if (visited.has(identity)) { - addDefinition(identity, definitions); - } else { - visited.add(identity); - - // TODO: Cache descriptor here, assert it's cached in jsonSchema (ditto all other types) - const { properties } = this.getDummyDescriptor(); - Object.values(properties).forEach((streamedType) => { - findDefinitions(streamedType, visited, definitions); - }); - } - } - - public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { - const { description, properties } = this.getDummyDescriptor(); - - const propertyNames = Object.keys(properties); - const schemaProperties: Record = {}; - propertyNames.forEach((fieldName) => { - schemaProperties[fieldName] = jsonSchemaFromStreamedType( - properties[fieldName]!, - root, - definitions, - ); - }); - - const schema: JsonObject = { - type: "object", - properties: schemaProperties, - }; - - if (description !== undefined) { - schema.description = description; - } - - schema.required = Object.keys(schemaProperties); - schema.additionalProperties = false; - - return schema; - } - - public invoke(parentPartial: PartialArg, partial: PartialArg): StreamedObjectHandler { - return new StreamedObjectHandlerImpl( - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - partial, - this.getDescriptor(this.getInput?.(parentPartial) as Input), - ); - } - - public get properties(): FieldTypes { - // TODO-AnyOf: Expose this more gracefully - return this.getDummyDescriptor().properties; - } - - public delayedInvoke(parentPartial: PartialArg): StreamedObjectDescriptor { - // TODO-AnyOf: Expose this more gracefully - return this.getDescriptor(this.getInput?.(parentPartial) as Input); - } - - private getDummyDescriptor(): StreamedObjectDescriptor { - if (this.dummyDescriptor === undefined) { - this.dummyDescriptor = this.getDescriptor(guaranteedErrorObject as Input); - } - - return this.dummyDescriptor; - } - - private dummyDescriptor?: StreamedObjectDescriptor; -} - -class StreamedObjectHandlerImpl implements StreamedObjectHandler { - public constructor( - private partial: PartialObject, - private descriptor?: StreamedObjectDescriptor, - private readonly streamedAnyOf?: StreamedAnyOf, - ) {} - - public addObject(key: string): StreamedObjectHandler { - this.attemptResolution(key, StreamedObject); - - if (this.descriptor) { - let streamedType: StreamedType | undefined = this.descriptor.properties[key]; - - if (streamedType === undefined) { - throw new Error(`Unhandled key ${key}`); - } - - if (streamedType instanceof StreamedOptional) { - streamedType = streamedType.optionalType; - } - - if (streamedType instanceof StreamedAnyOf) { - const streamedAnyOf = streamedType; - if (!(streamedType = streamedType.streamedTypeIfSingleMatch(StreamedObject))) { - // The type is ambiguous, so create an "unbound" StreamedObjectHandler and wait for more input - const childPartial: PartialObject = {}; - this.partial[key] = childPartial; - this.handlers[key] = new StreamedObjectHandlerImpl( - childPartial, - undefined, - streamedAnyOf, - ); - return this.handlers[key] as StreamedObjectHandler; - } - } - - if (streamedType instanceof StreamedObject) { - const childPartial: PartialObject = {}; - this.partial[key] = childPartial; - this.handlers[key] = streamedType.invoke(this.partial, this.partial[key]); - return this.handlers[key] as StreamedObjectHandler; - } - } - - throw new Error(`Expected object for key ${key}`); - } - - public addArray(key: string): StreamedArrayHandler { - this.attemptResolution(key, StreamedArray); - - if (this.descriptor) { - let streamedType: StreamedType | undefined = this.descriptor.properties[key]; - - if (streamedType === undefined) { - throw new Error(`Unhandled key ${key}`); - } - - if (streamedType instanceof StreamedOptional) { - streamedType = streamedType.optionalType; - } - - if (streamedType instanceof StreamedAnyOf) { - const streamedAnyOf = streamedType; - if (!(streamedType = streamedType.streamedTypeIfSingleMatch(StreamedArray))) { - // The type is ambiguous, so create an "unbound" StreamedObjectHandler and wait for more input - const childPartial: PartialArray = []; - this.partial[key] = childPartial; - this.handlers[key] = new StreamedArrayHandlerImpl( - childPartial, - undefined, - streamedAnyOf, - ); - return this.handlers[key] as StreamedArrayHandler; - } - } - - if (streamedType instanceof StreamedArray) { - const childPartial = [] as PartialArray; - this.partial[key] = childPartial; - this.handlers[key] = streamedType.invoke(this.partial, childPartial); - return this.handlers[key] as StreamedArrayHandler; - } - } - - throw new Error(`Expected array for key ${key}`); - } - - // TODO: Return boolean requesting throttling if StreamedString (also in StreamedArrayHandlerImpl) - public addPrimitive(value: JsonPrimitive, key: string): void { - if (!this.descriptor) { - this.partial[key] = value; - return; - } - - let streamedType: StreamedType | undefined = this.descriptor.properties[key]; - - if (streamedType === undefined) { - throw new Error(`Unhandled key ${key}`); - } - - this.partial[key] = value; - - if (streamedType instanceof StreamedOptional) { - if (value === null) { - // Don't call the (non-null) handler, as the optional value wasn't present - return; - } - streamedType = streamedType.optionalType; - } - - if (streamedType instanceof StreamedAnyOf) { - streamedType = streamedType.streamedTypeOfFirstMatch(value); - } - - if (primitiveMatchesStreamedType(value, streamedType!)) { - this.handlers[key] = ( - streamedType as InvocableStreamedType - ).invoke(this.partial, undefined); - return; - } - - // Shouldn't happen with Structured Outputs - throw new Error(`Unexpected ${typeof value} for key ${key}`); - } - - public appendText(chars: string, key: string): void { - assert(typeof this.partial[key] === "string"); - (this.partial[key] as string) += chars; - if ( - this.handlers[key] instanceof StreamedStringPropertyHandlerImpl || - this.handlers[key] instanceof StreamedStringHandlerImpl - ) { - (this.handlers[key] as { append: (chars: string) => void }).append(chars); - } - } - - public completeProperty(key: string): void { - const value = this.partial[key]; - if (isPrimitiveValue(value!)) { - this.attemptResolution(key, value as PrimitiveType); - - // Objects and Arrays will have their complete() handler called directly - completePrimitive(this.handlers[key]!, value as PrimitiveType, this.partial); - } - } - - public complete(): void { - // TODO-AnyOf: - this.descriptor!.complete?.(this.partial); - } - - private attemptResolution( - key: string, - typeOrValue: typeof StreamedObject | typeof StreamedArray | PrimitiveType, - ): void { - if (!this.descriptor) { - assert(this.streamedAnyOf !== undefined); - for (const option of this.streamedAnyOf!.options) { - if (option instanceof StreamedObject) { - const property = option.properties[key]; - if (streamedTypeMatches(property!, typeOrValue)) { - // We now know which option in the AnyOf to use - this.descriptor = option.delayedInvoke(this.partial); - } - } - } - } - } - - private handlers: FieldHandlers = {}; // TODO: Overkill, since only one needed at a time? -} - -type PartialArray = JsonArray; - -// eslint-disable-next-line @rushstack/no-new-null -type ArrayAppendHandler = StreamedValueHandler | null; - -/** - * TBD - */ -export interface StreamedArrayDescriptor { - description?: string; - items: StreamedType; - complete?: (result: JsonArray) => void; -} - -class StreamedArray extends InvocableStreamedType { - public constructor( - private readonly getDescriptor: (input: Input) => StreamedArrayDescriptor, - private readonly identity: StreamedTypeIdentity, - private readonly getInput?: (partial: PartialArg) => Input, - ) { - super(); - } - - public override getIdentity(): StreamedTypeIdentity { - return this.identity; - } - - public findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void { - const identity = this.getIdentity(); - if (visited.has(identity)) { - addDefinition(identity, definitions); - } else { - visited.add(identity); - - const { items } = this.getDummyDescriptor(); - findDefinitions(items, visited, definitions); - } - } - - public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { - const { description, items } = this.getDummyDescriptor(); - - const schema: JsonObject = { - type: "array", - items: jsonSchemaFromStreamedType(items, root, definitions), - }; - - if (description !== undefined) { - schema.description = description; - } - - return schema; - } - - public invoke(parentPartial: PartialArg, partial: PartialArg): StreamedArrayHandler { - return new StreamedArrayHandlerImpl( - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - partial, - this.getDescriptor(this.getInput?.(parentPartial) as Input), - ); - } - - public get items(): StreamedType { - // TODO-AnyOf: Expose this more gracefully - return this.getDummyDescriptor().items; - } - - public delayedInvoke(parentPartial: PartialArg): StreamedArrayDescriptor { - // TODO-AnyOf: Expose this more gracefully - return this.getDescriptor(this.getInput?.(parentPartial) as Input); - } - - private getDummyDescriptor(): StreamedArrayDescriptor { - if (this.dummyDescriptor === undefined) { - this.dummyDescriptor = this.getDescriptor(guaranteedErrorObject as Input); - } - - return this.dummyDescriptor; - } - - private dummyDescriptor?: StreamedArrayDescriptor; -} - -class StreamedArrayHandlerImpl implements StreamedArrayHandler { - public constructor( - private readonly partial: PartialArray, - private descriptor?: StreamedArrayDescriptor, - private readonly streamedAnyOf?: StreamedAnyOf, - ) {} - - public addObject(): StreamedObjectHandler { - this.attemptResolution(StreamedObject); - - if (this.descriptor) { - let streamedType: StreamedType | undefined = this.descriptor.items; - - if (streamedType instanceof StreamedAnyOf) { - const streamedAnyOf = streamedType; - if (!(streamedType = streamedType.streamedTypeIfSingleMatch(StreamedObject))) { - const childPartial: PartialObject = {}; - this.partial.push(childPartial); - this.lastHandler = new StreamedObjectHandlerImpl( - childPartial, - undefined, - streamedAnyOf, - ); - return this.lastHandler as StreamedObjectHandler; - } - } - - if (streamedType instanceof StreamedObject) { - const childPartial: PartialObject = {}; - this.partial.push(childPartial); - this.lastHandler = streamedType.invoke(this.partial, childPartial); - return this.lastHandler as StreamedObjectHandler; - } - } - - throw new Error("Expected object for items"); - } - - public addArray(): StreamedArrayHandler { - this.attemptResolution(StreamedArray); - - if (this.descriptor) { - const streamedType = this.descriptor.items; - - if (streamedType instanceof StreamedObject) { - const childPartial = [] as PartialArray; - this.partial.push(childPartial); - this.lastHandler = streamedType.invoke(this.partial, childPartial); - return this.lastHandler as StreamedArrayHandler; - } - } - - throw new Error("Expected array for items"); - } - - public addPrimitive(value: JsonPrimitive): void { - if (!this.descriptor) { - this.partial.push(value); - return; - } - const streamedType = this.descriptor.items; - - this.partial.push(value); - - if (primitiveMatchesStreamedType(value, streamedType)) { - this.lastHandler = (streamedType as InvocableStreamedType).invoke( - this.partial, - undefined, - ); - return; - } - - // Shouldn't happen with Structured Outputs - throw new Error(`Unexpected ${typeof value}`); - } - - public appendText(chars: string): void { - assert(typeof this.partial[this.partial.length - 1] === "string"); - - (this.partial[this.partial.length - 1] as string) += chars; - if (this.lastHandler instanceof StreamedStringPropertyHandlerImpl) { - this.lastHandler.append(chars); - } - } - - public completeLast(): void { - const value = this.partial[this.partial.length - 1]; - - if (isPrimitiveValue(value!)) { - this.attemptResolution(value as PrimitiveType); - - // Objects and Arrays will have their complete() handler called directly - completePrimitive(this.lastHandler!, value as PrimitiveType, this.partial); - } - } - - public complete(): void { - // TODO-AnyOf: - this.descriptor!.complete?.(this.partial); - } - - private attemptResolution( - typeOrValue: typeof StreamedObject | typeof StreamedArray | PrimitiveType, - ): void { - if (!this.descriptor) { - assert(this.streamedAnyOf !== undefined); - for (const option of this.streamedAnyOf!.options) { - if (option instanceof StreamedArray) { - const property = option.items; - if (streamedTypeMatches(property, typeOrValue)) { - // We now know which option in the AnyOf to use - this.descriptor = option.delayedInvoke(this.partial); - } - } - } - } - } - - private lastHandler?: ArrayAppendHandler; -} - -const primitiveMatchesStreamedType = ( - value: JsonPrimitive, - streamedType: StreamedType, -): boolean => { - if (value === null) { - return streamedType instanceof AtomicNull; - } else { - switch (typeof value) { - case "string": - return ( - streamedType instanceof StreamedStringProperty || - streamedType instanceof StreamedString || - streamedType instanceof AtomicString || - streamedType instanceof AtomicEnum - ); - case "number": - return streamedType instanceof AtomicNumber; - case "boolean": - return streamedType instanceof AtomicBoolean; - default: - assert(false); - return false; - } - } -}; - -const isPrimitiveValue = (value: JsonValue): value is JsonPrimitive => { - return ( - value === null || - typeof value === "string" || - typeof value === "number" || - typeof value === "boolean" - ); -}; - -const completePrimitive = ( - handler: StreamedValueHandler, - value: PrimitiveType, - partialParent: PartialArg, -): void => { - if ( - handler instanceof StreamedStringPropertyHandlerImpl || - handler instanceof StreamedStringHandlerImpl || - handler instanceof AtomicStringHandlerImpl - ) { - handler.complete(value as string, partialParent); - } else if (handler instanceof AtomicNumberHandlerImpl) { - handler.complete(value as number, partialParent); - } else if (handler instanceof AtomicBooleanHandlerImpl) { - handler.complete(value as boolean, partialParent); - } else if (handler instanceof AtomicNullHandlerImpl) { - handler.complete(value as null, partialParent); - } -}; - -const streamedTypeMatches = ( - streamedType: StreamedType, - typeOrValue: typeof StreamedObject | typeof StreamedArray | PrimitiveType, -): boolean => { - if (typeOrValue === StreamedObject || typeOrValue === StreamedArray) { - return streamedType instanceof typeOrValue; - } else { - if (typeOrValue === null) { - return streamedType instanceof AtomicNull; - } else { - switch (typeof typeOrValue) { - case "string": - return ( - streamedType instanceof AtomicString || - (streamedType instanceof AtomicEnum && streamedType.values.includes(typeOrValue)) - ); - case "number": - return streamedType instanceof AtomicNumber; - case "boolean": - return streamedType instanceof AtomicBoolean; - default: - assert(false); - return false; - } - } - } -}; - -interface SchemaArgs { - description?: string; -} - -// TODO: Also need StreamedStringElementHandler, usable only under array items:? - implementation just appends to last item in array? -interface StreamedStringPropertyHandler { - append(chars: string): void; - complete?(value: string, partial: PartialArg): void; -} - -type StreamedStringPropertyDescriptor< - T extends Record, - P extends keyof T, -> = SchemaArgs & { - target: (partial: PartialArg) => T; - key: P; - complete?: (value: string, partial: PartialArg) => void; -}; - -class StreamedStringProperty< - T extends Record, - P extends keyof T, -> extends InvocableStreamedType { - public constructor(private readonly args: StreamedStringPropertyDescriptor) { - super(); - } - - public findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void { - if (visited.has(this)) { - addDefinition(this, definitions); - } else { - visited.add(this); - } - } - - public jsonSchema(): JsonObject { - const { description } = this.args; - - const schema: { type: string; description?: string } = { - type: "string", - }; - if (description !== undefined) { - schema.description = description; - } - return schema; - } - - public invoke(parentPartial: PartialArg): StreamedStringPropertyHandler { - const { target, key, complete } = this.args; - - const item = target(parentPartial); - item[key] = "" as T[P]; - const append = (chars: string): void => { - item[key] = (item[key] + chars) as T[P]; - }; - - return new StreamedStringPropertyHandlerImpl(append, complete); - } -} - -class StreamedStringPropertyHandlerImpl< - T extends Record, - P extends keyof T, -> implements StreamedStringPropertyHandler -{ - public constructor( - private readonly onAppend: (chars: string) => void, - private readonly onComplete?: (value: string, partial: PartialArg) => void, - ) {} - - public append(chars: string): void { - return this.onAppend(chars); - } - - public complete(value: string, partial: PartialArg): void { - this.onComplete?.(value, partial); - } -} - -interface StreamedStringHandler { - append(chars: string): void; - complete?(value: string, partial: PartialArg): void; -} - -type StreamedStringDescriptor = SchemaArgs & { - target: (partial: PartialArg) => Parent; - append: (chars: string, parent: Parent) => void; - complete?: (value: string, partial: PartialArg) => void; -}; - -class StreamedString< - Parent extends object, -> extends InvocableStreamedType { - public constructor(private readonly args: StreamedStringDescriptor) { - super(); - } - - public findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void { - if (visited.has(this)) { - addDefinition(this, definitions); - } else { - visited.add(this); - } - } - - public jsonSchema(): JsonObject { - const { description } = this.args; - - const schema: { type: string; description?: string } = { - type: "string", - }; - if (description !== undefined) { - schema.description = description; - } - return schema; - } - - public invoke(parentPartial: PartialArg): StreamedStringHandler { - const { target, append, complete } = this.args; - - const parent = target?.(parentPartial); - - return new StreamedStringHandlerImpl(parent, append, complete); - } -} - -class StreamedStringHandlerImpl implements StreamedStringHandler { - public constructor( - private readonly parent: Parent, - private readonly onAppend: (chars: string, parent: Parent) => void, - private readonly onComplete?: (value: string, partial: PartialArg) => void, - ) {} - - public append(chars: string): void { - return this.onAppend(chars, this.parent); - } - - public complete(value: string, partial: PartialArg): void { - this.onComplete?.(value, partial); - } -} - -// eslint-disable-next-line @rushstack/no-new-null -type PrimitiveType = string | number | boolean | null; - -interface AtomicPrimitiveHandler { - complete(value: T, partial: PartialArg): void; -} - -type AtomicPrimitiveDescriptor = SchemaArgs & { - values?: string[]; - complete?: (value: T, partial: PartialArg) => void; -}; - -abstract class AtomicPrimitive extends InvocableStreamedType< - AtomicPrimitiveHandler -> { - public constructor(protected descriptor?: AtomicPrimitiveDescriptor) { - super(); - } - - public findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void { - if (visited.has(this)) { - addDefinition(this, definitions); - } else { - visited.add(this); - } - } - - public jsonSchema(): JsonObject { - const description = this.descriptor?.description; - - const schema: { type: string; enum?: string[]; description?: string } = { - type: this.typeName, - }; - if (this.descriptor?.values !== undefined) { - schema.enum = this.descriptor.values; - } - if (this.descriptor?.description !== undefined) { - schema.description = description; - } - return schema; - } - - public abstract override invoke(): AtomicPrimitiveHandler; - - protected abstract typeName: string; -} - -class AtomicPrimitiveHandlerImpl - implements AtomicPrimitiveHandler -{ - public constructor(private readonly onComplete?: (value: T, partial: PartialArg) => void) {} - - public complete(value: T, partial: PartialArg): void { - if (this.onComplete) { - this.onComplete(value, partial); - } - } -} - -type AtomicStringHandler = AtomicPrimitiveHandler; -class AtomicString extends AtomicPrimitive { - public override invoke(): AtomicStringHandler { - return new AtomicStringHandlerImpl(this.descriptor?.complete); - } - - public override typeName = "string"; -} -class AtomicStringHandlerImpl extends AtomicPrimitiveHandlerImpl {} - -type AtomicEnumHandler = AtomicPrimitiveHandler; -class AtomicEnum extends AtomicPrimitive { - public override invoke(): AtomicEnumHandler { - return new AtomicEnumHandlerImpl(this.descriptor?.complete); - } - - public override typeName = "string"; - - public get values(): string[] { - // TODO-AnyOf: Expose this more cleanly - return this.descriptor!.values!; - } -} -class AtomicEnumHandlerImpl extends AtomicPrimitiveHandlerImpl {} - -type AtomicNumberHandler = AtomicPrimitiveHandler; -class AtomicNumber extends AtomicPrimitive { - public override invoke(): AtomicNumberHandler { - return new AtomicNumberHandlerImpl(this.descriptor?.complete); - } - - public override typeName = "number"; -} -class AtomicNumberHandlerImpl extends AtomicPrimitiveHandlerImpl {} - -type AtomicBooleanHandler = AtomicPrimitiveHandler; -class AtomicBoolean extends AtomicPrimitive { - public override invoke(): AtomicBooleanHandler { - return new AtomicBooleanHandlerImpl(this.descriptor?.complete); - } - - public override typeName = "boolean"; -} -class AtomicBooleanHandlerImpl extends AtomicPrimitiveHandlerImpl {} - -// eslint-disable-next-line @rushstack/no-new-null -type AtomicNullHandler = AtomicPrimitiveHandler; -class AtomicNull extends AtomicPrimitive { - public override invoke(): AtomicNullHandler { - return new AtomicNullHandlerImpl(this.descriptor?.complete); - } - - public override typeName = "null"; -} -class AtomicNullHandlerImpl extends AtomicPrimitiveHandlerImpl {} - -// TODO: Only make this legal under object properties, not array items -class StreamedOptional extends SchemaGeneratingStreamedType { - public constructor(optionalType: SchemaGeneratingStreamedType) { - assert(!(optionalType instanceof AtomicNull)); - - super(); - this.optionalType = optionalType; - } - - public findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void { - if (visited.has(this)) { - addDefinition(this, definitions); - } else { - visited.add(this); - - findDefinitions(this.optionalType, visited, definitions); - } - } - - public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { - const schema = jsonSchemaFromStreamedType(this.optionalType, root, definitions); - if (root === this.optionalType || definitions.has(this.optionalType)) { - return { anyOf: [schema, { type: "null" }] }; - } else { - assert(typeof schema.type === "string"); - schema.type = [schema.type!, "null"]; - return schema; - } - } - - public optionalType: SchemaGeneratingStreamedType; -} - -class StreamedAnyOf extends SchemaGeneratingStreamedType { - public constructor(options: SchemaGeneratingStreamedType[]) { - super(); - this.options = options; - } - - public findDefinitions( - visited: Set, - definitions: DefinitionMap, - ): void { - if (visited.has(this)) { - addDefinition(this, definitions); - } else { - visited.add(this); - - for (const streamedType of this.options) { - findDefinitions(streamedType, visited, definitions); - } - } - } - - public jsonSchema(root: StreamedTypeIdentity, definitions: DefinitionMap): JsonObject { - return { - anyOf: this.options.map((streamedType) => - jsonSchemaFromStreamedType(streamedType, root, definitions), - ), - }; - } - - public streamedTypeIfSingleMatch( - classType: typeof StreamedObject | typeof StreamedArray, - ): StreamedType | undefined { - // If there is exactly one child StreamedType that is of the given type, return it - let streamedType: StreamedType | undefined; - for (const option of this.options) { - // TODO-AnyOf: Must also consider Optional and AnyOf - if (option instanceof classType) { - if (streamedType) { - return undefined; - } - streamedType = option; - } - } - return streamedType; - } - - public streamedTypeOfFirstMatch( - // eslint-disable-next-line @rushstack/no-new-null - value: string | number | boolean | null, - ): StreamedType | undefined { - for (const option of this.options) { - // TODO-AnyOf: Must also consider Optional and AnyOf - if (value === null && option instanceof AtomicNull) { - return option; - } else { - switch (typeof value) { - case "string": - if (option instanceof AtomicString || option instanceof AtomicEnum) { - return option; - } - break; - case "number": - if (option instanceof AtomicNumber) { - return option; - } - break; - case "boolean": - if (option instanceof AtomicBoolean) { - return option; - } - break; - default: - break; - } - } - } - return undefined; - } - - public options: SchemaGeneratingStreamedType[]; -} diff --git a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts b/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts deleted file mode 100644 index ba83fa522dab..000000000000 --- a/packages/framework/ai-collab/src/explicit-strategy/json-handler/jsonParser.ts +++ /dev/null @@ -1,565 +0,0 @@ -/*! - * Copyright (c) Microsoft Corporation and contributors. All rights reserved. - * Licensed under the MIT License. - */ - -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -import { assert } from "./debug.js"; - -/** - * TBD - */ -// eslint-disable-next-line @rushstack/no-new-null -export type JsonPrimitive = string | number | boolean | null; - -/** - * TBD - */ -// eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style -export interface JsonObject { - [key: string]: JsonValue; -} -/** - * TBD - */ -export type JsonArray = JsonValue[]; -/** - * TBD - */ -export type JsonValue = JsonPrimitive | JsonObject | JsonArray; - -/** - * TBD - */ -export type JsonBuilderContext = - | { parentObject: ObjectHandle; key: string } - | { parentArray: ArrayHandle }; - -/** - * TBD - */ -export interface JsonBuilder { - addObject(context?: JsonBuilderContext): ObjectHandle; - addArray(context?: JsonBuilderContext): ArrayHandle; - addPrimitive( - value: JsonPrimitive, - context?: JsonBuilderContext, - ): void; - appendText(chars: string, context?: JsonBuilderContext): void; - completeContext(context?: JsonBuilderContext): void; - completeContainer(container: ObjectHandle | ArrayHandle): void; -} - -/** - * TBD - */ -export function contextIsObject( - context?: JsonBuilderContext, -): context is { parentObject: ObjectHandle; key: string } { - return context !== undefined && "parentObject" in context; -} - -/** - * TBD - */ -export interface StreamedJsonParser { - addChars(text: string): void; -} - -/** - * TBD - */ -export function createStreamedJsonParser( - builder: JsonBuilder, - abortController: AbortController, -): StreamedJsonParser { - return new JsonParserImpl(builder, abortController); -} - -// Implementation - -const smoothStreaming = false; - -// prettier-ignore -enum State { - Start, - End, - - InsideObjectAtStart, - InsideObjectAfterKey, - InsideObjectAfterColon, - InsideObjectAfterProperty, - InsideObjectAfterComma, - InsideArrayAtStart, - InsideArrayAfterElement, - InsideArrayAfterComma, - InsideMarkdownAtStart, - InsideMarkdownAtEnd, - - // Special states while processing multi-character tokens (which may not arrive all at once) - InsideKeyword, - InsideNumber, - InsideKey, - InsideString, - InsideLeadingMarkdownDelimiter, - InsideTrailingMarkdownDelimiter, - - // Special momentary state - Pop, -} - -// Grammar productions - includes individual tokens -// prettier-ignore -enum Production { - Value, - Key, - Colon, - Comma, - CloseBrace, - CloseBracket, - LeadingMarkdownDelimiter, - TrailingMarkdownDelimiter, -} - -type StateTransition = [Production, State]; - -// prettier-ignore -const stateTransitionTable = new Map([ - [ - State.Start, - [ - [Production.Value, State.End], - [Production.LeadingMarkdownDelimiter, State.InsideMarkdownAtStart], - ], - ], - [ - State.InsideObjectAtStart, - [ - [Production.Key, State.InsideObjectAfterKey], - [Production.CloseBrace, State.Pop], - ], - ], - [State.InsideObjectAfterKey, [[Production.Colon, State.InsideObjectAfterColon]]], - [State.InsideObjectAfterColon, [[Production.Value, State.InsideObjectAfterProperty]]], - [ - State.InsideObjectAfterProperty, - [ - [Production.Comma, State.InsideObjectAfterComma], - [Production.CloseBrace, State.Pop], - ], - ], - [State.InsideObjectAfterComma, [[Production.Key, State.InsideObjectAfterKey]]], - [ - State.InsideArrayAtStart, - [ - [Production.Value, State.InsideArrayAfterElement], - [Production.CloseBracket, State.Pop], - ], - ], - [ - State.InsideArrayAfterElement, - [ - [Production.Comma, State.InsideArrayAfterComma], - [Production.CloseBracket, State.Pop], - ], - ], - [State.InsideArrayAfterComma, [[Production.Value, State.InsideArrayAfterElement]]], - [State.InsideMarkdownAtStart, [[Production.Value, State.InsideMarkdownAtEnd]]], - [State.InsideMarkdownAtEnd, [[Production.TrailingMarkdownDelimiter, State.End]]], -]); - -const keywords = ["true", "false", "null"]; -// eslint-disable-next-line unicorn/no-null -const keywordValues = [true, false, null]; - -interface ParserContext { - state: State; - firstToken: string; - parentObject?: ObjectHandle; - key?: string; - parentArray?: ArrayHandle; -} - -class JsonParserImpl implements StreamedJsonParser { - public constructor( - private readonly builder: JsonBuilder, - private readonly abortController: AbortController, - ) {} - - public addChars(text: string): void { - this.buffer += text; - - if (!this.throttled) { - while (this.processJsonText()) { - // Process as much of the buffer as possible - } - } - } - - // Implementation - - private buffer: string = ""; // This could be something more efficient - private throttled = false; - private readonly contexts: ParserContext[] = [ - { state: State.Start, firstToken: "" }, - ]; - - // Returns true if another token should be processed - private processJsonText(): boolean { - // Exit if there's nothing to process or the fetch has been aborted - if (this.buffer.length === 0 || this.abortController.signal.aborted) { - return false; - } - - const state = this.contexts[this.contexts.length - 1]!.state; - - // Are we in the midst of a multi-character token? - switch (state) { - case State.InsideKeyword: - return this.processJsonKeyword(); - - case State.InsideNumber: - return this.processJsonNumber(); - - case State.InsideKey: - return this.processJsonKey(); - - case State.InsideString: - return this.processJsonStringCharacters(); - - case State.InsideLeadingMarkdownDelimiter: - return this.processLeadingMarkdownDelimiter(); - - case State.InsideTrailingMarkdownDelimiter: - return this.processTrailingMarkdownDelimiter(); - - default: - break; - } - - // We're between tokens, so trim leading whitespace - // this.buffer = this.buffer.trimStart(); // REVIEW: Requires es2019 or later - this.buffer = this.buffer.replace(/^\s+/, ""); - - // Again, exit if there's nothing left to process - if (this.buffer.length === 0) { - return false; - } - - // If we're already done, there shouldn't be anything left to process - if (state === State.End) { - // REVIEW: Shouldn't be necessary with GPT4o, especially with Structured Output - this.buffer = ""; - return false; - // throw new Error("JSON already complete"); - } - - const builderContext = this.builderContextFromParserContext( - this.contexts[this.contexts.length - 1]!, - )!; - - // Start a new token - const char = this.buffer[0]!; - // eslint-disable-next-line unicorn/prefer-code-point - const charCode = char.charCodeAt(0); - - switch (charCode) { - case 123: // '{' - this.consumeCharAndPush(State.InsideObjectAtStart, { - parentObject: this.builder.addObject(builderContext), - }); - break; - case 58: // ':' - this.consumeCharAndEnterNextState(Production.Colon); - break; - case 44: // ',' - this.consumeCharAndEnterNextState(Production.Comma); - break; - case 125: // '}' - this.consumeCharAndEnterNextState(Production.CloseBrace); - break; - case 91: // '[' - this.consumeCharAndPush(State.InsideArrayAtStart, { - parentArray: this.builder.addArray(builderContext), - }); - break; - case 93: // ']' - this.consumeCharAndEnterNextState(Production.CloseBracket); - break; - case 34: // '"' - if (state === State.InsideObjectAtStart || state === State.InsideObjectAfterComma) { - // Keys shouldn't be updated incrementally, so wait until the complete key has arrived - this.pushContext(State.InsideKey, char); - } else { - this.builder.addPrimitive("", builderContext); - this.consumeCharAndPush(State.InsideString); - } - break; - case 116: // 't' - case 102: // 'f' - case 110: // 'n' - this.pushContext(State.InsideKeyword, char); - break; - case 45: // '-' - this.pushContext(State.InsideNumber, char); - break; - default: - if (charCode >= 48 && charCode <= 57) { - // '0' - '9' - this.pushContext(State.InsideNumber, char); - } else if (charCode === 96) { - // '`' - if (state === State.Start) { - this.pushContext(State.InsideLeadingMarkdownDelimiter, char); - } else if (state === State.InsideMarkdownAtEnd) { - this.pushContext(State.InsideTrailingMarkdownDelimiter, char); - } else { - this.unexpectedTokenError(char); - } - } else { - this.unexpectedTokenError(char); - } - break; - } - - return this.buffer.length > 0; - } - - private processLeadingMarkdownDelimiter(): boolean { - const leadingMarkdownDelimiter = "```json"; - if (this.buffer.startsWith(leadingMarkdownDelimiter)) { - this.buffer = this.buffer.slice(leadingMarkdownDelimiter.length); - this.popContext(Production.LeadingMarkdownDelimiter); - return this.buffer.length > 0; - } - - return false; - } - - private processTrailingMarkdownDelimiter(): boolean { - const trailingMarkdownDelimiter = "```"; - if (this.buffer.startsWith(trailingMarkdownDelimiter)) { - this.buffer = this.buffer.slice(trailingMarkdownDelimiter.length); - this.popContext(Production.TrailingMarkdownDelimiter); - return this.buffer.length > 0; - } - - return false; - } - - private processJsonKeyword(): boolean { - // Just match the keyword, let the next iteration handle the next characters - for (let i = 0; i < keywords.length; i++) { - const keyword = keywords[i]!; - if (this.buffer.startsWith(keyword)) { - this.buffer = this.buffer.slice(keyword.length); - this.setPrimitiveValueAndPop(keywordValues[i]!); - return true; - } else if (keyword.startsWith(this.buffer)) { - return false; - } - } - - this.unexpectedTokenError(this.buffer); - return false; - } - - private processJsonNumber(): boolean { - // Match the number plus a single non-number character (so we know the number is complete) - const jsonNumber = /^-?(0|([1-9]\d*))(\.\d+)?([Ee][+-]?\d+)?(?=\s|\D)/; - const match = this.buffer.match(jsonNumber); - if (match) { - const numberText = match[0]; - this.buffer = this.buffer.slice(numberText.length); - this.setPrimitiveValueAndPop(+numberText); // Unary + parses the numeric string - return true; - } - - return false; - } - - private processJsonKey(): boolean { - // Match the complete string, including start and end quotes - assert(this.buffer.startsWith('"')); - const jsonStringRegex = /^"((?:[^"\\]|\\.)*)"/; - const match = this.buffer.match(jsonStringRegex); - - if (match) { - const keyText = match[0]; - this.buffer = this.buffer.slice(keyText.length); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const key = JSON.parse(keyText); - - assert(this.contexts.length > 1); - const parentContext = this.contexts[this.contexts.length - 2]!; - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - parentContext.key = key; - - this.popContext(Production.Key); - return true; - } - - return false; - } - - // String values are special because we might stream them - private processJsonStringCharacters(): boolean { - let maxCount = Number.POSITIVE_INFINITY; - - if (smoothStreaming) { - maxCount = 5; - } - - this.appendText(this.convertJsonStringCharacters(maxCount)); - - if (this.buffer.startsWith('"')) { - // The end of the string was reached - this.buffer = this.buffer.slice(1); - this.completePrimitiveAndPop(); - return this.buffer.length > 0; - } else if (this.buffer.length > 0) { - this.throttled = true; - setTimeout(() => { - this.throttled = false; - while (this.processJsonText()) { - // Process characters until it's time to pause again - } - }, 15); - } - - return false; - } - - private convertJsonStringCharacters(maxCount: number): string { - let escapeNext = false; - - let i = 0; - for (; i < Math.min(maxCount, this.buffer.length); i++) { - const char = this.buffer[i]; - - if (escapeNext) { - escapeNext = false; // JSON.parse will ensure valid escape sequence - } else if (char === "\\") { - escapeNext = true; - } else if (char === '"') { - // Unescaped " is reached - break; - } - } - - if (escapeNext) { - // Buffer ends with a single '\' character - i--; - } - - const result = this.buffer.slice(0, i); - this.buffer = this.buffer.slice(i); - return JSON.parse(`"${result}"`) as string; - } - - private appendText(text: string): void { - assert(this.contexts.length > 1); - const builderContext = this.builderContextFromParserContext( - this.contexts[this.contexts.length - 2]!, - )!; - this.builder.appendText(text, builderContext); - } - - private consumeCharAndPush( - state: State, - parent?: { parentObject?: ObjectHandle; parentArray?: ArrayHandle }, - ): void { - const firstToken = this.buffer[0]!; - this.buffer = this.buffer.slice(1); - this.pushContext(state, firstToken, parent); - } - - private pushContext( - state: State, - firstToken: string, - parent?: { parentObject?: ObjectHandle; parentArray?: ArrayHandle }, - ): void { - this.contexts.push({ - state, - firstToken, - parentObject: parent?.parentObject, - parentArray: parent?.parentArray, - }); - } - - private setPrimitiveValueAndPop(value: JsonPrimitive): void { - this.builder.addPrimitive( - value, - this.builderContextFromParserContext(this.contexts[this.contexts.length - 2]!), - ); - this.completePrimitiveAndPop(); - } - - private completePrimitiveAndPop(): void { - this.builder.completeContext( - this.builderContextFromParserContext(this.contexts[this.contexts.length - 2]!), - ); - this.popContext(Production.Value); - } - - private completeAndPopContext(): void { - const context = this.contexts[this.contexts.length - 1]!; - if (context.parentObject !== undefined) { - this.builder.completeContainer?.(context.parentObject); - } else if (context.parentArray !== undefined) { - this.builder.completeContainer?.(context.parentArray); - } - this.builder.completeContext?.( - this.builderContextFromParserContext(this.contexts[this.contexts.length - 2]!), - ); - this.popContext(); - } - - private builderContextFromParserContext( - context: ParserContext, - ): JsonBuilderContext | undefined { - if (context.parentObject !== undefined) { - return { parentObject: context.parentObject, key: context.key! }; - // eslint-disable-next-line unicorn/no-negated-condition - } else if (context.parentArray !== undefined) { - return { parentArray: context.parentArray }; - } else { - return undefined; - } - } - - private popContext(production = Production.Value): void { - assert(this.contexts.length > 1); - const poppedContext = this.contexts.pop()!; - this.nextState(poppedContext.firstToken, production); - } - - private consumeCharAndEnterNextState(production: Production): void { - const token = this.buffer[0]!; - this.buffer = this.buffer.slice(1); - this.nextState(token, production); - } - - private nextState(token: string, production: Production): void { - const context = this.contexts[this.contexts.length - 1]!; - - const stateTransitions = stateTransitionTable.get(context.state); - assert(stateTransitions !== undefined); - for (const [productionCandidate, nextState] of stateTransitions!) { - if (productionCandidate === production) { - if (nextState === State.Pop) { - this.completeAndPopContext(); - } else { - context.state = nextState; - } - return; - } - } - - this.unexpectedTokenError(token); - } - - private unexpectedTokenError(token: string): void { - throw new Error(`Unexpected token ${token}`); - } -} diff --git a/packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts b/packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts new file mode 100644 index 000000000000..da8cbf14dfee --- /dev/null +++ b/packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts @@ -0,0 +1,26 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +/** + * TBD + */ +// eslint-disable-next-line @rushstack/no-new-null +export type JsonPrimitive = string | number | boolean | null; + +/** + * TBD + */ +// eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style +export interface JsonObject { + [key: string]: JsonValue; +} +/** + * TBD + */ +export type JsonArray = JsonValue[]; +/** + * TBD + */ +export type JsonValue = JsonPrimitive | JsonObject | JsonArray; diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index 4212969d0cab..e2d6e738e272 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -74,17 +74,17 @@ export function toDecoratedJson( * TBD */ export function getPlanningSystemPrompt( - userPrompt: string, view: TreeView, - appGuidance?: string, + userPrompt: string, + systemRoleContext?: string, ): string { const schema = normalizeFieldSchema(view.schema); const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); const role = `I'm an agent who makes plans for another agent to achieve a user-specified goal to update the state of an application.${ - appGuidance === undefined + systemRoleContext === undefined ? "" : ` - The other agent follows this guidance: ${appGuidance}` + The other agent follows this guidance: ${systemRoleContext}` }`; const systemPrompt = ` diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts index 796a78517289..6ab77ff6cf73 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts @@ -67,18 +67,6 @@ describe("toDecoratedJson", () => { y: 2, }), ); - - // const vector = new Vector({ x: 1, y: 2 }); - // // const hydratedObject = hydrate(Vector, vector); - - // assert.equal( - // toDecoratedJson(idGenerator, hydratedObject), - // JSON.stringify({ - // [objectIdKey]: "Vector0", - // x: 1, - // y: 2, - // }), - // ); }); it("handles nested objects", () => { diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts index 169c1754c1d0..d987b3b152c7 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts @@ -23,19 +23,16 @@ import { import { applyAgentEdit, - typeField, // eslint-disable-next-line import/no-internal-modules } from "../../explicit-strategy/agentEditReducer.js"; import { - objectIdKey, + typeField, // eslint-disable-next-line import/no-internal-modules } from "../../explicit-strategy/agentEditTypes.js"; -// eslint-disable-next-line import/order import type { TreeEdit, // eslint-disable-next-line import/no-internal-modules } from "../../explicit-strategy/agentEditTypes.js"; - // eslint-disable-next-line import/no-internal-modules import { IdGenerator } from "../../explicit-strategy/idGenerator.js"; @@ -202,7 +199,7 @@ describe("applyAgentEdit", () => { content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, destination: { type: "objectPlace", - [objectIdKey]: vectorId, + target: vectorId, place: "after", }, }; @@ -214,7 +211,7 @@ describe("applyAgentEdit", () => { content: { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, destination: { type: "objectPlace", - [objectIdKey]: vectorId, + target: vectorId, place: "after", }, }; @@ -282,7 +279,7 @@ describe("applyAgentEdit", () => { content: { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, destination: { type: "objectPlace", - [objectIdKey]: vectorId, + target: vectorId, place: "after", }, }; @@ -399,14 +396,14 @@ describe("applyAgentEdit", () => { content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, destination: { type: "objectPlace", - [objectIdKey]: vectorId, + target: vectorId, place: "after", }, }; assert.throws( () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), - validateUsageError(/invalid data provided for schema/), + validateUsageError(/provided data is incompatible/), ); }); @@ -436,7 +433,7 @@ describe("applyAgentEdit", () => { content: { [typeField]: Vector.identifier, x: 3, y: 4, z: 5 }, destination: { type: "objectPlace", - [objectIdKey]: vectorId, + target: vectorId, place: "before", }, }; @@ -468,7 +465,7 @@ describe("applyAgentEdit", () => { const modifyEdit: TreeEdit = { explanation: "Modify a vector", type: "modify", - target: { __fluid_objectId: vectorId }, + target: { target: vectorId }, field: "vectors", modification: [ { [typeField]: Vector.identifier, x: 2, y: 3, z: 4 }, @@ -480,7 +477,7 @@ describe("applyAgentEdit", () => { const modifyEdit2: TreeEdit = { explanation: "Modify a vector", type: "modify", - target: { __fluid_objectId: vectorId }, + target: { target: vectorId }, field: "bools", modification: [false], }; @@ -494,7 +491,7 @@ describe("applyAgentEdit", () => { const modifyEdit3: TreeEdit = { explanation: "Modify a vector", type: "modify", - target: { __fluid_objectId: vectorId2 }, + target: { target: vectorId2 }, field: "x", modification: 111, }; @@ -554,7 +551,7 @@ describe("applyAgentEdit", () => { const removeEdit: TreeEdit = { explanation: "remove a vector", type: "remove", - source: { [objectIdKey]: vectorId1 }, + source: { target: vectorId1 }, }; applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); @@ -595,7 +592,7 @@ describe("applyAgentEdit", () => { const removeEdit: TreeEdit = { explanation: "remove a vector", type: "remove", - source: { [objectIdKey]: singleVectorId }, + source: { target: singleVectorId }, }; applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); @@ -634,7 +631,7 @@ describe("applyAgentEdit", () => { const removeEdit: TreeEdit = { explanation: "remove the root", type: "remove", - source: { [objectIdKey]: rootId }, + source: { target: rootId }, }; applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); assert.equal(view.root, undefined); @@ -664,7 +661,7 @@ describe("applyAgentEdit", () => { const removeEdit: TreeEdit = { explanation: "remove the root", type: "remove", - source: { [objectIdKey]: rootId }, + source: { target: rootId }, }; assert.throws( @@ -705,12 +702,12 @@ describe("applyAgentEdit", () => { type: "remove", source: { from: { - [objectIdKey]: vectorId1, + target: vectorId1, type: "objectPlace", place: "before", }, to: { - [objectIdKey]: vectorId2, + target: vectorId2, type: "objectPlace", place: "after", }, @@ -759,12 +756,12 @@ describe("applyAgentEdit", () => { type: "remove", source: { from: { - [objectIdKey]: vectorId1, + target: vectorId1, type: "objectPlace", place: "before", }, to: { - [objectIdKey]: vectorId2, + target: vectorId2, type: "objectPlace", place: "after", }, @@ -808,7 +805,7 @@ describe("applyAgentEdit", () => { const moveEdit: TreeEdit = { explanation: "Move a vector", type: "move", - source: { [objectIdKey]: vectorId1 }, + source: { target: vectorId1 }, destination: { type: "arrayPlace", parentId: vectorId2, @@ -878,12 +875,12 @@ describe("applyAgentEdit", () => { type: "move", source: { from: { - [objectIdKey]: vectorId1, + target: vectorId1, type: "objectPlace", place: "before", }, to: { - [objectIdKey]: vectorId2, + target: vectorId2, type: "objectPlace", place: "after", }, @@ -965,12 +962,12 @@ describe("applyAgentEdit", () => { explanation: "Move a vector", source: { from: { - [objectIdKey]: vectorId1, + target: vectorId1, type: "objectPlace", place: "before", }, to: { - [objectIdKey]: vectorId2, + target: vectorId2, type: "objectPlace", place: "after", }, @@ -1019,12 +1016,12 @@ describe("applyAgentEdit", () => { explanation: "Move a vector", source: { from: { - [objectIdKey]: vectorId1, + target: vectorId1, type: "objectPlace", place: "before", }, to: { - [objectIdKey]: vectorId2, + target: vectorId2, type: "objectPlace", place: "after", }, @@ -1073,7 +1070,7 @@ describe("applyAgentEdit", () => { type: "move", explanation: "Move a vector", source: { - [objectIdKey]: strId, + target: strId, }, destination: { type: "arrayPlace", @@ -1117,7 +1114,7 @@ describe("applyAgentEdit", () => { type: "move", explanation: "Move a vector", source: { - [objectIdKey]: strId, + target: strId, }, destination: { type: "arrayPlace", @@ -1159,7 +1156,7 @@ describe("applyAgentEdit", () => { content: { [typeField]: Vector.identifier, x: 2, nonVectorField: "invalid", z: 4 }, destination: { type: "objectPlace", - [objectIdKey]: "testObjectId", + target: "testObjectId", place: "after", }, }; @@ -1191,12 +1188,12 @@ describe("applyAgentEdit", () => { explanation: "Move a vector", source: { from: { - [objectIdKey]: "testObjectId1", + target: "testObjectId1", type: "objectPlace", place: "before", }, to: { - [objectIdKey]: "testObjectId2", + target: "testObjectId2", type: "objectPlace", place: "after", }, @@ -1219,11 +1216,11 @@ describe("applyAgentEdit", () => { type: "move", explanation: "Move a vector", source: { - [objectIdKey]: "testObjectId1", + target: "testObjectId1", }, destination: { type: "objectPlace", - [objectIdKey]: "testObjectId2", + target: "testObjectId2", place: "before", }, }; @@ -1238,7 +1235,7 @@ describe("applyAgentEdit", () => { const modifyEdit: TreeEdit = { explanation: "Modify a vector", type: "modify", - target: { __fluid_objectId: "testObjectId" }, + target: { target: "testObjectId" }, field: "x", modification: 111, }; diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts index 4eb524cccfb5..d9c76d80c2c9 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts @@ -3,8 +3,6 @@ * Licensed under the MIT License. */ -import { strict as assert } from "node:assert"; - // eslint-disable-next-line import/no-internal-modules import { createIdCompressor } from "@fluidframework/id-compressor/internal"; // eslint-disable-next-line import/no-internal-modules @@ -16,7 +14,7 @@ import { // eslint-disable-next-line import/no-internal-modules } from "@fluidframework/tree/internal"; -import { generateSuggestions, generateTreeEdits } from "../../explicit-strategy/index.js"; +import { generateTreeEdits } from "../../explicit-strategy/index.js"; import { initializeOpenAIClient } from "./utils.js"; @@ -89,38 +87,6 @@ describe.skip("Agent Editing Integration", () => { process.env.AZURE_OPENAI_ENDPOINT = "TODO "; process.env.AZURE_OPENAI_DEPLOYMENT = "gpt-4o"; - it("Suggestion Test", async () => { - const tree = factory.create( - new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - "tree", - ); - const view = tree.viewWith(new TreeViewConfiguration({ schema: Conference })); - view.initialize({ name: "Plucky Penguins", sessions: [], days: [], sessionsPerDay: 3 }); - - const openAIClient = initializeOpenAIClient("azure"); - const abortController = new AbortController(); - const suggestions = await generateSuggestions( - { client: openAIClient, modelName: TEST_MODEL_NAME }, - view, - 3, - ); - for (const prompt of suggestions) { - const result = await generateTreeEdits({ - openAI: { client: openAIClient, modelName: TEST_MODEL_NAME }, - treeView: view, - prompt: { - userAsk: prompt, - systemRoleContext: "", - }, - limiters: { - abortController, - maxModelCalls: 15, - }, - }); - assert.equal(result, "success"); - } - }); - it("Roblox Test", async () => { const tree = factory.create( new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ccb65c7305a3..5691bd5c5a64 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10452,6 +10452,9 @@ importers: openai: specifier: ^4.67.3 version: 4.68.0(zod@3.23.8) + typechat: + specifier: ^0.1.1 + version: 0.1.1(typescript@5.4.5)(zod@3.23.8) zod: specifier: ^3.23.8 version: 3.23.8 @@ -10525,9 +10528,6 @@ importers: rimraf: specifier: ^4.4.0 version: 4.4.1 - typechat: - specifier: ^0.1.1 - version: 0.1.1(typescript@5.4.5)(zod@3.23.8) typescript: specifier: ~5.4.5 version: 5.4.5 @@ -30284,6 +30284,7 @@ packages: /iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} + requiresBuild: true dependencies: safer-buffer: 2.1.2 dev: true @@ -38403,7 +38404,6 @@ packages: dependencies: typescript: 5.4.5 zod: 3.23.8 - dev: true /typed-array-buffer@1.0.2: resolution: {integrity: sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ==} From 0cf8cfe7bf19c968bbcbe5b3f474e956c8211cc2 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 25 Oct 2024 19:42:13 +0000 Subject: [PATCH 13/28] Adds jsdoc, resolves build errors, work in progress integration tests --- .../api-report/ai-collab.alpha.api.md | 6 - packages/framework/ai-collab/package.json | 2 +- .../framework/ai-collab/src/aiCollabApi.ts | 35 ++ .../src/explicit-strategy/agentEditReducer.ts | 2 +- .../ai-collab/src/explicit-strategy/index.ts | 1 - .../src/explicit-strategy/typeGeneration.ts | 6 +- .../explicit-strategy/integration.spec.ts | 4 +- .../integration/planner.spec.ts | 358 ++++++++++++++++++ pnpm-lock.yaml | 137 ++++++- 9 files changed, 536 insertions(+), 15 deletions(-) create mode 100644 packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts diff --git a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md index 7747cdf21408..35d83cb869a6 100644 --- a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md +++ b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md @@ -19,11 +19,8 @@ export interface AiCollabErrorResponse { // @alpha export interface AiCollabOptions { - // (undocumented) dumpDebugLog?: boolean; - // (undocumented) finalReviewStep?: boolean; - // (undocumented) limiters?: { abortController?: AbortController; maxSequentialErrors?: number; @@ -32,16 +29,13 @@ export interface AiCollabOptions { }; // (undocumented) openAI: OpenAiClientOptions; - // (undocumented) planningStep?: boolean; // (undocumented) prompt: { systemRoleContext: string; userAsk: string; }; - // (undocumented) treeView: TreeView; - // (undocumented) validator?: (newContent: TreeNode) => void; } diff --git a/packages/framework/ai-collab/package.json b/packages/framework/ai-collab/package.json index f186f960958a..b588455b2d23 100644 --- a/packages/framework/ai-collab/package.json +++ b/packages/framework/ai-collab/package.json @@ -62,7 +62,7 @@ "test:coverage": "c8 npm test", "test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs", "test:mocha:cjs": "mocha --recursive \"dist/test/**/*.spec.js\"", - "test:mocha:esm": "mocha --recursive \"lib/test/**/*.spec.js\"", + "test:mocha:esm": "mocha --recursive \"lib/test/**/*.spec.js\" --timeout 60000", "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha", "tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && copyfiles -f ../../../common/build/build-common/src/cjs/package.json ./dist" }, diff --git a/packages/framework/ai-collab/src/aiCollabApi.ts b/packages/framework/ai-collab/src/aiCollabApi.ts index ca18c47d0e46..ecc9f74d44c1 100644 --- a/packages/framework/ai-collab/src/aiCollabApi.ts +++ b/packages/framework/ai-collab/src/aiCollabApi.ts @@ -24,20 +24,55 @@ export interface OpenAiClientOptions { */ export interface AiCollabOptions { openAI: OpenAiClientOptions; + /** + * The view of your Shared Tree. + * @remarks Its is recommended to pass a branch of your current tree view so the AI has a separate canvas to work on + * and merge said branch back to the main tree when the AI is done and the user accepts + */ treeView: TreeView; prompt: { + /** + * The context to give the LLM about its role in the collaboration. + */ systemRoleContext: string; + /** + * The request from the users to the LLM. + */ userAsk: string; }; + /** + * Limiters are various optional ways to limit this library's usage of the LLM. + */ limiters?: { abortController?: AbortController; + /** + * The maximum number of sequential errors the LLM can make before aborting the collaboration. + */ maxSequentialErrors?: number; + /** + * The maximum number of model calls the LLM can make before aborting the collaboration. + */ maxModelCalls?: number; + /** + * The maximum token usage limits for the LLM. + */ tokenLimits?: TokenUsage; }; + /** + * When enabled, the LLM will be prompted to first produce a plan based on the user's ask before generating changes to your applications data + */ planningStep?: boolean; + /** + * When enabled, the LLM will be prompted with a final review of the changes they made to confimr their validity. + */ finalReviewStep?: boolean; + /** + * An optional validator function that can be used to validate the new content produced by the LLM. + */ validator?: (newContent: TreeNode) => void; + /** + * When enabled, the library will console.log information useful for debugging the AI collaboration. + */ dumpDebugLog?: boolean; } diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts index 01303d492b1b..9fc8e0a53186 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts @@ -59,7 +59,7 @@ function populateDefaults( } else { assert( typeof json[typeField] === "string", - `${typeField} must be present in new JSON content`, + "The typeField must be present in new JSON content", ); const nodeSchema = definitionMap.get(json[typeField]); assert(nodeSchema?.kind === NodeKind.Object, "Expected object schema"); diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index 2c1a6545cd87..fe4a3ec28d2e 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -6,7 +6,6 @@ import { getSimpleSchema, normalizeFieldSchema, - // Tree, type ImplicitFieldSchema, type SimpleTreeSchema, type TreeNode, diff --git a/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts index 600ee9fa947d..be19034edf00 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts @@ -2,6 +2,7 @@ * Copyright (c) Microsoft Corporation and contributors. All rights reserved. * Licensed under the MIT License. */ + import { assert } from "@fluidframework/core-utils/internal"; import { FieldKind, NodeKind, ValueSchema } from "@fluidframework/tree/internal"; import type { @@ -284,7 +285,8 @@ function getOrCreateTypeForField( fieldSchema.allowedTypes, ); case FieldKind.Optional: - return z.optional( + return z.union([ + z.null(), getTypeForAllowedTypes( definitionMap, typeMap, @@ -293,7 +295,7 @@ function getOrCreateTypeForField( modifyTypeSet, fieldSchema.allowedTypes, ), - ); + ]); case FieldKind.Identifier: return undefined; default: diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts index d9c76d80c2c9..563479bc13b2 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts @@ -82,7 +82,7 @@ const factory = SharedTree.getFactory(); const TEST_MODEL_NAME = "gpt-4o"; describe.skip("Agent Editing Integration", () => { - process.env.OPENAI_API_KEY = "TODO "; // DON'T COMMIT THIS + process.env.OPENAI_API_KEY = ""; // DON'T COMMIT THIS process.env.AZURE_OPENAI_API_KEY = "TODO "; // DON'T COMMIT THIS process.env.AZURE_OPENAI_ENDPOINT = "TODO "; process.env.AZURE_OPENAI_DEPLOYMENT = "gpt-4o"; @@ -164,7 +164,7 @@ describe.skip("Agent Editing Integration", () => { ], sessionsPerDay: 2, }); - const openAIClient = initializeOpenAIClient("azure"); + const openAIClient = initializeOpenAIClient("openai"); const abortController = new AbortController(); await generateTreeEdits({ openAI: { client: openAIClient, modelName: TEST_MODEL_NAME }, diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts new file mode 100644 index 000000000000..575910ad2ba7 --- /dev/null +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts @@ -0,0 +1,358 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +/* eslint-disable jsdoc/require-jsdoc */ + +import { strict as assert } from "node:assert"; + +// eslint-disable-next-line import/no-internal-modules +import { createIdCompressor } from "@fluidframework/id-compressor/internal"; +// eslint-disable-next-line import/no-internal-modules +import { MockFluidDataStoreRuntime } from "@fluidframework/test-runtime-utils/internal"; +import { + SchemaFactory, + SharedTree, + TreeViewConfiguration, + // eslint-disable-next-line import/no-internal-modules +} from "@fluidframework/tree/internal"; +import { APIError, OpenAI } from "openai"; + +import { aiCollab } from "../../../index.js"; + +const sf = new SchemaFactory("ai-collab-sample-application"); + +export class SharedTreeTask extends sf.object("Task", { + title: sf.string, + id: sf.identifier, + description: sf.string, + priority: sf.string, + complexity: sf.number, + status: sf.string, + assignee: sf.optional(sf.string), +}) {} + +export class SharedTreeTaskList extends sf.array("TaskList", SharedTreeTask) {} + +export class SharedTreeEngineer extends sf.object("Engineer", { + name: sf.string, + id: sf.identifier, + skills: sf.string, + maxCapacity: sf.number, +}) {} + +export class SharedTreeEngineerList extends sf.array("EngineerList", SharedTreeEngineer) {} + +export class SharedTreeTaskGroup extends sf.object("TaskGroup", { + description: sf.string, + id: sf.identifier, + title: sf.string, + tasks: SharedTreeTaskList, + engineers: SharedTreeEngineerList, + // optionalInfo: sf.optional(sf.string), +}) {} + +export class SharedTreeTaskGroupList extends sf.array("TaskGroupList", SharedTreeTaskGroup) {} + +export class SharedTreeAppState extends sf.object("AppState", { + taskGroups: SharedTreeTaskGroupList, + optionalInfo: sf.optional(sf.string), +}) {} + +export const INITIAL_APP_STATE = { + taskGroups: [ + { + title: "My First Task Group", + description: "Placeholder for first task group", + tasks: [ + { + assignee: "Alice", + title: "Task #1", + description: + "This is the first task. Blah Blah blah Blah Blah blahBlah Blah blahBlah Blah blahBlah Blah blah", + priority: "low", + complexity: 1, + status: "todo", + }, + { + assignee: "Bob", + title: "Task #2", + description: + "This is the second task. Blah Blah blah Blah Blah blahBlah Blah blahBlah Blah blahBlah Blah blah", + priority: "medium", + complexity: 2, + status: "in-progress", + }, + { + assignee: "Charlie", + title: "Task #3", + description: + "This is the third task! Blah Blah blah Blah Blah blahBlah Blah blahBlah Blah blahBlah Blah blah", + priority: "high", + complexity: 3, + status: "done", + }, + ], + engineers: [ + { + name: "Alice", + maxCapacity: 15, + skills: + "Senior engineer capable of handling complex tasks. Versed in most languages", + }, + { + name: "Bob", + maxCapacity: 12, + skills: + "Mid-level engineer capable of handling medium complexity tasks. Versed in React, Node.JS", + }, + { + name: "Charlie", + maxCapacity: 7, + skills: "Junior engineer capable of handling simple tasks. Versed in Node.JS", + }, + ], + }, + { + title: "My Second Task Group", + description: "Placeholder for second task group", + tasks: [ + { + assignee: "Alice", + title: "Task #1", + description: + "This is the first task. Blah Blah blah Blah Blah blahBlah Blah blahBlah Blah blahBlah Blah blah", + priority: "low", + complexity: 1, + status: "todo", + }, + { + assignee: "Bob", + title: "Task #2", + description: + "This is the second task. Blah Blah blah Blah Blah blahBlah Blah blahBlah Blah blahBlah Blah blah", + priority: "medium", + complexity: 2, + status: "in-progress", + }, + { + assignee: "Charlie", + title: "Task #3", + description: + "This is the third task! Blah Blah blah Blah Blah blahBlah Blah blahBlah Blah blahBlah Blah blah", + priority: "high", + complexity: 3, + status: "done", + }, + ], + engineers: [ + { + name: "Alice", + maxCapacity: 15, + skills: + "Senior engineer capable of handling complex tasks. Versed in most languages", + }, + { + name: "Bob", + maxCapacity: 12, + skills: + "Mid-level engineer capable of handling medium complexity tasks. Versed in React, Node.JS", + }, + { + name: "Charlie", + maxCapacity: 7, + skills: "Junior engineer capable of handling simple tasks. Versed in Node.JS", + }, + ], + }, + ], +} as const; + +const factory = SharedTree.getFactory(); + +const OPENAI_API_KEY = ""; + +describe("Ai Planner App", () => { + it.skip("Simple test", async () => { + // mocha.setup({ + // timeout: 20000, + // }); + + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: SharedTreeAppState })); + view.initialize(INITIAL_APP_STATE); + + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + prompt: { + systemRoleContext: + "You are a project manager overseeing a team of engineers and assigning tasks within task groups to them.", + userAsk: "Change the first task group title to 'Hello World'", + }, + planningStep: true, + finalReviewStep: true, + dumpDebugLog: true, + }); + + // assert.equal(view.root.taskGroups[0]?.title, "Hello World"); + }); + + it.skip("VERY Simple test", async () => { + class TestAppSchema extends sf.object("PrioritySpecification", { + priority: sf.optional(sf.string), + }) {} + + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: TestAppSchema })); + view.initialize({ priority: "low" }); + + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + prompt: { + systemRoleContext: "You are a managing objects with a priority field.", + userAsk: "Change the priority from low to high", + }, + planningStep: true, + finalReviewStep: true, + }); + + assert.equal(view.root.priority, "high"); + }); + + it.skip("BUG: Invalid json schema produced when schema has no arrays", async () => { + class TestAppSchema extends sf.object("TestAppSchema", { + title: sf.string, + }) {} + + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: TestAppSchema })); + view.initialize({ title: "Sample Title" }); + + try { + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + prompt: { + systemRoleContext: "You are a managing json objects", + userAsk: "Change the `title` field to 'Hello World'", + }, + planningStep: true, + finalReviewStep: true, + }); + } catch (error) { + assert(error instanceof APIError); + assert(error.status === 400); + assert(error.type === "invalid_request_error"); + assert( + error.message === + "400 Invalid schema for response_format 'SharedTreeAI': In context=('properties', 'edit', 'anyOf', '1', 'properties', 'content', 'not'), schema must have a 'type' key.", + ); + } + }); + + it("BUG: OpenAI structured output fails when json schema with psuedo optional field is used in response format", async () => { + class TestAppSchemaWithOptionalProp extends sf.object("TestAppSchemaWithOptionalProp", { + nonOptionalProp: sf.string, + taskList: SharedTreeTaskList, + optionalProp: sf.optional(sf.string), + }) {} + + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith( + new TreeViewConfiguration({ schema: TestAppSchemaWithOptionalProp }), + ); + view.initialize({ nonOptionalProp: "Hello", taskList: [] }); + + try { + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + prompt: { + systemRoleContext: "You are a managing json objects", + userAsk: "Change the `optionalProp` field to 'Hello World'", + }, + planningStep: true, + finalReviewStep: true, + }); + } catch (error) { + assert(error instanceof APIError); + assert(error.status === 400); + assert(error.type === "invalid_request_error"); + assert( + error.message === + "Invalid schema for response_format 'SharedTreeAI'. Please ensure it is a valid JSON Schema.", + ); + } + + class TestAppSchemaWithoutOptionalProp extends sf.object( + "TestAppSchemaWithoutOptionalProp", + { + nonOptionalProp: sf.string, + }, + ) {} + + const tree2 = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree2", + ); + const view2 = tree2.viewWith( + new TreeViewConfiguration({ schema: TestAppSchemaWithoutOptionalProp }), + ); + view2.initialize({ nonOptionalProp: "Hello" }); + + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + prompt: { + systemRoleContext: "You are a managing json objects", + userAsk: "Change the `optionalProp` field to 'Hello World'", + }, + planningStep: true, + finalReviewStep: true, + }); + + const jsonified = JSON.stringify(view.root); + + assert.equal(view.root.nonOptionalProp, "Hello World"); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5691bd5c5a64..33a24e225801 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10470,7 +10470,7 @@ importers: version: link:../../test/mocha-test-setup '@fluid-tools/build-cli': specifier: ^0.49.0 - version: 0.49.0(@types/node@18.19.54)(typescript@5.4.5)(webpack-cli@5.1.4) + version: 0.49.0(@types/node@18.19.54)(typescript@5.4.5) '@fluidframework/build-common': specifier: ^2.0.3 version: 2.0.3 @@ -19314,6 +19314,85 @@ packages: transitivePeerDependencies: - supports-color + /@fluid-tools/build-cli@0.49.0(@types/node@18.19.54)(typescript@5.4.5): + resolution: {integrity: sha512-V9h8OCJDvSz8m4zmeCO6y8DJi972BSFp3YO6S/R1v7J/CpaG5A6v1Di0Kp5+JYf+sQ2ILoBaEvdjCp3ii+eYTw==} + engines: {node: '>=18.17.1'} + hasBin: true + dependencies: + '@andrewbranch/untar.js': 1.0.3 + '@fluid-tools/version-tools': 0.49.0 + '@fluidframework/build-tools': 0.49.0 + '@fluidframework/bundle-size-tools': 0.49.0 + '@microsoft/api-extractor': 7.47.8(patch_hash=ldzfpsbo3oeejrejk775zxplmi)(@types/node@18.19.54) + '@oclif/core': 4.0.25 + '@oclif/plugin-autocomplete': 3.2.5 + '@oclif/plugin-commands': 4.0.16 + '@oclif/plugin-help': 6.2.13 + '@oclif/plugin-not-found': 3.2.22 + '@octokit/core': 4.2.4 + '@octokit/rest': 21.0.2 + '@rushstack/node-core-library': 3.66.1(@types/node@18.19.54) + async: 3.2.6 + azure-devops-node-api: 11.2.0 + chalk: 5.3.0 + change-case: 3.1.0 + cosmiconfig: 8.3.6(typescript@5.4.5) + danger: 11.3.1 + date-fns: 2.30.0 + debug: 4.3.7(supports-color@8.1.1) + execa: 5.1.1 + fflate: 0.8.2 + fs-extra: 11.2.0 + github-slugger: 2.0.0 + globby: 11.1.0 + gray-matter: 4.0.3 + human-id: 4.1.1 + inquirer: 8.2.6 + issue-parser: 7.0.1 + json5: 2.2.3 + jssm: 5.98.2 + jszip: 3.10.1 + latest-version: 5.1.0 + mdast: 3.0.0 + mdast-util-heading-range: 4.0.0 + mdast-util-to-string: 4.0.0 + minimatch: 7.4.6 + node-fetch: 2.7.0 + npm-check-updates: 16.14.20 + oclif: 4.15.0 + prettier: 3.2.5 + prompts: 2.4.2 + read-pkg-up: 7.0.1 + remark: 15.0.1 + remark-gfm: 4.0.0 + remark-github: 12.0.0 + remark-github-beta-blockquote-admonitions: 3.1.1 + remark-toc: 9.0.0 + replace-in-file: 7.2.0 + resolve.exports: 2.0.2 + semver: 7.6.3 + semver-utils: 1.1.4 + simple-git: 3.27.0 + sort-json: 2.0.1 + sort-package-json: 1.57.0 + strip-ansi: 6.0.1 + table: 6.8.2 + ts-morph: 22.0.0 + type-fest: 2.19.0 + unist-util-visit: 5.0.0 + xml2js: 0.5.0 + transitivePeerDependencies: + - '@swc/core' + - '@types/node' + - bluebird + - encoding + - esbuild + - supports-color + - typescript + - uglify-js + - webpack-cli + dev: true + /@fluid-tools/build-cli@0.49.0(@types/node@18.19.54)(typescript@5.4.5)(webpack-cli@5.1.4): resolution: {integrity: sha512-V9h8OCJDvSz8m4zmeCO6y8DJi972BSFp3YO6S/R1v7J/CpaG5A6v1Di0Kp5+JYf+sQ2ILoBaEvdjCp3ii+eYTw==} engines: {node: '>=18.17.1'} @@ -19681,6 +19760,22 @@ packages: transitivePeerDependencies: - supports-color + /@fluidframework/bundle-size-tools@0.49.0: + resolution: {integrity: sha512-SUrWc931wwOkwIERX282SmHUVjXz0mRhlYIoY68DkYVZ3XuUrKaVvHbJB6a3ek+TIX33zg90HKFNkp9K56m0SQ==} + dependencies: + azure-devops-node-api: 11.2.0 + jszip: 3.10.1 + msgpack-lite: 0.1.26 + pako: 2.1.0 + typescript: 5.4.5 + webpack: 5.95.0 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + - webpack-cli + dev: true + /@fluidframework/bundle-size-tools@0.49.0(webpack-cli@5.1.4): resolution: {integrity: sha512-SUrWc931wwOkwIERX282SmHUVjXz0mRhlYIoY68DkYVZ3XuUrKaVvHbJB6a3ek+TIX33zg90HKFNkp9K56m0SQ==} dependencies: @@ -37820,7 +37915,7 @@ packages: schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.34.1 - webpack: 5.95.0(webpack-cli@5.1.4) + webpack: 5.95.0 /terser@5.34.1: resolution: {integrity: sha512-FsJZ7iZLd/BXkz+4xrRTGJ26o/6VTjQytUk8b8OxkwcD2I+79VPJlz7qss1+zE7h8GNIScFqXcDyJ/KqBYZFVA==} @@ -39445,6 +39540,44 @@ packages: resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} engines: {node: '>=10.13.0'} + /webpack@5.95.0: + resolution: {integrity: sha512-2t3XstrKULz41MNMBF+cJ97TyHdyQ8HCt//pqErqDvNjU9YQBnZxIHa11VXsi7F3mb5/aO2tuDxdeTPdU7xu9Q==} + engines: {node: '>=10.13.0'} + hasBin: true + peerDependencies: + webpack-cli: '*' + peerDependenciesMeta: + webpack-cli: + optional: true + dependencies: + '@types/estree': 1.0.6 + '@webassemblyjs/ast': 1.12.1 + '@webassemblyjs/wasm-edit': 1.12.1 + '@webassemblyjs/wasm-parser': 1.12.1 + acorn: 8.12.1 + acorn-import-attributes: 1.9.5(acorn@8.12.1) + browserslist: 4.24.0 + chrome-trace-event: 1.0.4 + enhanced-resolve: 5.17.1 + es-module-lexer: 1.5.4 + eslint-scope: 5.1.1 + events: 3.3.0 + glob-to-regexp: 0.4.1 + graceful-fs: 4.2.11 + json-parse-even-better-errors: 2.3.1 + loader-runner: 4.3.0 + mime-types: 2.1.35 + neo-async: 2.6.2 + schema-utils: 3.3.0 + tapable: 2.2.1 + terser-webpack-plugin: 5.3.10(webpack@5.95.0) + watchpack: 2.4.2 + webpack-sources: 3.2.3 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + /webpack@5.95.0(webpack-cli@5.1.4): resolution: {integrity: sha512-2t3XstrKULz41MNMBF+cJ97TyHdyQ8HCt//pqErqDvNjU9YQBnZxIHa11VXsi7F3mb5/aO2tuDxdeTPdU7xu9Q==} engines: {node: '>=10.13.0'} From 232d335bbce0810b1ab1eb7c5a610d5a23193a89 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 25 Oct 2024 19:44:05 +0000 Subject: [PATCH 14/28] skips integ tests --- .../src/test/explicit-strategy/integration/planner.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts index 575910ad2ba7..1754f93fb76c 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts @@ -277,7 +277,7 @@ describe("Ai Planner App", () => { } }); - it("BUG: OpenAI structured output fails when json schema with psuedo optional field is used in response format", async () => { + it.skip("BUG: OpenAI structured output fails when json schema with psuedo optional field is used in response format", async () => { class TestAppSchemaWithOptionalProp extends sf.object("TestAppSchemaWithOptionalProp", { nonOptionalProp: sf.string, taskList: SharedTreeTaskList, From e5e3b2ddd7a5376087ca70e67367ee047e9283e7 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Fri, 25 Oct 2024 20:02:48 +0000 Subject: [PATCH 15/28] Adds more documented bug cases to integration testing --- .../integration/planner.spec.ts | 118 +++++++++++------- 1 file changed, 71 insertions(+), 47 deletions(-) diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts index 1754f93fb76c..c11edab5c4db 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts @@ -174,42 +174,54 @@ const factory = SharedTree.getFactory(); const OPENAI_API_KEY = ""; describe("Ai Planner App", () => { - it.skip("Simple test", async () => { - // mocha.setup({ - // timeout: 20000, - // }); + it.skip("BUG: Invalid json schema produced when schema has no arrays at all", async () => { + class TestAppSchema extends sf.object("PrioritySpecification", { + priority: sf.string, + }) {} const tree = factory.create( new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "tree", ); - const view = tree.viewWith(new TreeViewConfiguration({ schema: SharedTreeAppState })); - view.initialize(INITIAL_APP_STATE); - - await aiCollab({ - openAI: { - client: new OpenAI({ - apiKey: OPENAI_API_KEY, - }), - modelName: "gpt-4o", - }, - treeView: view, - prompt: { - systemRoleContext: - "You are a project manager overseeing a team of engineers and assigning tasks within task groups to them.", - userAsk: "Change the first task group title to 'Hello World'", - }, - planningStep: true, - finalReviewStep: true, - dumpDebugLog: true, - }); + const view = tree.viewWith(new TreeViewConfiguration({ schema: TestAppSchema })); + view.initialize({ priority: "low" }); - // assert.equal(view.root.taskGroups[0]?.title, "Hello World"); + try { + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + prompt: { + systemRoleContext: "You are a managing objects with a priority field.", + userAsk: "Change the priority from low to high", + }, + planningStep: true, + finalReviewStep: true, + }); + } catch (error) { + assert(error instanceof APIError); + assert(error.status === 400); + assert(error.type === "invalid_request_error"); + assert( + error.message === + "400 Invalid schema for response_format 'SharedTreeAI': In context=('properties', 'edit', 'anyOf', '1', 'properties', 'content', 'not'), schema must have a 'type' key.", + ); + } }); - it.skip("VERY Simple test", async () => { - class TestAppSchema extends sf.object("PrioritySpecification", { - priority: sf.optional(sf.string), + it.skip("BUG: Invalid json schema produced when schema has multiple keys with the same name and order", async () => { + class TestInnerAppSchema extends sf.object("TestInnerAppSchema", { + title: sf.string, + }) {} + + class TestAppSchema extends sf.object("TestAppSchema", { + title: sf.string, + taskList: SharedTreeTaskList, + appData: TestInnerAppSchema, }) {} const tree = factory.create( @@ -217,25 +229,37 @@ describe("Ai Planner App", () => { "tree", ); const view = tree.viewWith(new TreeViewConfiguration({ schema: TestAppSchema })); - view.initialize({ priority: "low" }); - - await aiCollab({ - openAI: { - client: new OpenAI({ - apiKey: OPENAI_API_KEY, - }), - modelName: "gpt-4o", - }, - treeView: view, - prompt: { - systemRoleContext: "You are a managing objects with a priority field.", - userAsk: "Change the priority from low to high", - }, - planningStep: true, - finalReviewStep: true, + view.initialize({ + title: "Sample Title", + taskList: [], + appData: { title: "Inner App Data" }, }); - assert.equal(view.root.priority, "high"); + try { + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + prompt: { + systemRoleContext: "You are a managing json objects", + userAsk: "Change the `title` field of the outer object to 'Hello World'", + }, + planningStep: true, + finalReviewStep: true, + }); + } catch (error) { + assert(error instanceof APIError); + assert(error.status === 400); + assert(error.type === "invalid_request_error"); + assert( + error.message === + "400 Invalid schema for response_format 'SharedTreeAI'. Please ensure it is a valid JSON Schema.", + ); + } }); it.skip("BUG: Invalid json schema produced when schema has no arrays", async () => { @@ -315,7 +339,7 @@ describe("Ai Planner App", () => { assert(error.type === "invalid_request_error"); assert( error.message === - "Invalid schema for response_format 'SharedTreeAI'. Please ensure it is a valid JSON Schema.", + "400 Invalid schema for response_format 'SharedTreeAI'. Please ensure it is a valid JSON Schema.", ); } @@ -342,7 +366,7 @@ describe("Ai Planner App", () => { }), modelName: "gpt-4o", }, - treeView: view, + treeView: view2, prompt: { systemRoleContext: "You are a managing json objects", userAsk: "Change the `optionalProp` field to 'Hello World'", From b60ecf3ce2ffe6a64658205867a91ba1dea57762 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:16:24 +0000 Subject: [PATCH 16/28] API Updated to accept a treeNode instead of the whole tree --- .../api-report/ai-collab.alpha.api.md | 1 + packages/framework/ai-collab/src/aiCollab.ts | 1 + .../framework/ai-collab/src/aiCollabApi.ts | 8 + .../src/explicit-strategy/agentEditReducer.ts | 45 +++- .../ai-collab/src/explicit-strategy/index.ts | 33 ++- .../src/explicit-strategy/promptGeneration.ts | 29 ++- .../agentEditingReducer.spec.ts | 224 ++++++++++-------- .../explicit-strategy/integration.spec.ts | 1 + .../integration/planner.spec.ts | 90 ++++--- 9 files changed, 254 insertions(+), 178 deletions(-) diff --git a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md index 35d83cb869a6..a604c5a321c6 100644 --- a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md +++ b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md @@ -35,6 +35,7 @@ export interface AiCollabOptions { systemRoleContext: string; userAsk: string; }; + treeNode: TreeNode; treeView: TreeView; validator?: (newContent: TreeNode) => void; } diff --git a/packages/framework/ai-collab/src/aiCollab.ts b/packages/framework/ai-collab/src/aiCollab.ts index 850c675155d9..d042f1125836 100644 --- a/packages/framework/ai-collab/src/aiCollab.ts +++ b/packages/framework/ai-collab/src/aiCollab.ts @@ -23,6 +23,7 @@ export async function aiCollab( ): Promise { const response = await generateTreeEdits({ treeView: options.treeView, + treeNode: options.treeNode, validator: options.validator, openAI: options.openAI, prompt: options.prompt, diff --git a/packages/framework/ai-collab/src/aiCollabApi.ts b/packages/framework/ai-collab/src/aiCollabApi.ts index ecc9f74d44c1..e211a1b9c2fe 100644 --- a/packages/framework/ai-collab/src/aiCollabApi.ts +++ b/packages/framework/ai-collab/src/aiCollabApi.ts @@ -30,6 +30,14 @@ export interface AiCollabOptions { * and merge said branch back to the main tree when the AI is done and the user accepts */ treeView: TreeView; + /** + * The specific tree node you want the AI to collaborate on. Pass the root node of your tree if you intend + * for the AI to work on the entire tree. + * @remarks + * - Optional root nodes are not supported + * - Primitive root nodes are not supported + */ + treeNode: TreeNode; prompt: { /** * The context to give the LLM about its role in the collaboration. diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts index 9fc8e0a53186..46e79f6b799a 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts @@ -114,23 +114,33 @@ function contentWithIds(content: TreeNode, idGenerator: IdGenerator): TreeEditOb * TBD */ export function applyAgentEdit( - tree: TreeView, + treeView: TreeView, + treeNode: TreeNode, treeEdit: TreeEdit, idGenerator: IdGenerator, definitionMap: ReadonlyMap, validator?: (edit: TreeNode) => void, ): TreeEdit { objectIdsExist(treeEdit, idGenerator); + const isRootNode = Tree.parent(treeNode) === undefined; switch (treeEdit.type) { case "setRoot": { populateDefaults(treeEdit.content, definitionMap); - const treeSchema = normalizeFieldSchema(tree.schema); + // const treeSchema = normalizeFieldSchema(tree.schema); + const treeSchema = isRootNode + ? normalizeFieldSchema(treeView.schema) + : normalizeFieldSchema(Tree.schema(treeNode)); const schemaIdentifier = getSchemaIdentifier(treeEdit.content); let insertedObject: TreeNode | undefined; if (treeSchema.kind === FieldKind.Optional && treeEdit.content === undefined) { - tree.root = treeEdit.content; + if (isRootNode) { + treeView.root = treeEdit.content; + } else { + // eslint-disable-next-line no-param-reassign, @typescript-eslint/no-explicit-any + (treeNode as any) = treeEdit.content; + } } else { for (const allowedType of treeSchema.allowedTypeSet.values()) { if (schemaIdentifier === allowedType.identifier) { @@ -140,12 +150,22 @@ export function applyAgentEdit( ) => TreeNode; const rootNode = new simpleNodeSchema(treeEdit.content); validator?.(rootNode); - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access - (tree as any).root = rootNode; + if (isRootNode) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + (treeView as any).root = rootNode; + } else { + // eslint-disable-next-line no-param-reassign + treeNode = rootNode; + } insertedObject = rootNode; } else { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access - (tree as any).root = treeEdit.content; + if (isRootNode) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + (treeView as any).root = treeEdit.content; + } else { + // eslint-disable-next-line no-param-reassign, @typescript-eslint/no-explicit-any + (treeNode as any) = treeEdit.content; + } } } } @@ -192,10 +212,15 @@ export function applyAgentEdit( const parentNode = Tree.parent(node); // Case for deleting rootNode if (parentNode === undefined) { - const treeSchema = tree.schema; + const treeSchema = isRootNode ? treeView.schema : Tree.schema(treeNode); if (treeSchema instanceof FieldSchema && treeSchema.kind === FieldKind.Optional) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access - (tree as any).root = undefined; + if (isRootNode) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any + (treeView as any).root = undefined; + } else { + // eslint-disable-next-line no-param-reassign, @typescript-eslint/no-explicit-any + (treeNode as any) = undefined; + } } else { throw new UsageError( "The root is required, and cannot be removed. Please use modify edit instead.", diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index fe4a3ec28d2e..1eb4a602b456 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -6,6 +6,7 @@ import { getSimpleSchema, normalizeFieldSchema, + Tree, type ImplicitFieldSchema, type SimpleTreeSchema, type TreeNode, @@ -44,6 +45,7 @@ const DEBUG_LOG: string[] = []; export interface GenerateTreeEditsOptions { openAI: OpenAiClientOptions; treeView: TreeView; + treeNode: TreeNode; prompt: { systemRoleContext: string; userAsk: string; @@ -75,6 +77,10 @@ interface GenerateTreeEditsErrorResponse { * Prompts the provided LLM client to generate valid tree edits. * Applies those edits to the provided tree branch before returning. * + * @remarks + * - Optional root nodes are not supported + * - Primitive root nodes are not supported + * * @internal */ export async function generateTreeEdits( @@ -84,11 +90,11 @@ export async function generateTreeEdits( const editLog: EditLog = []; let editCount = 0; let sequentialErrorCount = 0; - const simpleSchema = getSimpleSchema( - normalizeFieldSchema(options.treeView.schema).allowedTypes, - ); - // const simpleSchema = getSimpleSchema(Tree.schema(options.treeNode)); + const isRootNode = Tree.parent(options.treeNode) === undefined; + const simpleSchema = isRootNode + ? getSimpleSchema(normalizeFieldSchema(options.treeView.schema).allowedTypes) + : getSimpleSchema(Tree.schema(options.treeNode)); const tokenUsage = { inputTokens: 0, outputTokens: 0 }; @@ -104,6 +110,7 @@ export async function generateTreeEdits( try { const result = applyAgentEdit( options.treeView, + options.treeNode, edit, idGenerator, simpleSchema.definitions, @@ -190,19 +197,19 @@ async function* generateEdits( let plan: string | undefined; if (options.planningStep !== undefined) { - plan = await getStringFromLlm( - getPlanningSystemPrompt( - options.treeView, - options.prompt.userAsk, - options.prompt.systemRoleContext, - ), - options.openAI, + const planningPromt = getPlanningSystemPrompt( + options.treeView, + options.treeNode, + options.prompt.userAsk, + options.prompt.systemRoleContext, ); + DEBUG_LOG?.push(planningPromt); + plan = await getStringFromLlm(planningPromt, options.openAI); } const originalDecoratedJson = (options.finalReviewStep ?? false) - ? toDecoratedJson(idGenerator, options.treeView.root) + ? toDecoratedJson(idGenerator, options.treeNode) : undefined; // reviewed is implicitly true if finalReviewStep is false let hasReviewed = (options.finalReviewStep ?? false) ? false : true; @@ -211,6 +218,7 @@ async function* generateEdits( options.prompt.userAsk, idGenerator, options.treeView, + options.treeNode, editLog, options.prompt.systemRoleContext, plan, @@ -261,6 +269,7 @@ async function* generateEdits( options.prompt.userAsk, idGenerator, options.treeView, + options.treeNode, originalDecoratedJson ?? fail("Original decorated tree not provided."), options.prompt.systemRoleContext, ); diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index e2d6e738e272..eedb84f0e040 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -17,8 +17,9 @@ import { type JsonTreeSchema, getSimpleSchema, Tree, + type TreeNode, } from "@fluidframework/tree/internal"; -// eslint-disable-next-line import/no-extraneous-dependencies, import/no-internal-modules +// eslint-disable-next-line import/no-internal-modules import { createZodJsonValidator } from "typechat/zod"; import { @@ -75,10 +76,15 @@ export function toDecoratedJson( */ export function getPlanningSystemPrompt( view: TreeView, + treeNode: TreeNode, userPrompt: string, systemRoleContext?: string, ): string { - const schema = normalizeFieldSchema(view.schema); + const isRootNode = Tree.parent(treeNode) === undefined; + const schema = isRootNode + ? normalizeFieldSchema(view.schema) + : normalizeFieldSchema(Tree.schema(treeNode)); + const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); const role = `I'm an agent who makes plans for another agent to achieve a user-specified goal to update the state of an application.${ systemRoleContext === undefined @@ -90,7 +96,7 @@ export function getPlanningSystemPrompt( const systemPrompt = ` ${role} The application state tree is a JSON object with the following schema: ${promptFriendlySchema} - The current state is: ${JSON.stringify(view.root)}. + The current state is: ${JSON.stringify(treeNode)}. The user requested that I accomplish the following goal: "${userPrompt}" I've made a plan to accomplish this goal by doing a sequence of edits to the tree. @@ -107,13 +113,17 @@ export function getEditingSystemPrompt( userPrompt: string, idGenerator: IdGenerator, view: TreeView, + treeNode: TreeNode, log: EditLog, appGuidance?: string, plan?: string, ): string { - const schema = normalizeFieldSchema(view.schema); + const isRootNode = Tree.parent(treeNode) === undefined; + const schema = isRootNode + ? normalizeFieldSchema(view.schema) + : normalizeFieldSchema(Tree.schema(treeNode)); const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); - const decoratedTreeJson = toDecoratedJson(idGenerator, view.root); + const decoratedTreeJson = toDecoratedJson(idGenerator, treeNode); function createEditList(edits: EditLog): string { return edits @@ -134,7 +144,6 @@ export function getEditingSystemPrompt( The application that owns the JSON tree has the following guidance about your role: ${appGuidance}` }`; - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call const treeSchemaString = createZodJsonValidator( ...generateGenericEditTypes(getSimpleSchema(schema), false), ).getSchemaText(); @@ -169,12 +178,16 @@ export function getReviewSystemPrompt( userPrompt: string, idGenerator: IdGenerator, view: TreeView, + treeNode: TreeNode, originalDecoratedJson: string, appGuidance?: string, ): string { - const schema = normalizeFieldSchema(view.schema); + const isRootNode = Tree.parent(treeNode) === undefined; + const schema = isRootNode + ? normalizeFieldSchema(view.schema) + : normalizeFieldSchema(Tree.schema(treeNode)); const promptFriendlySchema = getPromptFriendlyTreeSchema(getJsonSchema(schema.allowedTypes)); - const decoratedTreeJson = toDecoratedJson(idGenerator, view.root); + const decoratedTreeJson = toDecoratedJson(idGenerator, treeNode); const role = `You are a collaborative agent who interacts with a JSON tree by performing edits to achieve a user-specified goal.${ appGuidance === undefined diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts index d987b3b152c7..0e80a8c86f32 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts @@ -55,7 +55,7 @@ class Vector2 extends sf.object("Vector2", { z2: sf.optional(sf.number), }) {} -class RootObjectPolymorphic extends sf.object("RootObject", { +class RootObjectPolymorphic extends sf.object("RootObjectPolymorphic", { str: sf.string, // Two different vector types to handle the polymorphic case vectors: sf.array([Vector, Vector2]), @@ -69,30 +69,37 @@ class RootObject extends sf.object("RootObject", { bools: sf.array(sf.boolean), }) {} -class RootObjectWithMultipleVectorArrays extends sf.object("RootObject", { - str: sf.string, - // Two different vector types to handle the polymorphic case - vectors: sf.array([Vector]), - vectors2: sf.array([Vector]), - bools: sf.array(sf.boolean), -}) {} - -class RootObjectWithDifferentVectorArrayTypes extends sf.object("RootObject", { - str: sf.string, - // Two different vector types to handle the polymorphic case - vectors: sf.array([Vector]), - vectors2: sf.array([Vector2]), - bools: sf.array(sf.boolean), -}) {} - -class RootObjectWithNonArrayVectorField extends sf.object("RootObject", { - singleVector: sf.optional(Vector), - // Two different vector types to handle the polymorphic case - vectors: sf.array([Vector]), - bools: sf.array(sf.boolean), -}) {} - -const config = new TreeViewConfiguration({ schema: [sf.number, RootObjectPolymorphic] }); +class RootObjectWithMultipleVectorArrays extends sf.object( + "RootObjectWithMultipleVectorArrays", + { + str: sf.string, + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector]), + vectors2: sf.array([Vector]), + bools: sf.array(sf.boolean), + }, +) {} + +class RootObjectWithDifferentVectorArrayTypes extends sf.object( + "RootObjectWithDifferentVectorArrayTypes", + { + str: sf.string, + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector]), + vectors2: sf.array([Vector2]), + bools: sf.array(sf.boolean), + }, +) {} + +class RootObjectWithNonArrayVectorField extends sf.object( + "RootObjectWithNonArrayVectorField", + { + singleVector: sf.optional(Vector), + // Two different vector types to handle the polymorphic case + vectors: sf.array([Vector]), + bools: sf.array(sf.boolean), + }, +) {} const factory = SharedTree.getFactory(); @@ -107,6 +114,12 @@ describe("applyAgentEdit", () => { new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "tree", ); + class DifferentRootObject extends sf.object("DifferentRootObject", { + testProperty: sf.string, + }) {} + const config = new TreeViewConfiguration({ + schema: [DifferentRootObject, RootObjectPolymorphic], + }); const view = tree.viewWith(config); const schema = normalizeFieldSchema(view.schema); const simpleSchema = getSimpleSchema(schema.allowedTypes); @@ -127,7 +140,7 @@ describe("applyAgentEdit", () => { }, }; - applyAgentEdit(view, setRootEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, setRootEdit, idGenerator, simpleSchema.definitions); const expected = { str: "rootStr", @@ -135,50 +148,55 @@ describe("applyAgentEdit", () => { bools: [], }; + const jsonRoot = JSON.stringify(view.root, undefined, 2); + const jsonExpected = JSON.stringify(expected, undefined, 2); + assert.deepEqual( JSON.stringify(view.root, undefined, 2), JSON.stringify(expected, undefined, 2), ); }); - it("optional root", () => { - const tree = factory.create( - new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - "tree", - ); - const configOptionalRoot = new TreeViewConfiguration({ schema: sf.optional(sf.number) }); - const view = tree.viewWith(configOptionalRoot); - const schema = normalizeFieldSchema(view.schema); - const simpleSchema = getSimpleSchema(schema.allowedTypes); - view.initialize(1); - - const setRootEdit: TreeEdit = { - explanation: "Set root to 2", - type: "setRoot", - content: 2, - }; - - applyAgentEdit(view, setRootEdit, idGenerator, simpleSchema.definitions); - - const expectedTreeView = factory - .create( - new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - "expectedTree", - ) - .viewWith(configOptionalRoot); - expectedTreeView.initialize(2); - - assert.deepEqual(view.root, expectedTreeView.root); - }); + // // TODO: optional roots are not supported due to the way the schema is generated, differentiating the + // // root from non root nodes. When undefined is passed as the root, we cannot determine if it is a root. + // it.skip("optional root", () => { + // const tree = factory.create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "tree", + // ); + // const configOptionalRoot = new TreeViewConfiguration({ schema: sf.optional(sf.number) }); + // const view = tree.viewWith(configOptionalRoot); + // const schema = normalizeFieldSchema(view.schema); + // const simpleSchema = getSimpleSchema(schema.allowedTypes); + // view.initialize(1); + + // const setRootEdit: TreeEdit = { + // explanation: "Set root to 2", + // type: "setRoot", + // content: 2, + // }; + + // applyAgentEdit(view, view.root, setRootEdit, idGenerator, simpleSchema.definitions); + + // const expectedTreeView = factory + // .create( + // new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + // "expectedTree", + // ) + // .viewWith(configOptionalRoot); + // expectedTreeView.initialize(2); + + // assert.deepEqual(view.root, expectedTreeView.root); + // }); }); describe("insert edits", () => { - it("polymorphic insert edits", () => { + it("inner polymorphic tree node insert edits", () => { const tree = factory.create( new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "tree", ); - const view = tree.viewWith(config); + const view = tree.viewWith(new TreeViewConfiguration({ schema: RootObjectPolymorphic })); const schema = normalizeFieldSchema(view.schema); const simpleSchema = getSimpleSchema(schema.allowedTypes); @@ -190,8 +208,7 @@ describe("applyAgentEdit", () => { idGenerator.assignIds(view.root); const vectorId = // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0]!) ?? - fail("ID expected."); + idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); const insertEdit: TreeEdit = { explanation: "Insert a vector", @@ -203,7 +220,7 @@ describe("applyAgentEdit", () => { place: "after", }, }; - applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, insertEdit, idGenerator, simpleSchema.definitions); const insertEdit2: TreeEdit = { explanation: "Insert a vector", @@ -215,11 +232,11 @@ describe("applyAgentEdit", () => { place: "after", }, }; - applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, insertEdit2, idGenerator, simpleSchema.definitions); - const identifier1 = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; - const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector).id; - const identifier3 = ((view.root as RootObjectPolymorphic).vectors[2] as Vector).id; + const identifier1 = (view.root.vectors[0] as Vector).id; + const identifier2 = (view.root.vectors[1] as Vector).id; + const identifier3 = (view.root.vectors[2] as Vector).id; const expected = { "str": "testStr", @@ -257,7 +274,7 @@ describe("applyAgentEdit", () => { new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "tree", ); - const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + const config2 = new TreeViewConfiguration({ schema: RootObject }); const view = tree.viewWith(config2); const schema = normalizeFieldSchema(view.schema); const simpleSchema = getSimpleSchema(schema.allowedTypes); @@ -271,7 +288,7 @@ describe("applyAgentEdit", () => { idGenerator.assignIds(view.root); const vectorId = // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - idGenerator.getId((view.root as RootObject).vectors[0]!) ?? fail("ID expected."); + idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); const insertEdit: TreeEdit = { explanation: "Insert a vector", @@ -283,12 +300,12 @@ describe("applyAgentEdit", () => { place: "after", }, }; - applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, insertEdit, idGenerator, simpleSchema.definitions); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const identifier1 = (view.root as RootObject).vectors[0]!.id; + const identifier1 = view.root.vectors[0]!.id; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const identifier2 = (view.root as RootObject).vectors[1]!.id; + const identifier2 = view.root.vectors[1]!.id; const expected = { "str": "testStr", @@ -320,7 +337,7 @@ describe("applyAgentEdit", () => { new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "tree", ); - const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + const config2 = new TreeViewConfiguration({ schema: RootObject }); const view = tree.viewWith(config2); const schema = normalizeFieldSchema(view.schema); const simpleSchema = getSimpleSchema(schema.allowedTypes); @@ -332,7 +349,7 @@ describe("applyAgentEdit", () => { }); idGenerator.assignIds(view.root); - const vectorId = idGenerator.getId(view.root as RootObject) ?? fail("ID expected."); + const vectorId = idGenerator.getId(view.root) ?? fail("ID expected."); const insertEdit: TreeEdit = { explanation: "Insert a vector", @@ -345,10 +362,10 @@ describe("applyAgentEdit", () => { location: "start", }, }; - applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, insertEdit, idGenerator, simpleSchema.definitions); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const identifier1 = (view.root as RootObject).vectors[0]!.id; + const identifier1 = view.root.vectors[0]!.id; const expected = { "str": "testStr", @@ -374,7 +391,7 @@ describe("applyAgentEdit", () => { new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "tree", ); - const config2 = new TreeViewConfiguration({ schema: [sf.number, RootObject] }); + const config2 = new TreeViewConfiguration({ schema: RootObject }); const view = tree.viewWith(config2); const schema = normalizeFieldSchema(view.schema); const simpleSchema = getSimpleSchema(schema.allowedTypes); @@ -388,7 +405,7 @@ describe("applyAgentEdit", () => { idGenerator.assignIds(view.root); const vectorId = // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - idGenerator.getId((view.root as RootObject).vectors[0]!) ?? fail("ID expected."); + idGenerator.getId(view.root.vectors[0]!) ?? fail("ID expected."); const insertEdit: TreeEdit = { explanation: "Insert a vector", @@ -402,7 +419,8 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + () => + applyAgentEdit(view, view.root, insertEdit, idGenerator, simpleSchema.definitions), validateUsageError(/provided data is incompatible/), ); }); @@ -438,7 +456,8 @@ describe("applyAgentEdit", () => { }, }; assert.throws( - () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + () => + applyAgentEdit(view, view.root, insertEdit, idGenerator, simpleSchema.definitions), validateUsageError(/Expected child to be in an array node/), ); }); @@ -449,6 +468,7 @@ describe("applyAgentEdit", () => { new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), "tree", ); + const config = new TreeViewConfiguration({ schema: RootObjectPolymorphic }); const view = tree.viewWith(config); const schema = normalizeFieldSchema(view.schema); const simpleSchema = getSimpleSchema(schema.allowedTypes); @@ -472,7 +492,7 @@ describe("applyAgentEdit", () => { { [typeField]: Vector2.identifier, x2: 3, y2: 4, z2: 5 }, ], }; - applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, modifyEdit, idGenerator, simpleSchema.definitions); const modifyEdit2: TreeEdit = { explanation: "Modify a vector", @@ -481,12 +501,11 @@ describe("applyAgentEdit", () => { field: "bools", modification: [false], }; - applyAgentEdit(view, modifyEdit2, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, modifyEdit2, idGenerator, simpleSchema.definitions); idGenerator.assignIds(view.root); const vectorId2 = - idGenerator.getId((view.root as RootObjectPolymorphic).vectors[0] as Vector) ?? - fail("ID expected."); + idGenerator.getId(view.root.vectors[0] as Vector) ?? fail("ID expected."); const modifyEdit3: TreeEdit = { explanation: "Modify a vector", @@ -495,10 +514,10 @@ describe("applyAgentEdit", () => { field: "x", modification: 111, }; - applyAgentEdit(view, modifyEdit3, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, modifyEdit3, idGenerator, simpleSchema.definitions); - const identifier = ((view.root as RootObjectPolymorphic).vectors[0] as Vector).id; - const identifier2 = ((view.root as RootObjectPolymorphic).vectors[1] as Vector2).id; + const identifier = (view.root.vectors[0] as Vector).id; + const identifier2 = (view.root.vectors[1] as Vector2).id; const expected = { "str": "testStr", @@ -553,7 +572,7 @@ describe("applyAgentEdit", () => { type: "remove", source: { target: vectorId1 }, }; - applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, removeEdit, idGenerator, simpleSchema.definitions); const expected = { "str": "testStr", @@ -594,7 +613,7 @@ describe("applyAgentEdit", () => { type: "remove", source: { target: singleVectorId }, }; - applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, removeEdit, idGenerator, simpleSchema.definitions); const expected = { "vectors": [], @@ -633,7 +652,7 @@ describe("applyAgentEdit", () => { type: "remove", source: { target: rootId }, }; - applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, removeEdit, idGenerator, simpleSchema.definitions); assert.equal(view.root, undefined); }); @@ -665,7 +684,8 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), + () => + applyAgentEdit(view, view.root, removeEdit, idGenerator, simpleSchema.definitions), validateUsageError( /The root is required, and cannot be removed. Please use modify edit instead./, @@ -713,7 +733,7 @@ describe("applyAgentEdit", () => { }, }, }; - applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, removeEdit, idGenerator, simpleSchema.definitions); const expected = { "str": "testStr", @@ -769,7 +789,8 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, removeEdit, idGenerator, simpleSchema.definitions), + () => + applyAgentEdit(view, view.root, removeEdit, idGenerator, simpleSchema.definitions), validateUsageError( /The "from" node and "to" nodes of the range must be in the same parent array./, ), @@ -813,7 +834,7 @@ describe("applyAgentEdit", () => { location: "start", }, }; - applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, moveEdit, idGenerator, simpleSchema.definitions); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const identifier = view.root.vectors2[0]!.id; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -892,7 +913,7 @@ describe("applyAgentEdit", () => { location: "start", }, }; - applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions); + applyAgentEdit(view, view.root, moveEdit, idGenerator, simpleSchema.definitions); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const identifier = view.root.vectors2[0]!.id; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -980,7 +1001,7 @@ describe("applyAgentEdit", () => { }, }; assert.throws( - () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, moveEdit, idGenerator, simpleSchema.definitions), validateUsageError(/Illegal node type in destination array/), ); }); @@ -1035,7 +1056,7 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, moveEdit, idGenerator, simpleSchema.definitions), validateUsageError( /The "from" node and "to" nodes of the range must be in the same parent array./, ), @@ -1081,7 +1102,7 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, moveEdit, idGenerator, simpleSchema.definitions), validateUsageError(/the source node must be within an arrayNode/), ); }); @@ -1125,7 +1146,7 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, moveEdit, idGenerator, simpleSchema.definitions), validateUsageError(/No child under field field/), ); }); @@ -1162,7 +1183,7 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, insertEdit, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, insertEdit, idGenerator, simpleSchema.definitions), validateUsageError(/objectIdKey testObjectId does not exist/), ); @@ -1179,7 +1200,8 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, insertEdit2, idGenerator, simpleSchema.definitions), + () => + applyAgentEdit(view, view.root, insertEdit2, idGenerator, simpleSchema.definitions), validateUsageError(/objectIdKey testObjectId does not exist/), ); @@ -1208,7 +1230,7 @@ describe("applyAgentEdit", () => { const objectIdKeys = ["testObjectId1", "testObjectId2", "testObjectId3"]; const errorMessage = `objectIdKeys [${objectIdKeys.join(",")}] does not exist`; assert.throws( - () => applyAgentEdit(view, moveEdit, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, moveEdit, idGenerator, simpleSchema.definitions), validateUsageError(errorMessage), ); @@ -1228,7 +1250,7 @@ describe("applyAgentEdit", () => { const objectIdKeys2 = ["testObjectId1", "testObjectId2"]; const errorMessage2 = `objectIdKeys [${objectIdKeys2.join(",")}] does not exist`; assert.throws( - () => applyAgentEdit(view, moveEdit2, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, moveEdit2, idGenerator, simpleSchema.definitions), validateUsageError(errorMessage2), ); @@ -1241,7 +1263,7 @@ describe("applyAgentEdit", () => { }; assert.throws( - () => applyAgentEdit(view, modifyEdit, idGenerator, simpleSchema.definitions), + () => applyAgentEdit(view, view.root, modifyEdit, idGenerator, simpleSchema.definitions), validateUsageError(/objectIdKey testObjectId does not exist/), ); }); diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts index 563479bc13b2..a725725af041 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration.spec.ts @@ -169,6 +169,7 @@ describe.skip("Agent Editing Integration", () => { await generateTreeEdits({ openAI: { client: openAIClient, modelName: TEST_MODEL_NAME }, treeView: view, + treeNode: view.root, prompt: { userAsk: "Please alphabetize the sessions.", systemRoleContext: "", diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts index c11edab5c4db..098c491bfd48 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts @@ -30,7 +30,7 @@ export class SharedTreeTask extends sf.object("Task", { priority: sf.string, complexity: sf.number, status: sf.string, - assignee: sf.optional(sf.string), + assignee: sf.string, }) {} export class SharedTreeTaskList extends sf.array("TaskList", SharedTreeTask) {} @@ -50,14 +50,12 @@ export class SharedTreeTaskGroup extends sf.object("TaskGroup", { title: sf.string, tasks: SharedTreeTaskList, engineers: SharedTreeEngineerList, - // optionalInfo: sf.optional(sf.string), }) {} export class SharedTreeTaskGroupList extends sf.array("TaskGroupList", SharedTreeTaskGroup) {} export class SharedTreeAppState extends sf.object("AppState", { taskGroups: SharedTreeTaskGroupList, - optionalInfo: sf.optional(sf.string), }) {} export const INITIAL_APP_STATE = { @@ -173,7 +171,38 @@ const factory = SharedTree.getFactory(); const OPENAI_API_KEY = ""; -describe("Ai Planner App", () => { +describe.skip("Ai Planner App", () => { + it("should be able to change the priority of a task", async () => { + const tree = factory.create( + new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), + "tree", + ); + const view = tree.viewWith(new TreeViewConfiguration({ schema: SharedTreeAppState })); + view.initialize(INITIAL_APP_STATE); + + await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + treeNode: view.root.taskGroups[0]!, + prompt: { + systemRoleContext: "You are a managing objects with a priority field.", + userAsk: + "Change the priority of the first task within the first task group from low to high", + }, + planningStep: true, + finalReviewStep: true, + dumpDebugLog: true, + }); + + assert(view.root.taskGroups[0]?.tasks[0]?.priority === "high"); + }); + it.skip("BUG: Invalid json schema produced when schema has no arrays at all", async () => { class TestAppSchema extends sf.object("PrioritySpecification", { priority: sf.string, @@ -195,6 +224,7 @@ describe("Ai Planner App", () => { modelName: "gpt-4o", }, treeView: view, + treeNode: view.root, prompt: { systemRoleContext: "You are a managing objects with a priority field.", userAsk: "Change the priority from low to high", @@ -214,13 +244,15 @@ describe("Ai Planner App", () => { }); it.skip("BUG: Invalid json schema produced when schema has multiple keys with the same name and order", async () => { + class TaskList extends sf.array("taskList", sf.string) {} + class TestInnerAppSchema extends sf.object("TestInnerAppSchema", { title: sf.string, }) {} class TestAppSchema extends sf.object("TestAppSchema", { title: sf.string, - taskList: SharedTreeTaskList, + taskList: TaskList, appData: TestInnerAppSchema, }) {} @@ -244,6 +276,7 @@ describe("Ai Planner App", () => { modelName: "gpt-4o", }, treeView: view, + treeNode: view.root, prompt: { systemRoleContext: "You are a managing json objects", userAsk: "Change the `title` field of the outer object to 'Hello World'", @@ -262,49 +295,12 @@ describe("Ai Planner App", () => { } }); - it.skip("BUG: Invalid json schema produced when schema has no arrays", async () => { - class TestAppSchema extends sf.object("TestAppSchema", { - title: sf.string, - }) {} - - const tree = factory.create( - new MockFluidDataStoreRuntime({ idCompressor: createIdCompressor() }), - "tree", - ); - const view = tree.viewWith(new TreeViewConfiguration({ schema: TestAppSchema })); - view.initialize({ title: "Sample Title" }); - - try { - await aiCollab({ - openAI: { - client: new OpenAI({ - apiKey: OPENAI_API_KEY, - }), - modelName: "gpt-4o", - }, - treeView: view, - prompt: { - systemRoleContext: "You are a managing json objects", - userAsk: "Change the `title` field to 'Hello World'", - }, - planningStep: true, - finalReviewStep: true, - }); - } catch (error) { - assert(error instanceof APIError); - assert(error.status === 400); - assert(error.type === "invalid_request_error"); - assert( - error.message === - "400 Invalid schema for response_format 'SharedTreeAI': In context=('properties', 'edit', 'anyOf', '1', 'properties', 'content', 'not'), schema must have a 'type' key.", - ); - } - }); - it.skip("BUG: OpenAI structured output fails when json schema with psuedo optional field is used in response format", async () => { + class TaskList extends sf.array("taskList", sf.string) {} + class TestAppSchemaWithOptionalProp extends sf.object("TestAppSchemaWithOptionalProp", { nonOptionalProp: sf.string, - taskList: SharedTreeTaskList, + taskList: TaskList, optionalProp: sf.optional(sf.string), }) {} @@ -326,6 +322,7 @@ describe("Ai Planner App", () => { modelName: "gpt-4o", }, treeView: view, + treeNode: view.root, prompt: { systemRoleContext: "You are a managing json objects", userAsk: "Change the `optionalProp` field to 'Hello World'", @@ -367,6 +364,7 @@ describe("Ai Planner App", () => { modelName: "gpt-4o", }, treeView: view2, + treeNode: view.root, prompt: { systemRoleContext: "You are a managing json objects", userAsk: "Change the `optionalProp` field to 'Hello World'", @@ -375,8 +373,6 @@ describe("Ai Planner App", () => { finalReviewStep: true, }); - const jsonified = JSON.stringify(view.root); - assert.equal(view.root.nonOptionalProp, "Hello World"); }); }); From c3c50d849a2b301d83b9c40d58baa00ba9ce2f7b Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:14:45 +0000 Subject: [PATCH 17/28] Small integ test update --- .../src/test/explicit-strategy/integration/planner.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts index 098c491bfd48..735b0dbfa8fd 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts @@ -191,9 +191,9 @@ describe.skip("Ai Planner App", () => { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion treeNode: view.root.taskGroups[0]!, prompt: { - systemRoleContext: "You are a managing objects with a priority field.", - userAsk: - "Change the priority of the first task within the first task group from low to high", + systemRoleContext: + "You are a manager that is helping out with a project management tool. You have been asked to edit a group of tasks.", + userAsk: "Change the priority of the first task from low to high", }, planningStep: true, finalReviewStep: true, From 3c1934c32a8820a9a4699cdd8b24b0cea8b6b442 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:40:47 +0000 Subject: [PATCH 18/28] Merge conflict resolutions and small api update to take TSchema type parameter --- packages/dds/tree/src/simple-tree/index.ts | 1 + .../api-report/ai-collab.alpha.api.md | 2 +- packages/framework/ai-collab/src/aiCollab.ts | 4 +- .../src/explicit-strategy/idGenerator.ts | 2 +- .../ai-collab/src/explicit-strategy/index.ts | 4 +- .../src/explicit-strategy/promptGeneration.ts | 12 +- .../integration/planner.spec.ts | 8 +- pnpm-lock.yaml | 141 +----------------- 8 files changed, 23 insertions(+), 151 deletions(-) diff --git a/packages/dds/tree/src/simple-tree/index.ts b/packages/dds/tree/src/simple-tree/index.ts index f913c7c418fa..196629e0efa0 100644 --- a/packages/dds/tree/src/simple-tree/index.ts +++ b/packages/dds/tree/src/simple-tree/index.ts @@ -128,6 +128,7 @@ export { type InsertableField, type Insertable, type UnsafeUnknownSchema, + normalizeAllowedTypes, } from "./schemaTypes.js"; export { getTreeNodeForField, diff --git a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md index e39e7b2fa462..a9b9aa5b33f8 100644 --- a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md +++ b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md @@ -5,7 +5,7 @@ ```ts // @alpha -export function aiCollab(options: AiCollabOptions): Promise; +export function aiCollab(options: AiCollabOptions): Promise; // @alpha export interface AiCollabErrorResponse { diff --git a/packages/framework/ai-collab/src/aiCollab.ts b/packages/framework/ai-collab/src/aiCollab.ts index d042f1125836..f7943ffffe02 100644 --- a/packages/framework/ai-collab/src/aiCollab.ts +++ b/packages/framework/ai-collab/src/aiCollab.ts @@ -18,8 +18,8 @@ import { generateTreeEdits } from "./explicit-strategy/index.js"; * * @alpha */ -export async function aiCollab( - options: AiCollabOptions, +export async function aiCollab( + options: AiCollabOptions, ): Promise { const response = await generateTreeEdits({ treeView: options.treeView, diff --git a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts index e0afb3245589..e983e9105c56 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts @@ -59,7 +59,7 @@ export class IdGenerator { // assert(isTreeNode(node), "Non-TreeNode value in tree."); const objId = this.getOrCreateId(node as TreeNode); Object.keys(node).forEach((key) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument this.assignIds((node as unknown as any)[key]); }); return objId; diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index 1eb4a602b456..17f8c5edda86 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -83,8 +83,8 @@ interface GenerateTreeEditsErrorResponse { * * @internal */ -export async function generateTreeEdits( - options: GenerateTreeEditsOptions, +export async function generateTreeEdits( + options: GenerateTreeEditsOptions, ): Promise { const idGenerator = new IdGenerator(); const editLog: EditLog = []; diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index eedb84f0e040..901fc6f142e9 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -74,8 +74,8 @@ export function toDecoratedJson( /** * TBD */ -export function getPlanningSystemPrompt( - view: TreeView, +export function getPlanningSystemPrompt( + view: TreeView, treeNode: TreeNode, userPrompt: string, systemRoleContext?: string, @@ -109,10 +109,10 @@ export function getPlanningSystemPrompt( /** * TBD */ -export function getEditingSystemPrompt( +export function getEditingSystemPrompt( userPrompt: string, idGenerator: IdGenerator, - view: TreeView, + view: TreeView, treeNode: TreeNode, log: EditLog, appGuidance?: string, @@ -174,10 +174,10 @@ export function getEditingSystemPrompt( /** * TBD */ -export function getReviewSystemPrompt( +export function getReviewSystemPrompt( userPrompt: string, idGenerator: IdGenerator, - view: TreeView, + view: TreeView, treeNode: TreeNode, originalDecoratedJson: string, appGuidance?: string, diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts index 735b0dbfa8fd..05ef72f6eb70 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts @@ -180,7 +180,7 @@ describe.skip("Ai Planner App", () => { const view = tree.viewWith(new TreeViewConfiguration({ schema: SharedTreeAppState })); view.initialize(INITIAL_APP_STATE); - await aiCollab({ + await aiCollab({ openAI: { client: new OpenAI({ apiKey: OPENAI_API_KEY, @@ -216,7 +216,7 @@ describe.skip("Ai Planner App", () => { view.initialize({ priority: "low" }); try { - await aiCollab({ + await aiCollab({ openAI: { client: new OpenAI({ apiKey: OPENAI_API_KEY, @@ -268,7 +268,7 @@ describe.skip("Ai Planner App", () => { }); try { - await aiCollab({ + await aiCollab({ openAI: { client: new OpenAI({ apiKey: OPENAI_API_KEY, @@ -356,7 +356,7 @@ describe.skip("Ai Planner App", () => { ); view2.initialize({ nonOptionalProp: "Hello" }); - await aiCollab({ + await aiCollab({ openAI: { client: new OpenAI({ apiKey: OPENAI_API_KEY, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5936063c0629..d1f4ec4c6d97 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10476,7 +10476,7 @@ importers: version: link:../../test/mocha-test-setup '@fluid-tools/build-cli': specifier: ^0.49.0 - version: 0.49.0(@types/node@18.19.54)(typescript@5.4.5) + version: 0.49.0(@types/node@18.19.54)(typescript@5.4.5)(webpack-cli@5.1.4) '@fluidframework/build-common': specifier: ^2.0.3 version: 2.0.3 @@ -19320,85 +19320,6 @@ packages: transitivePeerDependencies: - supports-color - /@fluid-tools/build-cli@0.49.0(@types/node@18.19.54)(typescript@5.4.5): - resolution: {integrity: sha512-V9h8OCJDvSz8m4zmeCO6y8DJi972BSFp3YO6S/R1v7J/CpaG5A6v1Di0Kp5+JYf+sQ2ILoBaEvdjCp3ii+eYTw==} - engines: {node: '>=18.17.1'} - hasBin: true - dependencies: - '@andrewbranch/untar.js': 1.0.3 - '@fluid-tools/version-tools': 0.49.0 - '@fluidframework/build-tools': 0.49.0 - '@fluidframework/bundle-size-tools': 0.49.0 - '@microsoft/api-extractor': 7.47.8(patch_hash=ldzfpsbo3oeejrejk775zxplmi)(@types/node@18.19.54) - '@oclif/core': 4.0.25 - '@oclif/plugin-autocomplete': 3.2.5 - '@oclif/plugin-commands': 4.0.16 - '@oclif/plugin-help': 6.2.13 - '@oclif/plugin-not-found': 3.2.22 - '@octokit/core': 4.2.4 - '@octokit/rest': 21.0.2 - '@rushstack/node-core-library': 3.66.1(@types/node@18.19.54) - async: 3.2.6 - azure-devops-node-api: 11.2.0 - chalk: 5.3.0 - change-case: 3.1.0 - cosmiconfig: 8.3.6(typescript@5.4.5) - danger: 11.3.1 - date-fns: 2.30.0 - debug: 4.3.7(supports-color@8.1.1) - execa: 5.1.1 - fflate: 0.8.2 - fs-extra: 11.2.0 - github-slugger: 2.0.0 - globby: 11.1.0 - gray-matter: 4.0.3 - human-id: 4.1.1 - inquirer: 8.2.6 - issue-parser: 7.0.1 - json5: 2.2.3 - jssm: 5.98.2 - jszip: 3.10.1 - latest-version: 5.1.0 - mdast: 3.0.0 - mdast-util-heading-range: 4.0.0 - mdast-util-to-string: 4.0.0 - minimatch: 7.4.6 - node-fetch: 2.7.0 - npm-check-updates: 16.14.20 - oclif: 4.15.0 - prettier: 3.2.5 - prompts: 2.4.2 - read-pkg-up: 7.0.1 - remark: 15.0.1 - remark-gfm: 4.0.0 - remark-github: 12.0.0 - remark-github-beta-blockquote-admonitions: 3.1.1 - remark-toc: 9.0.0 - replace-in-file: 7.2.0 - resolve.exports: 2.0.2 - semver: 7.6.3 - semver-utils: 1.1.4 - simple-git: 3.27.0 - sort-json: 2.0.1 - sort-package-json: 1.57.0 - strip-ansi: 6.0.1 - table: 6.8.2 - ts-morph: 22.0.0 - type-fest: 2.19.0 - unist-util-visit: 5.0.0 - xml2js: 0.5.0 - transitivePeerDependencies: - - '@swc/core' - - '@types/node' - - bluebird - - encoding - - esbuild - - supports-color - - typescript - - uglify-js - - webpack-cli - dev: true - /@fluid-tools/build-cli@0.49.0(@types/node@18.19.54)(typescript@5.4.5)(webpack-cli@5.1.4): resolution: {integrity: sha512-V9h8OCJDvSz8m4zmeCO6y8DJi972BSFp3YO6S/R1v7J/CpaG5A6v1Di0Kp5+JYf+sQ2ILoBaEvdjCp3ii+eYTw==} engines: {node: '>=18.17.1'} @@ -19766,22 +19687,6 @@ packages: transitivePeerDependencies: - supports-color - /@fluidframework/bundle-size-tools@0.49.0: - resolution: {integrity: sha512-SUrWc931wwOkwIERX282SmHUVjXz0mRhlYIoY68DkYVZ3XuUrKaVvHbJB6a3ek+TIX33zg90HKFNkp9K56m0SQ==} - dependencies: - azure-devops-node-api: 11.2.0 - jszip: 3.10.1 - msgpack-lite: 0.1.26 - pako: 2.1.0 - typescript: 5.4.5 - webpack: 5.95.0 - transitivePeerDependencies: - - '@swc/core' - - esbuild - - uglify-js - - webpack-cli - dev: true - /@fluidframework/bundle-size-tools@0.49.0(webpack-cli@5.1.4): resolution: {integrity: sha512-SUrWc931wwOkwIERX282SmHUVjXz0mRhlYIoY68DkYVZ3XuUrKaVvHbJB6a3ek+TIX33zg90HKFNkp9K56m0SQ==} dependencies: @@ -29188,6 +29093,10 @@ packages: signal-exit: 4.1.0 dev: true + /form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + dev: false + /form-data-encoder@2.1.4: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} @@ -37685,7 +37594,7 @@ packages: schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.34.1 - webpack: 5.95.0 + webpack: 5.95.0(webpack-cli@5.1.4) /terser@5.34.1: resolution: {integrity: sha512-FsJZ7iZLd/BXkz+4xrRTGJ26o/6VTjQytUk8b8OxkwcD2I+79VPJlz7qss1+zE7h8GNIScFqXcDyJ/KqBYZFVA==} @@ -39283,44 +39192,6 @@ packages: resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} engines: {node: '>=10.13.0'} - /webpack@5.95.0: - resolution: {integrity: sha512-2t3XstrKULz41MNMBF+cJ97TyHdyQ8HCt//pqErqDvNjU9YQBnZxIHa11VXsi7F3mb5/aO2tuDxdeTPdU7xu9Q==} - engines: {node: '>=10.13.0'} - hasBin: true - peerDependencies: - webpack-cli: '*' - peerDependenciesMeta: - webpack-cli: - optional: true - dependencies: - '@types/estree': 1.0.6 - '@webassemblyjs/ast': 1.12.1 - '@webassemblyjs/wasm-edit': 1.12.1 - '@webassemblyjs/wasm-parser': 1.12.1 - acorn: 8.12.1 - acorn-import-attributes: 1.9.5(acorn@8.12.1) - browserslist: 4.24.0 - chrome-trace-event: 1.0.4 - enhanced-resolve: 5.17.1 - es-module-lexer: 1.5.4 - eslint-scope: 5.1.1 - events: 3.3.0 - glob-to-regexp: 0.4.1 - graceful-fs: 4.2.11 - json-parse-even-better-errors: 2.3.1 - loader-runner: 4.3.0 - mime-types: 2.1.35 - neo-async: 2.6.2 - schema-utils: 3.3.0 - tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(webpack@5.95.0) - watchpack: 2.4.2 - webpack-sources: 3.2.3 - transitivePeerDependencies: - - '@swc/core' - - esbuild - - uglify-js - /webpack@5.95.0(webpack-cli@5.1.4): resolution: {integrity: sha512-2t3XstrKULz41MNMBF+cJ97TyHdyQ8HCt//pqErqDvNjU9YQBnZxIHa11VXsi7F3mb5/aO2tuDxdeTPdU7xu9Q==} engines: {node: '>=10.13.0'} From 6c3b76d447395d7c3113b1bf8e429b43ed553b88 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:59:12 +0000 Subject: [PATCH 19/28] Adds a new README.md file to the /src directory and fixes a SharedTree test to account for new metadata property --- .../test/simple-tree/getSimpleSchema.spec.ts | 1 + packages/framework/ai-collab/src/README.md | 187 ++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 packages/framework/ai-collab/src/README.md diff --git a/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts index 8ea48173ed23..1d550a61749c 100644 --- a/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/getSimpleSchema.spec.ts @@ -33,6 +33,7 @@ describe("getSimpleSchema", () => { }, ], ]), + metadata: { description: "An optional string." }, allowedTypes: new Set(["com.fluidframework.leaf.string"]), }; assert.deepEqual(actual, expected); diff --git a/packages/framework/ai-collab/src/README.md b/packages/framework/ai-collab/src/README.md new file mode 100644 index 000000000000..34c870b538e1 --- /dev/null +++ b/packages/framework/ai-collab/src/README.md @@ -0,0 +1,187 @@ +## Description +The ai-collab client library makes adding complex, human-like collaboration with LLM's built directly in your application as simple as one function call. Simply pass your SharedTree and ask AI to collaborate. For example, +- Task Management App: "Reorder this list of tasks in order from least to highest complexity." +- Job Board App: "Create a new job listing and add it to this job board" +- Calender App: "Manage my calender to slot in a new 2:30 appointment" + +## Usage + +### Your SharedTree types file +```ts +// --------- File name: "types.ts" --------- +import { SchemaFactory } from "@fluidframework/tree"; + +const sf = new SchemaFactory("ai-collab-sample-application"); + +export class Task extends sf.object("Task", { + title: sf.required(sf.string, { + metadata: { description: `The title of the task` }, + }), + id: sf.identifier, + description: sf.required(sf.string, { + metadata: { description: `The description of the task` }, + }), + priority: sf.required(sf.string, { + metadata: { description: `The priority of the task in three levels, "low", "medium", "high"` }, + }), + complexity: sf.required(sf.number, { + metadata: { description: `The complexity of the task as a fibonacci number` }, + }), + status: sf.required(sf.string, { + metadata: { description: `The status of the task as either "todo", "in-progress", or "done"` }, + }), + assignee: sf.required(sf.string, { + metadata: { description: `The name of the tasks assignee e.g. "Bob" or "Alice"` }, + }), +}) {} + +export class TaskList extends sf.array("TaskList", SharedTreeTask) {} + +export class Engineer extends sf.object("Engineer", { + name: sf.required(sf.string, { + metadata: { description: `The name of an engineer whom can be assigned to a task` }, + }), + id: sf.identifier, + skills: sf.required(sf.string, { + metadata: { description: `A description of the engineers skills which influence what types of tasks they should be assigned to.` }, + }), + maxCapacity: sf.required(sf.number, { + metadata: { description: `The maximum capacity of tasks this engineer can handle measured in in task complexity points.` }, + }), +}) {} + +export class EngineerList extends sf.array("EngineerList", SharedTreeEngineer) {} + +export class TaskGroup extends sf.object("TaskGroup", { + description: sf.required(sf.string, { + metadata: { description: `The description of the task group, which is a collection of tasks and engineers that can be assigned to said tasks.` }, + }), + id: sf.identifier, + title: sf.required(sf.string, { + metadata: { description: `The title of the task group.` }, + }), + tasks: sf.required(SharedTreeTaskList, { + metadata: { description: `The lists of tasks within this task group.` }, + }), + engineers: sf.required(SharedTreeEngineerList, { + metadata: { description: `The lists of engineers within this task group which can be assigned to tasks.` }, + }), +}) {} + +export class TaskGroupList extends sf.array("TaskGroupList", SharedTreeTaskGroup) {} + +export class PlannerAppState extends sf.object("PlannerAppState", { + taskGroups: sf.required(SharedTreeTaskGroupList, { + metadata: { description: `The list of task groups that are being managed by this task management application.` }, + }), +}) {} +``` + +### Example 1: Collaborate with AI +```ts +import { aiCollab } from "@fluid-experimental/ai-collab"; +import { PlannerAppState } from "./types.ts" +// This is not a real file, this is meant to represent how you initialize your app data. +import { initializeAppState } from "./yourAppInitializationFile.ts" + +// --------- File name: "app.ts" --------- + +// Initialize your app state somehow +const appState: PlannerAppState = initializeAppState({ + taskGroups: [ + { + title: "My First Task Group", + description: "Placeholder for first task group", + tasks: [ + { + assignee: "Alice", + title: "Task #1", + description: + "This is the first Sample task.", + priority: "low", + complexity: 1, + status: "todo", + }, + ], + engineers: [ + { + name: "Alice", + maxCapacity: 15, + skills: + "Senior engineer capable of handling complex tasks. Versed in most languages", + }, + { + name: "Charlie", + maxCapacity: 7, + skills: "Junior engineer capable of handling simple tasks. Versed in Node.JS", + }, + ], + }, + ], +}) + +// Typically, the user would input this through a UI form/input of some sort. +const userAsk = "Update the task group description to be a about creating a new Todo list application. Create a set of tasks to accomplish this and assign them to the available engineers. Keep in mind the max capacity of each engineer as you assign tasks." + +// Collaborate with AI one function call. +const response = await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + treeNode: view.root.taskGroups[0], + prompt: { + systemRoleContext: + "You are a manager that is helping out with a project management tool. You have been asked to edit a group of tasks.", + userAsk: userAsk, + }, + planningStep: true, + finalReviewStep: true, + dumpDebugLog: true, + }); + +if (response.status === 'sucess') { + // Render the UI view of your task groups. + window.alert(`The AI has successfully completed your request.`); +} else { + window.alert(`Something went wrong! response status: ${response.status}, error message: ${response.errorMessage}`); +} + + +``` + +Once the `aiCollab` function call is initiated, an LLM will immediately begin attempting to make changes to your Shared Tree using the provided user prompt, the types of your SharedTree and the provided app guidance. The LLM produces multiple changes, in a loop asynchronously. Meaning, you will immediatley see changes if your UI's render loop is connected to your SharedTree App State. + +### Example 2: Collaborate with AI onto a branched state and let the user merge the review and merge the branch back manually +- **Coming Soon** + + +## Folder Structure + +- `/explicit-strategy`: The new explicit strategy, utilizing the prototype built during the fall FHL, with a few adjustments. + - `agentEditReducer`: This file houses the logic for taking in a `TreeEdit`, which the LLM produces, and applying said edit to the + - actual SharedTree. + - `agentEditTypes.ts`: The types of edits an LLM is prompted to produce in order to modify a SharedTree. + - `idGenerator.ts`: `A manager for producing and mapping simple id's in place of UUID hashes when generating prompts for an LLM + - `jsonTypes.ts`: utility JSON related types used in parsing LLM response and generating LLM prompts. + - `promptGeneration.ts`: Logic for producing the different types of prompts sent to an LLM in order to edit a SharedTree. + - `typeGeneration.ts`: Generates serialized(/able) representations of a SharedTree Schema which is used within prompts and the generated of the structured output JSON schema + - `utils.ts`: Utilities for interacting with a SharedTree +- `/implicit-strategy`: The original implicit strategy, currently not used under the exported aiCollab API surface. + +## Known Issues & limitations +1. Union types for a TreeNode are not present when generating App Schema. This will require extracting a field schema instead of TreeNodeSchema when passed a non root node. +1. The Editing System prompt & structured out schema currently provide array related edits even when there are no arrays. This forces you to have an array in your schema to produce a valid json schema +1. Optional roots are not allowed, This is because if you pass undefined as your treeNode to the API, we cannot disambiguate whether you passed the root or not. +1. Primitive root nodes are not allowed to be passed to the API. You must use an object or array as your root. +1. Optional nodes are not supported -- when we use optional nodes, the OpenAI API returns an error complaining that the structured output JSON schema is invalid. I have introduced a fix that should work upon manual validation of the json schema, but there looks to be an issue with their API. I have filed a ticket with OpenAI to address this +1. The current scheme does not allow manipulation of arrays of primitive values because you cannot refer to them. We could accomplish this via a path (probably JSON Pointer or JSONPath) from a possibly-null objectId, or wrap arrays in an identified object. +1. Only 100 object fields total are allowed by OpenAI right now, so larger schemas will fail faster if we have a bunch of schema types generated for type-specific edits. +1. We don't support nested arrays yet. +1. Handle 429 rate limit error in streamFromLlm. +1. Top level arrays are not supported with current DSL. +1. Structured Output fails when multiple schema types have the same first field name (e.g. id: sf.identifier on multiple types). +1. Pass descriptions from schema metadata to the generated TS types that we put in the prompt. From f3bd57bb536a60cea4cf9e0f5e6716497d5bfc4c Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 29 Oct 2024 03:03:55 +0000 Subject: [PATCH 20/28] README update --- packages/framework/ai-collab/src/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/framework/ai-collab/src/README.md b/packages/framework/ai-collab/src/README.md index 34c870b538e1..c39df0f227ab 100644 --- a/packages/framework/ai-collab/src/README.md +++ b/packages/framework/ai-collab/src/README.md @@ -7,6 +7,7 @@ The ai-collab client library makes adding complex, human-like collaboration with ## Usage ### Your SharedTree types file +This file is where we define the types of our task management application's SharedTree data ```ts // --------- File name: "types.ts" --------- import { SchemaFactory } from "@fluidframework/tree"; @@ -124,7 +125,7 @@ const appState: PlannerAppState = initializeAppState({ const userAsk = "Update the task group description to be a about creating a new Todo list application. Create a set of tasks to accomplish this and assign them to the available engineers. Keep in mind the max capacity of each engineer as you assign tasks." // Collaborate with AI one function call. -const response = await aiCollab({ +const response = await aiCollab({ openAI: { client: new OpenAI({ apiKey: OPENAI_API_KEY, From b0e2984bd363baf1453a861deee60323f77bd528 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:13:46 +0000 Subject: [PATCH 21/28] Adds more jsdoc comments to internal explicit strategy functions and removes some commented out imports --- .../src/explicit-strategy/agentEditReducer.ts | 2 +- .../ai-collab/src/explicit-strategy/index.ts | 10 ++++++++++ .../ai-collab/src/explicit-strategy/jsonTypes.ts | 9 +++++---- .../src/explicit-strategy/promptGeneration.ts | 12 ++++++++---- .../explicit-strategy/agentEditingReducer.spec.ts | 3 --- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts index 46e79f6b799a..ecd8880a79ea 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditReducer.ts @@ -111,7 +111,7 @@ function contentWithIds(content: TreeNode, idGenerator: IdGenerator): TreeEditOb } /** - * TBD + * Manages applying the various types of {@link TreeEdit}'s to a a given {@link TreeNode}. */ export function applyAgentEdit( treeView: TreeView, diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index 17f8c5edda86..ceffc071ba3a 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -185,6 +185,13 @@ interface ReviewResult { goalAccomplished: "yes" | "no"; } +/** + * Generates a single {@link TreeEdit} from an LLM. + * The design of this async generator function is such that which each iteration of this functions values, + * an LLM will be prompted to generate the next value (a {@link TreeEdit}) based on the users ask. + * Once the LLM believes it has completed the user's ask, it will no longer return an edit and as a result + * this generator will no longer yield a next value. + */ async function* generateEdits( options: GenerateTreeEditsOptions, simpleSchema: SimpleTreeSchema, @@ -297,6 +304,9 @@ async function* generateEdits( } } +/** + * Calls an LLM to generate a response based on the provided prompt. + */ async function getFromLlm( prompt: string, openAi: OpenAiClientOptions, diff --git a/packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts b/packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts index da8cbf14dfee..34551c4fb560 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/jsonTypes.ts @@ -4,23 +4,24 @@ */ /** - * TBD + * Primitive JSON Types */ // eslint-disable-next-line @rushstack/no-new-null export type JsonPrimitive = string | number | boolean | null; /** - * TBD + * A JSON Object, a collection of key to {@link JsonValue} pairs */ // eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style export interface JsonObject { [key: string]: JsonValue; } /** - * TBD + * An Array of {@link JsonValue} */ export type JsonArray = JsonValue[]; + /** - * TBD + * A union type of all possible JSON values, including primitives, objects, and arrays */ export type JsonValue = JsonPrimitive | JsonObject | JsonArray; diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index 901fc6f142e9..4168f1d523fa 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -34,7 +34,8 @@ import { generateGenericEditTypes } from "./typeGeneration.js"; import { fail } from "./utils.js"; /** - * + * A log of edits that have been made to a tree. + * @remarks This is primarily used to help an LLM keep track of the active changes it has made. */ export type EditLog = { edit: TreeEdit; @@ -72,7 +73,7 @@ export function toDecoratedJson( } /** - * TBD + * Generates a prompt designed to make an LLM produce a plan to edit the SharedTree to accomplish a user-specified goal. */ export function getPlanningSystemPrompt( view: TreeView, @@ -107,7 +108,9 @@ export function getPlanningSystemPrompt( } /** - * TBD + * Generates the main prompt of this explicit strategy. + * This prompt is designed to give an LLM instructions on how it can modify a SharedTree using specific types of {@link TreeEdit}'s + * and provides with both a serialized version of the current state of the provided tree node as well as the interfaces that compromise said tree nodes data. */ export function getEditingSystemPrompt( userPrompt: string, @@ -172,7 +175,8 @@ export function getEditingSystemPrompt( } /** - * TBD + * Generates a prompt designed to make an LLM review the edits it created and applied to a SharedTree based + * on a user-specified goal. This prompt is designed to give the LLM's ability to correct for mistakes and improve the accuracy/fidelity of its final set of tree edits */ export function getReviewSystemPrompt( userPrompt: string, diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts index 0e80a8c86f32..e6e7573dcdda 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditingReducer.spec.ts @@ -14,8 +14,6 @@ import { normalizeFieldSchema, SchemaFactory, TreeViewConfiguration, - // type TreeNode, - // jsonableTreeFromForest, SharedTree, type TreeNode, // eslint-disable-next-line import/no-internal-modules @@ -37,7 +35,6 @@ import type { import { IdGenerator } from "../../explicit-strategy/idGenerator.js"; import { validateUsageError } from "./utils.js"; -// import { validateUsageError } from "./utils.js"; const sf = new SchemaFactory("agentSchema"); From c9d79b5101326f262142dfc8be4cff8b2d166140 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:32:35 +0000 Subject: [PATCH 22/28] Adds more JSDOC comments and renames ambigious function --- .../ai-collab/src/explicit-strategy/index.ts | 11 +++++--- .../src/explicit-strategy/typeGeneration.ts | 26 ++++++++++++++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index ceffc071ba3a..c24de4f3ee27 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -234,7 +234,7 @@ async function* generateEdits( DEBUG_LOG?.push(systemPrompt); const schema = types[rootTypeName] ?? fail("Root type not found."); - const wrapper = await getFromLlm( + const wrapper = await getStructuredOutputFromLlm( systemPrompt, options.openAI, schema, @@ -288,7 +288,7 @@ async function* generateEdits( .enum(["yes", "no"]) .describe('Whether the user\'s goal was met in the "after" tree.'), }); - return getFromLlm(systemPrompt, options.openAI, schema); + return getStructuredOutputFromLlm(systemPrompt, options.openAI, schema); } let edit = await getNextEdit(); @@ -305,9 +305,9 @@ async function* generateEdits( } /** - * Calls an LLM to generate a response based on the provided prompt. + * Calls an LLM to generate a structured output response based on the provided prompt. */ -async function getFromLlm( +async function getStructuredOutputFromLlm( prompt: string, openAi: OpenAiClientOptions, structuredOutputSchema: Zod.ZodTypeAny, @@ -336,6 +336,9 @@ async function getFromLlm( return result.choices[0]?.message.parsed as T | undefined; } +/** + * Calls an LLM to generate a response based on the provided prompt. + */ async function getStringFromLlm( prompt: string, openAi: OpenAiClientOptions, diff --git a/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts index be19034edf00..baa6c1a300dd 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts @@ -15,6 +15,10 @@ import { z } from "zod"; import { objectIdKey, typeField } from "./agentEditTypes.js"; import { fail, getOrCreate, mapIterable } from "./utils.js"; +/** + * Zod Object type used to represent & validate the ObjectTarget type within a {@link TreeEdit}. + * @remarks this is used as a component with {@link generateGenericEditTypes} to produce the final zod validation objects. + */ const objectTarget = z.object({ target: z .string() @@ -22,6 +26,10 @@ const objectTarget = z.object({ `The id of the object (as specified by the object's ${objectIdKey} property) that is being referenced`, ), }); +/** + * Zod Object type used to represent & validate the ObjectPlace type within a {@link TreeEdit}. + * @remarks this is used as a component with {@link generateGenericEditTypes} to produce the final zod validation objects. + */ const objectPlace = z .object({ type: z.enum(["objectPlace"]), @@ -39,6 +47,10 @@ const objectPlace = z .describe( "A pointer to a location either just before or just after an object that is in an array", ); +/** + * Zod Object type used to represent & validate the ArrayPlace type within a {@link TreeEdit}. + * @remarks this is used as a component with {@link generateGenericEditTypes} to produce the final zod validation objects. + */ const arrayPlace = z .object({ type: z.enum(["arrayPlace"]), @@ -55,6 +67,10 @@ const arrayPlace = z .describe( `either the "start" or "end" of an array, as specified by a "parent" ObjectTarget and a "field" name under which the array is stored (useful for prepending or appending)`, ); +/** + * Zod Object type used to represent & validate the Range type within a {@link TreeEdit}. + * @remarks this is used as a component with {@link generateGenericEditTypes} to produce the final zod validation objects. + */ const range = z .object({ from: objectPlace, @@ -63,10 +79,18 @@ const range = z .describe( 'A range of objects in the same array specified by a "from" and "to" Place. The "to" and "from" objects MUST be in the same array.', ); +/** + * Cache used to prevent repeatedly generating the same Zod validation objects for the same {@link SimpleTreeSchema} as generate propts for repeated calls to an LLM + */ const cache = new WeakMap>(); /** - * TBD + * Generates a set of ZOD validation objects for the various types of data that can be put into the provided {@link SimpleTreeSchema} + * and then uses those sets to generate an all-encompassing ZOD object for each type of {@link TreeEdit} that can validate any of the types of data that can be put into the tree. + * + * @returns a Record of schema names to Zod validation objects, and the name of the root schema used to encompass all of the other schemas. + * + * @remarks - The return type of this function is designed to work with Typechat's createZodJsonValidator as well as be used as the JSON schema for OpenAi's structured output response format. */ export function generateGenericEditTypes( schema: SimpleTreeSchema, From 8f1f36f084fb72663cd61f09a3fa6b886e9f4712 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:12:38 +0000 Subject: [PATCH 23/28] Removes old unused functions from promptGeneration, removes exports from integ test, adds baseline comments to agentEditTypes.ts --- .../src/explicit-strategy/agentEditTypes.ts | 49 ++++++---- .../ai-collab/src/explicit-strategy/index.ts | 1 + .../src/explicit-strategy/promptGeneration.ts | 91 +------------------ .../integration/planner.spec.ts | 18 ++-- 4 files changed, 41 insertions(+), 118 deletions(-) diff --git a/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts index 1ebed48a9818..26115a2d91ca 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/agentEditTypes.ts @@ -54,58 +54,66 @@ export const typeField = "__fluid_type"; export const objectIdKey = "__fluid_objectId"; /** - * TBD + * Describes an edit to a field within a node. + * @remarks what is the [key: string] for? */ export interface TreeEditObject { [key: string]: TreeEditValue; [typeField]: string; } /** - * TBD + * An array of {@link TreeEditValue}'s, allowing a single TreeEdit to contain edits to multiple fields. */ export type TreeEditArray = TreeEditValue[]; + /** - * TBD + * The potential values for a given TreeEdit. These values are typically a field within a node or an entire node, + * represented in JSON because they are expected to be returned by an LLM. */ export type TreeEditValue = JsonPrimitive | TreeEditObject | TreeEditArray; /** - * TBD + * This is the the final object we expected from an LLM response. + * @remarks Because TreeEdit can be multiple different types (polymorphic), + * we need to wrap to avoid anyOf at the root level when generating the necessary JSON Schema. */ -// For polymorphic edits, we need to wrap the edit in an object to avoid anyOf at the root level. export interface EditWrapper { // eslint-disable-next-line @rushstack/no-new-null edit: TreeEdit | null; } /** - * TBD + * Union type representing all possible types of edits that can be made to a tree. */ export type TreeEdit = SetRoot | Insert | Modify | Remove | Move; /** - * TBD + * The base interface for all types of {@link TreeEdit}. */ export interface Edit { explanation: string; type: "setRoot" | "insert" | "modify" | "remove" | "move"; } + /** - * TBD + * This object provides a way to 'select' either a given node or a range of nodes in an array. */ export type Selection = ObjectTarget | Range; /** - * TBD + * A Target object for an {@link TreeEdit}, identified by the target object's Id */ export interface ObjectTarget { target: string; } /** - * TBD + * Desribes where an object can be inserted into an array. + * For example, if an you have an array with 5 objects, and you insert an object at index 3, this differentiates whether you want + * the existing item at index 3 should be shifted forward (if the 'place' is 'before') or shifted backwards (if the 'place' is 'after') + * + * @remarks TODO: Allow support for nested arrays */ -// TODO: Allow support for nested arrays export interface ArrayPlace { type: "arrayPlace"; parentId: string; @@ -114,7 +122,11 @@ export interface ArrayPlace { } /** - * TBD + * Desribes where an object can be inserted into an array. + * For example, if an you have an array with 5 objects, and you insert an object at index 3, this differentiates whether you want + * the existing item at index 3 should be shifted forward (if the 'place' is 'before') or shifted backwards (if the 'place' is 'after') + * + * @remarks Why does this and {@link ArrayPlace} exist together? */ export interface ObjectPlace extends ObjectTarget { type: "objectPlace"; @@ -123,7 +135,8 @@ export interface ObjectPlace extends ObjectTarget { } /** - * TBD + * A range of objects within an array. This allows the LLM to select multiple nodes at once, + * for example during an {@link Remove} operation to remove a range of nodes. */ export interface Range { from: ObjectPlace; @@ -131,7 +144,7 @@ export interface Range { } /** - * TBD + * Describes an operation to set the root of the tree. This is the only edit that can change the root object. */ export interface SetRoot extends Edit { type: "setRoot"; @@ -139,7 +152,7 @@ export interface SetRoot extends Edit { } /** - * TBD + * Describes an operation to insert a new node into the tree. */ export interface Insert extends Edit { type: "insert"; @@ -148,7 +161,7 @@ export interface Insert extends Edit { } /** - * TBD + * Describes an operation to modify an existing node in the tree. */ export interface Modify extends Edit { type: "modify"; @@ -158,7 +171,7 @@ export interface Modify extends Edit { } /** - * TBD + * Describes an operation to remove either a specific node or a range of nodes in an array. */ export interface Remove extends Edit { type: "remove"; @@ -166,7 +179,7 @@ export interface Remove extends Edit { } /** - * TBD + * Describes an operation to move a node within an array */ export interface Move extends Edit { type: "move"; diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index c24de4f3ee27..e5e27f777411 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -212,6 +212,7 @@ async function* generateEdits( ); DEBUG_LOG?.push(planningPromt); plan = await getStringFromLlm(planningPromt, options.openAI); + DEBUG_LOG?.push(`AI Generated the following plan: ${planningPromt}`); } const originalDecoratedJson = diff --git a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts index 4168f1d523fa..b6fa5fa5873a 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/promptGeneration.ts @@ -22,13 +22,7 @@ import { // eslint-disable-next-line import/no-internal-modules import { createZodJsonValidator } from "typechat/zod"; -import { - objectIdKey, - type ObjectTarget, - type TreeEdit, - type TreeEditValue, - type Range, -} from "./agentEditTypes.js"; +import { objectIdKey, type TreeEdit } from "./agentEditTypes.js"; import type { IdGenerator } from "./idGenerator.js"; import { generateGenericEditTypes } from "./typeGeneration.js"; import { fail } from "./utils.js"; @@ -248,81 +242,6 @@ export function getPromptFriendlyTreeSchema(jsonSchema: JsonTreeSchema): string return stringifiedSchema; } -function printContent(content: TreeEditValue, idGenerator: IdGenerator): string { - switch (typeof content) { - case "boolean": - return content ? "true" : "false"; - case "number": - return content.toString(); - case "string": - return `"${truncateString(content, 32)}"`; - case "object": { - if (Array.isArray(content)) { - // TODO: Describe the types of the array contents - return "a new array"; - } - if (content === null) { - return "null"; - } - const id = content[objectIdKey]; - assert(typeof id === "string", "Object content has no id."); - const node = idGenerator.getNode(id) ?? fail("Node not found."); - const schema = Tree.schema(node); - return `a new ${getFriendlySchemaName(schema.identifier)}`; - } - default: - fail("Unexpected content type."); - } -} - -/** - * TBD - */ -export function describeEdit(edit: TreeEdit, idGenerator: IdGenerator): string { - switch (edit.type) { - case "setRoot": - return `Set the root of the tree to ${printContent(edit.content, idGenerator)}.`; - case "insert": { - if (edit.destination.type === "arrayPlace") { - return `Insert ${printContent(edit.content, idGenerator)} at the ${edit.destination.location} of the array that is under the "${edit.destination.field}" property of ${edit.destination.parentId}.`; - } else { - const target = - idGenerator.getNode(edit.destination.target) ?? fail("Target node not found."); - const array = Tree.parent(target) ?? fail("Target node has no parent."); - const container = Tree.parent(array); - if (container === undefined) { - return `Insert ${printContent(edit.content, idGenerator)} into the array at the root of the tree. Insert it ${edit.destination.place} ${edit.destination.target}.`; - } - return `Insert ${printContent(edit.content, idGenerator)} into the array that is under the "${Tree.key(array)}" property of ${idGenerator.getId(container)}. Insert it ${edit.destination.place} ${edit.destination.target}.`; - } - } - case "modify": - return `Set the "${edit.field}" field of ${edit.target.target} to ${printContent(edit.modification, idGenerator)}.`; - case "remove": - return isObjectTarget(edit.source) - ? `Remove "${edit.source.target}" from the containing array.` - : `Remove all elements from ${edit.source.from.place} ${edit.source.from.target} to ${edit.source.to.place} ${edit.source.to.target} in their containing array.`; - case "move": - if (edit.destination.type === "arrayPlace") { - const suffix = `to the ${edit.destination.location} of the array that is under the "${edit.destination.field}" property of ${edit.destination.parentId}`; - return isObjectTarget(edit.source) - ? `Move ${edit.source.target} ${suffix}.` - : `Move all elements from ${edit.source.from.place} ${edit.source.from.target} to ${edit.source.to.place} ${edit.source.to.target} ${suffix}.`; - } else { - const suffix = `to ${edit.destination.place} ${edit.destination.target}`; - return isObjectTarget(edit.source) - ? `Move ${edit.source.target} ${suffix}.` - : `Move all elements from ${edit.source.from.place} ${edit.source.from.target} to ${edit.source.to.place} ${edit.source.to.target} ${suffix}.`; - } - default: - return "Unknown edit type."; - } -} - -function isObjectTarget(value: ObjectTarget | Range): value is ObjectTarget { - return (value as Partial).target !== undefined; -} - function getTypeString( defs: Record, [name, currentDef]: [string, JsonNodeSchema], @@ -380,11 +299,3 @@ export function getFriendlySchemaName(schemaName: string): string { } return matches[0]; } - -function truncateString(str: string, maxLength: number): string { - if (str.length > maxLength) { - // eslint-disable-next-line unicorn/prefer-string-slice - return `${str.substring(0, maxLength - 3)}...`; - } - return str; -} diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts index 05ef72f6eb70..981bb18b9a3f 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/integration/planner.spec.ts @@ -3,8 +3,6 @@ * Licensed under the MIT License. */ -/* eslint-disable jsdoc/require-jsdoc */ - import { strict as assert } from "node:assert"; // eslint-disable-next-line import/no-internal-modules @@ -23,7 +21,7 @@ import { aiCollab } from "../../../index.js"; const sf = new SchemaFactory("ai-collab-sample-application"); -export class SharedTreeTask extends sf.object("Task", { +class SharedTreeTask extends sf.object("Task", { title: sf.string, id: sf.identifier, description: sf.string, @@ -33,18 +31,18 @@ export class SharedTreeTask extends sf.object("Task", { assignee: sf.string, }) {} -export class SharedTreeTaskList extends sf.array("TaskList", SharedTreeTask) {} +class SharedTreeTaskList extends sf.array("TaskList", SharedTreeTask) {} -export class SharedTreeEngineer extends sf.object("Engineer", { +class SharedTreeEngineer extends sf.object("Engineer", { name: sf.string, id: sf.identifier, skills: sf.string, maxCapacity: sf.number, }) {} -export class SharedTreeEngineerList extends sf.array("EngineerList", SharedTreeEngineer) {} +class SharedTreeEngineerList extends sf.array("EngineerList", SharedTreeEngineer) {} -export class SharedTreeTaskGroup extends sf.object("TaskGroup", { +class SharedTreeTaskGroup extends sf.object("TaskGroup", { description: sf.string, id: sf.identifier, title: sf.string, @@ -52,13 +50,13 @@ export class SharedTreeTaskGroup extends sf.object("TaskGroup", { engineers: SharedTreeEngineerList, }) {} -export class SharedTreeTaskGroupList extends sf.array("TaskGroupList", SharedTreeTaskGroup) {} +class SharedTreeTaskGroupList extends sf.array("TaskGroupList", SharedTreeTaskGroup) {} -export class SharedTreeAppState extends sf.object("AppState", { +class SharedTreeAppState extends sf.object("AppState", { taskGroups: SharedTreeTaskGroupList, }) {} -export const INITIAL_APP_STATE = { +const INITIAL_APP_STATE = { taskGroups: [ { title: "My First Task Group", From bac309434408ff1fd2adfbc0dcacf19c89f40ee8 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:54:26 +0000 Subject: [PATCH 24/28] Updates some JSOC, removes commented imports from tests, exports and unused tests --- .../tree/src/simple-tree/api/simpleSchema.ts | 1 + packages/framework/ai-collab/eslintrc.cjs | 3 -- packages/framework/ai-collab/package.json | 2 +- .../ai-collab/src/explicit-strategy/utils.ts | 6 +-- .../explicit-strategy/agentEditing.spec.ts | 52 ------------------- 5 files changed, 5 insertions(+), 59 deletions(-) diff --git a/packages/dds/tree/src/simple-tree/api/simpleSchema.ts b/packages/dds/tree/src/simple-tree/api/simpleSchema.ts index d4b8a51a0dbb..e13e1e3c372f 100644 --- a/packages/dds/tree/src/simple-tree/api/simpleSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/simpleSchema.ts @@ -132,6 +132,7 @@ export interface SimpleFieldSchema { * which are represented inline with identifiers. * * @internal + * @sealed */ export interface SimpleTreeSchema extends SimpleFieldSchema { /** diff --git a/packages/framework/ai-collab/eslintrc.cjs b/packages/framework/ai-collab/eslintrc.cjs index 021c6bc37cb0..f1aaefc67a58 100644 --- a/packages/framework/ai-collab/eslintrc.cjs +++ b/packages/framework/ai-collab/eslintrc.cjs @@ -8,7 +8,4 @@ module.exports = { parserOptions: { project: ["./tsconfig.json", "./src/test/tsconfig.json"], }, - rules: { - "unicorn/no-null": "off", - }, }; diff --git a/packages/framework/ai-collab/package.json b/packages/framework/ai-collab/package.json index b588455b2d23..f186f960958a 100644 --- a/packages/framework/ai-collab/package.json +++ b/packages/framework/ai-collab/package.json @@ -62,7 +62,7 @@ "test:coverage": "c8 npm test", "test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs", "test:mocha:cjs": "mocha --recursive \"dist/test/**/*.spec.js\"", - "test:mocha:esm": "mocha --recursive \"lib/test/**/*.spec.js\" --timeout 60000", + "test:mocha:esm": "mocha --recursive \"lib/test/**/*.spec.js\"", "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha", "tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && copyfiles -f ../../../common/build/build-common/src/cjs/package.json ./dist" }, diff --git a/packages/framework/ai-collab/src/explicit-strategy/utils.ts b/packages/framework/ai-collab/src/explicit-strategy/utils.ts index cbd716f9d630..49a230f8a9d0 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/utils.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/utils.ts @@ -6,7 +6,7 @@ /** * Subset of Map interface. * - * @remarks - originally from tree/src/utils.ts + * @remarks originally from tree/src/util/utils.ts */ export interface MapGetSet { get(key: K): V | undefined; @@ -26,7 +26,7 @@ export function fail(message: string): never { * @param map - the transformation function to run on each element of the iterable * @returns a new iterable of elements which have been transformed by the `map` function * - * @remarks - originally from tree/src/utils.ts + * @remarks originally from tree/src/util/utils.ts */ export function* mapIterable( iterable: Iterable, @@ -44,7 +44,7 @@ export function* mapIterable( * @param defaultValue - a function which returns a default value. This is called and used to set an initial value for the given key in the map if none exists * @returns either the existing value for the given key, or the newly-created value (the result of `defaultValue`) * - * @remarks - originally from tree/src/utils.ts + * @remarks originally from tree/src/util/utils.ts */ export function getOrCreate( map: MapGetSet, diff --git a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts index 6ab77ff6cf73..d9805ddb5cf4 100644 --- a/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts +++ b/packages/framework/ai-collab/src/test/explicit-strategy/agentEditing.spec.ts @@ -16,14 +16,11 @@ import { TreeViewConfiguration, // eslint-disable-next-line import/no-internal-modules } from "@fluidframework/tree/internal"; -// eslint-disable-next-line import/no-internal-modules -import type { ResponseFormatJSONSchema } from "openai/resources/shared.mjs"; // eslint-disable-next-line import/no-internal-modules import { objectIdKey } from "../../explicit-strategy/agentEditTypes.js"; // eslint-disable-next-line import/no-internal-modules import { IdGenerator } from "../../explicit-strategy/idGenerator.js"; -// import { getResponse } from "../../explicit-strategy/llmClient.js"; import { getPromptFriendlyTreeSchema, toDecoratedJson, @@ -193,7 +190,6 @@ describe("Makes TS type strings from schema", () => { class Foo extends testSf.object("Foo", { y: demoSf.array(demoSf.number), }) {} - const stringified = JSON.stringify(getJsonSchema(Foo)); assert.equal( getPromptFriendlyTreeSchema(getJsonSchema(Foo)), "interface Foo { y: number[]; }", @@ -221,51 +217,3 @@ describe("Makes TS type strings from schema", () => { ); }); }); - -describe.skip("llmClient", () => { - it("can accept a structured schema prompt", async () => { - const userPrompt = - "I need a catalog listing for a product. Please extract this info into the required schema. The product is a Red Ryder bicycle, which is a particularly fast bicycle, and which should be listed for one hundred dollars."; - - // const testSf = new SchemaFactory("test"); - // class CatalogEntry extends testSf.object("CatalogEntry", { - // itemTitle: testSf.string, - // itemDescription: testSf.string, - // itemPrice: testSf.number, - // }) {} - - // const jsonSchema = getJsonSchema(CatalogEntry); - - const responseSchema: ResponseFormatJSONSchema = { - type: "json_schema", - json_schema: { - name: "Catalog_Entry", - description: "An entry for an item in a product catalog", - strict: true, - schema: { - "type": "object", - "properties": { - "title": { - "type": "string", - "description": "a title which must be in all caps", - }, - "description": { - "type": "string", - "description": "the description of the item, which must be in CaMeLcAsE", - }, - "price": { - "type": "number", - "description": "the price, which must be expressed with one decimal place.", - }, - }, - "required": ["title", "description", "price"], - "additionalProperties": false, - }, - }, - }; - - // const response = await getResponse(userPrompt, responseSchema); - - // console.log(response); - }); -}); From 6eb662389384cc2360c519307d3b33d3a7a02309 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:59:50 +0000 Subject: [PATCH 25/28] Another round of JSDoc updates --- .../ai-collab/src/explicit-strategy/idGenerator.ts | 4 ++-- .../framework/ai-collab/src/explicit-strategy/index.ts | 8 +++++--- .../ai-collab/src/explicit-strategy/typeGeneration.ts | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts index e983e9105c56..122ecab2c7c9 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/idGenerator.ts @@ -13,8 +13,8 @@ import type { } from "@fluidframework/tree/internal"; /** - * Given a tree node, generates a set of LLM friendly, unique ids for each node in a given Shared Tree. - * @remarks - simple id's are important for the LLM and this library to create and distinguish between different types certain TreeEdits + * Given a tree node, generates a set of LLM -friendly, unique IDs for each node in a given Shared Tree. + * @remarks simple id's are important for the LLM and this library to create and distinguish between different types certain TreeEdits */ export class IdGenerator { private readonly idCountMap = new Map(); diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index e5e27f777411..f2adbae87151 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -116,7 +116,7 @@ export async function generateTreeEdits( simpleSchema.definitions, options.validator, ); - const explanation = result.explanation; // TODO: describeEdit(result, idGenerator); + const explanation = result.explanation; editLog.push({ edit: { ...result, explanation } }); sequentialErrorCount = 0; } catch (error: unknown) { @@ -187,6 +187,8 @@ interface ReviewResult { /** * Generates a single {@link TreeEdit} from an LLM. + * + * @remarks * The design of this async generator function is such that which each iteration of this functions values, * an LLM will be prompted to generate the next value (a {@link TreeEdit}) based on the users ask. * Once the LLM believes it has completed the user's ask, it will no longer return an edit and as a result @@ -306,7 +308,7 @@ async function* generateEdits( } /** - * Calls an LLM to generate a structured output response based on the provided prompt. + * Calls the LLM to generate a structured output response based on the provided prompt. */ async function getStructuredOutputFromLlm( prompt: string, @@ -338,7 +340,7 @@ async function getStructuredOutputFromLlm( } /** - * Calls an LLM to generate a response based on the provided prompt. + * Calls the LLM to generate a response based on the provided prompt. */ async function getStringFromLlm( prompt: string, diff --git a/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts index baa6c1a300dd..621fa4e37978 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/typeGeneration.ts @@ -90,7 +90,7 @@ const cache = new WeakMap Date: Thu, 31 Oct 2024 16:06:50 +0000 Subject: [PATCH 26/28] Removes SharedTree simple schema omitFromJson capability from FieldSchemaMetadata --- packages/dds/tree/api-report/tree.alpha.api.md | 1 - packages/dds/tree/api-report/tree.beta.api.md | 1 - packages/dds/tree/api-report/tree.legacy.alpha.api.md | 1 - packages/dds/tree/api-report/tree.legacy.public.api.md | 1 - packages/dds/tree/api-report/tree.public.api.md | 1 - .../tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts | 4 ---- packages/dds/tree/src/simple-tree/schemaTypes.ts | 6 ------ .../test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts | 2 -- .../fluid-framework/api-report/fluid-framework.alpha.api.md | 1 - .../fluid-framework/api-report/fluid-framework.beta.api.md | 1 - .../api-report/fluid-framework.legacy.alpha.api.md | 1 - .../api-report/fluid-framework.legacy.public.api.md | 1 - .../api-report/fluid-framework.public.api.md | 1 - 13 files changed, 22 deletions(-) diff --git a/packages/dds/tree/api-report/tree.alpha.api.md b/packages/dds/tree/api-report/tree.alpha.api.md index 6ad6475422a5..0c6dda982aa4 100644 --- a/packages/dds/tree/api-report/tree.alpha.api.md +++ b/packages/dds/tree/api-report/tree.alpha.api.md @@ -104,7 +104,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.beta.api.md b/packages/dds/tree/api-report/tree.beta.api.md index d5e8b49b0da1..c8cf8b575972 100644 --- a/packages/dds/tree/api-report/tree.beta.api.md +++ b/packages/dds/tree/api-report/tree.beta.api.md @@ -72,7 +72,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.legacy.alpha.api.md b/packages/dds/tree/api-report/tree.legacy.alpha.api.md index 85906d539927..1e72cf551819 100644 --- a/packages/dds/tree/api-report/tree.legacy.alpha.api.md +++ b/packages/dds/tree/api-report/tree.legacy.alpha.api.md @@ -72,7 +72,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.legacy.public.api.md b/packages/dds/tree/api-report/tree.legacy.public.api.md index 48d41030f3de..8e2510fd001d 100644 --- a/packages/dds/tree/api-report/tree.legacy.public.api.md +++ b/packages/dds/tree/api-report/tree.legacy.public.api.md @@ -72,7 +72,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/api-report/tree.public.api.md b/packages/dds/tree/api-report/tree.public.api.md index 48d41030f3de..8e2510fd001d 100644 --- a/packages/dds/tree/api-report/tree.public.api.md +++ b/packages/dds/tree/api-report/tree.public.api.md @@ -72,7 +72,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts b/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts index 6c585d74c1fa..0e81be8260d1 100644 --- a/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts +++ b/packages/dds/tree/src/simple-tree/api/simpleSchemaToJsonSchema.ts @@ -137,10 +137,6 @@ function convertObjectNodeSchema(schema: SimpleObjectNodeSchema): JsonObjectNode const properties: Record = {}; const required: string[] = []; for (const [key, value] of Object.entries(schema.fields)) { - if (value.metadata?.omitFromJson === true) { - // Don't emit JSON Schema for fields which specify they should be excluded. - continue; - } const allowedTypes: JsonSchemaRef[] = []; for (const allowedType of value.allowedTypes) { allowedTypes.push(createSchemaRef(allowedType)); diff --git a/packages/dds/tree/src/simple-tree/schemaTypes.ts b/packages/dds/tree/src/simple-tree/schemaTypes.ts index 0486536bcd43..b49a313ee94f 100644 --- a/packages/dds/tree/src/simple-tree/schemaTypes.ts +++ b/packages/dds/tree/src/simple-tree/schemaTypes.ts @@ -231,12 +231,6 @@ export interface FieldSchemaMetadata { * used as the `description` field. */ readonly description?: string | undefined; - - /** - * Whether or not to include the field in JSON output generated by `getJsonSchema` (experimental). - * @defaultValue `false` - */ - readonly omitFromJson?: boolean | undefined; } /** diff --git a/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts b/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts index c53eb3389453..5ff5a37a997a 100644 --- a/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts +++ b/packages/dds/tree/src/test/simple-tree/api/simpleSchemaToJsonSchema.spec.ts @@ -184,8 +184,6 @@ describe("simpleSchemaToJsonSchema", () => { allowedTypes: new Set(["test.string"]), metadata: { description: "Unique identifier for the test object.", - // IDs should be generated by the system. Hide from the JSON Schema. - omitFromJson: true, }, }, }, diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md index 33d58913d657..9093aa5c59c3 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.alpha.api.md @@ -148,7 +148,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md index f84788c16bf4..faf63f1d1f98 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.beta.api.md @@ -113,7 +113,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md index a476c9fce12a..8e17609b6597 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.alpha.api.md @@ -116,7 +116,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md index f68fc3a141c4..cfc94e326b90 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.legacy.public.api.md @@ -113,7 +113,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public diff --git a/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md b/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md index d7bfda5963d1..b39b99f18221 100644 --- a/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md +++ b/packages/framework/fluid-framework/api-report/fluid-framework.public.api.md @@ -113,7 +113,6 @@ export class FieldSchema { readonly custom?: TCustomMetadata; readonly description?: string | undefined; - readonly omitFromJson?: boolean | undefined; } // @public From a79e33c651f4084ad3e7f7c04689ef61d95f4cd7 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 31 Oct 2024 16:36:21 +0000 Subject: [PATCH 27/28] adds comments for undocumented aiCollabApi members and adds partial-failure error message as intented to generateTreeEdits --- packages/framework/ai-collab/README.md | 216 +++++++++++++++--- .../api-report/ai-collab.alpha.api.md | 3 - .../framework/ai-collab/src/aiCollabApi.ts | 15 ++ .../ai-collab/src/explicit-strategy/index.ts | 12 +- 4 files changed, 211 insertions(+), 35 deletions(-) diff --git a/packages/framework/ai-collab/README.md b/packages/framework/ai-collab/README.md index 05e553b58b53..06fed0a6de8d 100644 --- a/packages/framework/ai-collab/README.md +++ b/packages/framework/ai-collab/README.md @@ -1,34 +1,194 @@ # @fluid-experimental/ai-collab -Utilities for using SharedTree with LLMs. -This package aims to assist SharedTree-based apps that want to leverage LLMs. +## Description +The ai-collab client library makes adding complex, human-like collaboration with LLM's built directly in your application as simple as one function call. Simply pass your SharedTree and ask AI to collaborate. For example, +- Task Management App: "Reorder this list of tasks in order from least to highest complexity." +- Job Board App: "Create a new job listing and add it to this job board" +- Calender App: "Manage my calender to slot in a new 2:30 appointment" + +## Usage + +### Your SharedTree types file +This file is where we define the types of our task management application's SharedTree data +```ts +// --------- File name: "types.ts" --------- +import { SchemaFactory } from "@fluidframework/tree"; + +const sf = new SchemaFactory("ai-collab-sample-application"); + +export class Task extends sf.object("Task", { + title: sf.required(sf.string, { + metadata: { description: `The title of the task` }, + }), + id: sf.identifier, + description: sf.required(sf.string, { + metadata: { description: `The description of the task` }, + }), + priority: sf.required(sf.string, { + metadata: { description: `The priority of the task in three levels, "low", "medium", "high"` }, + }), + complexity: sf.required(sf.number, { + metadata: { description: `The complexity of the task as a fibonacci number` }, + }), + status: sf.required(sf.string, { + metadata: { description: `The status of the task as either "todo", "in-progress", or "done"` }, + }), + assignee: sf.required(sf.string, { + metadata: { description: `The name of the tasks assignee e.g. "Bob" or "Alice"` }, + }), +}) {} + +export class TaskList extends sf.array("TaskList", SharedTreeTask) {} + +export class Engineer extends sf.object("Engineer", { + name: sf.required(sf.string, { + metadata: { description: `The name of an engineer whom can be assigned to a task` }, + }), + id: sf.identifier, + skills: sf.required(sf.string, { + metadata: { description: `A description of the engineers skills which influence what types of tasks they should be assigned to.` }, + }), + maxCapacity: sf.required(sf.number, { + metadata: { description: `The maximum capacity of tasks this engineer can handle measured in in task complexity points.` }, + }), +}) {} + +export class EngineerList extends sf.array("EngineerList", SharedTreeEngineer) {} + +export class TaskGroup extends sf.object("TaskGroup", { + description: sf.required(sf.string, { + metadata: { description: `The description of the task group, which is a collection of tasks and engineers that can be assigned to said tasks.` }, + }), + id: sf.identifier, + title: sf.required(sf.string, { + metadata: { description: `The title of the task group.` }, + }), + tasks: sf.required(SharedTreeTaskList, { + metadata: { description: `The lists of tasks within this task group.` }, + }), + engineers: sf.required(SharedTreeEngineerList, { + metadata: { description: `The lists of engineers within this task group which can be assigned to tasks.` }, + }), +}) {} + +export class TaskGroupList extends sf.array("TaskGroupList", SharedTreeTaskGroup) {} + +export class PlannerAppState extends sf.object("PlannerAppState", { + taskGroups: sf.required(SharedTreeTaskGroupList, { + metadata: { description: `The list of task groups that are being managed by this task management application.` }, + }), +}) {} +``` + +### Example 1: Collaborate with AI +```ts +import { aiCollab } from "@fluid-experimental/ai-collab"; +import { PlannerAppState } from "./types.ts" +// This is not a real file, this is meant to represent how you initialize your app data. +import { initializeAppState } from "./yourAppInitializationFile.ts" + +// --------- File name: "app.ts" --------- + +// Initialize your app state somehow +const appState: PlannerAppState = initializeAppState({ + taskGroups: [ + { + title: "My First Task Group", + description: "Placeholder for first task group", + tasks: [ + { + assignee: "Alice", + title: "Task #1", + description: + "This is the first Sample task.", + priority: "low", + complexity: 1, + status: "todo", + }, + ], + engineers: [ + { + name: "Alice", + maxCapacity: 15, + skills: + "Senior engineer capable of handling complex tasks. Versed in most languages", + }, + { + name: "Charlie", + maxCapacity: 7, + skills: "Junior engineer capable of handling simple tasks. Versed in Node.JS", + }, + ], + }, + ], +}) + +// Typically, the user would input this through a UI form/input of some sort. +const userAsk = "Update the task group description to be a about creating a new Todo list application. Create a set of tasks to accomplish this and assign them to the available engineers. Keep in mind the max capacity of each engineer as you assign tasks." + +// Collaborate with AI one function call. +const response = await aiCollab({ + openAI: { + client: new OpenAI({ + apiKey: OPENAI_API_KEY, + }), + modelName: "gpt-4o", + }, + treeView: view, + treeNode: view.root.taskGroups[0], + prompt: { + systemRoleContext: + "You are a manager that is helping out with a project management tool. You have been asked to edit a group of tasks.", + userAsk: userAsk, + }, + planningStep: true, + finalReviewStep: true, + dumpDebugLog: true, + }); + +if (response.status === 'sucess') { + // Render the UI view of your task groups. + window.alert(`The AI has successfully completed your request.`); +} else { + window.alert(`Something went wrong! response status: ${response.status}, error message: ${response.errorMessage}`); +} + + +``` + +Once the `aiCollab` function call is initiated, an LLM will immediately begin attempting to make changes to your Shared Tree using the provided user prompt, the types of your SharedTree and the provided app guidance. The LLM produces multiple changes, in a loop asynchronously. Meaning, you will immediatley see changes if your UI's render loop is connected to your SharedTree App State. + +### Example 2: Collaborate with AI onto a branched state and let the user merge the review and merge the branch back manually +- **Coming Soon** + + +## Folder Structure + +- `/explicit-strategy`: The new explicit strategy, utilizing the prototype built during the fall FHL, with a few adjustments. + - `agentEditReducer`: This file houses the logic for taking in a `TreeEdit`, which the LLM produces, and applying said edit to the + - actual SharedTree. + - `agentEditTypes.ts`: The types of edits an LLM is prompted to produce in order to modify a SharedTree. + - `idGenerator.ts`: `A manager for producing and mapping simple id's in place of UUID hashes when generating prompts for an LLM + - `jsonTypes.ts`: utility JSON related types used in parsing LLM response and generating LLM prompts. + - `promptGeneration.ts`: Logic for producing the different types of prompts sent to an LLM in order to edit a SharedTree. + - `typeGeneration.ts`: Generates serialized(/able) representations of a SharedTree Schema which is used within prompts and the generated of the structured output JSON schema + - `utils.ts`: Utilities for interacting with a SharedTree +- `/implicit-strategy`: The original implicit strategy, currently not used under the exported aiCollab API surface. + +## Known Issues & limitations +1. Union types for a TreeNode are not present when generating App Schema. This will require extracting a field schema instead of TreeNodeSchema when passed a non root node. +1. The Editing System prompt & structured out schema currently provide array related edits even when there are no arrays. This forces you to have an array in your schema to produce a valid json schema +1. Optional roots are not allowed, This is because if you pass undefined as your treeNode to the API, we cannot disambiguate whether you passed the root or not. +1. Primitive root nodes are not allowed to be passed to the API. You must use an object or array as your root. +1. Optional nodes are not supported -- when we use optional nodes, the OpenAI API returns an error complaining that the structured output JSON schema is invalid. I have introduced a fix that should work upon manual validation of the json schema, but there looks to be an issue with their API. I have filed a ticket with OpenAI to address this +1. The current scheme does not allow manipulation of arrays of primitive values because you cannot refer to them. We could accomplish this via a path (probably JSON Pointer or JSONPath) from a possibly-null objectId, or wrap arrays in an identified object. +1. Only 100 object fields total are allowed by OpenAI right now, so larger schemas will fail faster if we have a bunch of schema types generated for type-specific edits. +1. We don't support nested arrays yet. +1. Handle 429 rate limit error in streamFromLlm. +1. Top level arrays are not supported with current DSL. +1. Structured Output fails when multiple schema types have the same first field name (e.g. id: sf.identifier on multiple types). +1. Pass descriptions from schema metadata to the generated TS types that we put in the prompt. -The next steps in LLM/AI-based collaborative experiences with applications involves allowing LLMs to propose updates -to application state directly. - -## The classic LLM developer experience & it's problems - -The classic LLM dev experience involves crafting a prompt for an an LLM with some information about the app, then -having the LLM response in a parseable format. - -From here the developer needs to: - -1. Translate & interpet the LLM response format so it can be applied to their application state -2. Deal with potentially invalid responses -3. Deal with merging LLM responses that use potentially stale state into their apps. - - This in particular comes into play with more dynamic application state, for example a list that users can - add and remove from. - You'll need to make sure the LLM isn't trying to delete something that doesn't exist or overwrite something that no - longer makes sense. -4. Try to preview LLM changes to the user before accepting them. - This requires maintaining a data structure before any LLM changes are applied, and another one where they are. - -### How this library fixes things - -Newer LLM developer tooling has solved issue #1 in a variety of ways, getting the LLM to respond with a format that you -can merge into your application and ensuring that the JSON response schema is valid. -However, problems 3-4 still exist and the current landscape requires bespoke, per-app solutions for dealing with this. -This library simplifies these issues. diff --git a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md index a9b9aa5b33f8..3f782c53a681 100644 --- a/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md +++ b/packages/framework/ai-collab/api-report/ai-collab.alpha.api.md @@ -9,11 +9,8 @@ export function aiCollab(options: AiCollabO // @alpha export interface AiCollabErrorResponse { - // (undocumented) errorMessage: "tokenLimitExceeded" | "tooManyErrors" | "tooManyModelCalls" | "aborted"; - // (undocumented) status: "failure" | "partial-failure"; - // (undocumented) tokenUsage: TokenUsage; } diff --git a/packages/framework/ai-collab/src/aiCollabApi.ts b/packages/framework/ai-collab/src/aiCollabApi.ts index e211a1b9c2fe..67805c5e45ce 100644 --- a/packages/framework/ai-collab/src/aiCollabApi.ts +++ b/packages/framework/ai-collab/src/aiCollabApi.ts @@ -100,8 +100,23 @@ export interface AiCollabSuccessResponse { * @alpha */ export interface AiCollabErrorResponse { + /** + * The status of the Ai Collaboration. + * - A 'partial-failure' status indicates that the AI collaboration was partially successful, but was aborted due to a limiter or other error + * - A "failure" status indicates that the AI collaboration was not successful at creating any changes. + */ status: "failure" | "partial-failure"; + /** + * The type of known error that occured + * - 'tokenLimitExceeded' indicates that the LLM exceeded the token limits set by the user + * - 'tooManyErrors' indicates that the LLM made too many errors in a row + * - 'tooManyModelCalls' indicates that the LLM made too many model calls + * - 'aborted' indicates that the AI collaboration was aborted by the user or a limiter + */ errorMessage: "tokenLimitExceeded" | "tooManyErrors" | "tooManyModelCalls" | "aborted"; + /** + * The total token usage by the LLM. + */ tokenUsage: TokenUsage; } diff --git a/packages/framework/ai-collab/src/explicit-strategy/index.ts b/packages/framework/ai-collab/src/explicit-strategy/index.ts index f2adbae87151..1d2b80ff1237 100644 --- a/packages/framework/ai-collab/src/explicit-strategy/index.ts +++ b/packages/framework/ai-collab/src/explicit-strategy/index.ts @@ -129,9 +129,12 @@ export async function generateTreeEdits( } } + const responseStatus = + editCount > 0 && sequentialErrorCount < editCount ? "partial-failure" : "failure"; + if (options.limiters?.abortController?.signal.aborted === true) { return { - status: "failure", + status: responseStatus, errorMessage: "aborted", tokenUsage, }; @@ -142,7 +145,7 @@ export async function generateTreeEdits( (options.limiters?.maxSequentialErrors ?? Number.POSITIVE_INFINITY) ) { return { - status: "failure", + status: responseStatus, errorMessage: "tooManyErrors", tokenUsage, }; @@ -150,7 +153,7 @@ export async function generateTreeEdits( if (++editCount >= (options.limiters?.maxModelCalls ?? Number.POSITIVE_INFINITY)) { return { - status: "failure", + status: responseStatus, errorMessage: "tooManyModelCalls", tokenUsage, }; @@ -162,7 +165,8 @@ export async function generateTreeEdits( } if (error instanceof TokenLimitExceededError) { return { - status: "failure", + status: + editCount > 0 && sequentialErrorCount < editCount ? "partial-failure" : "failure", errorMessage: "tokenLimitExceeded", tokenUsage, }; From b6755728ce844e69b98b9dd01ebb510111b187d0 Mon Sep 17 00:00:00 2001 From: seanimam <105244057+seanimam@users.noreply.github.com> Date: Thu, 31 Oct 2024 16:39:14 +0000 Subject: [PATCH 28/28] Adds jsdoc for remaining aiCollab API members --- packages/framework/ai-collab/src/aiCollabApi.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/framework/ai-collab/src/aiCollabApi.ts b/packages/framework/ai-collab/src/aiCollabApi.ts index 67805c5e45ce..86975ee8e996 100644 --- a/packages/framework/ai-collab/src/aiCollabApi.ts +++ b/packages/framework/ai-collab/src/aiCollabApi.ts @@ -13,7 +13,13 @@ import type OpenAI from "openai"; * @alpha */ export interface OpenAiClientOptions { + /** + * The {@link OpenAI} client to use for the AI collaboration. + */ client: OpenAI; + /** + * The name of the target OpenAI model to use for the AI collaboration. + */ modelName?: string; } @@ -90,7 +96,14 @@ export interface AiCollabOptions { * @alpha */ export interface AiCollabSuccessResponse { + /** + * The status of the Ai Collaboration. + * A 'success' status indicates that the AI collaboration was successful at creating changes. + */ status: "success"; + /** + * {@inheritDoc TokenUsage} + */ tokenUsage: TokenUsage; } @@ -115,7 +128,7 @@ export interface AiCollabErrorResponse { */ errorMessage: "tokenLimitExceeded" | "tooManyErrors" | "tooManyModelCalls" | "aborted"; /** - * The total token usage by the LLM. + * {@inheritDoc TokenUsage} */ tokenUsage: TokenUsage; }