diff --git a/apps/api/src/yjs/v2/index.ts b/apps/api/src/yjs/v2/index.ts index 8394420d..6f3464e4 100644 --- a/apps/api/src/yjs/v2/index.ts +++ b/apps/api/src/yjs/v2/index.ts @@ -810,7 +810,17 @@ export class WSSharedDocV2 { loadStateResult, persistor ) - await doc.init() + logger().trace( + { + id: doc.id, + documentId: doc.documentId, + workspaceId: doc.workspaceId, + applyUpdateLatency: loadStateResult.applyUpdateLatency, + clockUpdatedAt: loadStateResult.clockUpdatedAt, + }, + 'Loadded YDoc' + ) + if ( loadStateResult.applyUpdateLatency > 1000 && Date.now() - loadStateResult.clockUpdatedAt.getTime() > @@ -818,9 +828,29 @@ export class WSSharedDocV2 { ) { // if the latency is more than 1 second and the clock was updated more than 24 hours ago // remove history to reduce the size of the document and improve load performance + logger().info( + { + id: doc.id, + documentId: doc.documentId, + workspaceId: doc.workspaceId, + applyUpdateLatency: loadStateResult.applyUpdateLatency, + clockUpdatedAt: loadStateResult.clockUpdatedAt, + }, + 'Removing history from YDoc to reduce size and improve load performance' + ) await doc.removeHistory() + logger().info( + { + id: doc.id, + documentId: doc.documentId, + workspaceId: doc.workspaceId, + }, + 'Removed history from YDoc to reduce size and improve load performance' + ) } + await doc.init() + return doc } } diff --git a/apps/web/src/hooks/useYDoc.ts b/apps/web/src/hooks/useYDoc.ts index 73c62e28..1a0ad7f4 100644 --- a/apps/web/src/hooks/useYDoc.ts +++ b/apps/web/src/hooks/useYDoc.ts @@ -16,23 +16,26 @@ import Dexie, { EntityTable } from 'dexie' import { useReusableComponents } from './useReusableComponents' const db = new Dexie('YjsDatabase') as Dexie & { - yDocs: EntityTable<{ id: string; data: Uint8Array }, 'id'> + yDocs: EntityTable<{ id: string; data: Uint8Array; clock: number }, 'id'> } db.version(1).stores({ - yDocs: 'id, data', + yDocs: 'id, data, clock', }) -function persistYDoc(id: string, yDoc: Y.Doc) { +function persistYDoc(id: string, yDoc: Y.Doc, clock: number) { const data = Y.encodeStateAsUpdate(yDoc) - db.yDocs.put({ id, data }) + db.yDocs.put({ id, data, clock }) } -function restoreYDoc(id: string): [Y.Doc, Promise] { +function restoreYDoc( + id: string, + clock: number +): [{ clock: number; yDoc: Y.Doc }, Promise] { const yDoc = new Y.Doc() const restore = db.yDocs - .get(id) + .get({ id, clock }) .then((item) => { if (item) { Y.applyUpdate(yDoc, item.data) @@ -48,13 +51,13 @@ function restoreYDoc(id: string): [Y.Doc, Promise] { } }) - return [yDoc, restore] + return [{ yDoc, clock }, restore] } -const cache = new LRUCache({ +const cache = new LRUCache({ max: 10, - dispose: (yDoc) => { + dispose: ({ yDoc }) => { yDoc.destroy() }, }) @@ -63,6 +66,7 @@ type GetYDocResult = { id: string cached: boolean yDoc: Y.Doc + clock: number restore: Promise } @@ -73,18 +77,18 @@ function getYDoc( publishedAt: string | null ): GetYDocResult { const id = getDocId(documentId, isDataApp, clock, publishedAt) - let yDoc = cache.get(id) - const cached = Boolean(yDoc) + let fromCache = cache.get(id) + const cached = Boolean(fromCache) let restore = Promise.resolve() - if (!yDoc) { - const restoreResult = restoreYDoc(id) - yDoc = restoreResult[0] + if (!fromCache) { + const restoreResult = restoreYDoc(id, clock) + fromCache = restoreResult[0] restore = restoreResult[1] - cache.set(id, yDoc) + cache.set(id, fromCache) } - return { id, cached, yDoc, restore } + return { id, cached, yDoc: fromCache.yDoc, clock: fromCache.clock, restore } } export function useYDoc( @@ -112,14 +116,14 @@ export function useYDoc( if (isFirst.current) { isFirst.current = false return () => { - persistYDoc(id, yDoc) + persistYDoc(id, yDoc, clock) } } const next = getYDoc(documentId, isDataApp, clock, publishedAt) setYDoc(next) return () => { - persistYDoc(next.id, next.yDoc) + persistYDoc(next.id, next.yDoc, next.clock) } }, [documentId, isDataApp, clock, publishedAt, userId]) diff --git a/packages/editor/src/blocks/richText.ts b/packages/editor/src/blocks/richText.ts index dfa2d342..d42e3b17 100644 --- a/packages/editor/src/blocks/richText.ts +++ b/packages/editor/src/blocks/richText.ts @@ -8,6 +8,7 @@ import { duplicateBaseAttributes, } from './index.js' import { ExecutionStatus } from '../execution/item.js' +import { duplicateYXmlFragment } from '../index.js' export type RichTextBlock = BaseBlock & { content: Y.XmlFragment @@ -71,49 +72,3 @@ export function getRichTextBlockExecStatus( ): ExecutionStatus { return 'completed' } - -function duplicateYXmlFragment(fragment: Y.XmlFragment): Y.XmlFragment { - const newFragment = new Y.XmlFragment() - - function cloneElement(element: Y.XmlElement) { - const newElement = new Y.XmlElement(element.nodeName) - const attrs = element.getAttributes() - for (const key in attrs) { - const value = attrs[key] - if (value === undefined) { - continue - } - - newElement.setAttribute(key, value) - } - - const children: Array = [] - let child = element.firstChild - while (child) { - children.push(cloneNode(child)) - child = child.nextSibling - } - - // @ts-ignore - newElement.insert(0, children) - - return newElement - } - - function cloneNode(node: Y.XmlElement | Y.XmlText | Y.XmlHook) { - if (node instanceof Y.XmlElement) { - return cloneElement(node) - } - - return node.clone() - } - - // adapted from https://github.com/yjs/yjs/blob/e348255bb125e992eb661889e64a10efd7319172/src/types/YXmlFragment.js#L168-L173 - newFragment.insert( - 0, - // @ts-ignore - fragment.toArray().map(cloneNode) - ) - - return newFragment -} diff --git a/packages/editor/src/index.ts b/packages/editor/src/index.ts index 124d91e1..2d9ceb7c 100644 --- a/packages/editor/src/index.ts +++ b/packages/editor/src/index.ts @@ -355,3 +355,49 @@ export function getDataframe( return df ?? null } + +export function duplicateYXmlFragment(fragment: Y.XmlFragment): Y.XmlFragment { + const newFragment = new Y.XmlFragment() + + function cloneElement(element: Y.XmlElement) { + const newElement = new Y.XmlElement(element.nodeName) + const attrs = element.getAttributes() + for (const key in attrs) { + const value = attrs[key] + if (value === undefined) { + continue + } + + newElement.setAttribute(key, value) + } + + const children: Array = [] + let child = element.firstChild + while (child) { + children.push(cloneNode(child)) + child = child.nextSibling + } + + // @ts-ignore + newElement.insert(0, children) + + return newElement + } + + function cloneNode(node: Y.XmlElement | Y.XmlText | Y.XmlHook) { + if (node instanceof Y.XmlElement) { + return cloneElement(node) + } + + return node.clone() + } + + // adapted from https://github.com/yjs/yjs/blob/e348255bb125e992eb661889e64a10efd7319172/src/types/YXmlFragment.js#L168-L173 + newFragment.insert( + 0, + // @ts-ignore + fragment.toArray().map(cloneNode) + ) + + return newFragment +}