fix remove history

briefercloud · Jan 14, 2025 · dfb860b · dfb860b
1 parent 242e07a
commit dfb860b
Show file tree

Hide file tree

Showing 4 changed files with 100 additions and 65 deletions.
diff --git a/apps/api/src/yjs/v2/index.ts b/apps/api/src/yjs/v2/index.ts
@@ -810,17 +810,47 @@ export class WSSharedDocV2 {
       loadStateResult,
       persistor
     )
-    await doc.init()
+    logger().trace(
+      {
+        id: doc.id,
+        documentId: doc.documentId,
+        workspaceId: doc.workspaceId,
+        applyUpdateLatency: loadStateResult.applyUpdateLatency,
+        clockUpdatedAt: loadStateResult.clockUpdatedAt,
+      },
+      'Loadded YDoc'
+    )
+
     if (
       loadStateResult.applyUpdateLatency > 1000 &&
       Date.now() - loadStateResult.clockUpdatedAt.getTime() >
         1000 * 60 * 60 * 24
     ) {
       // if the latency is more than 1 second and the clock was updated more than 24 hours ago
       // remove history to reduce the size of the document and improve load performance
+      logger().info(
+        {
+          id: doc.id,
+          documentId: doc.documentId,
+          workspaceId: doc.workspaceId,
+          applyUpdateLatency: loadStateResult.applyUpdateLatency,
+          clockUpdatedAt: loadStateResult.clockUpdatedAt,
+        },
+        'Removing history from YDoc to reduce size and improve load performance'
+      )
       await doc.removeHistory()
+      logger().info(
+        {
+          id: doc.id,
+          documentId: doc.documentId,
+          workspaceId: doc.workspaceId,
+        },
+        'Removed history from YDoc to reduce size and improve load performance'
+      )
     }
 
+    await doc.init()
+
     return doc
   }
 }

diff --git a/apps/web/src/hooks/useYDoc.ts b/apps/web/src/hooks/useYDoc.ts
@@ -16,23 +16,26 @@ import Dexie, { EntityTable } from 'dexie'
 import { useReusableComponents } from './useReusableComponents'
 
 const db = new Dexie('YjsDatabase') as Dexie & {
-  yDocs: EntityTable<{ id: string; data: Uint8Array }, 'id'>
+  yDocs: EntityTable<{ id: string; data: Uint8Array; clock: number }, 'id'>
 }
 
 db.version(1).stores({
-  yDocs: 'id, data',
+  yDocs: 'id, data, clock',
 })
 
-function persistYDoc(id: string, yDoc: Y.Doc) {
+function persistYDoc(id: string, yDoc: Y.Doc, clock: number) {
   const data = Y.encodeStateAsUpdate(yDoc)
-  db.yDocs.put({ id, data })
+  db.yDocs.put({ id, data, clock })
 }
 
-function restoreYDoc(id: string): [Y.Doc, Promise<void>] {
+function restoreYDoc(
+  id: string,
+  clock: number
+): [{ clock: number; yDoc: Y.Doc }, Promise<void>] {
   const yDoc = new Y.Doc()
 
   const restore = db.yDocs
-    .get(id)
+    .get({ id, clock })
     .then((item) => {
       if (item) {
         Y.applyUpdate(yDoc, item.data)
@@ -48,13 +51,13 @@ function restoreYDoc(id: string): [Y.Doc, Promise<void>] {
       }
     })
 
-  return [yDoc, restore]
+  return [{ yDoc, clock }, restore]
 }
 
-const cache = new LRUCache<string, Y.Doc>({
+const cache = new LRUCache<string, { clock: number; yDoc: Y.Doc }>({
   max: 10,
 
-  dispose: (yDoc) => {
+  dispose: ({ yDoc }) => {
     yDoc.destroy()
   },
 })
@@ -63,6 +66,7 @@ type GetYDocResult = {
   id: string
   cached: boolean
   yDoc: Y.Doc
+  clock: number
   restore: Promise<void>
 }
 
@@ -73,18 +77,18 @@ function getYDoc(
   publishedAt: string | null
 ): GetYDocResult {
   const id = getDocId(documentId, isDataApp, clock, publishedAt)
-  let yDoc = cache.get(id)
-  const cached = Boolean(yDoc)
+  let fromCache = cache.get(id)
+  const cached = Boolean(fromCache)
   let restore = Promise.resolve()
 
-  if (!yDoc) {
-    const restoreResult = restoreYDoc(id)
-    yDoc = restoreResult[0]
+  if (!fromCache) {
+    const restoreResult = restoreYDoc(id, clock)
+    fromCache = restoreResult[0]
     restore = restoreResult[1]
-    cache.set(id, yDoc)
+    cache.set(id, fromCache)
   }
 
-  return { id, cached, yDoc, restore }
+  return { id, cached, yDoc: fromCache.yDoc, clock: fromCache.clock, restore }
 }
 
 export function useYDoc(
@@ -112,14 +116,14 @@ export function useYDoc(
     if (isFirst.current) {
       isFirst.current = false
       return () => {
-        persistYDoc(id, yDoc)
+        persistYDoc(id, yDoc, clock)
       }
     }
 
     const next = getYDoc(documentId, isDataApp, clock, publishedAt)
     setYDoc(next)
     return () => {
-      persistYDoc(next.id, next.yDoc)
+      persistYDoc(next.id, next.yDoc, next.clock)
     }
   }, [documentId, isDataApp, clock, publishedAt, userId])
 

diff --git a/packages/editor/src/blocks/richText.ts b/packages/editor/src/blocks/richText.ts
@@ -8,6 +8,7 @@ import {
   duplicateBaseAttributes,
 } from './index.js'
 import { ExecutionStatus } from '../execution/item.js'
+import { duplicateYXmlFragment } from '../index.js'
 
 export type RichTextBlock = BaseBlock<BlockType.RichText> & {
   content: Y.XmlFragment
@@ -71,49 +72,3 @@ export function getRichTextBlockExecStatus(
 ): ExecutionStatus {
   return 'completed'
 }
-
-function duplicateYXmlFragment(fragment: Y.XmlFragment): Y.XmlFragment {
-  const newFragment = new Y.XmlFragment()
-
-  function cloneElement(element: Y.XmlElement) {
-    const newElement = new Y.XmlElement(element.nodeName)
-    const attrs = element.getAttributes()
-    for (const key in attrs) {
-      const value = attrs[key]
-      if (value === undefined) {
-        continue
-      }
-
-      newElement.setAttribute(key, value)
-    }
-
-    const children: Array<Y.XmlElement | Y.XmlText | Y.XmlHook> = []
-    let child = element.firstChild
-    while (child) {
-      children.push(cloneNode(child))
-      child = child.nextSibling
-    }
-
-    // @ts-ignore
-    newElement.insert(0, children)
-
-    return newElement
-  }
-
-  function cloneNode(node: Y.XmlElement | Y.XmlText | Y.XmlHook) {
-    if (node instanceof Y.XmlElement) {
-      return cloneElement(node)
-    }
-
-    return node.clone()
-  }
-
-  // adapted from https://github.com/yjs/yjs/blob/e348255bb125e992eb661889e64a10efd7319172/src/types/YXmlFragment.js#L168-L173
-  newFragment.insert(
-    0,
-    // @ts-ignore
-    fragment.toArray().map(cloneNode)
-  )
-
-  return newFragment
-}
diff --git a/packages/editor/src/index.ts b/packages/editor/src/index.ts
@@ -355,3 +355,49 @@ export function getDataframe(
 
   return df ?? null
 }
+
+export function duplicateYXmlFragment(fragment: Y.XmlFragment): Y.XmlFragment {
+  const newFragment = new Y.XmlFragment()
+
+  function cloneElement(element: Y.XmlElement) {
+    const newElement = new Y.XmlElement(element.nodeName)
+    const attrs = element.getAttributes()
+    for (const key in attrs) {
+      const value = attrs[key]
+      if (value === undefined) {
+        continue
+      }
+
+      newElement.setAttribute(key, value)
+    }
+
+    const children: Array<Y.XmlElement | Y.XmlText | Y.XmlHook> = []
+    let child = element.firstChild
+    while (child) {
+      children.push(cloneNode(child))
+      child = child.nextSibling
+    }
+
+    // @ts-ignore
+    newElement.insert(0, children)
+
+    return newElement
+  }
+
+  function cloneNode(node: Y.XmlElement | Y.XmlText | Y.XmlHook) {
+    if (node instanceof Y.XmlElement) {
+      return cloneElement(node)
+    }
+
+    return node.clone()
+  }
+
+  // adapted from https://github.com/yjs/yjs/blob/e348255bb125e992eb661889e64a10efd7319172/src/types/YXmlFragment.js#L168-L173
+  newFragment.insert(
+    0,
+    // @ts-ignore
+    fragment.toArray().map(cloneNode)
+  )
+
+  return newFragment
+}