Feature - Add Vector Database and Resurrection Memory Script

.gitignore

@@ -54,3 +54,6 @@ node_modules/
 resources/
 threads.json
 cookies.json
+
+*.bin
+data

package.json

@@ -19,7 +19,8 @@
         "lint": "eslint \"src/**/*.{js,ts}\"",
         "lint:fix": "eslint \"src/**/*.{js,ts}\" --fix",
         "lint:characters": "eslint \"src/agents/workflows/kol/characters/*.ts\"",
-        "lint:characters:fix": "eslint \"src/agents/workflows/kol/characters/*.ts\" --fix"
+        "lint:characters:fix": "eslint \"src/agents/workflows/kol/characters/*.ts\" --fix",
+        "resurrect": "tsx src/cli/resurrect.ts"
     },
     "dependencies": {
         "@autonomys/auto-dag-data": "1.2.1",
@@ -31,15 +32,19 @@
         "@langchain/ollama": "^0.1.4",
         "@langchain/openai": "0.3.16",
         "agent-twitter-client": "0.0.18",
+        "better-sqlite3": "^11.8.0",
         "dotenv": "^16.3.1",
         "ethers": "^6.13.4",
+        "sqlite3": "^5.1.7",
+        "vectorlite": "^0.2.0",
         "winston": "^3.11.0",
         "zod": "^3.22.4",
         "zod-to-json-schema": "^3.24.1"
     },
     "devDependencies": {
         "@eslint/js": "^9.18.0",
         "@tsconfig/node20": "^20.1.4",
+        "@types/better-sqlite3": "^7.6.12",
         "@types/jest": "^29.5.12",
         "@types/js-yaml": "^4.0.9",
         "@types/node": "22.10.0",

src/agents/tools/uploadToDsnTool.ts

@@ -3,7 +3,7 @@ import { z } from 'zod';
 import { createLogger } from '../../utils/logger.js';
 import { ToolNode } from '@langchain/langgraph/prebuilt';
 import { AIMessage } from '@langchain/core/messages';
-import { uploadToDsn } from './utils/dsnUpload.js';
+import { uploadToDsn } from './utils/dsn/dsnUpload.js';
 
 const logger = createLogger('upload-to-dsn-tool');
 

src/agents/tools/utils/agentMemoryContract.ts → src/agents/tools/utils/blockchain/agentMemoryContract.ts

@@ -1,10 +1,10 @@
 import { ethers } from 'ethers';
-import { MEMORY_ABI } from './abi/memory.js';
-import { config } from '../../../config/index.js';
+import { MEMORY_ABI } from '../abi/memory.js';
+import { config } from '../../../../config/index.js';
 import { wallet } from './agentWallet.js';
 import { cidFromBlakeHash, cidToString } from '@autonomys/auto-dag-data';
-import { getLocalHash, saveHashLocally } from './localHashStorage.js';
-import { createLogger } from '../../../utils/logger.js';
+import { getLocalHash, saveHashLocally } from '../localHashStorage.js';
+import { createLogger } from '../../../../utils/logger.js';
 
 const logger = createLogger('agent-memory-contract');
 const CONTRACT_ADDRESS = config.blockchainConfig.CONTRACT_ADDRESS as `0x${string}`;

src/agents/tools/utils/agentWallet.ts → src/agents/tools/utils/blockchain/agentWallet.ts

@@ -1,5 +1,5 @@
 import { ethers } from 'ethers';
-import { config } from '../../../config/index.js';
+import { config } from '../../../../config/index.js';
 
 const provider = new ethers.JsonRpcProvider(config.blockchainConfig.RPC_URL);
 

src/agents/tools/utils/dsn/dsnDownload.ts

@@ -0,0 +1,50 @@
+import { createAutoDriveApi, downloadFile } from '@autonomys/auto-drive';
+import { config } from '../../../../config/index.js';
+import { createLogger } from '../../../../utils/logger.js';
+import { withRetry } from './retry.js';
+
+const logger = createLogger('dsn-download');
+
+interface BaseMemory {
+  previousCid?: string;
+  timestamp?: string;
+  agentVersion?: string;
+  [key: string]: unknown;
+}
+
+export async function download(cid: string): Promise<BaseMemory> {
+  return withRetry(
+    async () => {
+      const api = createAutoDriveApi({
+        apiKey: config.autoDriveConfig.AUTO_DRIVE_API_KEY || '',
+      });
+      logger.info(`Downloading file: ${cid}`);
+      const stream = await downloadFile(api, cid);
+
+      const chunks: Uint8Array[] = [];
+      for await (const chunk of stream) {
+        chunks.push(chunk);
+      }
+
+      const allChunks = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0));
+      let position = 0;
+      for (const chunk of chunks) {
+        allChunks.set(chunk, position);
+        position += chunk.length;
+      }
+
+      const jsonString = new TextDecoder().decode(allChunks);
+      const data = JSON.parse(jsonString);
+
+      return data;
+    },
+    {
+      shouldRetry: error => {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return !(
+          errorMessage.includes('Not Found') || errorMessage.includes('incorrect header check')
+        );
+      },
+    },
+  );
+}

src/agents/tools/utils/dsnUpload.ts → src/agents/tools/utils/dsn/dsnUpload.ts

@@ -1,54 +1,16 @@
-import { createLogger } from '../../../utils/logger.js';
+import { createLogger } from '../../../../utils/logger.js';
 import { hexlify } from 'ethers';
 import { createAutoDriveApi, uploadFile, UploadFileOptions } from '@autonomys/auto-drive';
 import { blake3HashFromCid, stringToCid } from '@autonomys/auto-dag-data';
-import { agentVersion, config } from '../../../config/index.js';
-import { signMessage, wallet } from './agentWallet.js';
-import { getLastMemoryCid, setLastMemoryHash } from './agentMemoryContract.js';
-import { saveHashLocally } from './localHashStorage.js';
+import { agentVersion, config } from '../../../../config/index.js';
+import { signMessage, wallet } from '../blockchain/agentWallet.js';
+import { getLastMemoryCid, setLastMemoryHash } from '../blockchain/agentMemoryContract.js';
+import { withRetry } from './retry.js';
 
 const logger = createLogger('dsn-upload-tool');
 const dsnApi = createAutoDriveApi({ apiKey: config.autoDriveConfig.AUTO_DRIVE_API_KEY || '' });
-let currentNonce = await wallet.getNonce();
-
-// New retry utility function
-const withRetry = async <T>(
-  operation: () => Promise<T>,
-  {
-    maxRetries = 5,
-    initialDelayMs = 1000,
-    operationName = 'Operation',
-  }: {
-    maxRetries?: number;
-    initialDelayMs?: number;
-    operationName?: string;
-  } = {},
-): Promise<T> => {
-  const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-
-  const attempt = async (retriesLeft: number, currentDelay: number): Promise<T> => {
-    try {
-      return await operation();
-    } catch (error) {
-      if (retriesLeft <= 0) {
-        logger.error(`${operationName} failed after all retry attempts`, { error });
-        throw error;
-      }
 
-      logger.warn(`${operationName} failed, retrying... (${retriesLeft} attempts left)`, {
-        error,
-        nextDelayMs: currentDelay,
-      });
-      await delay(currentDelay);
-      // Exponential backoff with jitter
-      const jitter = Math.random() * 0.3 + 0.85; // Random value between 0.85 and 1.15
-      const nextDelay = Math.min(currentDelay * 2 * jitter, 30000); // Cap at 30 seconds
-      return attempt(retriesLeft - 1, nextDelay);
-    }
-  };
-
-  return attempt(maxRetries, initialDelayMs);
-};
+let currentNonce = await wallet.getNonce();
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const uploadFileToDsn = async (file: any, options: UploadFileOptions) =>
@@ -101,8 +63,6 @@ export async function uploadToDsn(data: object) {
     logger.info('Setting last memory hash', {
       blake3hash: hexlify(blake3hash),
     });
-    // Temporary fix
-    saveHashLocally(hexlify(blake3hash));
 
     const tx = await submitMemoryHash(hexlify(blake3hash), currentNonce++);
     logger.info('Memory hash transaction submitted', {

src/agents/tools/utils/dsn/retry.ts

@@ -0,0 +1,36 @@
+import { createLogger } from '../../../../utils/logger.js';
+
+const logger = createLogger('retry-utility');
+
+export async function withRetry<T>(
+  operation: () => Promise<T>,
+  {
+    maxRetries = 5,
+    initialDelayMs = 1000,
+    operationName = 'Operation',
+    shouldRetry = (_error: unknown): boolean => true,
+  } = {},
+): Promise<T> {
+  const attempt = async (retriesLeft: number, currentDelay: number): Promise<T> => {
+    try {
+      return await operation();
+    } catch (error) {
+      if (!shouldRetry(error) || retriesLeft <= 0) {
+        logger.error(`${operationName} failed after all retry attempts`, { error });
+        throw error;
+      }
+
+      logger.warn(`${operationName} failed, retrying... (${retriesLeft} attempts left)`, {
+        error,
+        nextDelayMs: currentDelay,
+      });
+
+      await new Promise(resolve => setTimeout(resolve, currentDelay));
+      const jitter = Math.random() * 0.3 + 0.85; // Random value between 0.85 and 1.15
+      const nextDelay = Math.min(currentDelay * 2 * jitter, 30000); // Cap at 30 seconds
+      return attempt(retriesLeft - 1, nextDelay);
+    }
+  };
+
+  return attempt(maxRetries, initialDelayMs);
+}

src/agents/workflows/kol/prompts.ts

@@ -8,7 +8,7 @@ import {
 } from './schemas.js';
 import { ChatPromptTemplate, PromptTemplate } from '@langchain/core/prompts';
 import { SystemMessage } from '@langchain/core/messages';
-import { wallet } from '../../tools/utils/agentWallet.js';
+import { wallet } from '../../tools/utils/blockchain/agentWallet.js';
 import { loadCharacter } from '../../../config/characters.js';
 
 const followFormatInstructions = `

src/cli/resurrect.ts

@@ -0,0 +1,27 @@
+import { join } from 'path';
+import { mkdirSync } from 'fs';
+import { createLogger } from '../utils/logger.js';
+import MemoryResurrector from '../utils/resurrection.js';
+
+const logger = createLogger('resurrect-cli');
+
+async function main() {
+  try {
+    const outputDir = join(process.cwd(), 'memories');
+    mkdirSync(outputDir, { recursive: true });
+
+    logger.info('Starting memory resurrection...');
+    const resurrector = new MemoryResurrector(outputDir);
+    const result = await resurrector.downloadAllMemories();
+
+    logger.info(
+      `Memory resurrection complete. Processed: ${result.processed}, Failed: ${result.failed}`,
+    );
+    logger.info(`Memories saved to ${outputDir}`);
+  } catch (error) {
+    logger.error('Failed to resurrect memories:', error);
+    process.exit(1);
+  }
+}
+
+main();

src/utils/resurrection.ts

@@ -0,0 +1,89 @@
+import { createLogger } from '../utils/logger.js';
+import { getLastMemoryCid } from '../agents/tools/utils/blockchain/agentMemoryContract.js';
+import { download } from '../agents/tools/utils/dsn/dsnDownload.js';
+import { existsSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+
+const logger = createLogger('memory-resurrector');
+const STATE_FILE = join(process.cwd(), 'memories', 'last-processed-cid.json');
+
+class MemoryResurrector {
+  private failedCids: Set<string> = new Set();
+  private processedCount: number = 0;
+
+  constructor(private outputDir: string) {}
+
+  private getLastProcessedCid(): string | null {
+    try {
+      if (!existsSync(STATE_FILE)) {
+        return null;
+      }
+      const data = JSON.parse(readFileSync(STATE_FILE, 'utf-8'));
+      return data.lastProcessedCid;
+    } catch (error) {
+      logger.error('Failed to read last processed CID:', error);
+      return null;
+    }
+  }
+
+  private saveLastProcessedCid(cid: string): void {
+    try {
+      writeFileSync(STATE_FILE, JSON.stringify({ lastProcessedCid: cid }, null, 2));
+      logger.info(`Saved last processed CID: ${cid}`);
+    } catch (error) {
+      logger.error('Failed to save last processed CID:', error);
+    }
+  }
+
+  private async fetchMemoryChain(currentCid: string, stopAtCid: string | null): Promise<void> {
+    if (!currentCid || this.failedCids.has(currentCid) || currentCid === stopAtCid) {
+      return;
+    }
+
+    try {
+      const content = await download(currentCid);
+
+      const filename = `${currentCid}.json`;
+      const filepath = join(this.outputDir, filename);
+      writeFileSync(filepath, JSON.stringify(content, null, 2));
+
+      this.processedCount++;
+      logger.info(`Successfully fetched and saved memory ${currentCid}`);
+
+      if (content.previousCid && content.previousCid !== stopAtCid) {
+        await this.fetchMemoryChain(content.previousCid, stopAtCid);
+      }
+    } catch (error) {
+      logger.error(`Failed to fetch memory ${currentCid}:`, error);
+      this.failedCids.add(currentCid);
+    }
+  }
+
+  async downloadAllMemories(): Promise<{ processed: number; failed: number }> {
+    const latestCid = await getLastMemoryCid();
+    if (!latestCid) {
+      logger.info('No memories found (empty CID)');
+      return { processed: 0, failed: 0 };
+    }
+
+    const lastProcessedCid = this.getLastProcessedCid();
+    if (lastProcessedCid === latestCid) {
+      logger.info('Already up to date with latest CID');
+      return { processed: 0, failed: 0 };
+    }
+
+    logger.info(`Starting download from ${latestCid} to ${lastProcessedCid || 'genesis'}`);
+    await this.fetchMemoryChain(latestCid, lastProcessedCid);
+
+    this.saveLastProcessedCid(latestCid);
+
+    logger.info(`Downloaded ${this.processedCount} memories, failed CIDs: ${this.failedCids.size}`);
+
+    return {
+      processed: this.processedCount,
+      failed: this.failedCids.size,
+    };
+  }
+}
+
+export default MemoryResurrector;