Feature - Add Vector Database and Resurrection Memory Script

.gitignore

@@ -53,3 +53,6 @@ node_modules/
 resources/
 threads.json
 cookies.json
+
+*.bin
+data

README.md

@@ -258,14 +258,36 @@ yarn dev my-agent
 
 Monitor the agent's activity in the console and configured log files.
 
+## Resurrection
+
+To resurrect memories from the Autonomys Network, run the following command:
+
+### Options
+
+- `-o, --output`: (Optional) The directory where memories will be saved. Defaults to `./memories`
+- `-n, --number`: (Optional) Number of memories to fetch. If not specified, fetches all memories
+- `--help`: Show help menu with all available options
+
+Examples:
+
+```bash
+yarn resurrect                                    # Fetch all memories to ./memories/
+yarn resurrect -n 1000                           # Fetch 1000 memories to ./memories/
+yarn resurrect -o ./memories/my-agent -n 1000    # Fetch 1000 memories to specified directory
+yarn resurrect --output ./custom/path            # Fetch all memories to custom directory
+yarn resurrect --help                            # Show help menu
+```
+
+```
+
 ## Testing
 
 To run tests:
 
 ```bash
 yarn test
-```
-
-## License
+```## License
 
 MIT
+
+

package.json

@@ -17,7 +17,8 @@
         "test:watch": "jest --watch",
         "test:coverage": "jest --coverage",
         "lint": "eslint \"src/**/*.{js,ts}\"",
-        "lint:fix": "eslint \"src/**/*.{js,ts}\" --fix"
+        "lint:fix": "eslint \"src/**/*.{js,ts}\" --fix",
+        "resurrect": "tsx src/cli/resurrect.ts"
     },
     "dependencies": {
         "@autonomys/auto-dag-data": "1.2.1",
@@ -29,8 +30,11 @@
         "@langchain/ollama": "^0.1.4",
         "@langchain/openai": "0.3.16",
         "agent-twitter-client": "0.0.18",
+        "better-sqlite3": "^11.8.0",
         "dotenv": "^16.3.1",
         "ethers": "^6.13.4",
+        "sqlite3": "^5.1.7",
+        "vectorlite": "^0.2.0",
         "inquirer": "^10.2.0",
         "winston": "^3.11.0",
         "zod": "^3.22.4",
@@ -39,6 +43,7 @@
     "devDependencies": {
         "@eslint/js": "^9.18.0",
         "@tsconfig/node20": "^20.1.4",
+        "@types/better-sqlite3": "^7.6.12",
         "@types/inquirer": "^9.0.7",
         "@types/jest": "^29.5.12",
         "@types/js-yaml": "^4.0.9",

src/agents/tools/uploadToDsnTool.ts

@@ -3,7 +3,7 @@ import { z } from 'zod';
 import { createLogger } from '../../utils/logger.js';
 import { ToolNode } from '@langchain/langgraph/prebuilt';
 import { AIMessage } from '@langchain/core/messages';
-import { uploadToDsn } from './utils/dsnUpload.js';
+import { uploadToDsn } from './utils/dsn/dsnUpload.js';
 
 const logger = createLogger('upload-to-dsn-tool');
 

src/agents/tools/utils/agentMemoryContract.ts → src/agents/tools/utils/blockchain/agentMemoryContract.ts

@@ -1,10 +1,10 @@
 import { ethers } from 'ethers';
-import { MEMORY_ABI } from './abi/memory.js';
-import { config } from '../../../config/index.js';
+import { MEMORY_ABI } from '../abi/memory.js';
+import { config } from '../../../../config/index.js';
 import { wallet } from './agentWallet.js';
 import { cidFromBlakeHash, cidToString } from '@autonomys/auto-dag-data';
-import { getLocalHash, saveHashLocally } from './localHashStorage.js';
-import { createLogger } from '../../../utils/logger.js';
+import { getLocalHash, saveHashLocally } from '../localHashStorage.js';
+import { createLogger } from '../../../../utils/logger.js';
 
 const logger = createLogger('agent-memory-contract');
 const CONTRACT_ADDRESS = config.blockchainConfig.CONTRACT_ADDRESS as `0x${string}`;

src/agents/tools/utils/agentWallet.ts → src/agents/tools/utils/blockchain/agentWallet.ts

@@ -1,5 +1,5 @@
 import { ethers } from 'ethers';
-import { config } from '../../../config/index.js';
+import { config } from '../../../../config/index.js';
 
 const provider = new ethers.JsonRpcProvider(config.blockchainConfig.RPC_URL);
 

src/agents/tools/utils/dsn/dsnDownload.ts

@@ -0,0 +1,50 @@
+import { createAutoDriveApi, downloadFile } from '@autonomys/auto-drive';
+import { config } from '../../../../config/index.js';
+import { createLogger } from '../../../../utils/logger.js';
+import { withRetry } from './retry.js';
+
+const logger = createLogger('dsn-download');
+
+interface BaseMemory {
+  previousCid?: string;
+  timestamp?: string;
+  agentVersion?: string;
+  [key: string]: unknown;
+}
+
+export const download = async (cid: string): Promise<BaseMemory> => {
+  return withRetry(
+    async () => {
+      const api = createAutoDriveApi({
+        apiKey: config.autoDriveConfig.AUTO_DRIVE_API_KEY || '',
+      });
+      logger.info(`Downloading file: ${cid}`);
+      const stream = await downloadFile(api, cid);
+
+      const chunks: Uint8Array[] = [];
+      for await (const chunk of stream) {
+        chunks.push(chunk);
+      }
+
+      const allChunks = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0));
+      let position = 0;
+      for (const chunk of chunks) {
+        allChunks.set(chunk, position);
+        position += chunk.length;
+      }
+
+      const jsonString = new TextDecoder().decode(allChunks);
+      const data = JSON.parse(jsonString);
+
+      return data;
+    },
+    {
+      shouldRetry: error => {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return !(
+          errorMessage.includes('Not Found') || errorMessage.includes('incorrect header check')
+        );
+      },
+    },
+  );
+};

src/agents/tools/utils/dsnUpload.ts → src/agents/tools/utils/dsn/dsnUpload.ts

@@ -1,54 +1,16 @@
-import { createLogger } from '../../../utils/logger.js';
+import { createLogger } from '../../../../utils/logger.js';
 import { hexlify } from 'ethers';
 import { createAutoDriveApi, uploadFile, UploadFileOptions } from '@autonomys/auto-drive';
 import { blake3HashFromCid, stringToCid } from '@autonomys/auto-dag-data';
-import { agentVersion, config } from '../../../config/index.js';
-import { signMessage, wallet } from './agentWallet.js';
-import { getLastMemoryCid, setLastMemoryHash } from './agentMemoryContract.js';
-import { saveHashLocally } from './localHashStorage.js';
+import { agentVersion, config } from '../../../../config/index.js';
+import { signMessage, wallet } from '../blockchain/agentWallet.js';
+import { getLastMemoryCid, setLastMemoryHash } from '../blockchain/agentMemoryContract.js';
+import { withRetry } from './retry.js';
 
 const logger = createLogger('dsn-upload-tool');
 const dsnApi = createAutoDriveApi({ apiKey: config.autoDriveConfig.AUTO_DRIVE_API_KEY || '' });
-let currentNonce = await wallet.getNonce();
-
-// New retry utility function
-const withRetry = async <T>(
-  operation: () => Promise<T>,
-  {
-    maxRetries = 5,
-    initialDelayMs = 1000,
-    operationName = 'Operation',
-  }: {
-    maxRetries?: number;
-    initialDelayMs?: number;
-    operationName?: string;
-  } = {},
-): Promise<T> => {
-  const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-
-  const attempt = async (retriesLeft: number, currentDelay: number): Promise<T> => {
-    try {
-      return await operation();
-    } catch (error) {
-      if (retriesLeft <= 0) {
-        logger.error(`${operationName} failed after all retry attempts`, { error });
-        throw error;
-      }
 
-      logger.warn(`${operationName} failed, retrying... (${retriesLeft} attempts left)`, {
-        error,
-        nextDelayMs: currentDelay,
-      });
-      await delay(currentDelay);
-      // Exponential backoff with jitter
-      const jitter = Math.random() * 0.3 + 0.85; // Random value between 0.85 and 1.15
-      const nextDelay = Math.min(currentDelay * 2 * jitter, 30000); // Cap at 30 seconds
-      return attempt(retriesLeft - 1, nextDelay);
-    }
-  };
-
-  return attempt(maxRetries, initialDelayMs);
-};
+let currentNonce = await wallet.getNonce();
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const uploadFileToDsn = async (file: any, options: UploadFileOptions) =>
@@ -101,8 +63,6 @@ export async function uploadToDsn(data: object) {
     logger.info('Setting last memory hash', {
       blake3hash: hexlify(blake3hash),
     });
-    // Temporary fix
-    saveHashLocally(hexlify(blake3hash));
 
     const tx = await submitMemoryHash(hexlify(blake3hash), currentNonce++);
     logger.info('Memory hash transaction submitted', {

src/agents/tools/utils/dsn/retry.ts

@@ -0,0 +1,36 @@
+import { createLogger } from '../../../../utils/logger.js';
+
+const logger = createLogger('retry-utility');
+
+export const withRetry = async <T>(
+  operation: () => Promise<T>,
+  {
+    maxRetries = 5,
+    initialDelayMs = 1000,
+    operationName = 'Operation',
+    shouldRetry = (_error: unknown): boolean => true,
+  } = {},
+): Promise<T> => {
+  const attempt = async (retriesLeft: number, currentDelay: number): Promise<T> => {
+    try {
+      return await operation();
+    } catch (error) {
+      if (!shouldRetry(error) || retriesLeft <= 0) {
+        logger.error(`${operationName} failed after all retry attempts`, { error });
+        throw error;
+      }
+
+      logger.warn(`${operationName} failed, retrying... (${retriesLeft} attempts left)`, {
+        error,
+        nextDelayMs: currentDelay,
+      });
+
+      await new Promise(resolve => setTimeout(resolve, currentDelay));
+      const jitter = Math.random() * 0.3 + 0.85; // Random value between 0.85 and 1.15
+      const nextDelay = Math.min(currentDelay * 2 * jitter, 30000); // Cap at 30 seconds
+      return attempt(retriesLeft - 1, nextDelay);
+    }
+  };
+
+  return attempt(maxRetries, initialDelayMs);
+};

src/agents/workflows/twitter/prompts.ts

@@ -8,7 +8,7 @@ import {
 } from './schemas.js';
 import { ChatPromptTemplate, PromptTemplate } from '@langchain/core/prompts';
 import { SystemMessage } from '@langchain/core/messages';
-import { wallet } from '../../tools/utils/agentWallet.js';
+import { wallet } from '../../tools/utils/blockchain/agentWallet.js';
 import { loadCharacter } from '../../../config/characters.js';
 
 const followFormatInstructions = `

src/cli/resurrect.ts

@@ -0,0 +1,68 @@
+import { join } from 'path';
+import { mkdirSync } from 'fs';
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import { createLogger } from '../utils/logger.js';
+import { downloadAllMemories } from './utils/resurrection.js';
+
+const logger = createLogger('resurrect-cli');
+
+interface CliOptions {
+  outputDir: string;
+  memoriesToFetch: number | null;
+}
+
+const parseArgs = (): CliOptions => {
+  const argv = yargs(hideBin(process.argv))
+    .option('output', {
+      alias: 'o',
+      type: 'string',
+      description: 'Output directory for memories',
+      default: 'memories',
+    })
+    .option('number', {
+      alias: 'n',
+      type: 'number',
+      description: 'Number of memories to fetch',
+      default: null,
+    })
+    .check(argv => {
+      if (argv.number !== null && (isNaN(argv.number) || argv.number <= 0)) {
+        throw new Error('Number of memories must be a positive integer');
+      }
+      return true;
+    })
+    .help()
+    .parseSync();
+
+  return {
+    outputDir: join(process.cwd(), argv.output as string),
+    memoriesToFetch: argv.number as number | null,
+  };
+};
+
+const main = async () => {
+  try {
+    const options = parseArgs();
+    mkdirSync(options.outputDir, { recursive: true });
+
+    logger.info(`Using output directory: ${options.outputDir}`);
+    logger.info(
+      options.memoriesToFetch
+        ? `Starting memory resurrection (fetching ${options.memoriesToFetch} memories)...`
+        : 'Starting memory resurrection (fetching all memories)...',
+    );
+
+    const result = await downloadAllMemories(options.outputDir, options.memoriesToFetch);
+
+    logger.info(
+      `Memory resurrection complete. Processed: ${result.processed}, Failed: ${result.failed}`,
+    );
+    logger.info(`Memories saved to ${options.outputDir}`);
+  } catch (error) {
+    logger.error('Failed to resurrect memories:', error);
+    process.exit(1);
+  }
+};
+
+main();