Tests (#19)

* setup for tests

* fetch small fix, more test setups

* todo tests within docker

* added `fetch` and `pull` test flows

* bump version

README.md

@@ -7,7 +7,7 @@
     Dria CLI
   </h1>
   <p align="center">
-    <i>Dria CLI is a command-line tool that can be used to interact with Dria.</i>
+    <i>Dria CLI is a command-line tool that can be used to interact with Dria locally.</i>
   </p>
 </p>
 
@@ -155,3 +155,7 @@ You can lint & check formatting with:
 yarn lint
 yarn format
 ```
+
+## License
+
+See [LICENSE](./LICENSE).

jest.config.ts

@@ -0,0 +1,22 @@
+import type { JestConfigWithTsJest } from "ts-jest";
+
+const config: JestConfigWithTsJest = {
+  // ts-jest defaults
+  preset: "ts-jest",
+  testEnvironment: "node",
+  transform: {
+    "^.+\\.(ts|js)$": "ts-jest",
+  },
+  // timeout should be rather large due to Docker stuff & sleeps
+  testTimeout: 600_000,
+  // docker containers may take some time to close
+  openHandlesTimeout: 10_000,
+  // print everything like Mocha
+  verbose: true,
+  // dont run in parallel
+  maxConcurrency: 1,
+  // ignore output directory
+  testPathIgnorePatterns: ["bin", "node_modules", "src"],
+};
+
+export default config;

package.json

@@ -1,12 +1,11 @@
 {
   "name": "dria-cli",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "description": "A command-line tool for Dria",
   "author": "FirstBatch Team <[email protected]>",
   "contributors": [
     "Erhan Tezcan <[email protected]> (https://github.com/erhant)"
   ],
-  "homepage": "https://github.com/firstbatchxyz/dria-cli#readme",
   "license": "Apache-2.0",
   "files": [
     "/bin",
@@ -17,12 +16,12 @@
     "dria": "./bin/index.js"
   },
   "scripts": {
-    "lint": "npx eslint './src/**/*.ts' && echo 'All good!'",
-    "format": "npx prettier --check ./src/**/*.ts",
-    "build": "npx tsc",
+    "build": "npx tsc -p tsconfig.build.json",
     "start": "node ./bin/index.js",
     "dria": "yarn build && yarn start",
-    "test": "npx jest"
+    "test": "npx jest",
+    "lint": "npx eslint './src/**/*.ts' && echo 'All good!'",
+    "format": "npx prettier --check ./src/**/*.ts"
   },
   "engines": {
     "node": ">=18.0.0"
@@ -49,6 +48,7 @@
     "jest": "^29.7.0",
     "prettier": "^3.2.5",
     "ts-jest": "^29.1.2",
+    "ts-node": "^10.9.2",
     "typescript": "^5.3.3"
   },
   "prettier": {
@@ -65,6 +65,27 @@
     ],
     "rules": {
       "@typescript-eslint/no-unused-vars": "warn"
-    }
-  }
+    },
+    "ignorePatterns": [
+      "bin",
+      "node_modules"
+    ]
+  },
+  "homepage": "https://github.com/firstbatchxyz/dria-cli#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/firstbatchxyz/dria-cli.git"
+  },
+  "bugs": {
+    "url": "https://github.com/firstbatchxyz/dria-cli/issues/"
+  },
+  "keywords": [
+    "dria",
+    "hollowdb",
+    "arweave",
+    "yargs",
+    "cli",
+    "rag",
+    "ai"
+  ]
 }

src/cli.ts

@@ -0,0 +1,153 @@
+import yargs from "yargs";
+import commands from "./commands/";
+import { checkDocker, checkNetwork, logger } from "./common";
+import { getConfig, setConfig } from "./configurations";
+
+const config = getConfig();
+
+const contractIdArg = {
+  id: "contract" as const,
+  opts: {
+    alias: "c",
+    describe: "Contract ID",
+    type: "string",
+    default: config.contract,
+  } as const,
+} as const;
+
+const verboseArg = {
+  id: "verbose" as const,
+  opts: {
+    alias: "v",
+    describe: "Show extra information",
+    boolean: true,
+    default: false,
+    coerce: (verbose: boolean) => {
+      logger.setLevel(verbose ? "DEBUG" : "INFO");
+      return verbose;
+    },
+  } as const,
+} as const;
+
+const txIdArg = {
+  id: "txid" as const,
+  opts: {
+    describe: "Transaction ID",
+    type: "string",
+    demandOption: true,
+  } as const,
+} as const;
+
+async function checkArgs(args: { contract?: string }, checks: { contract?: boolean; docker?: boolean }) {
+  if (checks.contract) {
+    if (args.contract === undefined) throw new Error("Contract not provided.");
+  }
+
+  if (checks.docker) {
+    await checkDocker();
+    await checkNetwork();
+  }
+
+  return true;
+}
+
+/**
+ * Use Dria CLI with arguments.
+ * @param args command-line arguments
+ * @example
+ * import { hideBin } from "yargs/helpers";
+ *
+ * driaCLI(hideBin(process.argv));
+ */
+export function driaCLI(args: string[]) {
+  yargs(args)
+    .scriptName("dria")
+    .option(verboseArg.id, verboseArg.opts)
+
+    .command(
+      "pull [contract]",
+      "Pull a knowledge to your local machine.",
+      (yargs) =>
+        yargs.positional(contractIdArg.id, contractIdArg.opts).check(async (args) => {
+          return await checkArgs(args, { contract: true, docker: true });
+        }),
+      async (args) => {
+        await commands.pull(args.contract!);
+      },
+    )
+
+    .command(
+      "serve [contract]",
+      "Serve a local knowledge.",
+      (yargs) =>
+        yargs.positional(contractIdArg.id, contractIdArg.opts).check(async (args) => {
+          return await checkArgs(args, { contract: true, docker: true });
+        }),
+      async (args) => {
+        await commands.serve(args.contract!);
+      },
+    )
+
+    .command(
+      "clear [contract]",
+      "Clear local knowledge.",
+      (yargs) =>
+        yargs.positional(contractIdArg.id, contractIdArg.opts).check(async (args) => {
+          return await checkArgs(args, { contract: true });
+        }),
+      async (args) => {
+        await commands.clear(args.contract!);
+      },
+    )
+
+    .command(
+      "fetch <txid>",
+      "Fetch an existing index at the given URL directly.",
+      (yargs) => yargs.positional(txIdArg.id, txIdArg.opts),
+      async (args) => {
+        await commands.fetch(args.txid!);
+      },
+    )
+
+    .command(
+      "set-contract <contract>",
+      "Set default contract.",
+      (yargs) => yargs.option(contractIdArg.id, { ...contractIdArg.opts, demandOption: true }),
+      (args) => {
+        setConfig({
+          contract: args.contract,
+        });
+      },
+    )
+
+    .command(
+      "config",
+      "Show default configurations.",
+      (yargs) => yargs,
+      () => {
+        const cfg = getConfig();
+        logger.info("Contract: ", cfg.contract ?? "not set.");
+      },
+    )
+
+    .command(
+      "list",
+      "List all local knowledge.",
+      (yargs) => yargs,
+      () => {
+        commands.list();
+      },
+    )
+
+    .command(
+      "stop",
+      "Stop serving knowledge.",
+      (yargs) => yargs,
+      async () => {
+        await commands.stop();
+      },
+    )
+
+    .demandCommand(1)
+    .parse();
+}

src/commands/fetch.ts

@@ -5,8 +5,18 @@ import constants from "../constants";
  * Fetch an existing unbundled data on Arweave, i.e. a zipped Rocksdb folder
  * that is bundled & uploaded to Arweave.
  *
- * @param contractId contract ID to download
+ * Note that this command takes in the txID, not a contract ID! When the downloaded
+ * knowledge is unzipped, it might have a different name; you can see it with the command:
+ *
+ * ```sh
+ * dria list
+ * ```
+ *
+ * Or, you can see the knowledge ID in the console, while the unzip operations
+ * prints its logs.
+ *
+ * @param txId Arweave txID to download
  */
-export default async function cmdFetch(contractId: string) {
-  await downloadAndUnzip(contractId, constants.DRIA.DATA);
+export default async function cmdFetch(txId: string) {
+  await downloadAndUnzip(txId, constants.DRIA.DATA);
 }

src/common/download.ts

@@ -54,13 +54,19 @@ export async function downloadAndUnzip(txId: string, outDir: string) {
     });
   });
 
-  // unzip to out directory
-  createReadStream(tmpPath)
-    .pipe(unzipper.Extract({ path: outDir, verbose: true }))
-    .on("close", () => {
-      logger.info("Knowledge extracted at", outDir);
-      logger.info("Cleaning up zip artifacts.");
-      rmSync(tmpPath);
-      logger.info("Done.");
-    });
+  await new Promise((resolve, reject) => {
+    createReadStream(tmpPath)
+      // unzips to out directory
+      .pipe(unzipper.Extract({ path: outDir, verbose: process.env.NODE_ENV !== "test" }))
+      .on("error", (err) => {
+        reject(err);
+      })
+      .on("close", () => {
+        logger.info("Knowledge extracted at", outDir);
+        logger.info("Cleaning up zip artifacts.");
+        rmSync(tmpPath);
+        logger.info("Done.");
+        resolve(true);
+      });
+  });
 }

src/common/index.ts

@@ -1,7 +1,6 @@
 import Docker from "dockerode";
 import loglevel from "loglevel";
 import constants from "../constants";
-import { existsSync, mkdirSync } from "fs";
 
 export * from "./download";
 export * from "./image";

src/configurations/index.ts

@@ -15,7 +15,10 @@ const CONFIG_PATH = constants.DRIA.CONFIG;
 
 /**
  * Returns the active Dria configurations.
- * @returns config
+ *
+ * If no existing config is found, writes the default configuration to the config destination and returns it.
+ *
+ * @returns Dria configuration
  */
 export function getConfig(): DriaCLIConfig {
   if (existsSync(CONFIG_PATH)) {