Skip to content

Commit

Permalink
[optimize] simplify CLI code based on ZX
Browse files Browse the repository at this point in the history
  • Loading branch information
TechQuery committed Jun 22, 2024
1 parent c9c63d2 commit 0d3ba7a
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 13 deletions.
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
"koajax": "^1.1.2",
"office-text-extractor": "^3.0.3",
"tsx": "^4.15.7",
"web-utility": "^4.4.0"
"web-utility": "^4.4.0",
"zx": "^8.1.3"
},
"devDependencies": {
"@parcel/config-default": "~2.12.0",
Expand Down
38 changes: 38 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 2 additions & 8 deletions tool/core.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { spawn } from "@tech_query/node-toolkit";
import { readFile, rename, writeFile } from "fs/promises";
import { basename, dirname, extname, join } from "path";

Expand All @@ -9,6 +8,7 @@ import pdf2md from "@opendocsg/pdf2md";
import { turndown } from "edkit";
import { HTTPClient } from "koajax";
import { getTextExtractor } from "office-text-extractor";
import { $ } from "zx/core";

const { FASTGPT_API_HOST, FASTGPT_API_TOKEN, FASTGPT_DATASET_ID } = process.env;

Expand All @@ -26,13 +26,7 @@ const uploader = new HTTPClient({
});

export async function download(source: string, target: string) {
await spawn("curl", [
"--create-dirs",
"--output-dir",
dirname(target),
"-O",
source,
]);
await $`curl --create-dirs --output-dir ${dirname(target)} -O ${source}`;

const targetPath = dirname(target),
sourceName = basename(new URL(source).pathname);
Expand Down
6 changes: 2 additions & 4 deletions tool/index.tsx
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#! /usr/bin/env ts-node

import { Command, createCommand } from "commander-jsx";
import { readFile } from "fs/promises";
import { basename, extname, join } from "path";
import { fs } from "zx";

import { download, transform, upload } from "./core";
import { changeExtensionName, makeAbsolutePath } from "./utility";
Expand Down Expand Up @@ -74,9 +74,7 @@ type ArticleMeta = Record<"name" | "description" | "author" | "link", string>;

async function batcher({}, metaFile: string, batchFolder = "downloads") {
const rootFolder = makeAbsolutePath(batchFolder),
metaData = await readFile(makeAbsolutePath(metaFile));

const meta: ArticleMeta[] = JSON.parse(metaData + "");
meta: ArticleMeta[] = await fs.readJSON(makeAbsolutePath(metaFile));

for (const { name, link } of meta)
try {
Expand Down

0 comments on commit 0d3ba7a

Please sign in to comment.