Merge pull request #32 from IdeaLeap/dev

Dev

build

@@ -19,7 +19,7 @@ npm exec prettier -- --loglevel=warn --write .
 
 # make sure that nothing crashes when we require the output CJS or
 # import the output ESM
-# (cd dist && node -e 'require("llm-ops")')
+(cd dist && node -e 'require("llm-ops")')
 (cd dist && node -e 'import("llm-ops")' --input-type=module)
 
 # Path: build-docs

docs/api/llm-ops.md

@@ -60,6 +60,7 @@
 |  [milvusVectorDBSchema](./llm-ops.milvusvectordbschema.md) |  |
 |  [milvusVectorDBSearchSchema](./llm-ops.milvusvectordbsearchschema.md) |  |
 |  [milvusVectorDBUploadSchema](./llm-ops.milvusvectordbuploadschema.md) |  |
+|  [milvusVectorDBUpsertSchema](./llm-ops.milvusvectordbupsertschema.md) |  |
 |  [MultiPromptSchema](./llm-ops.multipromptschema.md) | 多重提示模板数据结构。 |
 |  [polishPromptTemplateSchema](./llm-ops.polishprompttemplateschema.md) |  |
 |  [PromptsSchema](./llm-ops.promptsschema.md) | 主要的模板数据结构。 |
@@ -76,6 +77,7 @@
 |  --- | --- |
 |  [formatPromptTemplate](./llm-ops.formatprompttemplate.md) | <p>格式化提示模板函数。</p><p>此函数的主要目的是为了从提供的<code>prompts</code>对象中提取出一个格式化的提示模板。 这可以是一个单一的模板，也可以是一个模板数组。</p> |
 |  [genMermaid](./llm-ops.genmermaid.md) |  |
+|  [replaceContentWithSequence](./llm-ops.replacecontentwithsequence.md) |  |
 
 ## Type Aliases
 

docs/api/llm-ops.milvusvectordb.delete.md

@@ -0,0 +1,24 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [milvusVectorDB](./llm-ops.milvusvectordb.md) &gt; [delete](./llm-ops.milvusvectordb.delete.md)
+
+## milvusVectorDB.delete() method
+
+**Signature:**
+
+```typescript
+delete(params: {
+        expr: string;
+    }): Promise<import("@zilliz/milvus2-sdk-node").MutationResult>;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  params | { expr: string; } |  |
+
+**Returns:**
+
+Promise&lt;import("@zilliz/milvus2-sdk-node").MutationResult&gt;
+

docs/api/llm-ops.milvusvectordb.md

@@ -29,9 +29,11 @@ export declare class milvusVectorDB
 |  Method | Modifiers | Description |
 |  --- | --- | --- |
 |  [createCollection(params)](./llm-ops.milvusvectordb.createcollection.md) |  |  |
+|  [delete(params)](./llm-ops.milvusvectordb.delete.md) |  |  |
 |  [generatePromptTemplate(params)](./llm-ops.milvusvectordb.generateprompttemplate.md) |  |  |
 |  [generateVector(data)](./llm-ops.milvusvectordb.generatevector.md) |  |  |
 |  [query(params)](./llm-ops.milvusvectordb.query.md) |  |  |
 |  [search(params)](./llm-ops.milvusvectordb.search.md) |  |  |
 |  [upload(params)](./llm-ops.milvusvectordb.upload.md) |  |  |
+|  [upsert(params)](./llm-ops.milvusvectordb.upsert.md) |  |  |
 

docs/api/llm-ops.milvusvectordb.upload.md

@@ -7,7 +7,7 @@
 **Signature:**
 
 ```typescript
-upload(params: milvusVectorDBUploadSchema): Promise<import("@zilliz/milvus2-sdk-node").ResStatus>;
+upload(params: milvusVectorDBUploadSchema): Promise<import("@zilliz/milvus2-sdk-node").MutationResult | import("@zilliz/milvus2-sdk-node").ResStatus>;
 ```
 
 ## Parameters
@@ -18,5 +18,5 @@ upload(params: milvusVectorDBUploadSchema): Promise<import("@zilliz/milvus2-sdk-
 
 **Returns:**
 
-Promise&lt;import("@zilliz/milvus2-sdk-node").ResStatus&gt;
+Promise&lt;import("@zilliz/milvus2-sdk-node").MutationResult \| import("@zilliz/milvus2-sdk-node").ResStatus&gt;
 

docs/api/llm-ops.milvusvectordb.upsert.md

@@ -0,0 +1,22 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [milvusVectorDB](./llm-ops.milvusvectordb.md) &gt; [upsert](./llm-ops.milvusvectordb.upsert.md)
+
+## milvusVectorDB.upsert() method
+
+**Signature:**
+
+```typescript
+upsert(params: milvusVectorDBUpsertSchema): Promise<import("@zilliz/milvus2-sdk-node").MutationResult>;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  params | [milvusVectorDBUpsertSchema](./llm-ops.milvusvectordbupsertschema.md) |  |
+
+**Returns:**
+
+Promise&lt;import("@zilliz/milvus2-sdk-node").MutationResult&gt;
+

docs/api/llm-ops.milvusvectordbupsertschema.data.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [milvusVectorDBUpsertSchema](./llm-ops.milvusvectordbupsertschema.md) &gt; [data](./llm-ops.milvusvectordbupsertschema.data.md)
+
+## milvusVectorDBUpsertSchema.data property
+
+**Signature:**
+
+```typescript
+data?: {
+        [x: string]: any;
+    }[];
+```

docs/api/llm-ops.milvusvectordbupsertschema.fields_data.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [milvusVectorDBUpsertSchema](./llm-ops.milvusvectordbupsertschema.md) &gt; [fields\_data](./llm-ops.milvusvectordbupsertschema.fields_data.md)
+
+## milvusVectorDBUpsertSchema.fields\_data property
+
+**Signature:**
+
+```typescript
+fields_data?: {
+        [x: string]: any;
+    }[];
+```

docs/api/llm-ops.milvusvectordbupsertschema.hash_keys.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [milvusVectorDBUpsertSchema](./llm-ops.milvusvectordbupsertschema.md) &gt; [hash\_keys](./llm-ops.milvusvectordbupsertschema.hash_keys.md)
+
+## milvusVectorDBUpsertSchema.hash\_keys property
+
+**Signature:**
+
+```typescript
+hash_keys?: Number[];
+```

docs/api/llm-ops.milvusvectordbupsertschema.md

@@ -0,0 +1,21 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [milvusVectorDBUpsertSchema](./llm-ops.milvusvectordbupsertschema.md)
+
+## milvusVectorDBUpsertSchema interface
+
+**Signature:**
+
+```typescript
+export interface milvusVectorDBUpsertSchema 
+```
+
+## Properties
+
+|  Property | Modifiers | Type | Description |
+|  --- | --- | --- | --- |
+|  [data?](./llm-ops.milvusvectordbupsertschema.data.md) |  | { \[x: string\]: any; }\[\] | _(Optional)_ |
+|  [fields\_data?](./llm-ops.milvusvectordbupsertschema.fields_data.md) |  | { \[x: string\]: any; }\[\] | _(Optional)_ |
+|  [hash\_keys?](./llm-ops.milvusvectordbupsertschema.hash_keys.md) |  | Number\[\] | _(Optional)_ |
+|  [partition\_name?](./llm-ops.milvusvectordbupsertschema.partition_name.md) |  | string | _(Optional)_ |
+

docs/api/llm-ops.milvusvectordbupsertschema.partition_name.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [milvusVectorDBUpsertSchema](./llm-ops.milvusvectordbupsertschema.md) &gt; [partition\_name](./llm-ops.milvusvectordbupsertschema.partition_name.md)
+
+## milvusVectorDBUpsertSchema.partition\_name property
+
+**Signature:**
+
+```typescript
+partition_name?: string;
+```

docs/api/llm-ops.replacecontentwithsequence.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [llm-ops](./llm-ops.md) &gt; [replaceContentWithSequence](./llm-ops.replacecontentwithsequence.md)
+
+## replaceContentWithSequence variable
+
+**Signature:**
+
+```typescript
+replaceContentWithSequence: (text: string) => string
+```

jest.config.cjs

@@ -18,7 +18,7 @@ module.exports = {
   transformIgnorePatterns: [
     "/node_modules/",
     "node_modules/(?!@ngrx|(?!deck.gl)|ng-dynamic)",
-    "/node_modules/(?!mermaid)",
+    "/node_modules/(?!mermaid-ssr)",
   ],
   setupFiles: ["dotenv/config"],
   testTimeout: 20_000,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-ops",
-  "version": "0.0.6",
+  "version": "0.0.12",
   "description": "高效制作llm workflow的低代码框架",
   "main": "package/index.js",
   "author": "Marlene && IdeaLeap",
@@ -33,10 +33,10 @@
     "publish": "cd dist && npm publish --access public"
   },
   "dependencies": {
-    "@idealeap/pipeline": "^1.1.6",
-    "@zilliz/milvus2-sdk-node": "^2.2.24",
+    "@idealeap/pipeline": "^1.1.7",
+    "@zilliz/milvus2-sdk-node": "^2.3.2",
     "dotenv": "^16.3.1",
-    "mermaid-ssr": "^10.5.0",
+    "mermaid-ssr": "11.46.16",
     "openai": "4.8.0",
     "typescript": "^5.1.6"
   },

package/db/milvus.ts

@@ -14,6 +14,16 @@ export interface milvusVectorDBQuerySchema {
   output_fields: string[];
   limit?: number;
 }
+export interface milvusVectorDBUpsertSchema {
+  partition_name?: string;
+  fields_data?: {
+    [x: string]: any;
+  }[];
+  data?: {
+    [x: string]: any;
+  }[];
+  hash_keys?: Number[];
+}
 export interface milvusVectorDBSearchSchema {
   vector: number[];
   filter?: string;
@@ -61,8 +71,43 @@ export class milvusVectorDB {
       limit: limit || 100,
     });
     console.timeEnd("Query time");
+    // if (query.status.error_code != "Success") {
+    //   console.error(query.status.reason);
+    //   throw new Error(query.status.reason);
+    // }
     return query;
   }
+  async upsert(params: milvusVectorDBUpsertSchema) {
+    const { partition_name, fields_data, data, hash_keys } = params;
+    console.time("Upsert time");
+    const upsert = await this.milvusClient.upsert({
+      collection_name: this.COLLECTION_NAME,
+      partition_name: partition_name || undefined,
+      fields_data: fields_data || undefined,
+      data: data || undefined,
+      hash_keys: hash_keys || undefined,
+    });
+    console.timeEnd("Upsert time");
+    if (upsert.status.error_code != "Success") {
+      console.error(upsert.status.reason);
+      throw new Error(upsert.status.reason);
+    }
+    return upsert;
+  }
+  async delete(params: { expr: string }) {
+    const { expr } = params;
+    console.time("Upsert time");
+    const res = await this.milvusClient.deleteEntities({
+      collection_name: this.COLLECTION_NAME,
+      expr,
+    });
+    console.timeEnd("Upsert time");
+    // if (res.status.error_code != "Success") {
+    //   console.error(res.status.reason);
+    //   throw new Error(res.status.reason);
+    // }
+    return res;
+  }
   async search(params: milvusVectorDBSearchSchema) {
     const { vector, output_fields, limit, consistency_level, filter } = params;
     console.time("Search time");
@@ -81,6 +126,10 @@ export class milvusVectorDB {
       consistency_level: consistency_level || undefined,
     });
     console.timeEnd("Search time");
+    // if (search.status.error_code != "Success") {
+    //   console.error(search.status.reason);
+    //   throw new Error(search.status.reason);
+    // }
     return search;
   }
   async generatePromptTemplate(params: milvusVectorDBPromptTemplateSchema) {
@@ -128,11 +177,16 @@ export class milvusVectorDB {
   async upload(params: milvusVectorDBUploadSchema) {
     const { fields_data, index, partition_name } = params;
     console.time("Upload time");
-    await this.milvusClient.insert({
+    const insertRes = await this.milvusClient.insert({
       collection_name: this.COLLECTION_NAME,
       fields_data: fields_data,
       partition_name: partition_name || undefined,
     });
+    if (insertRes.status.error_code != "Success") {
+      // console.error(insertRes.status.reason);
+      // throw new Error(insertRes.status.reason);
+      return insertRes;
+    }
     await this.milvusClient.createIndex({
       collection_name: this.COLLECTION_NAME,
       field_name: index.field_name,
@@ -149,7 +203,7 @@ export class milvusVectorDB {
       this.llm = new LLM({});
     }
     const vector = await this.llm?.embedding(data);
-    if (!vector?.data[0]?.embedding) {
+    if (!vector || !vector?.data[0] || !vector?.data[0]?.embedding) {
       throw new Error("生成向量失败");
     }
     return vector?.data[0]?.embedding;

package/utils/gen/__tests__/mermaid.test.ts

@@ -1,8 +1,32 @@
 import { genMermaid } from "llm-ops";
-
+const replaceContentWithSequence = (text: string) => {
+  let index = 1;
+  return text.replace(
+    /(\[).*?(\])|(\().*?(\))|(\{).*?(\})|(\").*?(\")|(\|).*?(\|)/g,
+    (match, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10) => {
+      if (p1 && p2) {
+        return `${p1}#${index++}${p2}`;
+      }
+      if (p3 && p4) {
+        return `${p3}#${index++}${p4}`;
+      }
+      if (p5 && p6) {
+        return `${p5}#${index++}${p6}`;
+      }
+      if (p7 && p8) {
+        return `${p7}#${index++}${p8}`;
+      }
+      if (p9 && p10) {
+        return `${p9}#${index++}${p10}`;
+      }
+      return match;
+    },
+  );
+};
 genMermaid({
   request:
-    "在对数字化转型的探讨中，技术、策略和人是三大核心要素。技术为数字化转型提供了可能性，允许企业利用先进的工具和平台去优化、创新和扩展其业务范围。而策略则为数字化转型提供了方向，确保企业的技术部署与其长期的商业目标和愿景相一致。最后，人是数字化转型的中心，因为不论有多先进的技术或策略，如果员工和客户不能或不愿适应新的变革，那么转型就不可能成功。因此，在进行数字化转型时，企业必须确保技术的适用性、策略的明晰性和人的参与性，以确保转型的全面性和成功性。",
+    "这是一个公司介绍的文本，总分布局：VTRON作为一家中国公司成立于1998年，总部位于广东省广州市广州科学城开发区；是专业从事高清晰数字显示系统研发、生产、销售与服务的创新型科技企业，它是全球同行业瞩目的集中信息显示系统产品制造基地，它拥有了全系列产品的国际尖端技术和自主知识产权，它一直保持在同行业技术及产品的领先优势",
 }).then((res) => {
   console.log(res);
+  console.log(replaceContentWithSequence(res));
 });