feat: DocKit restore elasticsearch/opensearch index from json/csv file (

#138) feat: DocKit restore elasticsearch/opensearch index from json file - [x] restore index from JSON - [x] restore index from CSV - [x] The user should only be allowed to select JSON/CSV file - [x] validate input index, display warn popup when input index already exists, only perform ingest when the user confirms to overwrite - [x] display success/error messages for restore result - [x] show progress bar for ingestion progress Refs: #23 --------- Signed-off-by: seven <[email protected]>
geek-fun · Nov 23, 2024 · eb86e01 · eb86e01
1 parent 0675e6e
commit eb86e01
Show file tree

Hide file tree

Showing 6 changed files with 493 additions and 33 deletions.
diff --git a/src/lang/enUS.ts b/src/lang/enUS.ts
@@ -147,6 +147,7 @@ export const enUS = {
     openSuccess: 'Opened successfully',
     switchSuccess: 'Switched successfully',
     overwriteFile: 'File already exists, do you want to overwrite it?',
+    overwriteIndex: 'Index already exists, confirm to overwrite?',
   },
   editor: {
     establishedRequired: 'Select a DB instance before execute actions',
@@ -177,6 +178,7 @@ export const enUS = {
     restore: 'Restore',
     restoreSourceDesc: 'Click or drag a file to this area to upload your JSON/CSV file',
     backupToFileSuccess: 'Successfully backed up to file',
+    restoreFromFileSuccess: 'Successfully restored from file',
     backupForm: {
       connection: 'Connection',
       index: 'Index',

diff --git a/src/lang/zhCN.ts b/src/lang/zhCN.ts
@@ -147,6 +147,7 @@ export const zhCN = {
     openSuccess: '开启成功',
     switchSuccess: '切换成功',
     overwriteFile: '文件已存在，确定覆盖？',
+    overwriteIndex: '索引已存在，确定覆盖？',
   },
   editor: {
     establishedRequired: '请选择执行操作的数据库实例',
@@ -178,6 +179,7 @@ export const zhCN = {
     restore: '恢复',
     restoreSourceDesc: '点击或拖动文件到此区域上传您的 JSON/CSV 文件',
     backupToFileSuccess: '成功备份到文件',
+    restoreFromFileSuccess: '成功从文件恢复',
     backupForm: {
       connection: '选择连接',
       index: '选择索引',

diff --git a/src/store/backupRestoreStore.ts b/src/store/backupRestoreStore.ts
@@ -6,24 +6,34 @@ import { get } from 'lodash';
 import { Connection } from './connectionStore.ts';
 import { loadHttpClient, sourceFileApi } from '../datasources';
 
-export type typeBackupInput = {
+export type BackupInput = {
   connection: Connection;
   index: string;
   backupFolder: string;
   backupFileName: string;
   backupFileType: string;
 };
 
+export type RestoreInput = {
+  connection: Connection;
+  index: string;
+  restoreFile: string;
+};
+
 export const useBackupRestoreStore = defineStore('backupRestoreStore', {
   state(): {
     folderPath: string;
     fileName: string;
     backupProgress: { complete: number; total: number } | null;
+    restoreProgress: { complete: number; total: number } | null;
+    restoreFile: string;
   } {
     return {
       folderPath: '',
       fileName: '',
       backupProgress: null,
+      restoreProgress: null,
+      restoreFile: '',
     };
   },
   persist: true,
@@ -39,7 +49,26 @@ export const useBackupRestoreStore = defineStore('backupRestoreStore', {
         );
       }
     },
-    async checkFileExist(input: Omit<typeBackupInput, 'connection'>) {
+    async selectFile() {
+      try {
+        this.restoreFile = (await open({
+          multiple: false,
+          directory: false,
+          filters: [
+            {
+              name: 'Backup/Restore Files',
+              extensions: ['csv', 'json'],
+            },
+          ],
+        })) as string;
+      } catch (error) {
+        throw new CustomError(
+          get(error, 'status', 500),
+          get(error, 'details', get(error, 'message', '')),
+        );
+      }
+    },
+    async checkFileExist(input: Omit<BackupInput, 'connection'>) {
       const filePath = `/${input.backupFolder}/${input.backupFileName}.${input.backupFileType}`;
       try {
         return await sourceFileApi.exists(filePath);
@@ -50,7 +79,7 @@ export const useBackupRestoreStore = defineStore('backupRestoreStore', {
         );
       }
     },
-    async backupToFile(input: typeBackupInput) {
+    async backupToFile(input: BackupInput) {
       const client = loadHttpClient(input.connection);
       const filePath = `${input.backupFolder}/${input.backupFileName}.${input.backupFileType}`;
       let searchAfter: any[] | undefined = undefined;
@@ -111,9 +140,105 @@ export const useBackupRestoreStore = defineStore('backupRestoreStore', {
         );
       }
     },
+    async restoreFromFile(input: RestoreInput) {
+      const fileType = input.restoreFile.split('.').pop();
+      const client = loadHttpClient(input.connection);
+      const bulkSize = 1000;
+      let data: string;
+      try {
+        data = await sourceFileApi.readFile(input.restoreFile);
+      } catch (error) {
+        throw new CustomError(
+          get(error, 'status', 500),
+          get(error, 'details', get(error, 'message', '')),
+        );
+      }
+
+      try {
+        if (fileType === 'json') {
+          const hits: Array<{
+            _index: string;
+            _id: string;
+            _score: number;
+            _source: unknown;
+          }> = JSON.parse(data);
+          this.restoreProgress = {
+            complete: 0,
+            total: hits.length,
+          };
+          for (let i = 0; i < hits.length; i += bulkSize) {
+            const bulkData = hits
+              .slice(i, i + bulkSize)
+              .flatMap(hit => [{ index: { _index: input.index, _id: hit._id } }, hit._source])
+              .map(item => JSON.stringify(item));
+
+            await bulkRequest(client, bulkData);
+
+            this.restoreProgress.complete += bulkData.length / 2;
+          }
+        } else if (fileType === 'csv') {
+          const lines = data.split('\r\n');
+          const headers = lines[0].split(',');
+          this.restoreProgress = {
+            complete: 0,
+            total: lines.length - 1,
+          };
+
+          for (let i = 1; i < lines.length; i += bulkSize) {
+            const bulkData = lines
+              .slice(i, i + bulkSize)
+              .flatMap(line => {
+                const values = line.split(',');
+                const body = headers.reduce(
+                  (acc, header, index) => {
+                    let value = values[index];
+                    try {
+                      value = JSON.parse(value);
+                    } catch (e) {
+                      // value is not a JSON string, keep it as is
+                    }
+                    acc[header] = value;
+                    return acc;
+                  },
+                  {} as { [key: string]: unknown },
+                );
+
+                return [{ index: { _index: input.index } }, body];
+              })
+              .map(item => JSON.stringify(item));
+
+            await bulkRequest(client, bulkData);
+
+            this.restoreProgress.complete += bulkData.length / 2;
+          }
+        } else {
+          throw new CustomError(400, 'Unsupported file type');
+        }
+      } catch (error) {
+        throw new CustomError(
+          get(error, 'status', 500),
+          get(error, 'details', get(error, 'message', '')),
+        );
+      }
+    },
   },
 });
 
+const bulkRequest = async (client: { post: Function }, bulkData: Array<unknown>) => {
+  const response = await client.post(`/_bulk`, undefined, bulkData.join('\r\n') + '\r\n');
+
+  if (response.status && response.status !== 200) {
+    throw new CustomError(
+      response.status,
+      get(
+        response,
+        'details',
+        get(response, 'message', JSON.stringify(get(response, 'error.root_cause', ''))),
+      ),
+    );
+  }
+};
+
 const buildCsvHeaders = ({
   mappings,
 }: {

diff --git a/src/views/backup-restore/components/backup.vue b/src/views/backup-restore/components/backup.vue
@@ -6,6 +6,7 @@
       label-width="100"
       :model="backupFormData"
       :rules="backupFormRules"
+      style="width: 100%"
     >
       <div class="backup-form-container">
         <n-card title="Source Data">
@@ -116,7 +117,7 @@
 import { FormRules } from 'naive-ui';
 import { DocumentExport, FolderDetails, ZoomArea } from '@vicons/carbon';
 import { storeToRefs } from 'pinia';
-import { typeBackupInput, useBackupRestoreStore, useConnectionStore } from '../../../store';
+import { BackupInput, useBackupRestoreStore, useConnectionStore } from '../../../store';
 import { CustomError, inputProps } from '../../../common';
 import { useLang } from '../../../lang';
 
@@ -267,7 +268,7 @@ const handleValidate = () => {
       : message.success(lang.t('connection.validationPassed')),
   );
 };
-const saveBackup = async (backupInput: typeBackupInput) => {
+const saveBackup = async (backupInput: BackupInput) => {
   try {
     const filePath = await backupToFile(backupInput);
     message.success(lang.t('backup.backupToFileSuccess') + `: ${filePath}`);