upload progress enhancemant

README.md

@@ -71,11 +71,11 @@ If you encounter the can't open problem, please refer to [Open Mac App from an u
 - [x] [1.1.0]Delete object[⭐️⭐️⭐️⭐️⭐️]
 - [x] [1.1.0]Filter object with prefix[⭐️⭐️⭐️⭐️]
 - [x] [1.3.1]Multiple files upload[⭐️⭐️⭐️⭐️]
+- [x] [1.3.2]Uploading progress[⭐️⭐️⭐️]
 - [ ] Batch downloads[⭐️⭐️⭐️]
 - [ ] Download a floder[⭐️⭐️⭐️]
 - [ ] Upload a folder[⭐️⭐️⭐️]
 - [ ] Downloading progress[⭐️⭐️⭐️]
-- [ ] Uploading progress[⭐️⭐️⭐️]
 - [ ] Object icons beautification[⭐️⭐️⭐️]
 - [ ] Preview object[⭐️⭐️⭐️]
 - [ ] Mouse double-click optimization[⭐️⭐️]

README.zh-CN.md

@@ -70,11 +70,11 @@ Ceph Desktop Manager是开源的，安全的应用程序，但由于苹果的严
 - [x] [1.1.0]删除对象[⭐️⭐️⭐️⭐️⭐️]
 - [x] [1.1.0]通过前缀过滤[⭐️⭐️⭐️⭐️]
 - [x] [1.3.1]支持多文件上传[⭐️⭐️⭐️⭐️]
+- [x] [1.3.2]上传进度条[⭐️⭐️⭐️]
 - [ ] 支持批量下载[⭐️⭐️⭐️]
 - [ ] 上传一个文件夹[⭐️⭐️⭐️]
 - [ ] 下载一个文件夹[⭐️⭐️⭐️]
 - [ ] 下载进度条[⭐️⭐️⭐️]
-- [ ] 上传进度条[⭐️⭐️⭐️]
 - [ ] 对象列表图标美化[⭐️⭐️⭐️]
 - [ ] 预览对象[⭐️⭐️⭐️]
 - [ ] 鼠标双击优化[⭐️⭐️]

backend/app.go

@@ -128,6 +128,7 @@ func allBinds(app *App, extraBinds []Bind) []interface{} {
 
 func (a *App) onStart(ctx context.Context, binds ...Bind) error {
 	a.Ctx = ctx
+	a.Service.Ctx = ctx
 	//init directory for app
 	err := a.initDirectoryStructure()
 	if err != nil {

backend/base/service.go

@@ -20,6 +20,7 @@ type Service struct {
 	Log         *zap.SugaredLogger
 	S3ClientMap map[string]*s3.Client
 	DbClient    *sql.DB
+	Ctx         context.Context
 }
 
 type Paths struct {

backend/models/response.go

@@ -76,3 +76,8 @@ type PutMultipartUploadRes struct {
 	ETag       string `json:"eTag"`
 	PartNumber int32  `json:"partNumber"`
 }
+
+type UploadDetail struct {
+	FileNameKey string `json:"fileNameKey"`
+	PartSize    int    `json:"partSize"`
+}

backend/resource/constant.go

@@ -25,4 +25,5 @@ const (
 const (
 	CHECK_UPGRADE    = "CHECK_UPGRADE"
 	UPGRADE_PROGRESS = "UPGRADE_PROGRESS"
+	UPLOAD_PROGRESS  = "UPLOAD_PROGRESS"
 )

backend/service/object.go

@@ -2,6 +2,9 @@ package service
 
 import (
 	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
 	"mime/multipart"
 	"net/url"
 	"os"
@@ -16,10 +19,20 @@ import (
 	"github.com/cjhuaxin/CephDesktopManager/backend/base"
 	"github.com/cjhuaxin/CephDesktopManager/backend/errcode"
 	"github.com/cjhuaxin/CephDesktopManager/backend/models"
+	"github.com/cjhuaxin/CephDesktopManager/backend/resource"
 	"github.com/cjhuaxin/CephDesktopManager/backend/util"
 	"github.com/rs/xid"
+	"github.com/wailsapp/wails/v2/pkg/runtime"
 )
 
+type progressReader struct {
+	io.ReadSeeker
+	fileNameKey string
+	sent        int
+	atEOF       int8
+	ctx         context.Context
+}
+
 type Object struct {
 	*base.Service
 }
@@ -185,9 +198,10 @@ func (s *Object) CreateMultipartUpload(req *models.CreateMultipartUploadReq) *mo
 }
 
 func (s *Object) PutMultipartUpload(fileHeader *multipart.FileHeader, params url.Values) *models.BaseResponse {
+	fileName := params.Get("fileName")
 	file, err := fileHeader.Open()
 	if err != nil {
-		s.Log.Errorf("open file[%s] failed: %v", fileHeader.Filename, err)
+		s.Log.Errorf("open file[%s] failed: %v", fileName, err)
 		return s.BuildFailed(errcode.UnExpectedErr, err.Error())
 	}
 	connectionId := params.Get("connectionId")
@@ -205,12 +219,18 @@ func (s *Object) PutMultipartUpload(fileHeader *multipart.FileHeader, params url
 		s.Log.Errorf("get connection[%s] failed: %v", connectionId, err)
 		return s.BuildFailed(errcode.UnExpectedErr, err.Error())
 	}
+
+	pr := &progressReader{
+		ReadSeeker:  file,
+		fileNameKey: fmt.Sprintf("%x", md5.Sum([]byte(fileName))),
+		ctx:         s.Ctx,
+	}
 	out, err := s3Client.UploadPart(context.TODO(), &s3.UploadPartInput{
 		Bucket:     aws.String(bucket),
 		Key:        aws.String(key),
 		UploadId:   aws.String(uploadId),
 		PartNumber: int32(partNumber),
-		Body:       file,
+		Body:       pr,
 	})
 	if err != nil {
 		s.Log.Errorf("upload part [%s|%s|%d] failed: %v", bucket, key, partNumber, err)
@@ -323,3 +343,25 @@ func (s *Object) makeTargetDirectory(connectionName, bucket, key string) (string
 
 	return file, nil
 }
+
+func (pr *progressReader) Read(p []byte) (int, error) {
+	n, err := pr.ReadSeeker.Read(p)
+	if err != nil && err == io.EOF {
+		pr.atEOF++
+	}
+	if pr.atEOF > 0 {
+		pr.sent += n
+		pr.report(n)
+	}
+
+	return n, err
+}
+
+func (pr *progressReader) report(partSize int) {
+	if pr.sent > 0 {
+		runtime.EventsEmit(pr.ctx, resource.UPLOAD_PROGRESS, &models.UploadDetail{
+			FileNameKey: pr.fileNameKey,
+			PartSize:    partSize,
+		})
+	}
+}

frontend/src/components/UpgradeDialog.tsx

@@ -73,8 +73,6 @@ export default function UpgradeDialog() {
                 let currentVersionArray = currentVersion.current.split(".");
                 let latestVersionArray = body.tag_name.substring(1).split(".");
                 for (let i = 0; i < currentVersionArray.length; i++) {
-                    console.log("Number(currentVersionArray[i]", Number(currentVersionArray[i]));
-                    console.log("Number(latestVersionArray[i]", Number(latestVersionArray[i]));
                     // From left to right, compare the version number sizes in order.
                     if (Number(currentVersionArray[i]) < Number(latestVersionArray[i])) {
                         hasNewVersion.current = true;

frontend/src/components/UploadObject.tsx

@@ -6,10 +6,11 @@ import prettyBytes from 'pretty-bytes';
 import React from "react";
 import sparkMD5 from 'spark-md5';
 import { AbortMultipartUpload, CompleteMultipartUpload, CreateMultipartUpload, PutMultipartUpload } from "../../wailsjs/go/service/Object";
-import { ALERT_TYPE_ERROR, ALERT_TYPE_SUCCESS, TOPIC_ALERT, TOPIC_LIST_OBJECTS, TOPIC_LOADING } from "../constants/Pubsub";
-import { ConnectionDetail } from "../dto/BackendRes";
+import { ALERT_TYPE_ERROR, ALERT_TYPE_SUCCESS, TOPIC_ALERT, TOPIC_LIST_OBJECTS, TOPIC_LOADING, UPLOAD_PROGRESS } from "../constants/Pubsub";
 import axios from 'axios';
 import { models } from '../../wailsjs/go/models';
+import { EventsOff, EventsOn } from '../../wailsjs/runtime/runtime';
+import { UploadDetail } from '../dto/BackendRes';
 
 const columns: GridColDef[] = [
     {
@@ -55,6 +56,7 @@ export default function UploadObject({ bucket, connectionId, prefix, searchKeywo
     const uploadInputRef = React.useRef<HTMLInputElement>(null);
     const preparedUploadFileMap = React.useRef(new Map<string, File>());
     const totalFileSize = React.useRef(0);
+    const currentFileSize = React.useRef(0);
     const apiRef = useGridApiRef();
 
 
@@ -70,8 +72,24 @@ export default function UploadObject({ bucket, connectionId, prefix, searchKeywo
         setWholeProgress(0.1);
         let index = 1;
         let uploadId;
+        // upload progress
+        EventsOn(UPLOAD_PROGRESS, (result: UploadDetail) => {
+            let currentProgress = result.partSize / currentFileSize.current
+            let totalProgress = result.partSize / totalFileSize.current * 100
+            console.log("currentProgress", currentProgress);
+            console.log("totalProgress", totalProgress);
+            console.log(result);
+            // update row progress
+            setRows((prevRows) => {
+                return prevRows.map((row, index) =>
+                    row.id === result.fileNameKey ? { ...row, progress: row.progress + currentProgress } : row,
+                );
+            });
+            setWholeProgress(prefix => prefix + totalProgress);
+        });
         for (const [key, value] of preparedUploadFileMap.current) {
             const objKey = prefix + value.name;
+            currentFileSize.current = value.size;
             try {
                 // Multipart uploads require a minimum size of 5 MB per part.
                 const numberOfParts = Math.ceil(value.size / (1024 * 1024 * 5));
@@ -91,8 +109,6 @@ export default function UploadObject({ bucket, connectionId, prefix, searchKeywo
                 }
                 uploadId = createRes.data.uploadId;
 
-                const uploadPromises = [];
-
                 //each part size,5M
                 let chunk = 5 * 1024 * 1024
                 let start = 0;
@@ -114,42 +130,26 @@ export default function UploadObject({ bucket, connectionId, prefix, searchKeywo
                     data.append('partNumber', i + "");
                     data.append('fileName', value.name);
                     data.append('chunk', value.slice(start, end));
-                    uploadPromises.push(
-                        axios.put("http://localhost:56789/custom/upload", data, {
-                            onUploadProgress: ({ progress, rate }) => {
-                                console.log(`Upload [${(progress! * 100).toFixed(2)}%]: ${(rate! / 1024).toFixed(2)}KB/s`)
+                    await axios.put("http://localhost:56789/custom/upload", data)
+                        .then((res) => {
+                            if (res.data.err_msg != "") {
+                                PubSub.publish(TOPIC_ALERT, {
+                                    alertType: ALERT_TYPE_ERROR,
+                                    message: res.data.err_msg,
+                                })
+                                throw new Error(res.data.err_msg)
+                            } else {
+                                eTags.push({
+                                    part: res.data.data.partNumber,
+                                    value: res.data.data.eTag
+                                })
                             }
+                        }).catch(err => {
+                            console.log("upload error", err);
                         })
-                            .then((res) => {
-                                console.log("upload res", res);
-                                if (res.data.err_msg != "") {
-                                    PubSub.publish(TOPIC_ALERT, {
-                                        alertType: ALERT_TYPE_ERROR,
-                                        message: res.data.err_msg,
-                                    })
-                                    throw new Error(res.data.err_msg)
-                                } else {
-                                    // update row progress
-                                    setRows((prevRows) => {
-                                        return prevRows.map((row, index) =>
-                                            row.id === key ? { ...row, progress: row.progress + (chunkSize / value.size) } : row,
-                                        );
-                                    });
-                                    setWholeProgress(prefix => prefix + progress)
-                                    console.log("put result", res.data);
-                                    eTags.push({
-                                        part: res.data.data.partNumber,
-                                        value: res.data.data.eTag
-                                    })
-                                }
-                            }).catch(err => {
-                                console.log("upload error", err);
-                            })
-                    );
                     start = end;
                 }
 
-                await Promise.all(uploadPromises);
                 if (eTags.length == 0) {
                     PubSub.publish(TOPIC_ALERT, {
                         alertType: ALERT_TYPE_ERROR,
@@ -188,7 +188,6 @@ export default function UploadObject({ bucket, connectionId, prefix, searchKeywo
                         bucket: bucket,
                         key: objKey,
                     })
-                    console.log("uploadId failed", abortRes);
                 }
                 setUplaoding(false)
                 return
@@ -270,6 +269,7 @@ export default function UploadObject({ bucket, connectionId, prefix, searchKeywo
         totalFileSize.current = 0;
 
         preparedUploadFileMap.current.clear();
+        EventsOff(UPLOAD_PROGRESS);
     }
 
     return (

frontend/src/constants/Pubsub.tsx

@@ -17,4 +17,5 @@ export const ALERT_TYPE_ERROR = "error"
 
 //backend Event name
 export const CHECK_UPGRADE = "CHECK_UPGRADE"
-export const UPGRADE_PROGRESS = "UPGRADE_PROGRESS"
+export const UPGRADE_PROGRESS = "UPGRADE_PROGRESS"
+export const UPLOAD_PROGRESS = "UPLOAD_PROGRESS"

frontend/src/dto/BackendRes.tsx

@@ -45,4 +45,9 @@ export interface BucketInfo {
         permission: string;
         displayName: string;
     }[];
+}
+
+export interface UploadDetail {
+    fileNameKey: string;
+    partSize: number;
 }

wails.json

@@ -11,6 +11,6 @@
     "email": "[email protected]"
   },
   "info": {
-    "productVersion": "1.3.1"
+    "productVersion": "1.3.2"
   }
 }