From bc637229b907cd4ee235b7ed094555a24463a18a Mon Sep 17 00:00:00 2001 From: XZB-1248 Date: Tue, 20 Sep 2022 09:57:40 +0800 Subject: [PATCH] optimize: performance of desktop viewer on Windows --- CHANGELOG.md | 14 +++- client/service/desktop/desktop.go | 67 ++++++++++++-------- client/service/desktop/screenshot_others.go | 26 ++++++++ client/service/desktop/screenshot_windows.go | 66 +++++++++++++++++++ go.mod | 4 +- go.sum | 3 + server/handler/desktop/desktop.go | 46 ++++---------- web/src/components/desktop.js | 34 ++++------ 8 files changed, 177 insertions(+), 83 deletions(-) create mode 100644 client/service/desktop/screenshot_others.go create mode 100644 client/service/desktop/screenshot_windows.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e53ce79..501973b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.1.5 + +* Optimize: performance of desktop viewer on Windows. +* Remove: deprecated ioutil package. + +* 优化：Windows下的远程桌面性能表现。 +* 移除：已经废弃的ioutil包。 + + + ## v0.1.4 * Add: desktop viewer (experimental). @@ -46,8 +56,8 @@ ## v0.1.0 -* fix: don't refresh after file upload. -* fix: don't display error when screenshot fails. +* Fix: don't refresh after file upload. +* Fix: don't display error when screenshot fails. * 修复：文件上传成功后文件管理器不会自动刷新。 * 修复：截图失败时不会显示错误提示。 diff --git a/client/service/desktop/desktop.go b/client/service/desktop/desktop.go index 8edd94d..d060a9c 100644 --- a/client/service/desktop/desktop.go +++ b/client/service/desktop/desktop.go @@ -13,6 +13,7 @@ import ( "image" "image/jpeg" "reflect" + "runtime" "sync" "time" "unsafe" @@ -51,20 +52,24 @@ type message struct { // 0: raw image // 1: compressed image (jpeg) +const fpsLimit = 10 const compress = true const blockSize = 64 -const imgQuality = 70 +const displayIndex = 0 +const imageQuality = 70 var lock = &sync.Mutex{} var working = false var sessions = cmap.New() var prevDesktop *image.RGBA +var ErrNoImage = errors.New("no image yet") func init() { go healthCheck() } func worker() { + runtime.LockOSThread() lock.Lock() if working { lock.Unlock() @@ -73,10 +78,14 @@ func worker() { working = true lock.Unlock() var ( + screen screen + bounds image.Rectangle img *image.RGBA err error errors int ) + bounds = screenshot.GetDisplayBounds(displayIndex) + screen.init(displayIndex) for working { if sessions.Count() == 0 { lock.Lock() @@ -84,9 +93,12 @@ func worker() { lock.Unlock() break } - <-time.After(70 * time.Millisecond) - img, err = screenshot.CaptureDisplay(0) + img = image.NewRGBA(bounds) + err = screen.capture(img, bounds) if err != nil { + if err == ErrNoImage { + return + } errors++ if errors > 10 { break @@ -106,6 +118,7 @@ func worker() { return true }) } + <-time.After(time.Second / fpsLimit) } } prevDesktop = nil @@ -115,6 +128,8 @@ func worker() { lock.Lock() working = false lock.Unlock() + screen.release() + runtime.UnlockOSThread() } func quitAll(info string) { @@ -210,8 +225,8 @@ func getImageBlock(img *image.RGBA, rect image.Rectangle, compress bool) []byte Stride: width * 4, Rect: image.Rect(0, 0, width, height), } - writer := new(bytes.Buffer) - jpeg.Encode(writer, subImg, &jpeg.Options{Quality: imgQuality}) + writer := &bytes.Buffer{} + jpeg.Encode(writer, subImg, &jpeg.Options{Quality: imageQuality}) return writer.Bytes() } @@ -219,7 +234,17 @@ func getDiff(img, prev *image.RGBA) []image.Rectangle { imgWidth := img.Rect.Dx() imgHeight := img.Rect.Dy() result := make([]image.Rectangle, 0) - for y := 0; y < imgHeight; y += blockSize { + for y := 0; y < imgHeight; y += blockSize * 2 { + height := utils.If(y+blockSize > imgHeight, imgHeight-y, blockSize) + for x := 0; x < imgWidth; x += blockSize { + width := utils.If(x+blockSize > imgWidth, imgWidth-x, blockSize) + rect := image.Rect(x, y, x+width, y+height) + if isDiff(img, prev, rect) { + result = append(result, rect) + } + } + } + for y := blockSize; y < imgHeight; y += blockSize * 2 { height := utils.If(y+blockSize > imgHeight, imgHeight-y, blockSize) for x := 0; x < imgWidth; x += blockSize { width := utils.If(x+blockSize > imgWidth, imgWidth-x, blockSize) @@ -249,25 +274,13 @@ func isDiff(img, prev *image.RGBA, rect image.Rectangle) bool { if imgHeader.Len < end || prevHeader.Len < end { return true } - if rectWidth%2 == 0 { - for y := rect.Min.Y; y < rect.Max.Y; y++ { - cursor := uintptr((y*imgWidth + rect.Min.X) * 4) - for x := 0; x < rectWidth; x += 2 { - if *(*uint64)(unsafe.Pointer(imgPtr + cursor)) != *(*uint64)(unsafe.Pointer(prevPtr + cursor)) { - return true - } - cursor += 8 - } - } - } else { - for y := rect.Min.Y; y < rect.Max.Y; y++ { - cursor := uintptr((y*imgWidth + rect.Min.X) * 4) - for x := 0; x < rectWidth; x++ { - if *(*uint32)(unsafe.Pointer(imgPtr + cursor)) != *(*uint32)(unsafe.Pointer(prevPtr + cursor)) { - return true - } - cursor += 4 + for y := rect.Min.Y; y < rect.Max.Y; y += 2 { + cursor := uintptr((y*imgWidth + rect.Min.X) * 4) + for x := 0; x < rectWidth; x += 4 { + if *(*uint64)(unsafe.Pointer(imgPtr + cursor)) != *(*uint64)(unsafe.Pointer(prevPtr + cursor)) { + return true } + cursor += 16 } } return false @@ -287,7 +300,7 @@ func InitDesktop(pack modules.Packet) error { desktop := &session{ event: pack.Event, rawEvent: rawEvent, - lastPack: time.Now().Unix(), + lastPack: common.Unix, escape: false, channel: make(chan message, 4), lock: &sync.Mutex{}, @@ -332,7 +345,7 @@ func PingDesktop(pack modules.Packet) { return } else { desktop = val.(*session) - desktop.lastPack = time.Now().Unix() + desktop.lastPack = common.Unix } } @@ -371,7 +384,9 @@ func GetDesktop(pack modules.Packet) { desktop = val.(*session) } if !desktop.escape { + lock.Lock() img := splitFullImage(prevDesktop, compress) + lock.Unlock() desktop.lock.Lock() desktop.channel <- message{t: 0, data: &img} desktop.lock.Unlock() diff --git a/client/service/desktop/screenshot_others.go b/client/service/desktop/screenshot_others.go new file mode 100644 index 0000000..84b3111 --- /dev/null +++ b/client/service/desktop/screenshot_others.go @@ -0,0 +1,26 @@ +//go:build !windows +// +build !windows + +package desktop + +import ( + "github.com/kbinani/screenshot" + "image" +) + +type screen struct { + displayIndex int +} + +func (s *screen) init(displayIndex int) { + s.displayIndex = displayIndex +} + +func (s *screen) capture(img *image.RGBA, _ image.Rectangle) error { + var err error + img, err = screenshot.CaptureDisplay(displayIndex) + return err +} + +func (s *screen) release() { +} diff --git a/client/service/desktop/screenshot_windows.go b/client/service/desktop/screenshot_windows.go new file mode 100644 index 0000000..1ccabc0 --- /dev/null +++ b/client/service/desktop/screenshot_windows.go @@ -0,0 +1,66 @@ +//go:build windows +// +build windows + +package desktop + +import ( + "github.com/kirides/screencapture/d3d" + "github.com/kirides/screencapture/screenshot" + "github.com/kirides/screencapture/win" + "image" +) + +type screen struct { + dxgi bool + ddup *d3d.OutputDuplicator + device *d3d.ID3D11Device + deviceCtx *d3d.ID3D11DeviceContext + displayIndex int +} + +func (s *screen) init(displayIndex int) { + var err error + s.displayIndex = displayIndex + s.dxgi = false + return + if win.IsValidDpiAwarenessContext(win.DpiAwarenessContextPerMonitorAwareV2) { + _, err = win.SetThreadDpiAwarenessContext(win.DpiAwarenessContextPerMonitorAwareV2) + s.dxgi = err == nil + } + if s.dxgi { + s.device, s.deviceCtx, err = d3d.NewD3D11Device() + s.ddup, err = d3d.NewIDXGIOutputDuplication(s.device, s.deviceCtx, uint(displayIndex)) + if err != nil { + s.dxgi = false + s.device.Release() + s.deviceCtx.Release() + } + } +} + +func (s *screen) capture(img *image.RGBA, bounds image.Rectangle) error { + var err error + if s.dxgi { + err = s.ddup.GetImage(img, 100) + if err != nil { + if err == d3d.ErrNoImageYet { + return ErrNoImage + } + return err + } + return nil + } + return screenshot.CaptureImg(img, 0, 0, bounds.Dx(), bounds.Dy()) +} + +func (s *screen) release() { + if s.ddup != nil { + s.ddup.Release() + } + if s.device != nil { + s.device.Release() + } + if s.deviceCtx != nil { + s.deviceCtx.Release() + } +} diff --git a/go.mod b/go.mod index ef26b61..b01293a 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,10 @@ require ( github.com/shirou/gopsutil/v3 v3.22.2 ) +require github.com/kirides/screencapture v0.0.0-20211101142135-282f3f7e0f33 // indirect + require ( - github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect + github.com/gen2brain/shm v0.0.0-20210511105953-083dbc7d9d83 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.13.0 // indirect diff --git a/go.sum b/go.sum index f4a47d4..cc82e7e 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 h1:Y5Q2mEwfzjMt5+3u70Gtw93ZOu2UuPeeeTBDntF7FoY= github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo= +github.com/gen2brain/shm v0.0.0-20210511105953-083dbc7d9d83/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo= github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0= github.com/gin-contrib/pprof v1.3.0/go.mod h1:waMjT1H9b179t3CxuG1cV3DHpga6ybizwfBaM5OXaB0= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -50,6 +51,8 @@ github.com/kataras/pio v0.0.10 h1:b0qtPUqOpM2O+bqa5wr2O6dN4cQNwSmFd6HQqgVae0g= github.com/kataras/pio v0.0.10/go.mod h1:gS3ui9xSD+lAUpbYnjOGiQyY7sUMJO+EHpiRzhtZ5no= github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329 h1:qq2nCpSrXrmvDGRxW0ruW9BVEV1CN2a9YDOExdt+U0o= github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329/go.mod h1:2VPVQDR4wO7KXHwP+DAypEy67rXf+okUx2zjgpCxZw4= +github.com/kirides/screencapture v0.0.0-20211101142135-282f3f7e0f33 h1:DXPbp2f7LBZzkWnpPGUG+H1+82++20vNIpalTamWv6E= +github.com/kirides/screencapture v0.0.0-20211101142135-282f3f7e0f33/go.mod h1:fMSGsolzmMhah/U24dXBHSf/6Ue/mXVSIb4wRU7U4Ts= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= diff --git a/server/handler/desktop/desktop.go b/server/handler/desktop/desktop.go index 902ad2d..7d5c8ea 100644 --- a/server/handler/desktop/desktop.go +++ b/server/handler/desktop/desktop.go @@ -67,6 +67,15 @@ func InitDesktop(ctx *gin.Context) { // device need to send a packet to browser func desktopEventWrapper(desktop *desktop) common.EventCallback { return func(pack modules.Packet, device *melody.Session) { + if len(pack.Act) == 0 { + if pack.Data == nil { + return + } + if data, ok := pack.Data[`data`]; ok { + desktop.targetConn.WriteBinary(*data.(*[]byte)) + } + return + } if pack.Act == `initDesktop` { if pack.Code != 0 { msg := `${i18n|desktopSessionCreationFailed}` @@ -91,14 +100,6 @@ func desktopEventWrapper(desktop *desktop) common.EventCallback { desktop.targetConn.Close() return } - if len(pack.Act) == 0 { - if pack.Data == nil { - return - } - if data, ok := pack.Data[`data`]; ok { - desktop.targetConn.WriteBinary(*data.(*[]byte)) - } - } } } @@ -157,45 +158,24 @@ func onDesktopMessage(session *melody.Session, data []byte) { } desktop := val.(*desktop) session.Set(`LastPack`, common.Unix) - if pack.Act == `inputDesktop` { - if pack.Data == nil { - return - } - if input, ok := pack.Data[`input`]; ok { - common.SendPack(modules.Packet{Act: `inputDesktop`, Data: gin.H{ - `input`: input, - `desktop`: desktop.uuid, - }, Event: desktop.event}, desktop.deviceConn) - } + if pack.Act == `pingDesktop` { + common.SendPack(modules.Packet{Act: `pingDesktop`, Data: gin.H{ + `desktop`: desktop.uuid, + }, Event: desktop.event}, desktop.deviceConn) return } if pack.Act == `killDesktop` { - if pack.Data == nil { - return - } common.SendPack(modules.Packet{Act: `killDesktop`, Data: gin.H{ `desktop`: desktop.uuid, }, Event: desktop.event}, desktop.deviceConn) return } if pack.Act == `getDesktop` { - if pack.Data == nil { - return - } common.SendPack(modules.Packet{Act: `getDesktop`, Data: gin.H{ `desktop`: desktop.uuid, }, Event: desktop.event}, desktop.deviceConn) return } - if pack.Act == `ping` { - if pack.Data == nil { - return - } - common.SendPack(modules.Packet{Act: `pingDesktop`, Data: gin.H{ - `desktop`: desktop.uuid, - }, Event: desktop.event}, desktop.deviceConn) - return - } session.Close() } diff --git a/web/src/components/desktop.js b/web/src/components/desktop.js index f19f7c3..a8436da 100644 --- a/web/src/components/desktop.js +++ b/web/src/components/desktop.js @@ -81,7 +81,7 @@ function ScreenModal(props) { if (ticks > 10 && conn) { ticks = 0; ws.send(encrypt({ - act: 'ping' + act: 'pingDesktop' }, secret)); } }, 1000); @@ -129,38 +129,30 @@ function ScreenModal(props) { canvas.height = dv.getUint16(3, false); return; } + if (op === 0) frames++; bytes += ab.byteLength; - if (op === 0) { // 0 means this is first part of a frame. - frames++; - } let offset = 1; while (offset < ab.byteLength) { - // let it = dv.getUint16(offset + 0, false); // image type - // let bw = dv.getUint16(offset + 8, false); // image block width - // let bh = dv.getUint16(offset + 10, false); // image block height - let len = dv.getUint16(offset + 2, false); // image block length - updateImage(ab.slice(offset, offset + len + 12), canvasCtx); - offset += len + 12; + let it = dv.getUint16(offset + 0, false); // image type + let il = dv.getUint16(offset + 2, false); // image length + let dx = dv.getUint16(offset + 4, false); // image block x + let dy = dv.getUint16(offset + 6, false); // image block y + let bw = dv.getUint16(offset + 8, false); // image block width + let bh = dv.getUint16(offset + 10, false); // image block height + offset += 12; + updateImage(ab.slice(offset, offset + il), it, dx, dy, bw, bh, canvasCtx); + offset += il; } dv = null; } - function updateImage(ab, canvasCtx) { - let dv = new DataView(ab); - // let bl = dv.getUint16(2, false); // block length without header. - let it = dv.getUint16(0, false); // image type: 0: raw, 1: jpg. - let dx = dv.getUint16(4, false); - let dy = dv.getUint16(6, false); - let bw = dv.getUint16(8, false); - let bh = dv.getUint16(10, false); - ab = ab.slice(12); + function updateImage(ab, it, dx, dy, bw, bh, canvasCtx) { if (it === 0) { canvasCtx.putImageData(new ImageData(new Uint8ClampedArray(ab), bw, bh), dx, dy, 0, 0, bw, bh); - dv = null; } else { createImageBitmap(new Blob([ab]), 0, 0, bw, bh) .then((ib) => { canvasCtx.drawImage(ib, 0, 0, bw, bh, dx, dy, bw, bh); - }).finally(() => dv = null); + }); } } function handleJSON(ab) {