Merge pull request #51 from picosh/am/rsync-updates

Am/rsync updates

.vscode/launch.json

@@ -0,0 +1,30 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug App",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/${input:service}/${input:type}/main.go",
+            "envFile": "${workspaceFolder}/.env"
+        }
+    ],
+    "inputs": [
+        {
+            "id": "service",
+            "type": "promptString",
+            "description": "The service to debug",
+            "default": "pgs"
+        },
+        {
+            "id": "type",
+            "type": "promptString",
+            "description": "The service type to debug",
+            "default": "ssh"
+        }
+    ]
+}

cmd/scripts/webp/webp.go

@@ -9,6 +9,7 @@ import (
 	"github.com/picosh/pico/imgs"
 	"github.com/picosh/pico/shared"
 	"github.com/picosh/pico/shared/storage"
+	"github.com/picosh/pico/wish/send/utils"
 )
 
 func main() {
@@ -42,7 +43,7 @@ func main() {
 			continue
 		}
 
-		reader, err := st.GetFile(bucket, post.Filename)
+		reader, _, _, err := st.GetFile(bucket, post.Filename)
 		if err != nil {
 			cfg.Logger.Infof("file not found %s/%s", post.UserID, post.Filename)
 			continue
@@ -67,7 +68,8 @@ func main() {
 		_, err = st.PutFile(
 			bucket,
 			fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename)),
-			storage.NopReaderAtCloser(webpReader),
+			utils.NopReaderAtCloser(webpReader),
+			&utils.FileEntry{},
 		)
 		if err != nil {
 			cfg.Logger.Error(err)

db/postgres/storage.go

@@ -9,11 +9,12 @@ import (
 	"strings"
 	"time"
 
+	"slices"
+
 	_ "github.com/lib/pq"
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/shared"
 	"go.uber.org/zap"
-	"golang.org/x/exp/slices"
 )
 
 var PAGER_SIZE = 15

feeds/scp_hooks.go

@@ -5,11 +5,12 @@ import (
 	"strings"
 	"time"
 
+	"slices"
+
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/filehandlers"
 	"github.com/picosh/pico/imgs"
 	"github.com/picosh/pico/shared"
-	"golang.org/x/exp/slices"
 )
 
 type FeedHooks struct {

filehandlers/assets/asset.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/picosh/pico/shared"
-	"github.com/picosh/pico/shared/storage"
+	"github.com/picosh/pico/wish/send/utils"
 )
 
 func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) {
@@ -72,16 +72,19 @@ func (h *UploadAssetHandler) writeAsset(data *FileData) error {
 		}
 	} else {
 		reader := bytes.NewReader(data.Text)
+
 		h.Cfg.Logger.Infof(
 			"(%s) uploading to (bucket: %s) (%s)",
 			data.User.Name,
 			data.Bucket.Name,
 			assetFilename,
 		)
+
 		_, err := h.Storage.PutFile(
 			data.Bucket,
 			assetFilename,
-			storage.NopReaderAtCloser(reader),
+			utils.NopReaderAtCloser(reader),
+			data.FileEntry,
 		)
 		if err != nil {
 			return err

filehandlers/assets/handler.go

@@ -15,6 +15,7 @@ import (
 	"github.com/picosh/pico/shared/storage"
 	"github.com/picosh/pico/wish/cms/util"
 	"github.com/picosh/pico/wish/send/utils"
+	"go.uber.org/zap"
 )
 
 type ctxUserKey struct{}
@@ -73,7 +74,11 @@ func NewUploadAssetHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage
 	}
 }
 
-func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) {
+func (h *UploadAssetHandler) GetLogger() *zap.SugaredLogger {
+	return h.Cfg.Logger
+}
+
+func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
 	user, err := getUser(s)
 	if err != nil {
 		return nil, nil, err
@@ -92,17 +97,20 @@ func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.Fil
 	}
 
 	fname := shared.GetAssetFileName(entry)
-	contents, err := h.Storage.GetFile(bucket, fname)
+	contents, size, modTime, err := h.Storage.GetFile(bucket, fname)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	fileInfo.FSize = size
+	fileInfo.FModTime = modTime
+
 	reader := utils.NewAllReaderAt(contents)
 
 	return fileInfo, reader, nil
 }
 
-func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) {
+func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
 	var fileList []os.FileInfo
 
 	user, err := getUser(s)
@@ -135,7 +143,7 @@ func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool) ([]os
 			cleanFilename += "/"
 		}
 
-		foundList, err := h.Storage.ListFiles(bucket, cleanFilename, false)
+		foundList, err := h.Storage.ListFiles(bucket, cleanFilename, recursive)
 		if err != nil {
 			return fileList, err
 		}

filehandlers/imgs/handler.go

@@ -9,14 +9,16 @@ import (
 	"path/filepath"
 	"time"
 
+	"slices"
+
 	"github.com/charmbracelet/ssh"
 	exifremove "github.com/neurosnap/go-exif-remove"
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/shared"
 	"github.com/picosh/pico/shared/storage"
 	"github.com/picosh/pico/wish/cms/util"
 	"github.com/picosh/pico/wish/send/utils"
-	"golang.org/x/exp/slices"
+	"go.uber.org/zap"
 )
 
 var maxSize = 1 * shared.GB
@@ -72,7 +74,11 @@ func (h *UploadImgHandler) removePost(data *PostMetaData) error {
 	return nil
 }
 
-func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) {
+func (h *UploadImgHandler) GetLogger() *zap.SugaredLogger {
+	return h.Cfg.Logger
+}
+
+func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
 	user, err := getUser(s)
 	if err != nil {
 		return nil, nil, err
@@ -101,7 +107,7 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI
 		return nil, nil, err
 	}
 
-	contents, err := h.Storage.GetFile(bucket, post.Filename)
+	contents, _, _, err := h.Storage.GetFile(bucket, post.Filename)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -111,7 +117,7 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI
 	return fileInfo, reader, nil
 }
 
-func (h *UploadImgHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) {
+func (h *UploadImgHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
 	var fileList []os.FileInfo
 	user, err := getUser(s)
 	if err != nil {

filehandlers/imgs/img.go

@@ -9,7 +9,7 @@ import (
 	"github.com/charmbracelet/ssh"
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/shared"
-	"github.com/picosh/pico/shared/storage"
+	"github.com/picosh/pico/wish/send/utils"
 )
 
 func (h *UploadImgHandler) validateImg(data *PostMetaData) (bool, error) {
@@ -59,7 +59,8 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
 	fname, err := h.Storage.PutFile(
 		bucket,
 		data.Filename,
-		storage.NopReaderAtCloser(reader),
+		utils.NopReaderAtCloser(reader),
+		&utils.FileEntry{},
 	)
 	if err != nil {
 		return err
@@ -105,7 +106,8 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error {
 	_, err = h.Storage.PutFile(
 		bucket,
 		finalName,
-		storage.NopReaderAtCloser(webpReader),
+		utils.NopReaderAtCloser(webpReader),
+		&utils.FileEntry{},
 	)
 	if err != nil {
 		return err

filehandlers/post_handler.go

@@ -17,6 +17,7 @@ import (
 	"github.com/picosh/pico/shared/storage"
 	"github.com/picosh/pico/wish/cms/util"
 	"github.com/picosh/pico/wish/send/utils"
+	"go.uber.org/zap"
 )
 
 type ctxUserKey struct{}
@@ -61,7 +62,11 @@ func NewScpPostHandler(dbpool db.DB, cfg *shared.ConfigSite, hooks ScpFileHooks,
 	}
 }
 
-func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) {
+func (h *ScpUploadHandler) GetLogger() *zap.SugaredLogger {
+	return h.Cfg.Logger
+}
+
+func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
 	user, err := getUser(s)
 	if err != nil {
 		return nil, nil, err
@@ -84,12 +89,12 @@ func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI
 		FModTime: *post.UpdatedAt,
 	}
 
-	reader := strings.NewReader(post.Text)
+	reader := utils.NopReaderAtCloser(strings.NewReader(post.Text))
 
 	return fileInfo, reader, nil
 }
 
-func (h *ScpUploadHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) {
+func (h *ScpUploadHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) {
 	var fileList []os.FileInfo
 	user, err := getUser(s)
 	if err != nil {

go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/alecthomas/chroma v0.10.0
-	github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8
+	github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/charmbracelet/bubbles v0.16.1
 	github.com/charmbracelet/bubbletea v0.24.2
@@ -33,7 +33,7 @@ require (
 	go.abhg.dev/goldmark/anchor v0.1.1
 	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.15.0
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678
 	gopkg.in/yaml.v2 v2.4.0
 )
 

go.sum

@@ -11,6 +11,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8 h1:gR27C6N8s5b+ciBRymi0zhUx8TylKFO755z6yrBuMiI=
 github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8/go.mod h1:zmqePVIo1hp+WEKxERLLGHJBDOr8/z/T4eFqXgWIw1w=
+github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d h1:NyzUTxebDLLdtNu1gY5hn/amdAEnKG9DOawz82LwNTY=
+github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d/go.mod h1:zmqePVIo1hp+WEKxERLLGHJBDOr8/z/T4eFqXgWIw1w=
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
@@ -271,6 +273,8 @@ golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
 golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w=
+golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=

imgs/api.go

@@ -11,14 +11,16 @@ import (
 
 	_ "net/http/pprof"
 
+	"slices"
+
 	"github.com/gorilla/feeds"
 	gocache "github.com/patrickmn/go-cache"
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/db/postgres"
 	"github.com/picosh/pico/shared"
 	"github.com/picosh/pico/shared/storage"
+	"github.com/picosh/pico/wish/send/utils"
 	"go.uber.org/zap"
-	"golang.org/x/exp/slices"
 )
 
 type PageData struct {
@@ -199,7 +201,7 @@ type ImgHandler struct {
 
 type ImgResizer struct {
 	Key      string
-	contents storage.ReaderAtCloser
+	contents utils.ReaderAtCloser
 	writer   io.Writer
 	Img      *shared.ImgOptimizer
 	Cache    *gocache.Cache
@@ -282,7 +284,7 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) {
 		fname = fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename))
 	}
 
-	contents, err := h.Storage.GetFile(bucket, fname)
+	contents, _, _, err := h.Storage.GetFile(bucket, fname)
 	if err != nil {
 		h.Logger.Infof(
 			"file not found %s/%s in storage (bucket: %s, name: %s)",

lists/api.go

@@ -10,14 +10,15 @@ import (
 	"strconv"
 	"time"
 
+	"slices"
+
 	"github.com/gorilla/feeds"
 	gocache "github.com/patrickmn/go-cache"
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/db/postgres"
 	"github.com/picosh/pico/imgs"
 	"github.com/picosh/pico/shared"
 	"github.com/picosh/pico/shared/storage"
-	"golang.org/x/exp/slices"
 )
 
 type PostItemData struct {

lists/scp_hooks.go

@@ -4,11 +4,12 @@ import (
 	"fmt"
 	"strings"
 
+	"slices"
+
 	"github.com/picosh/pico/db"
 	"github.com/picosh/pico/filehandlers"
 	"github.com/picosh/pico/imgs"
 	"github.com/picosh/pico/shared"
-	"golang.org/x/exp/slices"
 )
 
 type ListHooks struct {