diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..5d08e6e3 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,30 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Debug App", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/${input:service}/${input:type}/main.go", + "envFile": "${workspaceFolder}/.env" + } + ], + "inputs": [ + { + "id": "service", + "type": "promptString", + "description": "The service to debug", + "default": "pgs" + }, + { + "id": "type", + "type": "promptString", + "description": "The service type to debug", + "default": "ssh" + } + ] +} diff --git a/cmd/scripts/webp/webp.go b/cmd/scripts/webp/webp.go index e1b8f3ab..4eac138a 100644 --- a/cmd/scripts/webp/webp.go +++ b/cmd/scripts/webp/webp.go @@ -9,6 +9,7 @@ import ( "github.com/picosh/pico/imgs" "github.com/picosh/pico/shared" "github.com/picosh/pico/shared/storage" + "github.com/picosh/pico/wish/send/utils" ) func main() { @@ -42,7 +43,7 @@ func main() { continue } - reader, err := st.GetFile(bucket, post.Filename) + reader, _, _, err := st.GetFile(bucket, post.Filename) if err != nil { cfg.Logger.Infof("file not found %s/%s", post.UserID, post.Filename) continue @@ -67,7 +68,8 @@ func main() { _, err = st.PutFile( bucket, fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename)), - storage.NopReaderAtCloser(webpReader), + utils.NopReaderAtCloser(webpReader), + &utils.FileEntry{}, ) if err != nil { cfg.Logger.Error(err) diff --git a/db/postgres/storage.go b/db/postgres/storage.go index 3c9bb004..71f81cb1 100644 --- a/db/postgres/storage.go +++ b/db/postgres/storage.go @@ -9,11 +9,12 @@ import ( "strings" "time" + "slices" + _ "github.com/lib/pq" "github.com/picosh/pico/db" "github.com/picosh/pico/shared" "go.uber.org/zap" - "golang.org/x/exp/slices" ) var PAGER_SIZE = 15 diff --git a/feeds/scp_hooks.go b/feeds/scp_hooks.go index a5c97d9f..f3ae9871 100644 --- a/feeds/scp_hooks.go +++ b/feeds/scp_hooks.go @@ -5,11 +5,12 @@ import ( "strings" "time" + "slices" + "github.com/picosh/pico/db" "github.com/picosh/pico/filehandlers" "github.com/picosh/pico/imgs" "github.com/picosh/pico/shared" - "golang.org/x/exp/slices" ) type FeedHooks struct { diff --git a/filehandlers/assets/asset.go b/filehandlers/assets/asset.go index 1b2af00e..93ddf7ad 100644 --- a/filehandlers/assets/asset.go +++ b/filehandlers/assets/asset.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/picosh/pico/shared" - "github.com/picosh/pico/shared/storage" + "github.com/picosh/pico/wish/send/utils" ) func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) { @@ -72,16 +72,19 @@ func (h *UploadAssetHandler) writeAsset(data *FileData) error { } } else { reader := bytes.NewReader(data.Text) + h.Cfg.Logger.Infof( "(%s) uploading to (bucket: %s) (%s)", data.User.Name, data.Bucket.Name, assetFilename, ) + _, err := h.Storage.PutFile( data.Bucket, assetFilename, - storage.NopReaderAtCloser(reader), + utils.NopReaderAtCloser(reader), + data.FileEntry, ) if err != nil { return err diff --git a/filehandlers/assets/handler.go b/filehandlers/assets/handler.go index 7432047d..24af656b 100644 --- a/filehandlers/assets/handler.go +++ b/filehandlers/assets/handler.go @@ -15,6 +15,7 @@ import ( "github.com/picosh/pico/shared/storage" "github.com/picosh/pico/wish/cms/util" "github.com/picosh/pico/wish/send/utils" + "go.uber.org/zap" ) type ctxUserKey struct{} @@ -73,7 +74,11 @@ func NewUploadAssetHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage } } -func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) { +func (h *UploadAssetHandler) GetLogger() *zap.SugaredLogger { + return h.Cfg.Logger +} + +func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { user, err := getUser(s) if err != nil { return nil, nil, err @@ -92,17 +97,20 @@ func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.Fil } fname := shared.GetAssetFileName(entry) - contents, err := h.Storage.GetFile(bucket, fname) + contents, size, modTime, err := h.Storage.GetFile(bucket, fname) if err != nil { return nil, nil, err } + fileInfo.FSize = size + fileInfo.FModTime = modTime + reader := utils.NewAllReaderAt(contents) return fileInfo, reader, nil } -func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) { +func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { var fileList []os.FileInfo user, err := getUser(s) @@ -135,7 +143,7 @@ func (h *UploadAssetHandler) List(s ssh.Session, fpath string, isDir bool) ([]os cleanFilename += "/" } - foundList, err := h.Storage.ListFiles(bucket, cleanFilename, false) + foundList, err := h.Storage.ListFiles(bucket, cleanFilename, recursive) if err != nil { return fileList, err } diff --git a/filehandlers/imgs/handler.go b/filehandlers/imgs/handler.go index ed97b985..49314a67 100644 --- a/filehandlers/imgs/handler.go +++ b/filehandlers/imgs/handler.go @@ -9,6 +9,8 @@ import ( "path/filepath" "time" + "slices" + "github.com/charmbracelet/ssh" exifremove "github.com/neurosnap/go-exif-remove" "github.com/picosh/pico/db" @@ -16,7 +18,7 @@ import ( "github.com/picosh/pico/shared/storage" "github.com/picosh/pico/wish/cms/util" "github.com/picosh/pico/wish/send/utils" - "golang.org/x/exp/slices" + "go.uber.org/zap" ) var maxSize = 1 * shared.GB @@ -72,7 +74,11 @@ func (h *UploadImgHandler) removePost(data *PostMetaData) error { return nil } -func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) { +func (h *UploadImgHandler) GetLogger() *zap.SugaredLogger { + return h.Cfg.Logger +} + +func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { user, err := getUser(s) if err != nil { return nil, nil, err @@ -101,7 +107,7 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI return nil, nil, err } - contents, err := h.Storage.GetFile(bucket, post.Filename) + contents, _, _, err := h.Storage.GetFile(bucket, post.Filename) if err != nil { return nil, nil, err } @@ -111,7 +117,7 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI return fileInfo, reader, nil } -func (h *UploadImgHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) { +func (h *UploadImgHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { var fileList []os.FileInfo user, err := getUser(s) if err != nil { diff --git a/filehandlers/imgs/img.go b/filehandlers/imgs/img.go index 2ae04210..58c02faa 100644 --- a/filehandlers/imgs/img.go +++ b/filehandlers/imgs/img.go @@ -9,7 +9,7 @@ import ( "github.com/charmbracelet/ssh" "github.com/picosh/pico/db" "github.com/picosh/pico/shared" - "github.com/picosh/pico/shared/storage" + "github.com/picosh/pico/wish/send/utils" ) func (h *UploadImgHandler) validateImg(data *PostMetaData) (bool, error) { @@ -59,7 +59,8 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error { fname, err := h.Storage.PutFile( bucket, data.Filename, - storage.NopReaderAtCloser(reader), + utils.NopReaderAtCloser(reader), + &utils.FileEntry{}, ) if err != nil { return err @@ -105,7 +106,8 @@ func (h *UploadImgHandler) metaImg(data *PostMetaData) error { _, err = h.Storage.PutFile( bucket, finalName, - storage.NopReaderAtCloser(webpReader), + utils.NopReaderAtCloser(webpReader), + &utils.FileEntry{}, ) if err != nil { return err diff --git a/filehandlers/post_handler.go b/filehandlers/post_handler.go index abaa0d6d..b97d7379 100644 --- a/filehandlers/post_handler.go +++ b/filehandlers/post_handler.go @@ -17,6 +17,7 @@ import ( "github.com/picosh/pico/shared/storage" "github.com/picosh/pico/wish/cms/util" "github.com/picosh/pico/wish/send/utils" + "go.uber.org/zap" ) type ctxUserKey struct{} @@ -61,7 +62,11 @@ func NewScpPostHandler(dbpool db.DB, cfg *shared.ConfigSite, hooks ScpFileHooks, } } -func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) { +func (h *ScpUploadHandler) GetLogger() *zap.SugaredLogger { + return h.Cfg.Logger +} + +func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { user, err := getUser(s) if err != nil { return nil, nil, err @@ -84,12 +89,12 @@ func (h *ScpUploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileI FModTime: *post.UpdatedAt, } - reader := strings.NewReader(post.Text) + reader := utils.NopReaderAtCloser(strings.NewReader(post.Text)) return fileInfo, reader, nil } -func (h *ScpUploadHandler) List(s ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) { +func (h *ScpUploadHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { var fileList []os.FileInfo user, err := getUser(s) if err != nil { diff --git a/go.mod b/go.mod index 14b6bca6..f11a0e35 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/alecthomas/chroma v0.10.0 - github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8 + github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/charmbracelet/bubbles v0.16.1 github.com/charmbracelet/bubbletea v0.24.2 @@ -33,7 +33,7 @@ require ( go.abhg.dev/goldmark/anchor v0.1.1 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.15.0 - golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 0fdf9c9c..98f78150 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8 h1:gR27C6N8s5b+ciBRymi0zhUx8TylKFO755z6yrBuMiI= github.com/antoniomika/go-rsync-receiver v0.0.0-20220901010427-e6494124f0c8/go.mod h1:zmqePVIo1hp+WEKxERLLGHJBDOr8/z/T4eFqXgWIw1w= +github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d h1:NyzUTxebDLLdtNu1gY5hn/amdAEnKG9DOawz82LwNTY= +github.com/antoniomika/go-rsync-receiver v0.0.0-20231110145728-c94949e1ab7d/go.mod h1:zmqePVIo1hp+WEKxERLLGHJBDOr8/z/T4eFqXgWIw1w= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -271,6 +273,8 @@ golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= diff --git a/imgs/api.go b/imgs/api.go index ac86c360..5a95437a 100644 --- a/imgs/api.go +++ b/imgs/api.go @@ -11,14 +11,16 @@ import ( _ "net/http/pprof" + "slices" + "github.com/gorilla/feeds" gocache "github.com/patrickmn/go-cache" "github.com/picosh/pico/db" "github.com/picosh/pico/db/postgres" "github.com/picosh/pico/shared" "github.com/picosh/pico/shared/storage" + "github.com/picosh/pico/wish/send/utils" "go.uber.org/zap" - "golang.org/x/exp/slices" ) type PageData struct { @@ -199,7 +201,7 @@ type ImgHandler struct { type ImgResizer struct { Key string - contents storage.ReaderAtCloser + contents utils.ReaderAtCloser writer io.Writer Img *shared.ImgOptimizer Cache *gocache.Cache @@ -282,7 +284,7 @@ func imgHandler(w http.ResponseWriter, h *ImgHandler) { fname = fmt.Sprintf("%s.webp", shared.SanitizeFileExt(post.Filename)) } - contents, err := h.Storage.GetFile(bucket, fname) + contents, _, _, err := h.Storage.GetFile(bucket, fname) if err != nil { h.Logger.Infof( "file not found %s/%s in storage (bucket: %s, name: %s)", diff --git a/lists/api.go b/lists/api.go index 29656ffc..47e3e166 100644 --- a/lists/api.go +++ b/lists/api.go @@ -10,6 +10,8 @@ import ( "strconv" "time" + "slices" + "github.com/gorilla/feeds" gocache "github.com/patrickmn/go-cache" "github.com/picosh/pico/db" @@ -17,7 +19,6 @@ import ( "github.com/picosh/pico/imgs" "github.com/picosh/pico/shared" "github.com/picosh/pico/shared/storage" - "golang.org/x/exp/slices" ) type PostItemData struct { diff --git a/lists/scp_hooks.go b/lists/scp_hooks.go index 0e904fca..bbfcc036 100644 --- a/lists/scp_hooks.go +++ b/lists/scp_hooks.go @@ -4,11 +4,12 @@ import ( "fmt" "strings" + "slices" + "github.com/picosh/pico/db" "github.com/picosh/pico/filehandlers" "github.com/picosh/pico/imgs" "github.com/picosh/pico/shared" - "golang.org/x/exp/slices" ) type ListHooks struct { diff --git a/pgs/api.go b/pgs/api.go index 3d20c738..331c9c26 100644 --- a/pgs/api.go +++ b/pgs/api.go @@ -210,7 +210,7 @@ func assetHandler(w http.ResponseWriter, h *AssetHandler) { } var redirects []*RedirectRule - redirectFp, err := h.Storage.GetFile(bucket, filepath.Join(h.ProjectDir, "_redirects")) + redirectFp, _, _, err := h.Storage.GetFile(bucket, filepath.Join(h.ProjectDir, "_redirects")) if err == nil { defer redirectFp.Close() buf := new(strings.Builder) @@ -228,11 +228,11 @@ func assetHandler(w http.ResponseWriter, h *AssetHandler) { } routes := calcPossibleRoutes(h.ProjectDir, h.Filepath, redirects) - var contents storage.ReaderAtCloser + var contents utils.ReaderAtCloser assetFilepath := "" status := 200 for _, fp := range routes { - c, err := h.Storage.GetFile(bucket, fp.Filepath) + c, _, _, err := h.Storage.GetFile(bucket, fp.Filepath) if err == nil { contents = c assetFilepath = fp.Filepath diff --git a/pgs/redirect.go b/pgs/redirect.go index 840755ad..3799b378 100644 --- a/pgs/redirect.go +++ b/pgs/redirect.go @@ -93,7 +93,7 @@ func parseRedirectText(text string) ([]*RedirectRule, error) { parts := reSplitWhitespace.Split(trimmed, -1) if len(parts) < 2 { - return rules, fmt.Errorf("Missing destination path/URL") + return rules, fmt.Errorf("missing destination path/URL") } from := parts[0] @@ -114,7 +114,7 @@ func parseRedirectText(text string) ([]*RedirectRule, error) { } if toIndex == -1 { - return rules, fmt.Errorf("The destination path/URL must start with '/', 'http:' or 'https:'") + return rules, fmt.Errorf("the destination path/URL must start with '/', 'http:' or 'https:'") } queryParts := parts[:toIndex] diff --git a/prose/api.go b/prose/api.go index f38adc9b..ea44099c 100644 --- a/prose/api.go +++ b/prose/api.go @@ -10,6 +10,8 @@ import ( "strconv" "time" + "slices" + "github.com/gorilla/feeds" gocache "github.com/patrickmn/go-cache" "github.com/picosh/pico/db" @@ -17,7 +19,6 @@ import ( "github.com/picosh/pico/imgs" "github.com/picosh/pico/shared" "github.com/picosh/pico/shared/storage" - "golang.org/x/exp/slices" ) type PageData struct { diff --git a/prose/scp_hooks.go b/prose/scp_hooks.go index d03e5a40..4d69d95c 100644 --- a/prose/scp_hooks.go +++ b/prose/scp_hooks.go @@ -4,11 +4,12 @@ import ( "fmt" "strings" + "slices" + "github.com/picosh/pico/db" "github.com/picosh/pico/filehandlers" "github.com/picosh/pico/imgs" "github.com/picosh/pico/shared" - "golang.org/x/exp/slices" ) type MarkdownHooks struct { diff --git a/shared/listparser.go b/shared/listparser.go index 97b0408d..6b256c4a 100644 --- a/shared/listparser.go +++ b/shared/listparser.go @@ -9,8 +9,9 @@ import ( "strings" "time" + "slices" + "github.com/araddon/dateparse" - "golang.org/x/exp/slices" ) var reIndent = regexp.MustCompile(`^[[:blank:]]+`) diff --git a/shared/storage/fs.go b/shared/storage/fs.go index 21e22979..a38af863 100644 --- a/shared/storage/fs.go +++ b/shared/storage/fs.go @@ -3,10 +3,12 @@ package storage import ( "fmt" "io" + "io/fs" "os" "path" "path/filepath" "strings" + "time" "github.com/picosh/pico/wish/send/utils" ) @@ -82,16 +84,21 @@ func (s *StorageFS) DeleteBucket(bucket Bucket) error { return os.RemoveAll(bucket.Path) } -func (s *StorageFS) GetFile(bucket Bucket, fpath string) (ReaderAtCloser, error) { +func (s *StorageFS) GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error) { dat, err := os.Open(filepath.Join(bucket.Path, fpath)) if err != nil { - return nil, err + return nil, 0, time.Time{}, err } - return dat, nil + info, err := dat.Stat() + if err != nil { + return nil, 0, time.Time{}, err + } + + return dat, info.Size(), info.ModTime(), nil } -func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents ReaderAtCloser) (string, error) { +func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) { loc := filepath.Join(bucket.Path, fpath) err := os.MkdirAll(filepath.Dir(loc), os.ModePerm) if err != nil { @@ -101,13 +108,19 @@ func (s *StorageFS) PutFile(bucket Bucket, fpath string, contents ReaderAtCloser if err != nil { return "", err } - defer f.Close() _, err = io.Copy(f, contents) if err != nil { return "", err } + f.Close() + + if entry.Mtime > 0 { + uTime := time.Unix(entry.Mtime, 0) + _ = os.Chtimes(loc, uTime, uTime) + } + return loc, nil } @@ -142,10 +155,26 @@ func (s *StorageFS) ListFiles(bucket Bucket, dir string, recursive bool) ([]os.F return fileList, err } - files, err := os.ReadDir(fpath) - if err != nil { - fileList = append(fileList, info) - return fileList, nil + var files []fs.DirEntry + + if recursive { + err = filepath.WalkDir(fpath, func(s string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + files = append(files, d) + return nil + }) + if err != nil { + fileList = append(fileList, info) + return fileList, nil + } + } else { + files, err = os.ReadDir(fpath) + if err != nil { + fileList = append(fileList, info) + return fileList, nil + } } for _, f := range files { diff --git a/shared/storage/minio.go b/shared/storage/minio.go index 70db5a01..7b1bd062 100644 --- a/shared/storage/minio.go +++ b/shared/storage/minio.go @@ -6,7 +6,9 @@ import ( "fmt" "net/url" "os" + "strconv" "strings" + "time" "github.com/minio/madmin-go/v3" "github.com/minio/minio-go/v7" @@ -110,11 +112,20 @@ func (s *StorageMinio) ListFiles(bucket Bucket, dir string, recursive bool) ([]o isDir = true } + modTime := time.Time{} + + if mtime, ok := obj.UserMetadata["Mtime"]; ok { + mtimeUnix, err := strconv.Atoi(mtime) + if err == nil { + modTime = time.Unix(int64(mtimeUnix), 0) + } + } + info := &utils.VirtualFile{ FName: strings.TrimSuffix(strings.TrimPrefix(obj.Key, resolved), "/"), FIsDir: isDir, FSize: obj.Size, - FModTime: obj.LastModified, + FModTime: modTime, } fileList = append(fileList, info) } @@ -126,24 +137,40 @@ func (s *StorageMinio) DeleteBucket(bucket Bucket) error { return s.Client.RemoveBucket(context.TODO(), bucket.Name) } -func (s *StorageMinio) GetFile(bucket Bucket, fpath string) (ReaderAtCloser, error) { - // we have to stat the object first to see if it exists - // https://github.com/minio/minio-go/issues/654 - _, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{}) +func (s *StorageMinio) GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error) { + modTime := time.Time{} + + info, err := s.Client.StatObject(context.Background(), bucket.Name, fpath, minio.StatObjectOptions{}) if err != nil { - return nil, err + return nil, 0, modTime, err } obj, err := s.Client.GetObject(context.Background(), bucket.Name, fpath, minio.GetObjectOptions{}) if err != nil { - return nil, err + return nil, 0, modTime, err } - return obj, nil + if mtime, ok := info.UserMetadata["Mtime"]; ok { + mtimeUnix, err := strconv.Atoi(mtime) + if err == nil { + modTime = time.Unix(int64(mtimeUnix), 0) + } + } + + return obj, info.Size, modTime, nil } -func (s *StorageMinio) PutFile(bucket Bucket, fpath string, contents ReaderAtCloser) (string, error) { - info, err := s.Client.PutObject(context.TODO(), bucket.Name, fpath, contents, -1, minio.PutObjectOptions{}) +func (s *StorageMinio) PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) { + opts := minio.PutObjectOptions{} + + if entry.Mtime > 0 { + opts.UserMetadata = map[string]string{ + "Mtime": fmt.Sprint(entry.Mtime), + } + } + + info, err := s.Client.PutObject(context.TODO(), bucket.Name, fpath, contents, -1, opts) + if err != nil { return "", err } diff --git a/shared/storage/storage.go b/shared/storage/storage.go index 41238f4d..cb9c9c01 100644 --- a/shared/storage/storage.go +++ b/shared/storage/storage.go @@ -1,8 +1,10 @@ package storage import ( - "io" "os" + "time" + + "github.com/picosh/pico/wish/send/utils" ) type Bucket struct { @@ -10,34 +12,14 @@ type Bucket struct { Path string } -type ReadAndReaderAt interface { - io.ReaderAt - io.Reader -} - -type ReaderAtCloser interface { - io.ReaderAt - io.ReadCloser -} - -func NopReaderAtCloser(r ReadAndReaderAt) ReaderAtCloser { - return nopReaderAtCloser{r} -} - -type nopReaderAtCloser struct { - ReadAndReaderAt -} - -func (nopReaderAtCloser) Close() error { return nil } - type ObjectStorage interface { GetBucket(name string) (Bucket, error) UpsertBucket(name string) (Bucket, error) DeleteBucket(bucket Bucket) error GetBucketQuota(bucket Bucket) (uint64, error) - GetFile(bucket Bucket, fpath string) (ReaderAtCloser, error) - PutFile(bucket Bucket, fpath string, contents ReaderAtCloser) (string, error) + GetFile(bucket Bucket, fpath string) (utils.ReaderAtCloser, int64, time.Time, error) + PutFile(bucket Bucket, fpath string, contents utils.ReaderAtCloser, entry *utils.FileEntry) (string, error) DeleteFile(bucket Bucket, fpath string) error ListFiles(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) } diff --git a/shared/util.go b/shared/util.go index cede5320..e00b7c52 100644 --- a/shared/util.go +++ b/shared/util.go @@ -15,8 +15,9 @@ import ( "unicode" "unicode/utf8" + "slices" + "github.com/charmbracelet/ssh" - "golang.org/x/exp/slices" ) var fnameRe = regexp.MustCompile(`[-_]+`) diff --git a/wish/cmd/server/main.go b/wish/cmd/server/main.go index 0a9a7279..3ffa924c 100644 --- a/wish/cmd/server/main.go +++ b/wish/cmd/server/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io" "log" "os" "strings" @@ -12,11 +11,16 @@ import ( "github.com/charmbracelet/wish" "github.com/picosh/pico/wish/send" "github.com/picosh/pico/wish/send/utils" + "go.uber.org/zap" ) type handler struct { } +func (h *handler) GetLogger() *zap.SugaredLogger { + return zap.NewNop().Sugar() +} + func (h *handler) Write(session ssh.Session, file *utils.FileEntry) (string, error) { str := fmt.Sprintf("Received file: %+v from session: %+v", file, session) log.Print(str) @@ -29,7 +33,7 @@ func (h *handler) Validate(session ssh.Session) error { return nil } -func (h *handler) Read(session ssh.Session, entry *utils.FileEntry) (os.FileInfo, io.ReaderAt, error) { +func (h *handler) Read(session ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { log.Printf("Received validate from session: %+v", session) data := strings.NewReader("lorem ipsum dolor") @@ -39,10 +43,10 @@ func (h *handler) Read(session ssh.Session, entry *utils.FileEntry) (os.FileInfo FIsDir: false, FSize: data.Size(), FModTime: time.Now(), - }, data, nil + }, utils.NopReaderAtCloser(data), nil } -func (h *handler) List(session ssh.Session, fpath string, isDir bool) ([]os.FileInfo, error) { +func (h *handler) List(session ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { return nil, nil } diff --git a/wish/list/list.go b/wish/list/list.go index 970ae25f..89c4b393 100644 --- a/wish/list/list.go +++ b/wish/list/list.go @@ -18,7 +18,7 @@ func Middleware(writeHandler utils.CopyFromClientHandler) wish.Middleware { return } - fileList, err := writeHandler.List(session, "/", true) + fileList, err := writeHandler.List(session, "/", true, false) if err != nil { utils.ErrorHandler(session, err) return diff --git a/wish/send/rsync/rsync.go b/wish/send/rsync/rsync.go index 24bfdb4e..2567648b 100644 --- a/wish/send/rsync/rsync.go +++ b/wish/send/rsync/rsync.go @@ -1,12 +1,14 @@ package rsync import ( + "errors" "fmt" "io" "io/fs" - "log" "os" - "path/filepath" + "path" + "slices" + "strings" "github.com/antoniomika/go-rsync-receiver/rsyncreceiver" "github.com/antoniomika/go-rsync-receiver/rsyncsender" @@ -19,41 +21,117 @@ import ( type handler struct { session ssh.Session writeHandler utils.CopyFromClientHandler + root string + recursive bool + ignoreTimes bool } func (h *handler) Skip(file *rsyncutils.ReceiverFile) bool { + if file.FileMode().IsDir() { + return true + } + + fI, _, err := h.writeHandler.Read(h.session, &utils.FileEntry{Filepath: path.Join("/", h.root, file.Name)}) + if err == nil && fI.ModTime().Equal(file.ModTime) && file.Length == fI.Size() { + return true + } + return false } -func (h *handler) List(path string) ([]fs.FileInfo, error) { - list, err := h.writeHandler.List(h.session, path, true) +func (h *handler) List(rPath string) ([]fs.FileInfo, error) { + isDir := false + if rPath == "." { + rPath = "/" + isDir = true + } + + list, err := h.writeHandler.List(h.session, rPath, isDir, h.recursive) if err != nil { return nil, err } - newList := list - if list[0].IsDir() { - newList = list[1:] + var dirs []string + + var newList []fs.FileInfo + + for _, f := range list { + fname := f.Name() + if strings.HasPrefix(f.Name(), "/") { + fname = path.Join(rPath, f.Name()) + } + + if fname == "" && !f.IsDir() { + fname = path.Base(rPath) + } + + newFile := &utils.VirtualFile{ + FName: fname, + FIsDir: f.IsDir(), + FSize: f.Size(), + FModTime: f.ModTime(), + FSys: f.Sys(), + } + + newList = append(newList, newFile) + + parts := strings.Split(newFile.Name(), string(os.PathSeparator)) + lastDir := newFile.Name() + for i := 0; i < len(parts); i++ { + lastDir, _ = path.Split(lastDir) + if lastDir == "" { + continue + } + + lastDir = lastDir[:len(lastDir)-1] + dirs = append(dirs, lastDir) + } + } + + for _, dir := range dirs { + newList = append(newList, &utils.VirtualFile{ + FName: dir, + FIsDir: true, + }) + } + + slices.Reverse(newList) + + onlyEmpty := true + for _, f := range newList { + if f.Name() != "" { + onlyEmpty = false + } + } + + if len(newList) == 0 || onlyEmpty { + return nil, errors.New("no files to send, the directory may not exist or could be empty") } return newList, nil } -func (h *handler) Read(path string) (os.FileInfo, io.ReaderAt, error) { - return h.writeHandler.Read(h.session, &utils.FileEntry{Filepath: path}) +func (h *handler) Read(file *rsyncutils.SenderFile) (os.FileInfo, io.ReaderAt, error) { + filePath := file.WPath + + if strings.HasSuffix(h.root, file.WPath) { + filePath = h.root + } else if !strings.HasPrefix(filePath, h.root) { + filePath = path.Join(h.root, file.Path, file.WPath) + } + + return h.writeHandler.Read(h.session, &utils.FileEntry{Filepath: filePath}) } -func (h *handler) Put(fileName string, content io.Reader, fileSize int64, mTime int64, aTime int64) (int64, error) { - cleanName := filepath.Base(fileName) - fpath := "/" +func (h *handler) Put(file *rsyncutils.ReceiverFile) (int64, error) { fileEntry := &utils.FileEntry{ - Filepath: filepath.Join(fpath, cleanName), + Filepath: path.Join("/", h.root, file.Name), Mode: fs.FileMode(0600), - Size: fileSize, - Mtime: mTime, - Atime: aTime, + Size: file.Length, + Mtime: file.ModTime.Unix(), + Atime: file.ModTime.Unix(), } - fileEntry.Reader = content + fileEntry.Reader = file.Buf msg, err := h.writeHandler.Write(h.session, fileEntry) if err != nil { @@ -79,19 +157,52 @@ func Middleware(writeHandler utils.CopyFromClientHandler) wish.Middleware { fileHandler := &handler{ session: session, writeHandler: writeHandler, + root: strings.TrimPrefix(cmd[len(cmd)-1], "/"), } + cmdFlags := session.Command() + for _, arg := range cmd { if arg == "--sender" { - if err := rsyncsender.ClientRun(nil, session, fileHandler, cmd[len(cmd)-1], true); err != nil { - log.Println("error running rsync:", err) + opts, parser := rsyncsender.NewGetOpt() + + compress := parser.Bool("z", false) + + _, _ = parser.Parse(cmdFlags[1:]) + + fileHandler.recursive = opts.Recurse + fileHandler.ignoreTimes = opts.IgnoreTimes + + if *compress { + _, _ = session.Stderr().Write([]byte("compression is currently unsupported\r\n")) + return + } + + if opts.PreserveUid { + _, _ = session.Stderr().Write([]byte("uid preservation will not work as we don't retain user information\r\n")) + return + } + + if opts.PreserveGid { + _, _ = session.Stderr().Write([]byte("gid preservation will not work as we don't retain user information\r\n")) + return + } + + if err := rsyncsender.ClientRun(opts, session, fileHandler, fileHandler.root, true); err != nil { + writeHandler.GetLogger().Error("error running rsync sender:", err) } return } } - if _, err := rsyncreceiver.ClientRun(nil, session, fileHandler, true); err != nil { - log.Println("error running rsync:", err) + opts, parser := rsyncreceiver.NewGetOpt() + _, _ = parser.Parse(cmdFlags[1:]) + + fileHandler.recursive = opts.Recurse + fileHandler.ignoreTimes = opts.IgnoreTimes + + if _, err := rsyncreceiver.ClientRun(opts, session, fileHandler, true); err != nil { + writeHandler.GetLogger().Error("error running rsync receiver:", err) } } } diff --git a/wish/send/sftp/handler.go b/wish/send/sftp/handler.go index f7faf77a..d41fcc7f 100644 --- a/wish/send/sftp/handler.go +++ b/wish/send/sftp/handler.go @@ -6,10 +6,11 @@ import ( "io" "os" + "slices" + "github.com/charmbracelet/ssh" "github.com/picosh/pico/wish/send/utils" "github.com/pkg/sftp" - "golang.org/x/exp/slices" ) type listerat []os.FileInfo @@ -51,7 +52,7 @@ func (f *handler) Filelist(r *sftp.Request) (sftp.ListerAt, error) { case "List", "Stat": list := r.Method == "List" - listData, err := f.writeHandler.List(f.session, r.Filepath, list) + listData, err := f.writeHandler.List(f.session, r.Filepath, list, false) if err != nil { return nil, err } diff --git a/wish/send/sftp/sftp.go b/wish/send/sftp/sftp.go index 85e3df79..db326faf 100644 --- a/wish/send/sftp/sftp.go +++ b/wish/send/sftp/sftp.go @@ -3,7 +3,6 @@ package sftp import ( "errors" "io" - "log" "github.com/charmbracelet/ssh" "github.com/picosh/pico/wish/send/utils" @@ -45,7 +44,7 @@ func SubsystemHandler(writeHandler utils.CopyFromClientHandler) ssh.SubsystemHan err = requestServer.Serve() if err != nil && !errors.Is(err, io.EOF) { - log.Println("Error serving sftp subsystem:", err) + writeHandler.GetLogger().Error("Error serving sftp subsystem:", err) } } } diff --git a/wish/send/utils/allreaderat.go b/wish/send/utils/allreaderat.go deleted file mode 100644 index 00e2e468..00000000 --- a/wish/send/utils/allreaderat.go +++ /dev/null @@ -1,33 +0,0 @@ -package utils - -import ( - "errors" - "io" - "net/http" - - "github.com/minio/minio-go/v7" -) - -type AllReaderAt struct { - Reader io.ReaderAt -} - -func NewAllReaderAt(reader io.ReaderAt) *AllReaderAt { - return &AllReaderAt{reader} -} - -func (a *AllReaderAt) ReadAt(p []byte, off int64) (n int, err error) { - n, err = a.Reader.ReadAt(p, off) - - if errors.Is(err, io.EOF) { - return - } - - resp := minio.ToErrorResponse(err) - - if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { - err = io.EOF - } - - return -} diff --git a/wish/send/utils/io.go b/wish/send/utils/io.go new file mode 100644 index 00000000..97c8e55d --- /dev/null +++ b/wish/send/utils/io.go @@ -0,0 +1,61 @@ +package utils + +import ( + "errors" + "io" + "net/http" + + "github.com/minio/minio-go/v7" +) + +type ReadAndReaderAt interface { + io.ReaderAt + io.Reader +} + +type ReaderAtCloser interface { + io.ReaderAt + io.ReadCloser +} + +func NopReaderAtCloser(r ReadAndReaderAt) ReaderAtCloser { + return nopReaderAtCloser{r} +} + +type nopReaderAtCloser struct { + ReadAndReaderAt +} + +func (nopReaderAtCloser) Close() error { return nil } + +type AllReaderAt struct { + Reader ReaderAtCloser +} + +func NewAllReaderAt(reader ReaderAtCloser) *AllReaderAt { + return &AllReaderAt{reader} +} + +func (a *AllReaderAt) ReadAt(p []byte, off int64) (n int, err error) { + n, err = a.Reader.ReadAt(p, off) + + if errors.Is(err, io.EOF) { + return + } + + resp := minio.ToErrorResponse(err) + + if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { + err = io.EOF + } + + return +} + +func (a *AllReaderAt) Read(p []byte) (int, error) { + return a.Reader.Read(p) +} + +func (a *AllReaderAt) Close() error { + return a.Reader.Close() +} diff --git a/wish/send/utils/utils.go b/wish/send/utils/utils.go index e37fa581..63ca194d 100644 --- a/wish/send/utils/utils.go +++ b/wish/send/utils/utils.go @@ -10,6 +10,7 @@ import ( "strconv" "github.com/charmbracelet/ssh" + "go.uber.org/zap" ) // NULL is an array with a single NULL byte. @@ -57,8 +58,9 @@ func octalPerms(info fs.FileMode) string { type CopyFromClientHandler interface { // Write should write the given file. Write(ssh.Session, *FileEntry) (string, error) - Read(ssh.Session, *FileEntry) (os.FileInfo, io.ReaderAt, error) - List(ssh.Session, string, bool) ([]os.FileInfo, error) + Read(ssh.Session, *FileEntry) (os.FileInfo, ReaderAtCloser, error) + List(ssh ssh.Session, path string, isDir bool, recursive bool) ([]os.FileInfo, error) + GetLogger() *zap.SugaredLogger Validate(ssh.Session) error }