diff --git a/backends/local.go b/backends/local.go index b5b8cd7..b7792cb 100644 --- a/backends/local.go +++ b/backends/local.go @@ -28,35 +28,21 @@ func (c *LocalClient) ListDir(fileChan chan *FileDetails, task *CopyTask, summar visit := func(localPath string, fi os.FileInfo, err error) error { localPath = filepath.ToSlash(localPath) - if !task.Hidden && strings.HasPrefix(fi.Name(), ".") { - if fi.IsDir() { - return filepath.SkipDir - } - return nil - } - if fi.IsDir() { relPath := strings.TrimPrefix(localPath, filepath.ToSlash(c.params.Path)) - if relPath != "" && !task.Recursive { + if (relPath != "" && !task.Recursive) || (!task.Hidden && strings.HasPrefix(fi.Name(), ".")) { return filepath.SkipDir } return nil } + if fi.Mode()&os.ModeSymlink != 0 { return nil } - if !task.Since.IsZero() && fi.ModTime().Before(task.Since) { - return nil - } - if (fi.Size() < task.MinSize) || (task.MaxSize > 0 && fi.Size() > task.MaxSize) { + + if !IsMatch(task, fi.Name(), fi.ModTime(), fi.Size()) { return nil } - if task.Filter != "" { - match, err := filepath.Match(task.Filter, fi.Name()) - if err != nil || !match { - return err - } - } fileDetails := &FileDetails{ Key: localPath, Size: fi.Size(), Mtime: fi.ModTime(), diff --git a/backends/s3.go b/backends/s3.go index 19e5a8f..f7dd794 100644 --- a/backends/s3.go +++ b/backends/s3.go @@ -54,7 +54,6 @@ func SplitPath(path string) (string, string) { } func (c *s3client) ListDir(fileChan chan *FileDetails, task *CopyTask, summary *ListSummary) error { - //bucket, keyPrefix := SplitPath(c.params.Path) doneCh := make(chan struct{}) defer close(doneCh) defer close(fileChan) @@ -70,25 +69,10 @@ func (c *s3client) ListDir(fileChan chan *FileDetails, task *CopyTask, summary * } _, name := filepath.Split(obj.Key) - if !task.Hidden && strings.HasPrefix(name, ".") { + if !IsMatch(task, name, obj.LastModified, obj.Size) { continue } - if !task.Since.IsZero() && obj.LastModified.Before(task.Since) { - continue - } - - if (obj.Size < task.MinSize) || (task.MaxSize > 0 && obj.Size > task.MaxSize) { - continue - } - - if task.Filter != "" { - match, err := filepath.Match(task.Filter, name) - if err != nil || !match { - continue - } - } - c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size) fileDetails := &FileDetails{ Key: c.params.Bucket + "/" + obj.Key, Size: obj.Size, Mtime: obj.LastModified, diff --git a/backends/types.go b/backends/types.go index e7863e0..41ffa9a 100644 --- a/backends/types.go +++ b/backends/types.go @@ -29,6 +29,7 @@ type CopyTask struct { MaxSize int64 Filter string Recursive bool + CopyEmpty bool Hidden bool } @@ -109,3 +110,30 @@ func defaultFromEnv(param string, envvar string) string { } return param } + +func IsMatch(task *CopyTask, name string, mtime time.Time, size int64) bool { + if !task.CopyEmpty && size == 0 { + return false + } + + if !task.Hidden && strings.HasPrefix(name, ".") { + return false + } + + if !task.Since.IsZero() && mtime.Before(task.Since) { + return false + } + + if (size < task.MinSize) || (task.MaxSize > 0 && size > task.MaxSize) { + return false + } + + if task.Filter != "" { + match, err := filepath.Match(task.Filter, name) + if err != nil || !match { + return false + } + } + + return true +} diff --git a/backends/v3io.go b/backends/v3io.go index 35982a1..72796f6 100644 --- a/backends/v3io.go +++ b/backends/v3io.go @@ -30,7 +30,6 @@ type V3ioClientOpts struct { SessionKey string `json:"sessionKey,omitempty"` // Logging level (for verbose output) - "debug" | "info" | "warn" | "error" LogLevel string `json:"logLevel,omitempty"` - // Number of parallel V3IO worker routines } type V3ioClient struct { @@ -90,29 +89,15 @@ func (c *V3ioClient) getDir(path string, fileChan chan *FileDetails, summary *Li continue } - _, name := filepath.Split(obj.Key) - if !c.task.Hidden && strings.HasPrefix(name, ".") { - continue - } - t, err := time.Parse(time.RFC3339, obj.LastModified+"Z") if err != nil { return errors.Wrap(err, "Invalid object time string - not an RFC 3339 time format.") } - if !c.task.Since.IsZero() && t.Before(c.task.Since) { - continue - } - size := int64(obj.Size) - if (size < c.task.MinSize) || (c.task.MaxSize > 0 && size > c.task.MaxSize) { - continue - } + _, name := filepath.Split(obj.Key) - if c.task.Filter != "" { - match, err := filepath.Match(c.task.Filter, name) - if err != nil || !match { - continue - } + if !IsMatch(c.task, name, t, size) { + continue } c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size)