consolidate filtering

backends/local.go

@@ -28,35 +28,21 @@ func (c *LocalClient) ListDir(fileChan chan *FileDetails, task *CopyTask, summar
 	visit := func(localPath string, fi os.FileInfo, err error) error {
 		localPath = filepath.ToSlash(localPath)
 
-		if !task.Hidden && strings.HasPrefix(fi.Name(), ".") {
-			if fi.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
 		if fi.IsDir() {
 			relPath := strings.TrimPrefix(localPath, filepath.ToSlash(c.params.Path))
-			if relPath != "" && !task.Recursive {
+			if (relPath != "" && !task.Recursive) || (!task.Hidden && strings.HasPrefix(fi.Name(), ".")) {
 				return filepath.SkipDir
 			}
 			return nil
 		}
+
 		if fi.Mode()&os.ModeSymlink != 0 {
 			return nil
 		}
-		if !task.Since.IsZero() && fi.ModTime().Before(task.Since) {
-			return nil
-		}
-		if (fi.Size() < task.MinSize) || (task.MaxSize > 0 && fi.Size() > task.MaxSize) {
+
+		if !IsMatch(task, fi.Name(), fi.ModTime(), fi.Size()) {
 			return nil
 		}
-		if task.Filter != "" {
-			match, err := filepath.Match(task.Filter, fi.Name())
-			if err != nil || !match {
-				return err
-			}
-		}
 
 		fileDetails := &FileDetails{
 			Key: localPath, Size: fi.Size(), Mtime: fi.ModTime(),

backends/s3.go

@@ -54,7 +54,6 @@ func SplitPath(path string) (string, string) {
 }
 
 func (c *s3client) ListDir(fileChan chan *FileDetails, task *CopyTask, summary *ListSummary) error {
-	//bucket, keyPrefix := SplitPath(c.params.Path)
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 	defer close(fileChan)
@@ -70,25 +69,10 @@ func (c *s3client) ListDir(fileChan chan *FileDetails, task *CopyTask, summary *
 		}
 
 		_, name := filepath.Split(obj.Key)
-		if !task.Hidden && strings.HasPrefix(name, ".") {
+		if !IsMatch(task, name, obj.LastModified, obj.Size) {
 			continue
 		}
 
-		if !task.Since.IsZero() && obj.LastModified.Before(task.Since) {
-			continue
-		}
-
-		if (obj.Size < task.MinSize) || (task.MaxSize > 0 && obj.Size > task.MaxSize) {
-			continue
-		}
-
-		if task.Filter != "" {
-			match, err := filepath.Match(task.Filter, name)
-			if err != nil || !match {
-				continue
-			}
-		}
-
 		c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size)
 		fileDetails := &FileDetails{
 			Key: c.params.Bucket + "/" + obj.Key, Size: obj.Size, Mtime: obj.LastModified,

backends/types.go

@@ -29,6 +29,7 @@ type CopyTask struct {
 	MaxSize   int64
 	Filter    string
 	Recursive bool
+	CopyEmpty bool
 	Hidden    bool
 }
 
@@ -109,3 +110,30 @@ func defaultFromEnv(param string, envvar string) string {
 	}
 	return param
 }
+
+func IsMatch(task *CopyTask, name string, mtime time.Time, size int64) bool {
+	if !task.CopyEmpty && size == 0 {
+		return false
+	}
+
+	if !task.Hidden && strings.HasPrefix(name, ".") {
+		return false
+	}
+
+	if !task.Since.IsZero() && mtime.Before(task.Since) {
+		return false
+	}
+
+	if (size < task.MinSize) || (task.MaxSize > 0 && size > task.MaxSize) {
+		return false
+	}
+
+	if task.Filter != "" {
+		match, err := filepath.Match(task.Filter, name)
+		if err != nil || !match {
+			return false
+		}
+	}
+
+	return true
+}

backends/v3io.go

@@ -30,7 +30,6 @@ type V3ioClientOpts struct {
 	SessionKey     string `json:"sessionKey,omitempty"`
 	// Logging level (for verbose output) - "debug" | "info" | "warn" | "error"
 	LogLevel string `json:"logLevel,omitempty"`
-	// Number of parallel V3IO worker routines
 }
 
 type V3ioClient struct {
@@ -90,29 +89,15 @@ func (c *V3ioClient) getDir(path string, fileChan chan *FileDetails, summary *Li
 			continue
 		}
 
-		_, name := filepath.Split(obj.Key)
-		if !c.task.Hidden && strings.HasPrefix(name, ".") {
-			continue
-		}
-
 		t, err := time.Parse(time.RFC3339, obj.LastModified+"Z")
 		if err != nil {
 			return errors.Wrap(err, "Invalid object time string - not an RFC 3339 time format.")
 		}
-		if !c.task.Since.IsZero() && t.Before(c.task.Since) {
-			continue
-		}
-
 		size := int64(obj.Size)
-		if (size < c.task.MinSize) || (c.task.MaxSize > 0 && size > c.task.MaxSize) {
-			continue
-		}
+		_, name := filepath.Split(obj.Key)
 
-		if c.task.Filter != "" {
-			match, err := filepath.Match(c.task.Filter, name)
-			if err != nil || !match {
-				continue
-			}
+		if !IsMatch(c.task, name, t, size) {
+			continue
 		}
 
 		c.logger.DebugWith("List dir:", "key", obj.Key, "modified", obj.LastModified, "size", obj.Size)