add directory check to condition

flow-php · Jan 9, 2024 · 9e337a8 · 9e337a8
1 parent 25f85ce
commit 9e337a8
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 11 deletions.
diff --git a/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFS.php b/src/adapter/etl-adapter-filesystem/src/Flow/ETL/Adapter/Filesystem/FlysystemFS.php
@@ -132,6 +132,10 @@ public function rm(Path $path) : void
      */
     public function scan(Path $path, PartitionFilter $partitionFilter = new NoopFilter()) : \Generator
     {
+        if (!$path->isPattern() && !$this->fileExists($path) && !$this->directoryExists($path)) {
+            throw new RuntimeException(\sprintf('Path "%s" does not exists', $path->uri()));
+        }
+
         $fs = $this->factory->create($path);
 
         if ($fs->fileExists($path->path())) {
@@ -140,17 +144,15 @@ public function scan(Path $path, PartitionFilter $partitionFilter = new NoopFilt
             return;
         }
 
-        if (!$path->isPattern()) {
-            throw new RuntimeException(\sprintf('Path "%s" does not exists', $path->uri()));
-        }
-
         $filter = function (FileAttributes|DirectoryAttributes $file) use ($path, $partitionFilter) : bool {
             if ($file instanceof DirectoryAttributes) {
                 return false;
             }
 
-            if (!$path->matches(new Path($path->scheme() . '://' . $file->path(), $path->options()))) {
-                return false;
+            if ($path->isPattern()) {
+                if (!$path->matches(new Path($path->scheme() . '://' . $file->path(), $path->options()))) {
+                    return false;
+                }
             }
 
             return $partitionFilter->keep(...(new Path(DIRECTORY_SEPARATOR . $file->path()))->partitions()->toArray());

diff --git a/src/core/etl/src/Flow/ETL/Filesystem/LocalFilesystem.php b/src/core/etl/src/Flow/ETL/Filesystem/LocalFilesystem.php
@@ -116,14 +116,28 @@ public function scan(Path $path, PartitionFilter $partitionFilter = new NoopFilt
             throw new RuntimeException(\sprintf('Path "%s" is not local', $path->uri()));
         }
 
-        if (!$path->isPattern()) {
-            if ($this->fileExists($path)) {
-                yield $path;
+        if (!$path->isPattern() && !$this->fileExists($path) && !$this->directoryExists($path)) {
+            throw new RuntimeException(\sprintf('Path "%s" does not exists', $path->uri()));
+        }
+
+        if (!$path->isPattern() && $this->fileExists($path)) {
+            yield $path;
+
+            return;
+        }
 
-                return;
+        if (!$path->isPattern()) {
+            foreach (Glob::glob(\rtrim($path->path(), '/') . '/**/*') as $filePath) {
+                if (\is_dir($filePath)) {
+                    continue;
+                }
+
+                if ($partitionFilter->keep(...(Path::realpath($filePath, $path->options()))->partitions()->toArray())) {
+                    yield Path::realpath($filePath, $path->options());
+                }
             }
 
-            throw new RuntimeException(\sprintf('Path "%s" does not exists', $path->uri()));
+            return;
         }
 
         foreach (Glob::glob($path->path()) as $filePath) {