Add way to run all examples

composer.json

@@ -49,6 +49,7 @@
         "ramsey/uuid": "^4.5",
         "symfony/cache": "^6.2",
         "symfony/dotenv": "^6.2",
+        "symfony/finder": "^6.3",
         "symfony/uid": "^6.3"
     },
     "autoload": {

examples/bootstrap.php

@@ -6,11 +6,13 @@
 
 \ini_set('memory_limit', -1);
 
-const __FLOW_DATA__ = __DIR__ . '/data';
-const __FLOW_OUTPUT__ = __DIR__ . '/output';
-const __FLOW_VAR__ = __DIR__ . '/var';
-const __FLOW_VAR_RUN__ = __DIR__ . '/var/run';
-const __FLOW_SRC__ = __DIR__ . '/../src';
+if (!\defined('__FLOW_DATA__')) {
+    \define('__FLOW_DATA__', __DIR__ . '/data');
+    \define('__FLOW_OUTPUT__', __DIR__ . '/output');
+    \define('__FLOW_VAR__', __DIR__ . '/var');
+    \define('__FLOW_VAR_RUN__', __DIR__ . '/var/run');
+    \define('__FLOW_SRC__', __DIR__ . '/../src');
+}
 
 if (!\is_dir(__FLOW_VAR__)) {
     \mkdir(__FLOW_VAR__);

examples/data/extractor.php

@@ -39,7 +39,7 @@ public function extract(FlowContext $context) : Generator
             }
         }
 
-        if (\count($rows) >= 0) {
+        if ([] !== $rows) {
             yield new Rows(...$rows);
         }
     }

examples/run_examples.php

@@ -0,0 +1,60 @@
+#!/usr/bin/env php
+<?php declare(strict_types=1);
+
+require __DIR__ . '/../vendor/autoload.php';
+
+use Symfony\Component\Finder\Finder;
+
+if ($_ENV['FLOW_PHAR_APP'] ?? false) {
+    print PHP_EOL . 'This script cannot be run in PHAR, please use CLI approach.' . PHP_EOL;
+
+    exit(1);
+}
+
+if (false === \in_array(PHP_SAPI, ['cli', 'phpdbg', 'embed'], true)) {
+    print PHP_EOL . 'This script may only be invoked from a command line, got "' . PHP_SAPI . '"' . PHP_EOL;
+
+    exit(1);
+}
+
+\ini_set('memory_limit', -1);
+
+print "Running all available examples.\n";
+print "Excluding: async, database & remote filesystem. Those examples require additional manual setup to be run properly.\n";
+print "Removing previously generated output files.\n";
+
+$finder = new Finder();
+
+foreach ($finder->in(__DIR__ . '/output')->exclude('.gitkeep') as $file) {
+    @\unlink($file->getRealPath());
+}
+
+print "Running setup scripts.\n";
+
+$finder = new Finder();
+$finder->in(__DIR__ . '/setup')
+    ->files()
+    ->name('*.php');
+
+foreach ($finder as $file) {
+    print "\nScript: {$file->getRelativePathname()}\n";
+
+    include $file->getRealPath();
+}
+
+$finder = new Finder();
+$finder->in(__DIR__ . '/topics')
+    // async, database & remote filesystem examples require additional manual setup to be run properly
+    ->exclude(['async', 'db', 'fs'])
+    ->files()
+    ->name('*.php');
+
+foreach ($finder as $file) {
+    print "\nExample: {$file->getRelativePathname()}\n";
+
+    try {
+        include $file->getRealPath();
+    } catch (\Exception $e) {
+        print "Example failed: {$e->getMessage()}\n";
+    }
+}

examples/topics/types/csv/php_to_csv_and_json.php → examples/setup/php_to_csv_and_json.php

@@ -10,7 +10,7 @@
 use Flow\ETL\Flow;
 use Flow\ETL\Monitoring\Memory\Consumption;
 
-require __DIR__ . '/../../../bootstrap.php';
+require __DIR__ . '/../bootstrap.php';
 
 $extractor = require __FLOW_DATA__ . '/extractor.php';
 

examples/topics/types/csv/README.md

@@ -6,6 +6,4 @@ Examples:
 - [CSV to Json](csv_to_json.php)
 - [CSV to Parquet - 10k per group](csv_to_parquet_10k.php)
 - [CSV to Parquet - 100k per group](csv_to_parquet_100k.php)
-- [PHP to CSV & Json](php_to_csv_and_json.php)
-- [PHP to CSV partitioned](php_to_csv_partition.php)
-- [CSV read partitioned](php_to_csv_partition.php)
+- [CSV read partitioned](php_to_csv_partition.php)

examples/topics/types/csv/csv_read.php

@@ -10,6 +10,12 @@
 
 require __DIR__ . '/../../../bootstrap.php';
 
+if (!\file_exists(__FLOW_OUTPUT__ . '/dataset.csv')) {
+    print "Data file is missing, please first run: php examples/setup/php_to_csv_and_json.php\n";
+
+    exit(1);
+}
+
 $flow = (new Flow())
     ->read(CSV::from(__FLOW_OUTPUT__ . '/dataset.csv', 1000))
     ->withEntry('unpacked', ref('row')->unpack())

examples/topics/types/csv/csv_to_avro.php

@@ -11,6 +11,12 @@
 
 require __DIR__ . '/../../../bootstrap.php';
 
+if (!\file_exists(__FLOW_OUTPUT__ . '/dataset.csv')) {
+    print "Data file is missing, please first run: php examples/setup/php_to_csv_and_json.php\n";
+
+    exit(1);
+}
+
 $flow = (new Flow())
     ->read(CSV::from(__FLOW_OUTPUT__ . '/dataset.csv', 10_000))
     ->withEntry('unpacked', ref('row')->unpack())

examples/topics/types/csv/csv_to_parquet_100k.php

@@ -11,6 +11,12 @@
 
 require __DIR__ . '/../../../bootstrap.php';
 
+if (!\file_exists(__FLOW_OUTPUT__ . '/dataset.csv')) {
+    print "Data file is missing, please first run: php examples/setup/php_to_csv_and_json.php\n";
+
+    exit(1);
+}
+
 $flow = (new Flow())
     ->read(CSV::from(__FLOW_OUTPUT__ . '/dataset.csv', 10_000))
     ->withEntry('unpacked', ref('row')->unpack())

examples/topics/types/csv/csv_to_parquet_10k.php

@@ -11,6 +11,12 @@
 
 require __DIR__ . '/../../../bootstrap.php';
 
+if (!\file_exists(__FLOW_OUTPUT__ . '/dataset.csv')) {
+    print "Data file is missing, please first run: php examples/setup/php_to_csv_and_json.php\n";
+
+    exit(1);
+}
+
 $flow = (new Flow())
     ->read(CSV::from(__FLOW_OUTPUT__ . '/dataset.csv', 10_000))
     ->withEntry('unpacked', ref('row')->unpack())