Skip to content

Commit

Permalink
Rework examples to allow running them outside the phar
Browse files Browse the repository at this point in the history
  • Loading branch information
stloyd committed Oct 11, 2023
1 parent d9c0ff4 commit 0a7fb80
Show file tree
Hide file tree
Showing 31 changed files with 287 additions and 110 deletions.
2 changes: 2 additions & 0 deletions build/runtime.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
exit(1);
}

\ini_set('memory_limit', -1);

(new SingleCommandApplication())
->setName('Flow-PHP - Extract Transform Load - Data processing framework')
->setVersion(FlowVersion::getVersion())
Expand Down
6 changes: 6 additions & 0 deletions examples/bootstrap.php
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
<?php declare(strict_types=1);

if ('' === \Phar::running(false)) {
require __DIR__ . '/../vendor/autoload.php';
}

\ini_set('memory_limit', -1);

const __FLOW_DATA__ = __DIR__ . '/data';
const __FLOW_OUTPUT__ = __DIR__ . '/output';
const __FLOW_VAR__ = __DIR__ . '/var';
Expand Down
8 changes: 7 additions & 1 deletion examples/topics/aggregations/power_plant.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(CSV::from(__FLOW_DATA__ . '/power-plant-daily.csv', 10, delimiter: ';'))
->withEntry('unpacked', ref('row')->unpack())
->renameAll('unpacked.', '')
Expand Down Expand Up @@ -45,3 +45,9 @@
->withEntry('consumption', ref('consumption')->multiply(lit(100))->round(lit(2)))
->withEntry('consumption', concat(ref('consumption'), lit('%')))
->write(To::output(truncate: false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/aggregations/power_plant_bar_chart.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow)
$flow = (new Flow)
->read(CSV::from(__FLOW_DATA__ . '/power-plant-daily.csv', 10, delimiter: ';'))
->withEntry('unpacked', ref('row')->unpack())
->renameAll('unpacked.', '')
Expand Down Expand Up @@ -50,3 +50,9 @@
output: __FLOW_OUTPUT__ . '/power_plant_bar_chart.html'
)
);

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/http/psr_http_dynamic.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,17 @@ public function create(?Message\ResponseInterface $previousResponse = null) : ?M
}
});

return (new Flow())
$flow = (new Flow())
->read($extractor)
->withEntry('unpacked', ref('response_body')->jsonDecode())
->select('unpacked')
->withEntry('unpacked', ref('unpacked')->unpack())
->renameAll('unpacked.', '')
->select('name', 'html_url', 'blog')
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/join/left_anti/left_anti_join.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* then it might become performance bottleneck.
* In that case please look at DataFrame::joinEach.
*/
return (new Flow())
$flow = (new Flow())
->process($externalProducts)
->join(
(new Flow())->process($internalProducts),
Expand All @@ -39,6 +39,12 @@
)
->write(To::output());

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();

// Output
//
// +--+---------+
Expand Down
8 changes: 7 additions & 1 deletion examples/topics/join/left_anti/left_anti_join_each.php
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ private function findRowsInDatabase(Rows $rows) : Rows
* right size is much bigger then a left side. In that case it's better to reduce the ride side
* by fetching from the storage only what is relevant for the left side.
*/
return (new Flow())
$flow = (new Flow())
->extract($apiExtractor)
->joinEach(
$dbDataFrameFactory,
Expand All @@ -78,6 +78,12 @@ private function findRowsInDatabase(Rows $rows) : Rows
)
->write(To::output());

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();

// Output:
//
// +-----+-------------+
Expand Down
8 changes: 7 additions & 1 deletion examples/topics/transformations/aggregate.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::with(Entry::int('a', 100)),
Expand All @@ -25,3 +25,9 @@
)
->aggregate(Aggregation::sum(ref('a')))
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/array_expand.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::with(Entry::int('id', 1), Entry::array('array', ['a' => 1, 'b' => 2, 'c' => 3])),
Expand All @@ -22,3 +22,9 @@
->write(To::output(false))
->withEntry('expanded', array_expand(ref('array')))
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/array_unpack.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::with(Entry::int('id', 1), Entry::array('array', ['a' => 1, 'b' => 2, 'c' => 3])),
Expand All @@ -22,3 +22,9 @@
->write(To::output(false))
->withEntry('unpacked', ref('array')->unpack())
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/filter_divide.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::with(Entry::int('a', 100), Entry::int('b', 100)),
Expand All @@ -23,3 +23,9 @@
->filter(ref('b')->divide(lit(2))->equals(lit('a')))
->withEntry('new_b', ref('b')->multiply(lit(2))->multiply(lit(5)))
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/filter_mod.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::with(Entry::int('a', 4), Entry::int('b', 5)),
Expand All @@ -22,3 +22,9 @@
)
->filter(ref('b')->mod(lit(2))->equals(lit(0)))
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/group_by.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::with(Entry::int('a', 100)),
Expand All @@ -24,3 +24,9 @@
)
->groupBy(ref('a'))
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/literals.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,17 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::create(Entry::string('name', 'Norbert'))
))
)
->withEntry('number', lit(1))
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/math.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::create(Entry::integer('a', 100), Entry::integer('b', 200))
Expand All @@ -22,3 +22,9 @@
->withEntry('c', ref('a')->plus(ref('b')))
->withEntry('d', ref('b')->minus(ref('a')))
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/size.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,17 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(
From::rows(new Rows(
Row::with(Entry::int('id', 1), Entry::array('array', ['a' => 1, 'b' => 2, 'c' => 3])),
))
)
->withEntry('array_size', ref('array')->size())
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/sort.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,13 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(From::sequence_number('id', 0, 10))
->sortBy(ref('id')->desc())
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/when_null.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(From::rows(new Rows(
Row::with(Entry::int('id', 1), Entry::int('value', 1)),
Row::with(Entry::int('id', 2), Entry::int('value', 1)),
Expand All @@ -27,3 +27,9 @@
when(ref('value')->isNull(), then: lit(0))
)
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
8 changes: 7 additions & 1 deletion examples/topics/transformations/when_odd.php
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

require __DIR__ . '/../../bootstrap.php';

return (new Flow())
$flow = (new Flow())
->read(From::sequence_number('number', 1, 100))
->collect()
->withEntry(
Expand All @@ -23,3 +23,9 @@
)
)
->write(To::output(false));

if ('' !== \Phar::running(false)) {
return $flow;
}

$flow->run();
20 changes: 12 additions & 8 deletions examples/topics/types/csv/csv_read.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,24 @@

require __DIR__ . '/../../../bootstrap.php';

$flow = (new Flow())
->read(CSV::from(__FLOW_OUTPUT__ . '/dataset.csv', 1000))
->withEntry('unpacked', ref('row')->unpack())
->renameAll('unpacked.', '')
->drop(col('row'))
->limit(10_000);

if ('' !== \Phar::running(false)) {
return $flow;
}

$csvFileSize = \round(\filesize(__FLOW_OUTPUT__ . '/dataset.csv') / 1024 / 1024);
print "Reading CSV {$csvFileSize}Mb file...\n";

$stopwatch = new Stopwatch();
$stopwatch->start();
$total = 0;

(new Flow())
->read(CSV::from(__FLOW_OUTPUT__ . '/dataset.csv', 1000))
->withEntry('unpacked', ref('row')->unpack())
->renameAll('unpacked.', '')
->drop(col('row'))
->limit(10_000)
->run();
$flow->run();

$stopwatch->stop();

Expand Down
Loading

0 comments on commit 0a7fb80

Please sign in to comment.