Skip to content

Commit

Permalink
Added cloud storage examples to homepage (#1065)
Browse files Browse the repository at this point in the history
  • Loading branch information
norberttech authored Apr 30, 2024
1 parent 1dd8a53 commit 44eec85
Show file tree
Hide file tree
Showing 12 changed files with 617 additions and 460 deletions.
4 changes: 0 additions & 4 deletions examples/.env.dist

This file was deleted.

1 change: 1 addition & 0 deletions examples/topics/cloud_storage/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
**/.env
2 changes: 2 additions & 0 deletions examples/topics/cloud_storage/aws_s3/.env.dist
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
AWS_S3_KEY="xxxx"
AWS_S3_SECRET="xxxx"
45 changes: 45 additions & 0 deletions examples/topics/cloud_storage/aws_s3/code.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
<?php

declare(strict_types=1);

use function Flow\ETL\Adapter\CSV\to_csv;
use function Flow\ETL\DSL\{data_frame, from_array, overwrite};
use Flow\ETL\Adapter\Filesystem\AwsS3Stream;
use Flow\ETL\Filesystem\Path;
use Symfony\Component\Dotenv\Dotenv;

require __DIR__ . '/../../../autoload.php';

if (!\file_exists(__DIR__ . '/.env')) {
print 'Example skipped. Please create .env file with AWS S3 credentials.' . PHP_EOL;

return;
}

$dotenv = new Dotenv();
$dotenv->load(__DIR__ . '/.env');

$s3_client_option = [
'client' => [
'credentials' => [
'key' => $_ENV['AWS_S3_KEY'],
'secret' => $_ENV['AWS_S3_SECRET'],
],
'region' => 'eu-west-2',
'version' => 'latest',
],
'bucket' => 'flow-php',
];

AwsS3Stream::register();

data_frame()
->read(from_array([
['id' => 1, 'name' => 'test'],
['id' => 2, 'name' => 'test'],
['id' => 3, 'name' => 'test'],
['id' => 4, 'name' => 'test'],
]))
->saveMode(overwrite())
->write(to_csv(new Path('flow-aws-s3://test.csv', $s3_client_option)))
->run();
2 changes: 2 additions & 0 deletions examples/topics/cloud_storage/azure/.env.dist
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
AZURE_CONNECTION_STRING="xxxx"
AZURE_CONTAINER="xxxx"
38 changes: 38 additions & 0 deletions examples/topics/cloud_storage/azure/code.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
<?php

declare(strict_types=1);

use function Flow\ETL\Adapter\CSV\to_csv;
use function Flow\ETL\DSL\{data_frame, from_array, overwrite};
use Flow\ETL\Adapter\Filesystem\AzureBlobStream;
use Flow\ETL\Filesystem\Path;
use Symfony\Component\Dotenv\Dotenv;

require __DIR__ . '/../../../autoload.php';

if (!\file_exists(__DIR__ . '/.env')) {
print 'Example skipped. Please create .env file with Azure Storage Account credentials.' . PHP_EOL;

return;
}

$dotenv = new Dotenv();
$dotenv->load(__DIR__ . '/.env');

$azure_client_option = [
'connection-string' => $_ENV['AZURE_CONNECTION_STRING'],
'container' => $_ENV['AZURE_CONTAINER'],
];

AzureBlobStream::register();

data_frame()
->read(from_array([
['id' => 1, 'name' => 'test'],
['id' => 2, 'name' => 'test'],
['id' => 3, 'name' => 'test'],
['id' => 4, 'name' => 'test'],
]))
->saveMode(overwrite())
->write(to_csv(new Path('flow-azure-blob://test.csv', $azure_client_option)))
->run();
11 changes: 5 additions & 6 deletions examples/topics/data_frame/reorder_entries/code.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,22 @@
type_map,
type_string,
uuid_entry};
use Ramsey\Uuid\Uuid;

data_frame()
->read(from_rows(rows(
row(
int_entry('int_a', 1),
int_entry('int_b', 1),
float_entry('float_a', \random_int(100, 100000) / 100),
float_entry('float_b', \random_int(100, 100000) / 100),
float_entry('float_a', 57291 / 100),
float_entry('float_b', 21021 / 100),
bool_entry('bool', false),
bool_entry('bool_a', false),
bool_entry('bool_c', false),
datetime_entry('datetime_d', new DateTimeImmutable('now')),
datetime_entry('datetime_z', new DateTimeImmutable('now')),
datetime_entry('datetime_d', new DateTimeImmutable('2024-04-01 00:00:00')),
datetime_entry('datetime_z', new DateTimeImmutable('2024-04-01 00:00:00')),
str_entry('string_a', 'string'),
str_entry('string_b', 'string'),
uuid_entry('uuid', new Flow\ETL\Row\Entry\Type\Uuid(Uuid::uuid4())),
uuid_entry('uuid', '06143adb-3009-45c8-a4f0-c7016f97cab7'),
json_entry('json', ['id' => 1, 'status' => 'NEW']),
array_entry(
'array',
Expand Down
2 changes: 1 addition & 1 deletion examples/topics/data_frame/reorder_entries/output.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
+--------------------------------------+-------+-------+-------+--------+--------+---------+---------+---------------------------+---------------------------+----------+----------+-------------------------------------------------------+---------+-------------------------+----------------------+------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
| uuid | int_a | int_b | bool | bool_a | bool_c | float_a | float_b | datetime_d | datetime_z | string_a | string_b | array | list | json | map | object | struct |
+--------------------------------------+-------+-------+-------+--------+--------+---------+---------+---------------------------+---------------------------+----------+----------+-------------------------------------------------------+---------+-------------------------+----------------------+------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
| 1fb2af91-332b-409f-b314-d26969651f1d | 1 | 1 | false | false | false | 468.02 | 75.21 | 2024-04-27T14:24:02+00:00 | 2024-04-27T14:24:02+00:00 | string | string | [{"id":1,"status":"NEW"},{"id":2,"status":"PENDING"}] | [1,2,3] | {"id":1,"status":"NEW"} | ["zero","one","two"] | ArrayIterator Object( [storage:ArrayIterator:private] => Array ( [0] => 1 [1] => 2 [2] => 3 )) | {"street":"street","city":"city","zip":"zip","country":"country","location":{"lat":1.5,"lon":1.5}} |
| 06143adb-3009-45c8-a4f0-c7016f97cab7 | 1 | 1 | false | false | false | 572.91 | 210.21 | 2024-04-01T00:00:00+00:00 | 2024-04-01T00:00:00+00:00 | string | string | [{"id":1,"status":"NEW"},{"id":2,"status":"PENDING"}] | [1,2,3] | {"id":1,"status":"NEW"} | ["zero","one","two"] | ArrayIterator Object( [storage:ArrayIterator:private] => Array ( [0] => 1 [1] => 2 [2] => 3 )) | {"street":"street","city":"city","zip":"zip","country":"country","location":{"lat":1.5,"lon":1.5}} |
+--------------------------------------+-------+-------+-------+--------+--------+---------+---------+---------------------------+---------------------------+----------+----------+-------------------------------------------------------+---------+-------------------------+----------------------+------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
1 rows
3 changes: 3 additions & 0 deletions web/landing/composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@
"tools:install": [
"composer install --working-dir=./tools/phpunit"
],
"tools:updated": [
"composer update --working-dir=./tools/phpunit"
],
"test": [
"tools/phpunit/vendor/bin/phpunit"
],
Expand Down
Loading

0 comments on commit 44eec85

Please sign in to comment.