From e041fbf615d21ce0fbcdfb06c6bc541da3955eec Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Thu, 18 Apr 2024 11:34:05 +0200 Subject: [PATCH 1/2] TEST --- .../overwrite/date=2024-04-01/file.txt | 1 + .../overwrite/date=2024-04-02/file.txt | 1 + .../Integration/DataFrame/PartitioningTest.php | 14 +++++++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt new file mode 100644 index 000000000..2d1142304 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt @@ -0,0 +1 @@ +2024-04-01 \ No newline at end of file diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt new file mode 100644 index 000000000..afeba1152 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt @@ -0,0 +1 @@ +2024-04-02 \ No newline at end of file diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php index 6afdbe31a..e1463af80 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php @@ -4,7 +4,7 @@ namespace Flow\ETL\Tests\Integration\DataFrame; -use function Flow\ETL\Adapter\Text\from_text; +use function Flow\ETL\Adapter\Text\{from_text, to_text}; use function Flow\ETL\DSL\{df, from_array, from_rows, int_entry, lit, partition, ref, row, rows, rows_partitioned, str_entry}; use Flow\ETL\Tests\Integration\IntegrationTestCase; use Flow\ETL\{Partition, Rows}; @@ -33,6 +33,18 @@ public function test_dropping_partitions() : void self::assertFalse($rows->isPartitioned()); } + public function test_overwrite_save_mode_not_dropping_old_partitions() : void + { + df() + ->read(from_array([ + ['date' => '2024-04-03'], + ['date' => '2024-04-04'], + ])) + ->partitionBy('date') + ->write(to_text(__DIR__ . '/Fixtures/Partitioning/overwrite/file.txt')) + ->run(); + } + public function test_partition_by() : void { $rows = df() From 122fd9dd22db0cd1b2116a2bb41492b085c5ad35 Mon Sep 17 00:00:00 2001 From: Norbert Orzechowicz Date: Thu, 18 Apr 2024 11:41:05 +0200 Subject: [PATCH 2/2] Covered partitioning overwrite with tests --- .../Partitioning/overwrite/.gitignore | 2 + .../DataFrame/PartitioningTest.php | 37 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore new file mode 100644 index 000000000..cb0ec18f2 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore @@ -0,0 +1,2 @@ +date=2024-04-03 +date=2024-04-04 \ No newline at end of file diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php index e1463af80..914fe3bbb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php @@ -5,7 +5,18 @@ namespace Flow\ETL\Tests\Integration\DataFrame; use function Flow\ETL\Adapter\Text\{from_text, to_text}; -use function Flow\ETL\DSL\{df, from_array, from_rows, int_entry, lit, partition, ref, row, rows, rows_partitioned, str_entry}; +use function Flow\ETL\DSL\{df, + from_array, + from_rows, + int_entry, + lit, + overwrite, + partition, + ref, + row, + rows, + rows_partitioned, + str_entry}; use Flow\ETL\Tests\Integration\IntegrationTestCase; use Flow\ETL\{Partition, Rows}; @@ -35,14 +46,38 @@ public function test_dropping_partitions() : void public function test_overwrite_save_mode_not_dropping_old_partitions() : void { + if (\file_exists(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-03')) { + \unlink(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-03/file.txt'); + \rmdir(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-03'); + } + + if (\file_exists(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-04')) { + \unlink(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-04/file.txt'); + \rmdir(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-04'); + } + df() ->read(from_array([ ['date' => '2024-04-03'], ['date' => '2024-04-04'], ])) ->partitionBy('date') + ->saveMode(overwrite()) ->write(to_text(__DIR__ . '/Fixtures/Partitioning/overwrite/file.txt')) ->run(); + + self::assertSame( + [ + ['text' => '2024-04-01', 'date' => '2024-04-01'], + ['text' => '2024-04-02', 'date' => '2024-04-02'], + ['text' => '2024-04-03', 'date' => '2024-04-03'], + ['text' => '2024-04-04', 'date' => '2024-04-04'], + ], + df() + ->read(from_text(__DIR__ . '/Fixtures/Partitioning/overwrite/**/*.txt')) + ->fetch() + ->toArray() + ); } public function test_partition_by() : void