diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore new file mode 100644 index 000000000..cb0ec18f2 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/.gitignore @@ -0,0 +1,2 @@ +date=2024-04-03 +date=2024-04-04 \ No newline at end of file diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt new file mode 100644 index 000000000..2d1142304 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt @@ -0,0 +1 @@ +2024-04-01 \ No newline at end of file diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt new file mode 100644 index 000000000..afeba1152 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt @@ -0,0 +1 @@ +2024-04-02 \ No newline at end of file diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php index 6afdbe31a..914fe3bbb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php @@ -4,8 +4,19 @@ namespace Flow\ETL\Tests\Integration\DataFrame; -use function Flow\ETL\Adapter\Text\from_text; -use function Flow\ETL\DSL\{df, from_array, from_rows, int_entry, lit, partition, ref, row, rows, rows_partitioned, str_entry}; +use function Flow\ETL\Adapter\Text\{from_text, to_text}; +use function Flow\ETL\DSL\{df, + from_array, + from_rows, + int_entry, + lit, + overwrite, + partition, + ref, + row, + rows, + rows_partitioned, + str_entry}; use Flow\ETL\Tests\Integration\IntegrationTestCase; use Flow\ETL\{Partition, Rows}; @@ -33,6 +44,42 @@ public function test_dropping_partitions() : void self::assertFalse($rows->isPartitioned()); } + public function test_overwrite_save_mode_not_dropping_old_partitions() : void + { + if (\file_exists(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-03')) { + \unlink(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-03/file.txt'); + \rmdir(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-03'); + } + + if (\file_exists(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-04')) { + \unlink(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-04/file.txt'); + \rmdir(__DIR__ . '/Fixtures/Partitioning/overwrite/date=2024-04-04'); + } + + df() + ->read(from_array([ + ['date' => '2024-04-03'], + ['date' => '2024-04-04'], + ])) + ->partitionBy('date') + ->saveMode(overwrite()) + ->write(to_text(__DIR__ . '/Fixtures/Partitioning/overwrite/file.txt')) + ->run(); + + self::assertSame( + [ + ['text' => '2024-04-01', 'date' => '2024-04-01'], + ['text' => '2024-04-02', 'date' => '2024-04-02'], + ['text' => '2024-04-03', 'date' => '2024-04-03'], + ['text' => '2024-04-04', 'date' => '2024-04-04'], + ], + df() + ->read(from_text(__DIR__ . '/Fixtures/Partitioning/overwrite/**/*.txt')) + ->fetch() + ->toArray() + ); + } + public function test_partition_by() : void { $rows = df()