Add UniqueFactory for creating random string or int from given range (#…

…1128) * Add UniqueFactory for creating random string or int from given range * CS fixes Co-authored-by: Joseph Bielawski <[email protected]> * Rename to RandomGenerator, introduce interface * Fix missing import after move * Use functions instead of static calls for tests requiring randomness * Delete src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleCacheTest.php * Use functions instead of static calls for tests requiring randomness * Add scalar function for random string --------- Co-authored-by: Joseph Bielawski <[email protected]>
flow-php · Jul 31, 2024 · 63ef619 · 63ef619
1 parent 042b5f8
commit 63ef619
Show file tree

Hide file tree

Showing 37 changed files with 421 additions and 166 deletions.
diff --git a/docs/components/libs/snappy.md b/docs/components/libs/snappy.md
@@ -37,7 +37,7 @@ $faker = \Faker\Factory::create();
 
 $texts = [];
 for ($i = 0; $i < 10_000; $i++) {
-    $textSize = \random_int(100, 5000);
+    $textSize = \Flow\ETL\UniqueFactory::int(100, 5000);
     $texts[] = $faker->text($textSize);
 }
 

diff --git a/...L/Adapter/Elasticsearch/Tests/Integration/ElasticsearchPHP/ElasticsearchExtractorTest.php b/...L/Adapter/Elasticsearch/Tests/Integration/ElasticsearchPHP/ElasticsearchExtractorTest.php
@@ -5,7 +5,7 @@
 namespace Flow\ETL\Adapter\Elasticsearch\Tests\Integration\ElasticsearchPHP;
 
 use function Flow\ETL\Adapter\Elasticsearch\{es_hits_to_rows, from_es, to_es_bulk_index};
-use function Flow\ETL\DSL\df;
+use function Flow\ETL\DSL\{df, generate_random_int};
 use Flow\ETL\Adapter\Elasticsearch\ElasticsearchPHP\DocumentDataSource;
 use Flow\ETL\Adapter\Elasticsearch\EntryIdFactory\EntryIdFactory;
 use Flow\ETL\Adapter\Elasticsearch\Tests\Integration\TestCase;
@@ -40,7 +40,7 @@ public function test_empty_extraction() : void
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 \range(1, 100)
             ),
@@ -80,7 +80,7 @@ public function test_extraction_index_with_from_and_size() : void
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 \range(1, 2000)
             ),
@@ -123,7 +123,7 @@ public function test_extraction_index_with_search_after() : void
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 \range(1, 2005)
             ),
@@ -159,7 +159,7 @@ public function test_extraction_index_with_search_after_with_point_in_time() : v
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 \range(1, 2005)
             ),
@@ -200,7 +200,7 @@ public function test_extraction_whole_index_with_point_in_time() : void
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 \range(1, 2005)
             ),

diff --git a/.../ETL/Adapter/Elasticsearch/Tests/Integration/ElasticsearchPHP/ElasticsearchLoaderTest.php b/.../ETL/Adapter/Elasticsearch/Tests/Integration/ElasticsearchPHP/ElasticsearchLoaderTest.php
@@ -5,6 +5,7 @@
 namespace Flow\ETL\Adapter\Elasticsearch\Tests\Integration\ElasticsearchPHP;
 
 use function Flow\ETL\Adapter\Elasticsearch\{to_es_bulk_index, to_es_bulk_update};
+use function Flow\ETL\DSL\generate_random_string;
 use Flow\ETL\Adapter\Elasticsearch\EntryIdFactory\{EntryIdFactory, HashIdFactory};
 use Flow\ETL\Adapter\Elasticsearch\Tests\Integration\TestCase;
 use Flow\ETL\{Config, FlowContext, Row, Rows};
@@ -53,19 +54,19 @@ public function test_integration_with_entry_factory() : void
 
         $loader->load(new Rows(
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Łukasz')
             ),
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . \Flow\ETL\DSL\generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Norbert')
             ),
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . \Flow\ETL\DSL\generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Dawid')
             ),
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . \Flow\ETL\DSL\generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Tomek')
             ),
         ), new FlowContext(Config::default()));

diff --git a/...low/ETL/Adapter/Meilisearch/Tests/Integration/MeilisearchPHP/MeilisearchExtractorTest.php b/...low/ETL/Adapter/Meilisearch/Tests/Integration/MeilisearchPHP/MeilisearchExtractorTest.php
@@ -5,6 +5,7 @@
 namespace Flow\ETL\Adapter\Meilisearch\Tests\Integration\MeilisearchPHP;
 
 use function Flow\ETL\Adapter\Meilisearch\{from_meilisearch, meilisearch_hits_to_rows, to_meilisearch_bulk_index};
+use function Flow\ETL\DSL\generate_random_int;
 use Flow\ETL\Adapter\Meilisearch\Tests\Context\MeilisearchContext;
 use Flow\ETL\{Config, Flow, FlowContext, Row, Rows};
 use PHPUnit\Framework\TestCase;
@@ -37,7 +38,7 @@ public function test_empty_extraction() : void
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 \range(1, 100)
             ),
@@ -63,7 +64,7 @@ public function test_extraction_index_with_from_and_size() : void
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 // Default limit for Meilisearch is 1000 documents: https://www.meilisearch.com/docs/reference/api/settings#pagination
                 \range(1, 999)
@@ -102,7 +103,7 @@ public function test_extraction_index_with_sort() : void
                     new Row\Entry\StringEntry('id', \sha1((string) $i)),
                     new Row\Entry\IntegerEntry('position', $i),
                     new Row\Entry\StringEntry('name', 'id_' . $i),
-                    new Row\Entry\BooleanEntry('active', (bool) \random_int(0, 1))
+                    new Row\Entry\BooleanEntry('active', (bool) generate_random_int(0, 1))
                 ),
                 // Default limit for Meilisearch is 1000 documents: https://www.meilisearch.com/docs/reference/api/settings#pagination
                 \range(1, 999)

diff --git a/...s/Flow/ETL/Adapter/Meilisearch/Tests/Integration/MeilisearchPHP/MeilisearchLoaderTest.php b/...s/Flow/ETL/Adapter/Meilisearch/Tests/Integration/MeilisearchPHP/MeilisearchLoaderTest.php
@@ -43,19 +43,19 @@ public function test_integration_with_entry_factory() : void
         $loader = to_meilisearch_bulk_index($this->meilisearchContext->clientConfig(), self::INDEX_NAME);
         $loader->load(new Rows(
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . \Flow\ETL\DSL\generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Łukasz')
             ),
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . \Flow\ETL\DSL\generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Norbert')
             ),
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . \Flow\ETL\DSL\generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Dawid')
             ),
             Row::create(
-                new Row\Entry\StringEntry('id', \sha1('id' . bin2hex(random_bytes(16)))),
+                new Row\Entry\StringEntry('id', \sha1('id' . \Flow\ETL\DSL\generate_random_string())),
                 new Row\Entry\StringEntry('name', 'Tomek')
             ),
         ), new FlowContext(Config::default()));

diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php
@@ -13,7 +13,7 @@ final class TextTest extends TestCase
 {
     public function test_loading_text_files() : void
     {
-        $path = __DIR__ . '/var/flow_php_etl_csv_loader' . bin2hex(random_bytes(16)) . '.csv';
+        $path = __DIR__ . '/var/flow_php_etl_csv_loader' . \Flow\ETL\DSL\generate_random_string() . '.csv';
 
         (new Flow())
             ->process(

diff --git a/...em/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/AzureBlobDestinationStreamTest.php b/...em/azure/tests/Flow/Filesystem/Bridge/Azure/Tests/Unit/AzureBlobDestinationStreamTest.php
@@ -24,7 +24,7 @@ public function test_using_put_blob_with_content_when_data_is_larger_than_block_
         $blockFactory->method('create')
             ->willReturnCallback(
                 function () use ($blockSize) {
-                    return new Block($id = \bin2hex(\random_bytes(16)), $blockSize, new Path(sys_get_temp_dir() . '/' . $id . '_block_01.txt'));
+                    return new Block($id = \Flow\ETL\DSL\generate_random_string(), $blockSize, new Path(sys_get_temp_dir() . '/' . $id . '_block_01.txt'));
                 }
             );
 
@@ -74,7 +74,7 @@ public function test_using_put_blob_with_content_when_data_is_smaller_than_block
         $blockFactory->method('create')
             ->willReturnCallback(
                 function () use ($blockSize) {
-                    return new Block($id = \bin2hex(\random_bytes(16)), $blockSize, new Path(sys_get_temp_dir() . '/' . $id . '_block_01.txt'));
+                    return new Block($id = \Flow\ETL\DSL\generate_random_string(), $blockSize, new Path(sys_get_temp_dir() . '/' . $id . '_block_01.txt'));
                 }
             );
         $stream = AzureBlobDestinationStream::openBlank(

diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php
@@ -13,7 +13,7 @@
 use Flow\ETL\PHP\Type\Caster;
 use Flow\ETL\Pipeline\Optimizer;
 use Flow\ETL\Row\Factory\NativeEntryFactory;
-use Flow\ETL\{Cache, Config};
+use Flow\ETL\{Cache, Config, NativePHPRandomValueGenerator, RandomValueGenerator};
 use Flow\Filesystem\{Filesystem, FilesystemTable};
 use Flow\Serializer\{Base64Serializer, NativePHPSerializer, Serializer};
 
@@ -33,6 +33,8 @@ final class ConfigBuilder
 
     private bool $putInputIntoRows;
 
+    private RandomValueGenerator $randomValueGenerator;
+
     private ?Serializer $serializer;
 
     public function __construct()
@@ -45,11 +47,12 @@ public function __construct()
         $this->caster = null;
         $this->cache = new CacheConfigBuilder();
         $this->sort = new SortConfigBuilder();
+        $this->randomValueGenerator = new NativePHPRandomValueGenerator();
     }
 
     public function build() : Config
     {
-        $this->id ??= 'flow_php_' . bin2hex(random_bytes(16));
+        $this->id ??= 'flow_php' . $this->randomValueGenerator->string(32);
         $entryFactory = new NativeEntryFactory();
         $this->serializer ??= new Base64Serializer(new NativePHPSerializer());
 

diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php
@@ -14,7 +14,65 @@
 use Flow\ETL\Function\ArraySort\Sort;
 use Flow\ETL\Function\Between\Boundary;
 use Flow\ETL\Function\StyleConverter\StringStyles;
-use Flow\ETL\Function\{All, Any, ArrayExists, ArrayGet, ArrayGetCollection, ArrayKeyRename, ArrayKeysStyleConvert, ArrayMerge, ArrayMergeCollection, ArrayReverse, ArraySort, ArrayUnpack, Average, Between, CallMethod, Capitalize, Cast, Collect, CollectUnique, Combine, Concat, Count, DateTimeFormat, DenseRank, Exists, First, Hash, Last, ListFunctions, Literal, Max, Min, Not, Now, NumberFormat, Optional, PregMatch, PregMatchAll, PregReplace, Rank, Round, RowNumber, Sanitize, ScalarFunction, Size, Split, Sprintf, StructureFunctions, Sum, ToDate, ToDateTime, ToLower, ToMoney, ToTimeZone, ToUpper, Ulid, Uuid, When};
+use Flow\ETL\Function\{All,
+    Any,
+    ArrayExists,
+    ArrayGet,
+    ArrayGetCollection,
+    ArrayKeyRename,
+    ArrayKeysStyleConvert,
+    ArrayMerge,
+    ArrayMergeCollection,
+    ArrayReverse,
+    ArraySort,
+    ArrayUnpack,
+    Average,
+    Between,
+    CallMethod,
+    Capitalize,
+    Cast,
+    Collect,
+    CollectUnique,
+    Combine,
+    Concat,
+    Count,
+    DateTimeFormat,
+    DenseRank,
+    Exists,
+    First,
+    Hash,
+    Last,
+    ListFunctions,
+    Literal,
+    Max,
+    Min,
+    Not,
+    Now,
+    NumberFormat,
+    Optional,
+    PregMatch,
+    PregMatchAll,
+    PregReplace,
+    RandomString,
+    Rank,
+    Round,
+    RowNumber,
+    Sanitize,
+    ScalarFunction,
+    Size,
+    Split,
+    Sprintf,
+    StructureFunctions,
+    Sum,
+    ToDate,
+    ToDateTime,
+    ToLower,
+    ToMoney,
+    ToTimeZone,
+    ToUpper,
+    Ulid,
+    Uuid,
+    When};
 use Flow\ETL\Loader\StreamLoader\Output;
 use Flow\ETL\Loader\{CallbackLoader, MemoryLoader, StreamLoader, TransformerLoader};
 use Flow\ETL\Memory\Memory;
@@ -50,7 +108,9 @@
     Join\Comparison\Identical,
     Join\Expression,
     Loader,
+    NativePHPRandomValueGenerator,
     Pipeline,
+    RandomValueGenerator,
     Row,
     Rows,
     Transformer,
@@ -1209,3 +1269,20 @@ function is_type(array $types, mixed $value) : bool
 
     return false;
 }
+
+function generate_random_string(int $length = 32, NativePHPRandomValueGenerator $generator = new NativePHPRandomValueGenerator()) : string
+{
+    return $generator->string($length);
+}
+
+function generate_random_int(int $start = PHP_INT_MIN, int $end = PHP_INT_MAX, NativePHPRandomValueGenerator $generator = new NativePHPRandomValueGenerator()) : int
+{
+    return $generator->int($start, $end);
+}
+
+function random_string(
+    int|ScalarFunction $length,
+    RandomValueGenerator $generator = new NativePHPRandomValueGenerator()
+) : RandomString {
+    return new RandomString($length, $generator);
+}
diff --git a/src/core/etl/src/Flow/ETL/Function/RandomString.php b/src/core/etl/src/Flow/ETL/Function/RandomString.php
@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Function;
+
+use Flow\ETL\{NativePHPRandomValueGenerator, RandomValueGenerator, Row};
+
+class RandomString implements ScalarFunction
+{
+    private RandomValueGenerator|NativePHPRandomValueGenerator $generator;
+
+    private int|ScalarFunction $length;
+
+    public function __construct(ScalarFunction|int $length, RandomValueGenerator $generator = new NativePHPRandomValueGenerator())
+    {
+        $this->length = $length;
+        $this->generator = $generator;
+    }
+
+    public function eval(Row $row) : string
+    {
+        return $this->generator->string(is_int($this->length) ? $this->length : $this->length->eval($row));
+    }
+}
diff --git a/src/core/etl/src/Flow/ETL/NativePHPRandomValueGenerator.php b/src/core/etl/src/Flow/ETL/NativePHPRandomValueGenerator.php
@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL;
+
+final class NativePHPRandomValueGenerator implements RandomValueGenerator
+{
+    public function int(int $min, int $max) : int
+    {
+        return \random_int($min, $max);
+    }
+
+    public function string(int $int) : string
+    {
+        $bytes = (int) \ceil($int / 2);
+        $bytes >= 1 ?: $bytes = 1;
+
+        return \substr(\bin2hex(\random_bytes($bytes)), 0, \max(0, $int));
+    }
+}
diff --git a/src/core/etl/src/Flow/ETL/RandomValueGenerator.php b/src/core/etl/src/Flow/ETL/RandomValueGenerator.php
@@ -0,0 +1,12 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL;
+
+interface RandomValueGenerator
+{
+    public function int(int $min, int $max) : int;
+
+    public function string(int $int) : string;
+}
diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeExtractor.php
@@ -9,6 +9,7 @@
     datetime_entry,
     enum_entry,
     float_entry,
+    generate_random_int,
     int_entry,
     json_entry,
     list_entry,
@@ -50,7 +51,7 @@ public function extract(FlowContext $context) : \Generator
             yield rows(
                 row(
                     int_entry('int', $id),
-                    float_entry('float', \random_int(100, 100000) / 100),
+                    float_entry('float', generate_random_int(100, 100000) / 100),
                     bool_entry('bool', false),
                     datetime_entry('datetime', new \DateTimeImmutable('now')),
                     str_entry('null', null),