Fixed converting list of structures to parquet schema

flow-php · Jan 1, 2024 · 269bf52 · 269bf52
1 parent 4ecbd64
commit 269bf52
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 1 deletion.
diff --git a/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php b/src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php
@@ -86,7 +86,7 @@ private function flowListToParquetList(ListType $type) : ListElement
                     $this->flowMapValueToParquetMapValue($element->value())
                 );
             case StructureType::class:
-                return ListElement::structure(...$this->flowStructureToParquetStructureElements($element));
+                return ListElement::structure($this->flowStructureToParquetStructureElements($element));
         }
 
         throw new RuntimeException($element::class . ' is not supported.');

diff --git a/...ter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/SchemaConverterTest.php b/...ter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/SchemaConverterTest.php
@@ -4,6 +4,8 @@
 
 namespace Flow\ETL\Adapter\Parquet\Tests\Unit;
 
+use function Flow\ETL\DSL\type_boolean;
+use function Flow\ETL\DSL\type_int;
 use function Flow\ETL\DSL\type_object;
 use function Flow\ETL\DSL\type_string;
 use Flow\ETL\Adapter\Parquet\SchemaConverter;
@@ -44,6 +46,12 @@ public function test_convert_etl_entries_to_parquet_fields() : void
                 FlatColumn::dateTime('datetime'),
                 FlatColumn::string('json'),
                 NestedColumn::list('list', ParquetSchema\ListElement::string()),
+                NestedColumn::list('list_of_structs', ParquetSchema\ListElement::structure(
+                    [
+                        FlatColumn::int64('integer'),
+                        FlatColumn::boolean('boolean'),
+                    ]
+                )),
                 NestedColumn::struct('structure', [FlatColumn::string('a')]),
                 NestedColumn::map('map', ParquetSchema\MapKey::string(), ParquetSchema\MapValue::int64()),
                 FlatColumn::time('time')
@@ -56,6 +64,12 @@ public function test_convert_etl_entries_to_parquet_fields() : void
                 Schema\Definition::dateTime('datetime'),
                 Schema\Definition::json('json'),
                 Schema\Definition::list('list', new ListType(ListElement::string())),
+                Schema\Definition::list('list_of_structs', new ListType(ListElement::structure(
+                    new StructureType(
+                        new StructureElement('integer', type_int()),
+                        new StructureElement('boolean', type_boolean())
+                    ),
+                ))),
                 Schema\Definition::structure('structure', new StructureType(new StructureElement('a', type_string()))),
                 Schema\Definition::map('map', new MapType(MapKey::string(), MapValue::integer())),
                 Schema\Definition::object('time', type_object(\DateInterval::class, false))

diff --git a/src/core/etl/src/Flow/ETL/PHP/Type/Logical/List/ListElement.php b/src/core/etl/src/Flow/ETL/PHP/Type/Logical/List/ListElement.php
@@ -9,6 +9,7 @@
 use function Flow\ETL\DSL\type_string;
 use Flow\ETL\PHP\Type\Logical\ListType;
 use Flow\ETL\PHP\Type\Logical\MapType;
+use Flow\ETL\PHP\Type\Logical\StructureType;
 use Flow\ETL\PHP\Type\Type;
 
 final class ListElement
@@ -60,6 +61,11 @@ public static function string() : self
         return new self(type_string(false));
     }
 
+    public static function structure(StructureType $structure) : self
+    {
+        return new self($structure);
+    }
+
     public function isEqual(mixed $value) : bool
     {
         return $this->value->isEqual($value);