Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix Avro adapter handling nullable values #603

Merged
merged 1 commit into from
Oct 16, 2023
Merged

Fix Avro adapter handling nullable values #603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/SchemaConverter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,7 @@ public function toAvroJsonSchema(Schema $schema) : string
}

/**
* @param class-string<Entry> $type
* @param Definition $definition
*
* @return array{name: string, type: string}
* @return array{name: string, type: string|string[]}
*/
private function convert(Definition $definition) : array
{
Expand Down Expand Up @@ -100,7 +97,7 @@ private function convert(Definition $definition) : array
return ['name' => $definition->entry()->name(), 'type' => ['name' => \ucfirst($definition->entry()->name()), 'type' => \AvroSchema::RECORD_SCHEMA, 'fields' => $structConverter($structureDefinitions)]];
}

return match ($type) {
$avroType = match ($type) {
StringEntry::class, JsonEntry::class, UuidEntry::class => ['name' => $definition->entry()->name(), 'type' => \AvroSchema::STRING_TYPE],
EnumEntry::class => [
'name' => $definition->entry()->name(),
Expand All @@ -118,8 +115,15 @@ private function convert(Definition $definition) : array
BooleanEntry::class => ['name' => $definition->entry()->name(), 'type' => \AvroSchema::BOOLEAN_TYPE],
ArrayEntry::class => throw new RuntimeException("ArrayEntry entry can't be saved in Avro file, try convert it to ListEntry"),
DateTimeEntry::class => ['name' => $definition->entry()->name(), 'type' => 'long', \AvroSchema::LOGICAL_TYPE_ATTR => 'timestamp-micros'],
NullEntry::class => ['name' => $definition->entry()->name(), 'type' => \AvroSchema::NULL_TYPE],
default => throw new RuntimeException($type . ' is not yet supported.')
};

if ($definition->isNullable()) {
$avroType['type'] = [$avroType['type'], \AvroSchema::NULL_TYPE];
}

return $avroType;
}

private function typeFromDefinition(Definition $definition) : string
Expand Down
8 changes: 3 additions & 5 deletions src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/ValueConverter.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,15 @@
$avroType = $this->type($entry);

if ($avroType !== null && \is_array($avroType[\AvroSchema::TYPE_ATTR])) {
if ($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] === \AvroSchema::LONG_TYPE
if (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::LONG_TYPE

Check warning on line 38 in src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/ValueConverter.php

View workflow job for this annotation

GitHub Actions / Mutation Tests (locked, 8.1, ubuntu-latest) 

Escaped Mutant for Mutator "Identical":

--- Original
+++ New
@@ @@
         foreach ($data as $entry => $value) {
             $avroType = $this->type($entry);
             if ($avroType !== null && \is_array($avroType[\AvroSchema::TYPE_ATTR])) {
-                if (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::LONG_TYPE && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
+                if (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) !== \AvroSchema::LONG_TYPE && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
                     $convertedData[$entry] = \DateTimeImmutable::createFromFormat('U.u', \implode('.', \str_split((string) $value, 10)));
                 } elseif (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::ARRAY_SCHEMA && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType[\AvroSchema::TYPE_ATTR]) && $avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
                     $convertedData[$entry] = \array_map(static fn(int $timestamp): \DateTimeImmutable => \DateTimeImmutable::createFromFormat('U.u', \implode('.', \str_split((string) $timestamp, 10))), $value);

Check warning on line 38 in src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/ValueConverter.php

View workflow job for this annotation

GitHub Actions / Mutation Tests (locked, 8.1, ubuntu-latest) 

Escaped Mutant for Mutator "LogicalAnd":

--- Original
+++ New
@@ @@
         foreach ($data as $entry => $value) {
             $avroType = $this->type($entry);
             if ($avroType !== null && \is_array($avroType[\AvroSchema::TYPE_ATTR])) {
-                if (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::LONG_TYPE && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
+                if ((($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::LONG_TYPE || \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType)) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
                     $convertedData[$entry] = \DateTimeImmutable::createFromFormat('U.u', \implode('.', \str_split((string) $value, 10)));
                 } elseif (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::ARRAY_SCHEMA && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType[\AvroSchema::TYPE_ATTR]) && $avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
                     $convertedData[$entry] = \array_map(static fn(int $timestamp): \DateTimeImmutable => \DateTimeImmutable::createFromFormat('U.u', \implode('.', \str_split((string) $timestamp, 10))), $value);
&& \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType)
&& $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros'

Check warning on line 40 in src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/ValueConverter.php

View workflow job for this annotation

GitHub Actions / Mutation Tests (locked, 8.1, ubuntu-latest) 

Escaped Mutant for Mutator "Identical":

--- Original
+++ New
@@ @@
         foreach ($data as $entry => $value) {
             $avroType = $this->type($entry);
             if ($avroType !== null && \is_array($avroType[\AvroSchema::TYPE_ATTR])) {
-                if (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::LONG_TYPE && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
+                if (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::LONG_TYPE && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] !== 'timestamp-micros') {
                     $convertedData[$entry] = \DateTimeImmutable::createFromFormat('U.u', \implode('.', \str_split((string) $value, 10)));
                 } elseif (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::ARRAY_SCHEMA && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType[\AvroSchema::TYPE_ATTR]) && $avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
                     $convertedData[$entry] = \array_map(static fn(int $timestamp): \DateTimeImmutable => \DateTimeImmutable::createFromFormat('U.u', \implode('.', \str_split((string) $timestamp, 10))), $value);
) {
$convertedData[$entry] = \DateTimeImmutable::createFromFormat(
'U.u',
\implode('.', \str_split((string) $value, 10))
);
} elseif ($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] === \AvroSchema::ARRAY_SCHEMA
} elseif (($avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::TYPE_ATTR] ?? null) === \AvroSchema::ARRAY_SCHEMA
&& \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType[\AvroSchema::TYPE_ATTR])
&& $avroType[\AvroSchema::TYPE_ATTR][\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros'
) {
Expand All @@ -55,7 +55,7 @@
$convertedData[$entry] = $value;
}
} else {
if ($avroType[\AvroSchema::TYPE_ATTR] === \AvroSchema::LONG_TYPE

Check warning on line 58 in src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/ValueConverter.php

View workflow job for this annotation

GitHub Actions / Mutation Tests (locked, 8.1, ubuntu-latest) 

Escaped Mutant for Mutator "LogicalAnd":

--- Original
+++ New
@@ @@
                     $convertedData[$entry] = $value;
                 }
             } else {
-                if ($avroType[\AvroSchema::TYPE_ATTR] === \AvroSchema::LONG_TYPE && \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
+                if (($avroType[\AvroSchema::TYPE_ATTR] === \AvroSchema::LONG_TYPE || \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType)) && $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros') {
                     $convertedData[$entry] = \DateTimeImmutable::createFromFormat('U.u', \implode('.', \str_split((string) $value, 10)));
                 } else {
                     $convertedData[$entry] = $value;
&& \array_key_exists(\AvroSchema::LOGICAL_TYPE_ATTR, $avroType)
&& $avroType[\AvroSchema::LOGICAL_TYPE_ATTR] === 'timestamp-micros'
) {
Expand All @@ -69,19 +69,17 @@
}
}

return $convertedData;

Check warning on line 72 in src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/FlixTech/ValueConverter.php

View workflow job for this annotation

GitHub Actions / Mutation Tests (locked, 8.1, ubuntu-latest) 

Escaped Mutant for Mutator "ArrayOneItem":

--- Original
+++ New
@@ @@
                 }
             }
         }
-        return $convertedData;
+        return count($convertedData) > 1 ? array_slice($convertedData, 0, 1, true) : $convertedData;
     }
     private function type(string $entry) : ?array
     {
}

private function type(string $entry) : ?array
{
$type = null;

foreach ($this->avroSchema[\AvroSchema::FIELDS_ATTR] as $avroType) {
if ($avroType[\AvroSchema::NAME_ATTR] === $entry) {
return $avroType;
}
}

return $type;
return null;
}
}
2 changes: 1 addition & 1 deletion src/adapter/etl-adapter-avro/tests/Flow/ETL/Adapter/Avro/Tests/Integration/AvroTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public function test_writing_and_reading_avro_with_all_supported_types() : void
return Row::create(
Entry::integer('integer', $i),
Entry::float('float', 1.5),
Entry::string('string', 'name_' . $i),
$i % 10 === 0 ? Entry::null('string') : Entry::string('string', 'name_' . $i),
Entry::boolean('boolean', true),
Entry::datetime('datetime', new \DateTimeImmutable()),
Entry::json_object('json_object', ['id' => 1, 'name' => 'test']),
Expand Down
Loading