Skip to content

Commit

Permalink
Added support for converting entries into attributes through naming c…
Browse files Browse the repository at this point in the history
…onvention
  • Loading branch information
norberttech committed Aug 4, 2024
1 parent f876355 commit a4b590b
Show file tree
Hide file tree
Showing 15 changed files with 297 additions and 178 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,15 @@ public static function nestedNode(string $name) : self
return new self($name, null, XMLNodeType::NESTED);
}

public function append(self|XMLAttribute $element) : self
{
if ($element instanceof XMLAttribute) {
return $this->appendAttribute($element);
}

return $this->appendChild($element);
}

public function appendAttribute(XMLAttribute $attribute) : self
{
return new self(
Expand All @@ -48,7 +57,7 @@ public function appendAttribute(XMLAttribute $attribute) : self
);
}

public function appendChild(self $node) : self
public function appendChild(self $child) : self
{
if ($this->type === XMLNodeType::FLAT) {
throw new InvalidArgumentException('XMLNode can not have children if it has value');
Expand All @@ -59,7 +68,7 @@ public function appendChild(self $node) : self
$this->value,
$this->type,
$this->attributes,
[...$this->children, $node]
[...$this->children, $child]
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ public function __construct(
private readonly Path $path,
private readonly string $rootElementName,
private readonly string $rowElementName,
private readonly string $attributePrefix,
private readonly string $dateTimeFormat,
private readonly XMLWriter $xmlWriter
) {
}
Expand All @@ -44,7 +46,14 @@ public function destination() : Path

public function load(Rows $rows, FlowContext $context) : void
{
$normalizer = new RowsNormalizer(new EntryNormalizer(new PHPValueNormalizer($context->config->caster())), $this->rowElementName);
$normalizer = new RowsNormalizer(
new EntryNormalizer(
new PHPValueNormalizer($context->config->caster(), $this->attributePrefix, $this->dateTimeFormat),
$this->attributePrefix,
$this->dateTimeFormat
),
$this->rowElementName
);

$this->write($rows, $rows->partitions()->toArray(), $context, $normalizer);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public function normalize(Rows $rows) : \Generator
$node = XMLNode::nestedNode($this->rowNodeName);

foreach ($row->entries() as $entry) {
$node = $node->appendChild($this->entryNormalizer->normalize($entry));
$node = $node->append($this->entryNormalizer->normalize($entry));
}

yield $node;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

namespace Flow\ETL\Adapter\XML\RowsNormalizer;

use Flow\ETL\Adapter\XML\Abstraction\{XMLNode};
use Flow\ETL\Adapter\XML\Abstraction\{XMLAttribute, XMLNode};
use Flow\ETL\Adapter\XML\RowsNormalizer\EntryNormalizer\PHPValueNormalizer;
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\PHP\Type\Logical\Structure\StructureElement;
Expand All @@ -16,14 +16,17 @@ final class EntryNormalizer
{
public function __construct(
private readonly PHPValueNormalizer $valueNormalizer,
private readonly string $attributePrefix = '_',
private readonly string $dateTimeFormat = PHPValueNormalizer::DATE_TIME_FORMAT
) {

}

public function normalize(Entry $entry) : XMLNode
public function normalize(Entry $entry) : XMLNode|XMLAttribute
{
// TODO: detect entries with specific name to turn them into attributes
if (\str_starts_with($entry->name(), $this->attributePrefix)) {
return new XMLAttribute(\substr($entry->name(), \strlen($this->attributePrefix)), $entry->toString());
}

if ($entry instanceof ListEntry) {
return $this->listToNode($entry);
Expand All @@ -37,8 +40,6 @@ public function normalize(Entry $entry) : XMLNode
return $this->structureToNode($entry);
}

// TODO: handle XMLElementEntry and XMLEntry

return match ($entry::class) {
StringEntry::class => XMLNode::flatNode($entry->name(), $entry->value()),
IntegerEntry::class => XMLNode::flatNode($entry->name(), (string) $entry->value()),
Expand Down Expand Up @@ -73,7 +74,7 @@ private function listToNode(ListEntry $entry) : XMLNode
}

foreach ($listValue as $value) {
$node = $node->appendChild($this->valueNormalizer->normalize('element', $type->element()->type(), $value));
$node = $node->append($this->valueNormalizer->normalize('element', $type->element()->type(), $value));
}

return $node;
Expand Down Expand Up @@ -124,8 +125,8 @@ private function mapToNode(MapEntry $entry) : XMLNode
$type = $entry->type();

foreach ($mapValue as $key => $value) {
$node = $node->appendChild($this->valueNormalizer->normalize('key', $type->key()->type(), $key));
$node = $node->appendChild($this->valueNormalizer->normalize('value', $type->value()->type(), $value));
$node = $node->append($this->valueNormalizer->normalize('key', $type->key()->type(), $key));
$node = $node->append($this->valueNormalizer->normalize('value', $type->value()->type(), $value));
}

return $node;
Expand Down Expand Up @@ -153,7 +154,7 @@ private function structureToNode(StructureEntry $entry) : XMLNode
$structureElement = $element['structure_element'];
$structureValue = $element['value_element'];

$node = $node->appendChild($this->valueNormalizer->normalize($structureElement->name(), $structureElement->type(), $structureValue));
$node = $node->append($this->valueNormalizer->normalize($structureElement->name(), $structureElement->type(), $structureValue));
}

return $node;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
namespace Flow\ETL\Adapter\XML\RowsNormalizer\EntryNormalizer;

use function Flow\ETL\DSL\{type_json, type_string};
use Flow\ETL\Adapter\XML\Abstraction\XMLNode;
use Flow\ETL\Adapter\XML\Abstraction\{XMLAttribute, XMLNode};
use Flow\ETL\Exception\InvalidArgumentException;
use Flow\ETL\PHP\Type\Logical\Structure\StructureElement;
use Flow\ETL\PHP\Type\Logical\{DateTimeType, JsonType, ListType, MapType, StructureType, UuidType};
Expand All @@ -18,13 +18,18 @@ final class PHPValueNormalizer

public function __construct(
private readonly Caster $caster,
private readonly string $attributePrefix = '_',
private readonly string $dateTimeFormat = self::DATE_TIME_FORMAT
) {

}

public function normalize(string $name, Type $type, mixed $value) : XMLNode
public function normalize(string $name, Type $type, mixed $value) : XMLNode|XMLAttribute
{
if (\str_starts_with($name, $this->attributePrefix)) {
return new XMLAttribute(\substr($name, \strlen($this->attributePrefix)), $this->caster->to(type_string())->value($value));
}

if ($value === null) {
return XMLNode::flatNode($name, '');
}
Expand All @@ -37,7 +42,7 @@ public function normalize(string $name, Type $type, mixed $value) : XMLNode
}

foreach ($value as $elementValue) {
$listNode = $listNode->appendChild($this->normalize('element', $type->element()->type(), $elementValue));
$listNode = $listNode->append($this->normalize('element', $type->element()->type(), $elementValue));
}

return $listNode;
Expand All @@ -51,10 +56,10 @@ public function normalize(string $name, Type $type, mixed $value) : XMLNode
}

foreach ($value as $key => $elementValue) {
$mapNode = $mapNode->appendChild(
$mapNode = $mapNode->append(
XMLNode::nestedNode('element')
->appendChild($this->normalize('key', $type->key()->type(), $key))
->appendChild($this->normalize('value', $type->value()->type(), $elementValue))
->append($this->normalize('key', $type->key()->type(), $key))
->append($this->normalize('value', $type->value()->type(), $elementValue))
);
}

Expand All @@ -77,7 +82,7 @@ public function normalize(string $name, Type $type, mixed $value) : XMLNode
$structureElement = $element['structure_element'];
$structureValue = $element['value_element'];

$structureNode = $structureNode->appendChild($this->normalize($structureElement->name(), $structureElement->type(), $structureValue));
$structureNode = $structureNode->append($this->normalize($structureElement->name(), $structureElement->type(), $structureValue));
}

return $structureNode;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

use function Flow\ETL\DSL\from_all;
use Flow\ETL\{Adapter\XML\Loader\XMLLoader,
Adapter\XML\RowsNormalizer\EntryNormalizer\PHPValueNormalizer,
Adapter\XML\XMLWriter\DOMDocumentWriter,
Extractor
};
Extractor};
use Flow\Filesystem\Path;

/**
Expand Down Expand Up @@ -42,12 +42,16 @@ function to_xml(
string|Path $path,
string $root_element_name = 'rows',
string $row_element_name = 'row',
string $attribute_prefix = '_',
string $date_time_format = PHPValueNormalizer::DATE_TIME_FORMAT,
XMLWriter $xml_writer = new DOMDocumentWriter()
) : XMLLoader {
return new XMLLoader(
\is_string($path) ? Path::realpath($path) : $path,
$root_element_name,
$row_element_name,
$attribute_prefix,
$date_time_format,
$xml_writer
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,27 +5,55 @@
namespace Flow\ETL\Adapter\XML\Tests\Integration\Loader;

use function Flow\ETL\Adapter\XML\{from_xml, to_xml};
use function Flow\ETL\DSL\{df, overwrite};
use function Flow\ETL\DSL\{df, from_array, overwrite};
use Flow\ETL\Tests\Double\FakeExtractor;
use Flow\ETL\Tests\Integration\IntegrationTestCase;

final class XMLLoaderTest extends IntegrationTestCase
{
public function test_xml_loader() : void
public function test_writing_empty_rows() : void
{
df()
->read(from_array([]))
->write(to_xml($path = $this->cacheDir->suffix('test_xml_loader.xml')))
->run();

self::assertFalse(\file_exists($path->path()));
}

public function test_writing_xml() : void
{
df()
->read(new FakeExtractor(100))
->saveMode(overwrite())
->write(to_xml($path = __DIR__ . '/var/test_xml_loader.xml'))
->write(to_xml($path = $this->cacheDir->suffix('test_xml_loader.xml')))
->run();

self::assertEquals(
100,
df()->read(from_xml($path, 'rows/row'))->count()
);
}

public function test_writing_xml_with_attributes() : void
{
df()
->read(from_array([
['_id' => 1, 'name' => 'John', 'address' => ['_id' => 1, 'city' => 'New York', 'street' => '5th Avenue']],
['_id' => 2, 'name' => 'Jane', 'address' => ['_id' => 2, 'city' => 'Los Angeles', 'street' => 'Hollywood Boulevard']],
]))
->write(to_xml($path = $this->cacheDir->suffix('test_xml_loader.xml')))
->run();

// if (\file_exists($path)) {
// \unlink($path);
// }
self::assertXmlStringEqualsXmlString(
<<<'XML'
<?xml version="1.0" encoding="UTF-8"?>
<rows>
<row id="1"><name>John</name><address id="1"><city>New York</city><street>5th Avenue</street></address></row>
<row id="2"><name>Jane</name><address id="2"><city>Los Angeles</city><street>Hollywood Boulevard</street></address></row>
</rows>
XML,
\file_get_contents($path->path())
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@ public function test_normalization_of_list_of_flat_structures() : void

self::assertEquals(
XMLNode::nestedNode('list')
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('name', 'John'))
->appendChild(XMLNode::flatNode('age', '30'))
->append(XMLNode::flatNode('name', 'John'))
->append(XMLNode::flatNode('age', '30'))
)
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('name', 'Jane'))
->appendChild(XMLNode::flatNode('age', '25'))
->append(XMLNode::flatNode('name', 'Jane'))
->append(XMLNode::flatNode('age', '25'))
),
$normalizer->normalize(
'list',
Expand Down Expand Up @@ -57,9 +57,9 @@ public function test_normalizing_list_of_integers() : void

self::assertEquals(
XMLNode::nestedNode('list')
->appendChild(XMLNode::flatNode('element', '1'))
->appendChild(XMLNode::flatNode('element', '2'))
->appendChild(XMLNode::flatNode('element', '3')),
->append(XMLNode::flatNode('element', '1'))
->append(XMLNode::flatNode('element', '2'))
->append(XMLNode::flatNode('element', '3')),
$normalizer->normalize('list', type_list(type_integer()), [1, 2, 3])
);
}
Expand All @@ -70,17 +70,17 @@ public function test_normalizing_list_of_list_of_integers() : void

self::assertEquals(
XMLNode::nestedNode('list')
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('element', '1'))
->appendChild(XMLNode::flatNode('element', '2'))
->appendChild(XMLNode::flatNode('element', '3'))
->append(XMLNode::flatNode('element', '1'))
->append(XMLNode::flatNode('element', '2'))
->append(XMLNode::flatNode('element', '3'))
)
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('element', '4'))
->appendChild(XMLNode::flatNode('element', '5'))
->appendChild(XMLNode::flatNode('element', '6'))
->append(XMLNode::flatNode('element', '4'))
->append(XMLNode::flatNode('element', '5'))
->append(XMLNode::flatNode('element', '6'))
),
$normalizer->normalize('list', type_list(type_list(type_integer())), [[1, 2, 3], [4, 5, 6]])
);
Expand All @@ -92,30 +92,30 @@ public function test_normalizing_list_of_map_of_str_to_int() : void

self::assertEquals(
XMLNode::nestedNode('list')
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('key', 'one'))
->appendChild(XMLNode::flatNode('value', '1'))
->append(XMLNode::flatNode('key', 'one'))
->append(XMLNode::flatNode('value', '1'))
)
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('key', 'two'))
->appendChild(XMLNode::flatNode('value', '2'))
->append(XMLNode::flatNode('key', 'two'))
->append(XMLNode::flatNode('value', '2'))
)
)
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('key', 'three'))
->appendChild(XMLNode::flatNode('value', '3'))
->append(XMLNode::flatNode('key', 'three'))
->append(XMLNode::flatNode('value', '3'))
)
->appendChild(
->append(
XMLNode::nestedNode('element')
->appendChild(XMLNode::flatNode('key', 'four'))
->appendChild(XMLNode::flatNode('value', '4'))
->append(XMLNode::flatNode('key', 'four'))
->append(XMLNode::flatNode('value', '4'))
)
),
$normalizer->normalize('list', type_list(type_map(type_integer(), type_string())), [['one' => 1, 'two' => 2], ['three' => 3, 'four' => 4]])
Expand Down
Loading

0 comments on commit a4b590b

Please sign in to comment.