Skip to content

Commit

Permalink
Add polyfill for Parquet compression codecs
Browse files Browse the repository at this point in the history
  • Loading branch information
stloyd committed Dec 1, 2023
1 parent 712d471 commit b043982
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 1 deletion.
1 change: 1 addition & 0 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
"src/core/etl/src/Flow/ETL/DSL/functions.php",
"src/lib/array-dot/src/Flow/ArrayDot/array_dot.php",
"src/lib/parquet/src/Flow/Parquet/functions.php",
"src/lib/parquet/src/Flow/Parquet/polyfill.php",
"src/lib/snappy/polifil.php"
],
"psr-4": {
Expand Down
3 changes: 2 additions & 1 deletion src/lib/parquet/composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@
]
},
"files": [
"src/Flow/Parquet/functions.php"
"src/Flow/Parquet/functions.php",
"src/Flow/Parquet/polyfill.php"
]
},
"autoload-dev": {
Expand Down
6 changes: 6 additions & 0 deletions src/lib/parquet/src/Flow/Parquet/ParquetFile/Codec.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ public function compress(string $data, Compressions $compression) : string
Compressions::SNAPPY => \snappy_compress($data),
/** @phpstan-ignore-next-line */
Compressions::GZIP => \gzencode($data, $this->options->get(Option::GZIP_COMPRESSION_LEVEL)),
Compressions::BROTLI => \brotli_compress($data),
Compressions::LZ4 => \lz4_compress($data),
Compressions::ZSTD => \zstd_compress($data),
default => throw new RuntimeException('Compression ' . $compression->name . ' is not supported yet')
};

Expand All @@ -42,6 +45,9 @@ public function decompress(string $data, Compressions $compression) : string
Compressions::UNCOMPRESSED => $data,
Compressions::SNAPPY => \snappy_uncompress($data),
Compressions::GZIP => \gzdecode($data),
Compressions::BROTLI => \brotli_uncompress($data),
Compressions::LZ4 => \lz4_uncompress($data),
Compressions::ZSTD => \zstd_uncompress($data),
default => throw new RuntimeException('Compression ' . $compression->name . ' is not supported yet')
};

Expand Down
3 changes: 3 additions & 0 deletions src/lib/parquet/src/Flow/Parquet/Writer.php
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ public function __construct(
case Compressions::UNCOMPRESSED:
case Compressions::SNAPPY:
case Compressions::GZIP:
case Compressions::BROTLI:
case Compressions::LZ4:
case Compressions::ZSTD:
break;

default:
Expand Down
47 changes: 47 additions & 0 deletions src/lib/parquet/src/Flow/Parquet/polyfill.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
<?php

declare(strict_types=1);

use Flow\ETL\Exception\RuntimeException;

if (!\function_exists('brotli_compress')) {
function brotli_compress(string $plainText) : string|false
{
throw new RuntimeException('Brotli PHP extension is not installed, please follow instructions from: https://github.com/kjdev/php-ext-brotli');
}
}

if (!\function_exists('brotli_uncompress')) {
function brotli_uncompress(string $compressedText) : string|false
{
throw new RuntimeException('Brotli PHP extension is not installed, please follow instructions from: https://github.com/kjdev/php-ext-brotli');
}
}

if (!\function_exists('lz4_compress')) {
function lz4_compress(string $plainText) : string|false
{
throw new RuntimeException('LZ4 PHP extension is not installed, please follow instructions from: https://github.com/kjdev/php-ext-lz4');
}
}

if (!\function_exists('lz4_uncompress')) {
function lz4_uncompress(string $compressedText) : string|false
{
throw new RuntimeException('Zstd PHP extension is not installed, please follow instructions from: https://github.com/kjdev/php-ext-zstd');
}
}

if (!\function_exists('zstd_compress')) {
function zstd_compress(string $plainText) : string|false
{
throw new RuntimeException('Zstd PHP extension is not installed, please follow instructions from: https://github.com/kjdev/php-ext-zstd');
}
}

if (!\function_exists('zstd_uncompress')) {
function zstd_uncompress(string $compressedText) : string|false
{
throw new RuntimeException('LZ4 PHP extension is not installed, please follow instructions from: https://github.com/kjdev/php-ext-zstd');
}
}

0 comments on commit b043982

Please sign in to comment.