From 0dc418863335f66d129f9a274230b97dc3cfe492 Mon Sep 17 00:00:00 2001 From: tobozo Date: Tue, 9 Nov 2021 15:09:00 +0100 Subject: [PATCH 1/3] stream progress + custom path fix --- src/ESP32-targz-lib.cpp | 71 +++++++++++++++++++++++++++++++++-------- src/ESP32-targz-lib.h | 4 ++- src/uzlib/tinfgzip.c | 6 ++-- 3 files changed, 64 insertions(+), 17 deletions(-) diff --git a/src/ESP32-targz-lib.cpp b/src/ESP32-targz-lib.cpp index 6730aba..10c686a 100644 --- a/src/ESP32-targz-lib.cpp +++ b/src/ESP32-targz-lib.cpp @@ -98,6 +98,7 @@ static int32_t untarredBytesCount = 0; static size_t totalFiles = 0; static size_t totalFolders = 0; static int64_t uzlib_bytesleft = 0; +static int64_t stream_bytesleft = 0; static uint32_t output_position = 0; //position in output_buffer static uint16_t blockmod = GZIP_BUFF_SIZE / TAR_BLOCK_SIZE; static uint16_t gzTarBlockPos = 0; @@ -151,6 +152,7 @@ BaseUnpacker::BaseUnpacker() uzlib_gzip_dict = nullptr; uzlib_bytesleft = 0; + stream_bytesleft = 0; output_position = 0; //position in output_buffer blockmod = GZIP_BUFF_SIZE / TAR_BLOCK_SIZE; gzTarBlockPos = 0; @@ -582,16 +584,26 @@ int TarUnpacker::tarHeaderCallBack( TAR::header_translated_t *header, CC_UNUSED if( !tarSkipThisEntry ) { char file_path[256] = {0}; + // check that TAR path does not start with "./" and truncate if necessary + if( header->filename[0] == '.' && header->filename[1] == '/' ) { + snprintf( file_path, 101, "%s", header->filename ); // TAR paths are limited to 100 chars + snprintf( header->filename, 101, "%s", &file_path[2] ); + } memset( file_path, 0, 256 ); - // avoid double slashing root path if( strcmp( tarDestFolder, FOLDER_SEPARATOR ) != 0 ) { + // destination folder isn't root folder, prefix ! strcat(file_path, tarDestFolder); + // append slash if missing + if( tarDestFolder[strlen(tarDestFolder)-1] != '/' ) { + strcat(file_path, FOLDER_SEPARATOR); + } } // only append slash if destination folder does not end with a slash if( file_path[strlen(file_path)-1] != FOLDER_SEPARATOR[0] ) { strcat(file_path, FOLDER_SEPARATOR); } - strcat(file_path, header->filename); + + strcat(file_path, header->filename ); if( tarFS->exists( file_path ) ) { // file will be truncated @@ -1129,6 +1141,9 @@ unsigned int GzUnpacker::gzReadSourceByte(CC_UNUSED struct GZ::TINF_DATA *data, } else { //log_v("read 1 byte: 0x%02x", out[0] ); } + if( tarGzIO.gz_size > 0 ) { + stream_bytesleft -= 1; + } return 0; } @@ -1262,9 +1277,17 @@ int GzUnpacker::gzUncompress( bool isupdate, bool stream_to_tar, bool use_dict, } if( show_progress ) { - uzlib_bytesleft = tarGzIO.output_size - outlen; - int32_t progress = 100*(tarGzIO.output_size-uzlib_bytesleft) / tarGzIO.output_size; - gzProgressCallback( progress ); + if( tarGzIO.output_size > 0 ) { + uzlib_bytesleft = tarGzIO.output_size - outlen; + int32_t progress = 100*(tarGzIO.output_size-uzlib_bytesleft) / tarGzIO.output_size; + gzProgressCallback( progress ); + } else if( tarGzIO.gz_size > 0 ) { + //uzlib_bytesleft = tarGzIO.output_size - outlen; + //stream_bytesleft = tarGzIO.gz_size - + int32_t progress = 100*(tarGzIO.gz_size-stream_bytesleft) / tarGzIO.gz_size; + gzProgressCallback( progress ); + } + } } while ( res == TINF_OK ); @@ -1455,9 +1478,10 @@ bool GzUnpacker::gzStreamExpander( Stream *stream, size_t gz_size ) if( int( gz_size ) < 1 || gz_size == 0 ) { tgzLogger("[GZStreamExpander] unknown binary size\n"); - + stream_bytesleft = 0; } else { tgzLogger("[GZStreamExpander] Unzipping\n"); + stream_bytesleft = gz_size; } // process with unzipping int ret = gzUncompress( isupdate, stream_to_tar, use_dict, show_progress ); @@ -1704,6 +1728,8 @@ TarGzUnpacker::TarGzUnpacker() : TarUnpacker(), GzUnpacker() // gzWriteCallback bool TarGzUnpacker::gzProcessTarBuffer( CC_UNUSED unsigned char* buff, CC_UNUSED size_t buffsize ) { + //stream_bytesleft -= buffsize; + if( lastblock ) { return true; } @@ -1759,8 +1785,15 @@ int TarGzUnpacker::tarReadGzStream( unsigned char* buff, size_t buffsize ) tarReadGzStreamBytes += i; uzlib_bytesleft = tarGzIO.output_size - tarReadGzStreamBytes; - int32_t progress = 100*(tarGzIO.output_size-uzlib_bytesleft) / tarGzIO.output_size; - gzProgressCallback( progress ); + // stream_bytesleft -= buffsize; + if( tarGzIO.output_size > 0 ) { + int32_t progress = 100*(tarGzIO.output_size-uzlib_bytesleft) / tarGzIO.output_size; + gzProgressCallback( progress ); + } + // else if( tarGzIO.gz_size>0 ) { + //int32_t progress = 100*(tarGzIO.gz_size-stream_bytesleft) / tarGzIO.gz_size; + //gzProgressCallback( progress ); + //} return i; } @@ -1774,6 +1807,7 @@ int TarGzUnpacker::gzFeedTarBuffer( unsigned char* buff, size_t buffsize ) setError( ESP32_TARGZ_TAR_ERR_GZDEFL_FAIL ); return 0; } + //stream_bytesleft -= buffsize; uint32_t blockpos = gzTarBlockPos%blockmod; memcpy( buff, output_buffer/*uzlib_buffer*/+(TAR_BLOCK_SIZE*blockpos), TAR_BLOCK_SIZE ); bytes_fed += TAR_BLOCK_SIZE; @@ -1979,9 +2013,22 @@ bool TarGzUnpacker::tarGzStreamUpdater( Stream *stream ) // uncompress tar+gz stream (file or HTTP) to filesystem without intermediate tar file -bool TarGzUnpacker::tarGzStreamExpander( Stream *stream, fs::FS &destFS, const char* destFolder ) +bool TarGzUnpacker::tarGzStreamExpander( Stream *stream, fs::FS &destFS, const char* destFolder, int64_t streamSize ) { + bool isupdate = false; + bool stream_to_tar = false; + bool use_dict = true; + bool show_progress = false; + + // size was provided when passing the stream, enable progress + if( streamSize > 0 ) { + tarGzIO.gz_size = streamSize; + stream_bytesleft = streamSize; + show_progress = true; + log_w("Enabling progress"); + } + if( !stream->available() ) { log_e("Bad stream, aborting"); setError( ESP32_TARGZ_STREAM_ERROR ); @@ -1989,6 +2036,7 @@ bool TarGzUnpacker::tarGzStreamExpander( Stream *stream, fs::FS &destFS, const c } if( !gzProgressCallback ) { + show_progress = false; setGzProgressCallback( defaultProgressCallback ); } @@ -2030,10 +2078,7 @@ bool TarGzUnpacker::tarGzStreamExpander( Stream *stream, fs::FS &destFS, const c firstblock = true; // trigger TAR setup from gzUncompress callback lastblock = false; - bool isupdate = false; - bool stream_to_tar = false; - bool use_dict = true; - bool show_progress = false; + #if defined ESP8266 diff --git a/src/ESP32-targz-lib.h b/src/ESP32-targz-lib.h index 742a075..0034fae 100644 --- a/src/ESP32-targz-lib.h +++ b/src/ESP32-targz-lib.h @@ -86,6 +86,8 @@ namespace TAR } } +#include + // Callbacks for getting free/total space left on *destination* device. // Optional but recommended to prevent SPIFFS/LittleFS/FFat partitions // to explode during stream writes. @@ -269,7 +271,7 @@ struct TarGzUnpacker : public TarUnpacker, public GzUnpacker // requirements: targz archive must contain files with names suffixed by ".ino.bin" and/or ".spiffs.bin" bool tarGzStreamUpdater( Stream *stream ); // unpack stream://fileName.tar.gz contents to destFS::/destFolder/ - bool tarGzStreamExpander( Stream *stream, fs::FS &destFs, const char* destFolder = "/" ); + bool tarGzStreamExpander( Stream *stream, fs::FS &destFs, const char* destFolder = "/", int64_t streamSize = -1 ); static bool gzProcessTarBuffer( unsigned char* buff, size_t buffsize ); static int tarReadGzStream( unsigned char* buff, size_t buffsize ); diff --git a/src/uzlib/tinfgzip.c b/src/uzlib/tinfgzip.c index 090b027..5529802 100644 --- a/src/uzlib/tinfgzip.c +++ b/src/uzlib/tinfgzip.c @@ -64,13 +64,13 @@ int uzlib_gzip_parse_header(TINF_DATA *d) /* check id bytes */ if (uzlib_get_byte(d) != 0x1f || uzlib_get_byte(d) != 0x8b) { - d->log("bad id bytes\n"); + d->log("uzlib_gzip_parse_header: bad id bytes (expecting 1f:8b\n"); return TINF_DATA_ERROR; } /* check method is deflate */ if (uzlib_get_byte(d) != 8) { - d->log("bad method\n"); + d->log("uzlib_gzip_parse_header: bad method\n"); return TINF_DATA_ERROR; } @@ -79,7 +79,7 @@ int uzlib_gzip_parse_header(TINF_DATA *d) /* check that reserved bits are zero */ if (flg & 0xe0) { - d->log("bad reserved bytes\n"); + d->log("uzlib_gzip_parse_header: bad reserved bytes\n"); return TINF_DATA_ERROR; } From 89f65a75b2003a562febed7735a2ce1571fe6cad Mon Sep 17 00:00:00 2001 From: tobozo Date: Tue, 9 Nov 2021 15:11:10 +0100 Subject: [PATCH 2/3] fix for HTTPS without cert --- .../ESP32/Unpack_tar_gz_stream/Unpack_tar_gz_stream.ino | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/ESP32/Unpack_tar_gz_stream/Unpack_tar_gz_stream.ino b/examples/ESP32/Unpack_tar_gz_stream/Unpack_tar_gz_stream.ino index 2041283..d210ada 100644 --- a/examples/ESP32/Unpack_tar_gz_stream/Unpack_tar_gz_stream.ino +++ b/examples/ESP32/Unpack_tar_gz_stream/Unpack_tar_gz_stream.ino @@ -64,7 +64,11 @@ void stubbornConnect() WiFiClient *getTarGzHTTPClientPtr( WiFiClientSecure *client, const char* url, const char *cert = NULL ) { - client->setCACert( cert ); + if( cert == NULL ) { + client->setInsecure(); + } else { + client->setCACert( cert ); + } const char* UserAgent = "ESP32-HTTP-GzUpdater-Client"; http.setReuse(true); // handle 301 redirects gracefully http.setUserAgent( UserAgent ); @@ -134,7 +138,7 @@ void setup() #if defined DEST_FS_USES_SPIFFS || defined DEST_FS_USES_LITTLEFS tarGzFS.format(); #endif - if( !tarGzFS.begin() ) { + if( !tarGzFS.begin(4) ) { Serial.println("Could not start filesystem"); while(1); } From e1215ead74002ed156fb38a719bb475aae840079 Mon Sep 17 00:00:00 2001 From: tobozo Date: Tue, 9 Nov 2021 15:11:26 +0100 Subject: [PATCH 3/3] raising version --- library.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library.properties b/library.properties index ddc7b6e..20f5e44 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=ESP32-targz -version=1.1.2 +version=1.1.3 author=tobozo maintainer=tobozo sentence=A library to unpack/uncompress tar, gz, and tar.gz files on ESP32 and ESP8266