From be35c3cc9342235958835077c46d06c639d5c993 Mon Sep 17 00:00:00 2001 From: Dion Date: Tue, 10 Sep 2024 14:46:37 +0200 Subject: [PATCH] add logger --- .../ArchiveFormats/TarBal.cs | 46 +- .../Services/ExifToolDownload.cs | 647 ++++++++++-------- .../ArchiveFormats/TarBalTest.cs | 88 +-- .../ExifToolHostStorageServiceTest.cs | 2 +- 4 files changed, 432 insertions(+), 351 deletions(-) diff --git a/starsky/starsky.foundation.storage/ArchiveFormats/TarBal.cs b/starsky/starsky.foundation.storage/ArchiveFormats/TarBal.cs index 38bcc0fff2..288a2ce2fb 100644 --- a/starsky/starsky.foundation.storage/ArchiveFormats/TarBal.cs +++ b/starsky/starsky.foundation.storage/ArchiveFormats/TarBal.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using starsky.foundation.platform.Helpers; +using starsky.foundation.platform.Interfaces; using starsky.foundation.storage.Interfaces; namespace starsky.foundation.storage.ArchiveFormats; @@ -14,13 +15,15 @@ namespace starsky.foundation.storage.ArchiveFormats; "CA1835:Prefer the 'Memory'-based overloads for 'ReadAsync' and 'WriteAsync")] public sealed class TarBal { + private readonly IWebLogger _logger; private readonly char _pathSeparator; private readonly IStorage _storage; - public TarBal(IStorage storage, char pathSeparator = '/') + public TarBal(IStorage storage, IWebLogger logger, char pathSeparator = '/') { _storage = storage; _pathSeparator = pathSeparator; + _logger = logger; } /// @@ -54,44 +57,61 @@ public async Task ExtractTarGz(Stream stream, string outputDir, /// /// The .tar to extract. /// Output directory to write the files. - /// cancel token + /// Cancel token public async Task ExtractTar(Stream stream, string outputDir, CancellationToken cancellationToken) { var buffer = new byte[100]; var longFileName = string.Empty; + while ( true ) { - await stream.ReadAsync(buffer, 0, 100, cancellationToken); + // Read the first 100 bytes to get the name + var bytesRead = await stream.ReadAsync(buffer, 0, 100, cancellationToken); + if ( bytesRead != 100 ) + { + _logger.LogError("[ExtractTar] less than 100 bytes read {outputDir}", outputDir); + // If fewer bytes are read, it indicates an incomplete read or end of stream. + break; + } + var name = string.IsNullOrEmpty(longFileName) ? Encoding.ASCII.GetString(buffer).Trim('\0') - : longFileName; //Use longFileName if we have one read + : longFileName; // Use longFileName if we have one read + if ( string.IsNullOrWhiteSpace(name) ) { break; } stream.Seek(24, SeekOrigin.Current); - await stream.ReadAsync(buffer, 0, 12, cancellationToken); + + // Read the size + bytesRead = await stream.ReadAsync(buffer, 0, 12, cancellationToken); + if ( bytesRead != 12 ) + { + break; + } + var size = Convert.ToInt64(Encoding.UTF8.GetString(buffer, 0, 12).Trim('\0').Trim(), 8); - stream.Seek(20, SeekOrigin.Current); //Move head to typeTag byte + stream.Seek(20, SeekOrigin.Current); // Move head to typeTag byte var typeTag = stream.ReadByte(); - stream.Seek(355L, SeekOrigin.Current); //Move head to beginning of data (byte 512) + stream.Seek(355L, SeekOrigin.Current); // Move head to beginning of data (byte 512) if ( typeTag == 'L' ) { - //We have a long file name + // We have a long file name longFileName = await CreateLongFileName(size, stream, cancellationToken); } else { - //We have a normal file or directory - longFileName = - string.Empty; //Reset longFileName if current entry is not indicating one + // We have a normal file or directory + longFileName = string.Empty; + // Reset longFileName if current entry is not indicating one await CreateFileOrDirectory(outputDir, name, size, stream, cancellationToken); } - //Move head to next 512 byte block + // Move head to next 512 byte block var pos = stream.Position; var offset = 512 - pos % 512; if ( offset == 512 ) @@ -128,7 +148,7 @@ private async Task CreateFileOrDirectory(string outputDir, string name, long siz var str = new MemoryStream(); var buf = new byte[size]; await stream.ReadAsync(buf, 0, buf.Length, cancellationToken); - str.Write(buf, 0, buf.Length); + await str.WriteAsync(buf, 0, buf.Length, cancellationToken); _storage.WriteStreamOpenOrCreate(str, output); } } diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs index 2c79bc9849..acb98196f1 100644 --- a/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs @@ -20,384 +20,441 @@ using starsky.foundation.writemeta.Interfaces; [assembly: InternalsVisibleTo("starskytest")] -namespace starsky.foundation.writemeta.Services -{ - [Service(typeof(IExifToolDownload), InjectionLifetime = InjectionLifetime.Singleton)] - [SuppressMessage("Usage", "S1075:Refactor your code not to use hardcoded absolute paths or URIs", Justification = "Source of files")] - public sealed class ExifToolDownload : IExifToolDownload - { - private readonly IHttpClientHelper _httpClientHelper; - private readonly AppSettings _appSettings; - private readonly IStorage _hostFileSystemStorage; - private readonly IWebLogger _logger; - - private const string CheckSumLocation = "https://exiftool.org/checksums.txt"; - private const string CheckSumLocationMirror = "https://qdraw.nl/special/mirror/exiftool/checksums.txt"; - private const string ExiftoolDownloadBasePath = "https://exiftool.org/"; // with slash at the end - private const string ExiftoolDownloadBasePathMirror = "https://qdraw.nl/special/mirror/exiftool/"; // with slash at the end - public ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, IWebLogger logger) - { - _httpClientHelper = httpClientHelper; - _appSettings = appSettings; - _hostFileSystemStorage = new StorageHostFullPathFilesystem(logger); - _logger = logger; - } +namespace starsky.foundation.writemeta.Services; - internal ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, IWebLogger logger, IStorage storage) - { - _httpClientHelper = httpClientHelper; - _appSettings = appSettings; - _hostFileSystemStorage = storage; - _logger = logger; - } +[Service(typeof(IExifToolDownload), InjectionLifetime = InjectionLifetime.Singleton)] +[SuppressMessage("Usage", "S1075:Refactor your code not to use hardcoded absolute paths or URIs", + Justification = "Source of files")] +public sealed class ExifToolDownload : IExifToolDownload +{ + private const string CheckSumLocation = "https://exiftool.org/checksums.txt"; - /// - /// Auto Download Exiftool - /// - /// download Windows version if true - /// check for min file size in bytes (Default = 30 bytes) - /// - public async Task DownloadExifTool(bool isWindows, int minimumSize = 30) - { - if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || _appSettings is { AddSwaggerExport: true, AddSwaggerExportExitAfter: true } ) - { - var name = _appSettings.ExiftoolSkipDownloadOnStartup == true - ? "ExiftoolSkipDownloadOnStartup" - : "AddSwaggerExport and AddSwaggerExportExitAfter"; - _logger.LogInformation($"[DownloadExifTool] Skipped due true of {name} setting"); - return false; - } + private const string CheckSumLocationMirror = + "https://qdraw.nl/special/mirror/exiftool/checksums.txt"; - CreateDirectoryDependenciesFolderIfNotExists(); + private const string + ExiftoolDownloadBasePath = "https://exiftool.org/"; // with slash at the end - if ( isWindows && - ( !_hostFileSystemStorage.ExistFile(ExeExifToolWindowsFullFilePath()) || - _hostFileSystemStorage.Info(ExeExifToolWindowsFullFilePath()).Size <= minimumSize ) ) - { - return await StartDownloadForWindows(); - } + private const string ExiftoolDownloadBasePathMirror = + "https://qdraw.nl/special/mirror/exiftool/"; // with slash at the end - if ( !isWindows && - ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) || - _hostFileSystemStorage.Info(ExeExifToolUnixFullFilePath()).Size <= minimumSize ) ) - { - return await StartDownloadForUnix(); - } + private readonly AppSettings _appSettings; + private readonly IStorage _hostFileSystemStorage; + private readonly IHttpClientHelper _httpClientHelper; + private readonly IWebLogger _logger; - if ( _appSettings.IsVerbose() ) - { - var debugPath = isWindows ? ExeExifToolWindowsFullFilePath() - : ExeExifToolUnixFullFilePath(); - _logger.LogInformation($"[DownloadExifTool] {debugPath}"); - } + public ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, + IWebLogger logger) + { + _httpClientHelper = httpClientHelper; + _appSettings = appSettings; + _hostFileSystemStorage = new StorageHostFullPathFilesystem(logger); + _logger = logger; + } - // When running deploy scripts rights might reset (only for unix) - if ( isWindows ) return true; + internal ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, + IWebLogger logger, IStorage storage) + { + _httpClientHelper = httpClientHelper; + _appSettings = appSettings; + _hostFileSystemStorage = storage; + _logger = logger; + } - return await RunChmodOnExifToolUnixExe(); + /// + /// Auto Download Exiftool + /// + /// download Windows version if true + /// check for min file size in bytes (Default = 30 bytes) + /// + public async Task DownloadExifTool(bool isWindows, int minimumSize = 30) + { + if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || _appSettings is + { AddSwaggerExport: true, AddSwaggerExportExitAfter: true } ) + { + var name = _appSettings.ExiftoolSkipDownloadOnStartup == true + ? "ExiftoolSkipDownloadOnStartup" + : "AddSwaggerExport and AddSwaggerExportExitAfter"; + _logger.LogInformation($"[DownloadExifTool] Skipped due true of {name} setting"); + return false; } - private void CreateDirectoryDependenciesFolderIfNotExists() + CreateDirectoryDependenciesFolderIfNotExists(); + + if ( isWindows && + ( !_hostFileSystemStorage.ExistFile(ExeExifToolWindowsFullFilePath()) || + _hostFileSystemStorage.Info(ExeExifToolWindowsFullFilePath()).Size <= minimumSize ) ) { - if ( _hostFileSystemStorage.ExistFolder( - _appSettings.DependenciesFolder) ) - { - return; - } - _logger.LogInformation("[DownloadExifTool] Create Directory: " + _appSettings.DependenciesFolder); - _hostFileSystemStorage.CreateDirectory(_appSettings.DependenciesFolder); + return await StartDownloadForWindows(); } - internal async Task?> DownloadCheckSums() + if ( !isWindows && + ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) || + _hostFileSystemStorage.Info(ExeExifToolUnixFullFilePath()).Size <= minimumSize ) ) { - var baseLocationResult = await DownloadCheckSums(CheckSumLocation); - if ( baseLocationResult == null ) - { - return await DownloadCheckSums(CheckSumLocationMirror); - } - return baseLocationResult; + return await StartDownloadForUnix(); } - internal async Task?> DownloadCheckSums(string checkSumUrl) + if ( _appSettings.IsVerbose() ) { - var checksums = await _httpClientHelper.ReadString(checkSumUrl); - if ( checksums.Key ) - { - return checksums; - } - - _logger.LogError($"Checksum loading failed {CheckSumLocation}, next retry from mirror ~ error > " + checksums.Value); - return null; + var debugPath = isWindows + ? ExeExifToolWindowsFullFilePath() + : ExeExifToolUnixFullFilePath(); + _logger.LogInformation($"[DownloadExifTool] {debugPath}"); } - internal async Task StartDownloadForUnix() + // When running deploy scripts rights might reset (only for unix) + if ( isWindows ) { - var checksums = await DownloadCheckSums(); - if ( checksums == null ) - { - return false; - } - var matchExifToolForUnixName = GetUnixTarGzFromChecksum(checksums.Value.Value); - return await DownloadForUnix(matchExifToolForUnixName, - GetChecksumsFromTextFile(checksums.Value.Value)); + return true; } - internal static string GetUnixTarGzFromChecksum(string checksumsValue) + return await RunChmodOnExifToolUnixExe(); + } + + private void CreateDirectoryDependenciesFolderIfNotExists() + { + if ( _hostFileSystemStorage.ExistFolder( + _appSettings.DependenciesFolder) ) { - // (?<=SHA1\()Image-ExifTool-[\d\.]+\.zip - var regexExifToolForWindowsName = new Regex(@"(?<=SHA256\()Image-ExifTool-[0-9\.]+\.tar.gz", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - return regexExifToolForWindowsName.Match(checksumsValue).Value; + return; } - private string ExeExifToolUnixFullFilePath() + _logger.LogInformation("[DownloadExifTool] Create Directory: " + + _appSettings.DependenciesFolder); + _hostFileSystemStorage.CreateDirectory(_appSettings.DependenciesFolder); + } + + internal async Task?> DownloadCheckSums() + { + var baseLocationResult = await DownloadCheckSums(CheckSumLocation); + if ( baseLocationResult == null ) { - var path = Path.Combine(_appSettings.DependenciesFolder, - "exiftool-unix", - "exiftool"); - return path; + return await DownloadCheckSums(CheckSumLocationMirror); } - internal async Task DownloadForUnix(string matchExifToolForUnixName, string[] getChecksumsFromTextFile) - { - var result = await DownloadForUnix(ExiftoolDownloadBasePath, matchExifToolForUnixName, - getChecksumsFromTextFile); - - if ( result ) - { - return true; - } + return baseLocationResult; + } - return await DownloadForUnix(ExiftoolDownloadBasePathMirror, matchExifToolForUnixName, - getChecksumsFromTextFile); + internal async Task?> DownloadCheckSums(string checkSumUrl) + { + var checksums = await _httpClientHelper.ReadString(checkSumUrl); + if ( checksums.Key ) + { + return checksums; } - - private async Task DownloadForUnix(string exiftoolDownloadBasePath, string matchExifToolForUnixName, - string[] getChecksumsFromTextFile) + _logger.LogError( + $"Checksum loading failed {CheckSumLocation}, next retry from mirror ~ error > " + + checksums.Value); + return null; + } + + internal async Task StartDownloadForUnix() + { + var checksums = await DownloadCheckSums(); + if ( checksums == null ) { + return false; + } - if ( _hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) - { - return true; - } + var matchExifToolForUnixName = GetUnixTarGzFromChecksum(checksums.Value.Value); + return await DownloadForUnix(matchExifToolForUnixName, + GetChecksumsFromTextFile(checksums.Value.Value)); + } - var tarGzArchiveFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool.tar.gz"); + internal static string GetUnixTarGzFromChecksum(string checksumsValue) + { + // (?<=SHA1\()Image-ExifTool-[\d\.]+\.zip + var regexExifToolForWindowsName = new Regex(@"(?<=SHA256\()Image-ExifTool-[0-9\.]+\.tar.gz", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + return regexExifToolForWindowsName.Match(checksumsValue).Value; + } - var url = $"{exiftoolDownloadBasePath}{matchExifToolForUnixName}"; + private string ExeExifToolUnixFullFilePath() + { + var path = Path.Combine(_appSettings.DependenciesFolder, + "exiftool-unix", + "exiftool"); + return path; + } - var unixDownloaded = await _httpClientHelper.Download(url, tarGzArchiveFullFilePath); - if ( !unixDownloaded ) - { - _logger.LogError($"file is not downloaded {matchExifToolForUnixName}"); - return false; - } - - if ( !CheckSha256(tarGzArchiveFullFilePath, getChecksumsFromTextFile) ) - { - _logger.LogError($"Checksum for {tarGzArchiveFullFilePath} is not valid"); - _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); - return false; - } + internal async Task DownloadForUnix(string matchExifToolForUnixName, + string[] getChecksumsFromTextFile) + { + var result = await DownloadForUnix(ExiftoolDownloadBasePath, matchExifToolForUnixName, + getChecksumsFromTextFile); - await new TarBal(_hostFileSystemStorage).ExtractTarGz( - _hostFileSystemStorage.ReadStream(tarGzArchiveFullFilePath), _appSettings.DependenciesFolder, CancellationToken.None); + if ( result ) + { + return true; + } - var imageExifToolVersionFolder = _hostFileSystemStorage.GetDirectories(_appSettings.DependenciesFolder) - .FirstOrDefault(p => p.StartsWith(Path.Combine(_appSettings.DependenciesFolder, "Image-ExifTool-"))); - if ( imageExifToolVersionFolder != null ) - { - var exifToolUnixFolderFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix"); - if ( _hostFileSystemStorage.ExistFolder(exifToolUnixFolderFullFilePath) ) - { - _hostFileSystemStorage.FolderDelete( - exifToolUnixFolderFullFilePath); - } - _hostFileSystemStorage.FolderMove(imageExifToolVersionFolder, exifToolUnixFolderFullFilePath); - } - else - { - _logger.LogError($"[DownloadForUnix] ExifTool folder does not exists"); - return false; - } + return await DownloadForUnix(ExiftoolDownloadBasePathMirror, matchExifToolForUnixName, + getChecksumsFromTextFile); + } - // remove tar.gz file afterwards - _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); - var exifToolExePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix", "exiftool"); - _logger.LogInformation($"[DownloadForUnix] ExifTool is just downloaded: {exifToolExePath} for {_appSettings.ApplicationType}"); - return await RunChmodOnExifToolUnixExe(); + private async Task DownloadForUnix(string exiftoolDownloadBasePath, + string matchExifToolForUnixName, + string[] getChecksumsFromTextFile) + { + if ( _hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) + { + return true; } - internal async Task RunChmodOnExifToolUnixExe() + var tarGzArchiveFullFilePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool.tar.gz"); + + var url = $"{exiftoolDownloadBasePath}{matchExifToolForUnixName}"; + + var unixDownloaded = await _httpClientHelper.Download(url, tarGzArchiveFullFilePath); + if ( !unixDownloaded ) { - // need to check again - // when not exist - if ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) return false; - if ( _appSettings.IsWindows ) return true; + _logger.LogError($"file is not downloaded {matchExifToolForUnixName}"); + return false; + } + + if ( !CheckSha256(tarGzArchiveFullFilePath, getChecksumsFromTextFile) ) + { + _logger.LogError($"Checksum for {tarGzArchiveFullFilePath} is not valid"); + _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); + return false; + } - if ( !_hostFileSystemStorage.ExistFile("/bin/chmod") ) + await new TarBal(_hostFileSystemStorage, _logger).ExtractTarGz( + _hostFileSystemStorage.ReadStream(tarGzArchiveFullFilePath), + _appSettings.DependenciesFolder, CancellationToken.None); + + var imageExifToolVersionFolder = _hostFileSystemStorage + .GetDirectories(_appSettings.DependenciesFolder) + .FirstOrDefault(p => + p.StartsWith(Path.Combine(_appSettings.DependenciesFolder, "Image-ExifTool-"))); + if ( imageExifToolVersionFolder != null ) + { + var exifToolUnixFolderFullFilePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix"); + if ( _hostFileSystemStorage.ExistFolder(exifToolUnixFolderFullFilePath) ) { - _logger.LogError("[RunChmodOnExifToolUnixExe] WARNING: /bin/chmod does not exist"); - return true; + _hostFileSystemStorage.FolderDelete( + exifToolUnixFolderFullFilePath); } - // command.run does not care about the $PATH - var result = await Command.Run("/bin/chmod", "0755", ExeExifToolUnixFullFilePath()).Task; - if ( result.Success ) return true; - - _logger.LogError($"command failed with exit code {result.ExitCode}: {result.StandardError}"); + _hostFileSystemStorage.FolderMove(imageExifToolVersionFolder, + exifToolUnixFolderFullFilePath); + } + else + { + _logger.LogError("[DownloadForUnix] ExifTool folder does not exists"); return false; } - internal async Task StartDownloadForWindows() - { - var checksums = await DownloadCheckSums(); - if ( checksums == null ) return false; + // remove tar.gz file afterwards + _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); + + var exifToolExePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix", "exiftool"); + _logger.LogInformation( + $"[DownloadForUnix] ExifTool is just downloaded: {exifToolExePath} for {_appSettings.ApplicationType}"); + return await RunChmodOnExifToolUnixExe(); + } - var matchExifToolForWindowsName = GetWindowsZipFromChecksum(checksums.Value.Value); - return await DownloadForWindows(matchExifToolForWindowsName, - GetChecksumsFromTextFile(checksums.Value.Value)); + internal async Task RunChmodOnExifToolUnixExe() + { + // need to check again + // when not exist + if ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) + { + return false; } - internal static string GetWindowsZipFromChecksum(string checksumsValue) + if ( _appSettings.IsWindows ) { - // (?<=SHA256\()exiftool-[\d\.]+_64\.zip - var regexExifToolForWindowsName = new Regex(@"(?<=SHA256\()exiftool-[0-9\.]+_64\.zip", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - return regexExifToolForWindowsName.Match(checksumsValue).Value; + return true; } - /// - /// Parse the content of checksum file - /// - /// input file: see test for example - /// max number of SHA256 results - /// - internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 20) + if ( !_hostFileSystemStorage.ExistFile("/bin/chmod") ) { - // SHA256 = 64 characters, SHA1 = 40 characters - var regexExifToolForWindowsName = new Regex("[a-z0-9]{64}", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - var results = regexExifToolForWindowsName.Matches(checksumsValue). - Select(m => m.Value). - ToArray(); - if ( results.Length < max ) - { - return results; - } + _logger.LogError("[RunChmodOnExifToolUnixExe] WARNING: /bin/chmod does not exist"); + return true; + } - _logger.LogError($"More than {max} checksums found, this is not expected, code stops now"); - return []; + // command.run does not care about the $PATH + var result = await Command.Run("/bin/chmod", "0755", ExeExifToolUnixFullFilePath()).Task; + if ( result.Success ) + { + return true; } - /// - /// Check if SHA256 hash is valid - /// Instead of SHA1CryptoServiceProvider, we use SHA256.Create - /// - /// path of exiftool.exe - /// list of SHA256 hashes - /// - internal bool CheckSha256(string fullFilePath, IEnumerable checkSumOptions) + _logger.LogError( + $"command failed with exit code {result.ExitCode}: {result.StandardError}"); + return false; + } + + internal async Task StartDownloadForWindows() + { + var checksums = await DownloadCheckSums(); + if ( checksums == null ) { - using var buffer = _hostFileSystemStorage.ReadStream(fullFilePath); - using var hashAlgorithm = SHA256.Create(); + return false; + } - var byteHash = hashAlgorithm.ComputeHash(buffer); - var hash = BitConverter.ToString(byteHash).Replace("-", string.Empty).ToLowerInvariant(); - return checkSumOptions.AsEnumerable().Any(p => p.Equals(hash, StringComparison.InvariantCultureIgnoreCase)); + var matchExifToolForWindowsName = GetWindowsZipFromChecksum(checksums.Value.Value); + return await DownloadForWindows(matchExifToolForWindowsName, + GetChecksumsFromTextFile(checksums.Value.Value)); + } + + internal static string GetWindowsZipFromChecksum(string checksumsValue) + { + // (?<=SHA256\()exiftool-[\d\.]+_64\.zip + var regexExifToolForWindowsName = new Regex(@"(?<=SHA256\()exiftool-[0-9\.]+_64\.zip", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + return regexExifToolForWindowsName.Match(checksumsValue).Value; + } + + /// + /// Parse the content of checksum file + /// + /// input file: see test for example + /// max number of SHA256 results + /// + internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 20) + { + // SHA256 = 64 characters, SHA1 = 40 characters + var regexExifToolForWindowsName = new Regex("[a-z0-9]{64}", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + var results = regexExifToolForWindowsName.Matches(checksumsValue).Select(m => m.Value) + .ToArray(); + if ( results.Length < max ) + { + return results; } - private string ExeExifToolWindowsFullFilePath() + _logger.LogError($"More than {max} checksums found, this is not expected, code stops now"); + return []; + } + + /// + /// Check if SHA256 hash is valid + /// Instead of SHA1CryptoServiceProvider, we use SHA256.Create + /// + /// path of exiftool.exe + /// list of SHA256 hashes + /// + internal bool CheckSha256(string fullFilePath, IEnumerable checkSumOptions) + { + using var buffer = _hostFileSystemStorage.ReadStream(fullFilePath); + using var hashAlgorithm = SHA256.Create(); + + var byteHash = hashAlgorithm.ComputeHash(buffer); + var hash = BitConverter.ToString(byteHash).Replace("-", string.Empty).ToLowerInvariant(); + return checkSumOptions.AsEnumerable() + .Any(p => p.Equals(hash, StringComparison.InvariantCultureIgnoreCase)); + } + + private string ExeExifToolWindowsFullFilePath() + { + return Path.Combine(Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"), + "exiftool.exe"); + } + + internal async Task DownloadForWindows(string matchExifToolForWindowsName, + string[] getChecksumsFromTextFile) + { + var result = await DownloadForWindows(ExiftoolDownloadBasePath, matchExifToolForWindowsName, + getChecksumsFromTextFile); + + if ( result ) { - return Path.Combine(Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"), "exiftool.exe"); + return true; } - internal async Task DownloadForWindows(string matchExifToolForWindowsName, string[] getChecksumsFromTextFile) + return await DownloadForWindows(ExiftoolDownloadBasePathMirror, matchExifToolForWindowsName, + getChecksumsFromTextFile); + } + + private async Task DownloadForWindows(string exiftoolDownloadBasePath, + string matchExifToolForWindowsName, + string[] getChecksumsFromTextFile) + { + if ( _hostFileSystemStorage.ExistFile( + ExeExifToolWindowsFullFilePath()) ) { - var result = await DownloadForWindows(ExiftoolDownloadBasePath, matchExifToolForWindowsName, - getChecksumsFromTextFile); - - if ( result ) - { - return true; - } + return true; + } + + var zipArchiveFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool.zip"); + var windowsExifToolFolder = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"); - return await DownloadForWindows(ExiftoolDownloadBasePathMirror, matchExifToolForWindowsName, - getChecksumsFromTextFile); + var url = $"{exiftoolDownloadBasePath}{matchExifToolForWindowsName}"; + var windowsDownloaded = await _httpClientHelper.Download(url, zipArchiveFullFilePath); + if ( !windowsDownloaded ) + { + _logger.LogError($"file is not downloaded {matchExifToolForWindowsName}"); + return false; } - - private async Task DownloadForWindows(string exiftoolDownloadBasePath, string matchExifToolForWindowsName, - string[] getChecksumsFromTextFile) + + if ( !CheckSha256(zipArchiveFullFilePath, getChecksumsFromTextFile) ) { - if ( _hostFileSystemStorage.ExistFile( - ExeExifToolWindowsFullFilePath()) ) - { - return true; - } + _logger.LogError($"Checksum for {zipArchiveFullFilePath} is not valid"); + return false; + } - var zipArchiveFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool.zip"); - var windowsExifToolFolder = Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"); + _hostFileSystemStorage.CreateDirectory(windowsExifToolFolder); - var url = $"{exiftoolDownloadBasePath}{matchExifToolForWindowsName}"; - var windowsDownloaded = await _httpClientHelper.Download(url, zipArchiveFullFilePath); - if ( !windowsDownloaded ) - { - _logger.LogError($"file is not downloaded {matchExifToolForWindowsName}"); - return false; - } + new Zipper(_logger).ExtractZip(zipArchiveFullFilePath, windowsExifToolFolder); - if ( !CheckSha256(zipArchiveFullFilePath, getChecksumsFromTextFile) ) - { - _logger.LogError($"Checksum for {zipArchiveFullFilePath} is not valid"); - return false; - } + MoveFileIfExist(windowsExifToolFolder, "exiftool(-k).exe", windowsExifToolFolder, + "exiftool.exe"); + MoveFileIfExist(windowsExifToolFolder, "exiftool_files", windowsExifToolFolder, + "exiftool_files"); - _hostFileSystemStorage.CreateDirectory(windowsExifToolFolder); + _logger.LogInformation( + $"[DownloadForWindows] ExifTool downloaded: {ExeExifToolWindowsFullFilePath()}"); + return _hostFileSystemStorage.ExistFile(Path.Combine(windowsExifToolFolder, + "exiftool.exe")); + } - new Zipper(_logger).ExtractZip(zipArchiveFullFilePath, windowsExifToolFolder); - - MoveFileIfExist(windowsExifToolFolder, "exiftool(-k).exe", windowsExifToolFolder, "exiftool.exe"); - MoveFileIfExist(windowsExifToolFolder, "exiftool_files", windowsExifToolFolder, "exiftool_files"); + internal void MoveFileIfExist(string searchFolder, string searchForFileOrFolder, + string destFolder, string destFileNameOrFolderName) + { + var files = _hostFileSystemStorage.GetAllFilesInDirectoryRecursive(searchFolder).ToList(); + var folders = _hostFileSystemStorage.GetDirectoryRecursive(searchFolder).Select(p => p.Key); + List folderAndFiles = [..files, ..folders]; - _logger.LogInformation($"[DownloadForWindows] ExifTool downloaded: {ExeExifToolWindowsFullFilePath()}"); - return _hostFileSystemStorage.ExistFile(Path.Combine(windowsExifToolFolder, - "exiftool.exe")); + var srcFullPaths = folderAndFiles.Where(p => p.EndsWith(searchForFileOrFolder)).ToList(); + if ( srcFullPaths.Count == 0 ) + { + _logger.LogError($"[MoveFileIfExist] Could not find {searchForFileOrFolder} file"); + return; } - internal void MoveFileIfExist(string searchFolder, string searchForFileOrFolder, string destFolder, string destFileNameOrFolderName) + foreach ( var srcFullPath in srcFullPaths ) { - var files = _hostFileSystemStorage.GetAllFilesInDirectoryRecursive(searchFolder).ToList(); - var folders = _hostFileSystemStorage.GetDirectoryRecursive(searchFolder).Select(p => p.Key); - List folderAndFiles = [..files, ..folders]; - - var srcFullPaths = folderAndFiles.Where(p => p.EndsWith(searchForFileOrFolder)).ToList(); - if ( srcFullPaths.Count == 0 ) - { - _logger.LogError($"[MoveFileIfExist] Could not find {searchForFileOrFolder} file"); - return; - } + var isFolderOrFile = _hostFileSystemStorage.Info(srcFullPath).IsFolderOrFile; + var destFullPath = Path.Combine(destFolder, destFileNameOrFolderName); - foreach ( var srcFullPath in srcFullPaths ) + switch ( isFolderOrFile ) { - var isFolderOrFile = _hostFileSystemStorage.Info(srcFullPath).IsFolderOrFile; - var destFullPath = Path.Combine(destFolder, destFileNameOrFolderName); - - switch ( isFolderOrFile ) - { - case FolderOrFileModel.FolderOrFileTypeList.Folder: - if ( !_hostFileSystemStorage.ExistFolder(srcFullPath) ) continue; - _hostFileSystemStorage.FolderMove(srcFullPath, destFullPath); - break; - case FolderOrFileModel.FolderOrFileTypeList.File: - if ( !_hostFileSystemStorage.ExistFile(srcFullPath) ) continue; - _hostFileSystemStorage.FileMove(srcFullPath, destFullPath); - break; - default: - break; - } - + case FolderOrFileModel.FolderOrFileTypeList.Folder: + if ( !_hostFileSystemStorage.ExistFolder(srcFullPath) ) + { + continue; + } + + _hostFileSystemStorage.FolderMove(srcFullPath, destFullPath); + break; + case FolderOrFileModel.FolderOrFileTypeList.File: + if ( !_hostFileSystemStorage.ExistFile(srcFullPath) ) + { + continue; + } + + _hostFileSystemStorage.FileMove(srcFullPath, destFullPath); + break; } } } diff --git a/starsky/starskytest/starsky.foundation.storage/ArchiveFormats/TarBalTest.cs b/starsky/starskytest/starsky.foundation.storage/ArchiveFormats/TarBalTest.cs index aa81f7dc11..98e7c4c4e8 100644 --- a/starsky/starskytest/starsky.foundation.storage/ArchiveFormats/TarBalTest.cs +++ b/starsky/starskytest/starsky.foundation.storage/ArchiveFormats/TarBalTest.cs @@ -10,49 +10,53 @@ using starskytest.FakeCreateAn.CreateAnTagGzLongerThan100CharsFileName; using starskytest.FakeMocks; -namespace starskytest.starsky.foundation.storage.ArchiveFormats +namespace starskytest.starsky.foundation.storage.ArchiveFormats; + +[TestClass] +public sealed class TarBalTest { - [TestClass] - public sealed class TarBalTest + [TestMethod] + public async Task ExtractTar() + { + // Non Gz Tar + var storage = new FakeIStorage(new List { "/" }, + new List()); + + var memoryStream = new MemoryStream(CreateAnExifToolTar.Bytes.ToArray()); + await new TarBal(storage, new FakeIWebLogger()).ExtractTar(memoryStream, "/test", + CancellationToken.None); + Assert.IsTrue(storage.ExistFile("/test/Image-ExifTool-11.99/exiftool")); + } + + [TestMethod] + public async Task ExtractTarGz() { - [TestMethod] - public async Task ExtractTar() - { - // Non Gz Tar - var storage = new FakeIStorage(new List {"/"}, - new List()); - - var memoryStream = new MemoryStream(CreateAnExifToolTar.Bytes.ToArray()); - await new TarBal(storage).ExtractTar(memoryStream,"/test", CancellationToken.None); - Assert.IsTrue(storage.ExistFile("/test/Image-ExifTool-11.99/exiftool")); - } - - [TestMethod] - public async Task ExtractTarGz() - { - // Gz Tar! - var storage = new FakeIStorage(new List {"/"}, - new List()); - - var memoryStream = new MemoryStream(CreateAnExifToolTarGz.Bytes.ToArray()); - await new TarBal(storage).ExtractTarGz(memoryStream,"/test", CancellationToken.None); - Assert.IsTrue(storage.ExistFile("/test/Image-ExifTool-11.99/exiftool")); - } - - - [TestMethod] - public async Task ExtractTarGz_LongerThan100Chars() - { - // Gz Tar! - var storage = new FakeIStorage(new List {"/"}, - new List()); - - var memoryStream = new MemoryStream(new CreateAnTagGzLongerThan100CharsFileName().Bytes); - await new TarBal(storage).ExtractTarGz(memoryStream,"/test", CancellationToken.None); - Assert.IsTrue(storage.ExistFile($"/test/{CreateAnTagGzLongerThan100CharsFileName.FileName}")); - var file = storage.ReadStream($"/test/{CreateAnTagGzLongerThan100CharsFileName.FileName}"); - // the filename is written as content in the file - Assert.AreEqual(CreateAnTagGzLongerThan100CharsFileName.FileName,(await StreamToStringHelper.StreamToStringAsync(file)).Trim()); - } + // Gz Tar! + var storage = new FakeIStorage(new List { "/" }, + new List()); + + var memoryStream = new MemoryStream(CreateAnExifToolTarGz.Bytes.ToArray()); + await new TarBal(storage, new FakeIWebLogger()).ExtractTarGz(memoryStream, "/test", + CancellationToken.None); + Assert.IsTrue(storage.ExistFile("/test/Image-ExifTool-11.99/exiftool")); + } + + + [TestMethod] + public async Task ExtractTarGz_LongerThan100Chars() + { + // Gz Tar! + var storage = new FakeIStorage(new List { "/" }, + new List()); + + var memoryStream = new MemoryStream(new CreateAnTagGzLongerThan100CharsFileName().Bytes); + await new TarBal(storage, new FakeIWebLogger()).ExtractTarGz(memoryStream, "/test", + CancellationToken.None); + Assert.IsTrue( + storage.ExistFile($"/test/{CreateAnTagGzLongerThan100CharsFileName.FileName}")); + var file = storage.ReadStream($"/test/{CreateAnTagGzLongerThan100CharsFileName.FileName}"); + // the filename is written as content in the file + Assert.AreEqual(CreateAnTagGzLongerThan100CharsFileName.FileName, + ( await StreamToStringHelper.StreamToStringAsync(file) ).Trim()); } } diff --git a/starsky/starskytest/starsky.foundation.writemeta/Services/ExifToolHostStorageServiceTest.cs b/starsky/starskytest/starsky.foundation.writemeta/Services/ExifToolHostStorageServiceTest.cs index 1c735b0736..a382280b88 100644 --- a/starsky/starskytest/starsky.foundation.writemeta/Services/ExifToolHostStorageServiceTest.cs +++ b/starsky/starskytest/starsky.foundation.writemeta/Services/ExifToolHostStorageServiceTest.cs @@ -77,7 +77,7 @@ public async Task WriteTagsAndRenameThumbnailAsync_FakeExifToolBashTest_UnixOnly hostFileSystemStorage.FolderDelete(outputPath); } - await new TarBal(hostFileSystemStorage).ExtractTarGz(memoryStream, outputPath, + await new TarBal(hostFileSystemStorage, new FakeIWebLogger()).ExtractTarGz(memoryStream, outputPath, CancellationToken.None); var imageExifToolVersionFolder = hostFileSystemStorage.GetDirectories(outputPath) .FirstOrDefault(p => p.StartsWith(Path.Combine(outputPath, "Image-ExifTool-")))?