From 0a477b2df61d6ddf7fe255704f67c515e7ee8113 Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Apr 2024 10:46:56 +0200 Subject: [PATCH] WIP --- .../Services/ExifToolDownload.cs | 372 ------------------ .../ExifToolDownloadBackgroundService.cs | 1 + .../Services/ExifToolDownloader/CheckSha1.cs | 86 ++++ .../ExifToolDownloader/ExifToolDownload.cs | 115 ++++++ .../ExifToolDownloadUnix.cs | 141 +++++++ .../ExifToolDownloadWindows.cs | 72 ++++ .../ExifToolDownloader/ExifToolLocations.cs | 39 ++ .../Helpers/ExifToolDownloadTest.cs | 1 + 8 files changed, 455 insertions(+), 372 deletions(-) delete mode 100644 starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs create mode 100644 starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/CheckSha1.cs create mode 100644 starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownload.cs create mode 100644 starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadUnix.cs create mode 100644 starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadWindows.cs create mode 100644 starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolLocations.cs diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs deleted file mode 100644 index 8f19c71420..0000000000 --- a/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs +++ /dev/null @@ -1,372 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Medallion.Shell; -using starsky.foundation.http.Interfaces; -using starsky.foundation.injection; -using starsky.foundation.platform.Interfaces; -using starsky.foundation.platform.Models; -using starsky.foundation.storage.ArchiveFormats; -using starsky.foundation.storage.Interfaces; -using starsky.foundation.storage.Storage; -using starsky.foundation.writemeta.Helpers; -using starsky.foundation.writemeta.Interfaces; - -[assembly: InternalsVisibleTo("starskytest")] - -namespace starsky.foundation.writemeta.Services -{ - [Service(typeof(IExifToolDownload), InjectionLifetime = InjectionLifetime.Singleton)] - [SuppressMessage("Usage", - "S1075:Refactor your code not to use hardcoded absolute paths or URIs", - Justification = "Source of files")] - [SuppressMessage("Usage", - "S4790:Make sure this weak hash algorithm is not used in a sensitive context here.", - Justification = "Safe")] - public sealed class ExifToolDownload : IExifToolDownload - { - private readonly IHttpClientHelper _httpClientHelper; - private readonly AppSettings _appSettings; - private readonly IStorage _hostFileSystemStorage; - private readonly IWebLogger _logger; - - private const string CheckSumLocation = "https://exiftool.org/checksums.txt"; - - private const string CheckSumLocationMirror = - "https://qdraw.nl/special/mirror/exiftool/checksums.txt"; - - private const string - ExiftoolDownloadBasePath = "https://exiftool.org/"; // with slash at the end - - private const string ExiftoolDownloadBasePathMirror = - "https://qdraw.nl/special/mirror/exiftool/"; // with slash at the end - - public ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, - IWebLogger logger) - { - _httpClientHelper = httpClientHelper; - _appSettings = appSettings; - _hostFileSystemStorage = new StorageHostFullPathFilesystem(logger); - _logger = logger; - } - - internal ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, - IWebLogger logger, IStorage storage) - { - _httpClientHelper = httpClientHelper; - _appSettings = appSettings; - _hostFileSystemStorage = storage; - _logger = logger; - } - - /// - /// Auto Download Exiftool - /// - /// download windows version if true - /// check for min file size in bytes (Default = 30 bytes) - /// - public async Task DownloadExifTool(bool isWindows, int minimumSize = 30) - { - if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || _appSettings is - { AddSwaggerExport: true, AddSwaggerExportExitAfter: true } ) - { - var name = _appSettings.ExiftoolSkipDownloadOnStartup == true - ? "ExiftoolSkipDownloadOnStartup" - : "AddSwaggerExport and AddSwaggerExportExitAfter"; - _logger.LogInformation($"[DownloadExifTool] Skipped due true of {name} setting"); - return false; - } - - new CreateFolderIfNotExists(_logger, _appSettings) - .CreateDirectoryDependenciesTempFolderIfNotExists(); - - if ( isWindows && - ( !_hostFileSystemStorage.ExistFile(ExeExifToolWindowsFullFilePath()) || - _hostFileSystemStorage.Info(ExeExifToolWindowsFullFilePath()).Size <= - minimumSize ) ) - { - return await StartDownloadForWindows(); - } - - if ( !isWindows && - ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) || - _hostFileSystemStorage.Info(ExeExifToolUnixFullFilePath()).Size <= - minimumSize ) ) - { - return await StartDownloadForUnix(); - } - - if ( _appSettings.IsVerbose() ) - { - var debugPath = isWindows - ? ExeExifToolWindowsFullFilePath() - : ExeExifToolUnixFullFilePath(); - _logger.LogInformation($"[DownloadExifTool] {debugPath}"); - } - - // When running deploy scripts rights might reset (only for unix) - if ( isWindows ) return true; - - return await RunChmodOnExifToolUnixExe(); - } - - - internal async Task?> DownloadCheckSums() - { - var checksums = await _httpClientHelper.ReadString(CheckSumLocation); - if ( checksums.Key ) - { - return checksums; - } - - _logger.LogError( - $"Checksum loading failed {CheckSumLocation}, next retry from mirror ~ error > " + - checksums.Value); - - checksums = await _httpClientHelper.ReadString(CheckSumLocationMirror); - if ( checksums.Key ) return new KeyValuePair(false, checksums.Value); - - _logger.LogError($"Checksum loading failed {CheckSumLocationMirror}" + - $", next stop; please connect to internet and restart app ~ error > " + - checksums.Value); - return null; - } - - internal async Task StartDownloadForUnix() - { - var checksums = await DownloadCheckSums(); - if ( checksums == null ) return false; - var matchExifToolForUnixName = GetUnixTarGzFromChecksum(checksums.Value.Value); - return await DownloadForUnix(matchExifToolForUnixName, - GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); - } - - internal static string GetUnixTarGzFromChecksum(string checksumsValue) - { - // (?<=SHA1\()Image-ExifTool-[\d\.]+\.zip - var regexExifToolForWindowsName = new Regex( - @"(?<=SHA1\()Image-ExifTool-[0-9\.]+\.tar.gz", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - return regexExifToolForWindowsName.Match(checksumsValue).Value; - } - - private string ExeExifToolUnixFullFilePath() - { - var path = Path.Combine(_appSettings.DependenciesFolder, - "exiftool-unix", - "exiftool"); - return path; - } - - internal async Task DownloadForUnix(string matchExifToolForUnixName, - IEnumerable getChecksumsFromTextFile, bool downloadFromMirror = false) - { - if ( _hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) - { - return true; - } - - var tarGzArchiveFullFilePath = - Path.Combine(_appSettings.TempFolder, "exiftool.tar.gz"); - - var url = $"{ExiftoolDownloadBasePath}{matchExifToolForUnixName}"; - if ( downloadFromMirror ) - url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForUnixName}"; - - var unixDownloaded = await _httpClientHelper.Download(url, tarGzArchiveFullFilePath); - if ( !unixDownloaded ) - { - throw new HttpRequestException( - $"file is not downloaded {matchExifToolForUnixName}"); - } - - if ( !CheckSha1(tarGzArchiveFullFilePath, getChecksumsFromTextFile) ) - { - throw new HttpRequestException( - $"checksum for {tarGzArchiveFullFilePath} is not valid"); - } - - await new TarBal(_hostFileSystemStorage).ExtractTarGz( - _hostFileSystemStorage.ReadStream(tarGzArchiveFullFilePath), - _appSettings.TempFolder, CancellationToken.None); - - var imageExifToolVersionFolder = _hostFileSystemStorage - .GetDirectories(_appSettings.TempFolder) - .FirstOrDefault(p => - p.StartsWith(Path.Combine(_appSettings.TempFolder, "Image-ExifTool-"))); - if ( imageExifToolVersionFolder != null ) - { - var exifToolUnixFolderFullFilePathTempFolder = - Path.Combine(_appSettings.TempFolder, "exiftool-unix"); - - if ( _hostFileSystemStorage.ExistFolder(exifToolUnixFolderFullFilePathTempFolder) ) - { - _hostFileSystemStorage.FolderDelete( - exifToolUnixFolderFullFilePathTempFolder); - } - - _hostFileSystemStorage.FolderMove(imageExifToolVersionFolder, - exifToolUnixFolderFullFilePathTempFolder); - - var exifToolUnixFolderFullFilePath = - Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix"); - - _hostFileSystemStorage.FileCopy(imageExifToolVersionFolder, - exifToolUnixFolderFullFilePathTempFolder); - } - else - { - _logger.LogError($"[DownloadForUnix] ExifTool folder does not exists"); - return false; - } - - // remove tar.gz file afterwards - _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); - - var exifToolExePath = - Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix", "exiftool"); - _logger.LogInformation( - $"[DownloadForUnix] ExifTool is just downloaded: {exifToolExePath} for {_appSettings.ApplicationType}"); - return await RunChmodOnExifToolUnixExe(); - } - - internal async Task RunChmodOnExifToolUnixExe() - { - // need to check again - // when not exist - if ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) return false; - if ( _appSettings.IsWindows ) return true; - - if ( !_hostFileSystemStorage.ExistFile("/bin/chmod") ) - { - _logger.LogError("[RunChmodOnExifToolUnixExe] WARNING: /bin/chmod does not exist"); - return true; - } - - // command.run does not care about the $PATH - var result = await Command.Run("/bin/chmod", "0755", ExeExifToolUnixFullFilePath()) - .Task; - if ( result.Success ) return true; - - _logger.LogError( - $"command failed with exit code {result.ExitCode}: {result.StandardError}"); - return false; - } - - internal async Task StartDownloadForWindows() - { - var checksums = await DownloadCheckSums(); - if ( checksums == null ) return false; - - var matchExifToolForWindowsName = GetWindowsZipFromChecksum(checksums.Value.Value); - return await DownloadForWindows(matchExifToolForWindowsName, - GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); - } - - internal static string GetWindowsZipFromChecksum(string checksumsValue) - { - // (?<=SHA1\()exiftool-[\d\.]+\.zip - var regexExifToolForWindowsName = new Regex(@"(?<=SHA1\()exiftool-[0-9\.]+\.zip", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - return regexExifToolForWindowsName.Match(checksumsValue).Value; - } - - /// - /// Parse the content of checksum file - /// - /// input file: see test for example - /// max number of SHA1 results - /// - internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 8) - { - var regexExifToolForWindowsName = new Regex("[a-z0-9]{40}", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - var results = regexExifToolForWindowsName.Matches(checksumsValue).Select(m => m.Value) - .ToArray(); - if ( results.Length < max ) return results; - - _logger.LogError( - $"More than {max} checksums found, this is not expected, code stops now"); - return []; - } - - /// - /// Check if SHA1 hash is valid - /// Instead of SHA1CryptoServiceProvider, we use SHA1.Create - /// - /// path of exiftool.exe - /// list of sha1 hashes - /// - internal bool CheckSha1(string fullFilePath, IEnumerable checkSumOptions) - { - using var buffer = _hostFileSystemStorage.ReadStream(fullFilePath); - using var hashAlgorithm = SHA1.Create(); - - var byteHash = hashAlgorithm.ComputeHash(buffer); - var hash = BitConverter.ToString(byteHash).Replace("-", string.Empty) - .ToLowerInvariant(); - return checkSumOptions.AsEnumerable().Any(p => - p.Equals(hash, StringComparison.InvariantCultureIgnoreCase)); - } - - private string ExeExifToolWindowsFullFilePath() - { - return Path.Combine(Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"), - "exiftool.exe"); - } - - internal async Task DownloadForWindows(string matchExifToolForWindowsName, - string[] getChecksumsFromTextFile, bool downloadFromMirror = false) - { - if ( _hostFileSystemStorage.ExistFile( - ExeExifToolWindowsFullFilePath()) ) return true; - - var zipArchiveFullFilePath = - Path.Combine(_appSettings.DependenciesFolder, "exiftool.zip"); - var windowsExifToolFolder = - Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"); - - var url = $"{ExiftoolDownloadBasePath}{matchExifToolForWindowsName}"; - if ( downloadFromMirror ) - url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForWindowsName}"; - - var windowsDownloaded = await _httpClientHelper.Download(url, zipArchiveFullFilePath); - if ( !windowsDownloaded ) - { - throw new HttpRequestException( - $"file is not downloaded {matchExifToolForWindowsName}"); - } - - if ( !CheckSha1(zipArchiveFullFilePath, getChecksumsFromTextFile) ) - { - throw new HttpRequestException( - $"checksum for {zipArchiveFullFilePath} is not valid"); - } - - _hostFileSystemStorage.CreateDirectory(windowsExifToolFolder); - - new Zipper().ExtractZip(zipArchiveFullFilePath, windowsExifToolFolder); - MoveFileIfExist(Path.Combine(windowsExifToolFolder, "exiftool(-k).exe"), - Path.Combine(windowsExifToolFolder, "exiftool.exe")); - - _logger.LogInformation( - $"[DownloadForWindows] ExifTool downloaded: {ExeExifToolWindowsFullFilePath()}"); - return _hostFileSystemStorage.ExistFile(Path.Combine(windowsExifToolFolder, - "exiftool.exe")); - } - - private void MoveFileIfExist(string srcFullPath, string toFullPath) - { - if ( !_hostFileSystemStorage.ExistFile(srcFullPath) ) return; - _hostFileSystemStorage.FileMove(srcFullPath, toFullPath); - } - } -} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs index a50862ed73..dea77dee14 100644 --- a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs @@ -6,6 +6,7 @@ using starsky.foundation.injection; using starsky.foundation.platform.Interfaces; using starsky.foundation.platform.Models; +using starsky.foundation.writemeta.Services.ExifToolDownloader; namespace starsky.foundation.writemeta.Services { diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/CheckSha1.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/CheckSha1.cs new file mode 100644 index 0000000000..05590b35cf --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/CheckSha1.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using starsky.foundation.http.Interfaces; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.storage.Interfaces; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class CheckSums(IStorage storage, IHttpClientHelper httpClientHelper, IWebLogger logger) +{ + internal async Task?> DownloadCheckSums(bool isMirror = false) + { + var url = !isMirror + ? ExifToolLocations.CheckSumLocation + : ExifToolLocations.CheckSumLocationMirror; + + var checksums = await httpClientHelper.ReadString(url); + if ( checksums.Key ) + { + return checksums; + } + + return null; + } + + /// + /// Check if SHA1 hash is valid + /// Instead of SHA1CryptoServiceProvider, we use SHA1.Create + /// + /// path of exiftool.exe + /// list of sha1 hashes + /// + internal bool CheckSha1(string fullFilePath, IEnumerable checkSumOptions) + { + using var buffer = storage.ReadStream(fullFilePath); + using var hashAlgorithm = SHA1.Create(); + + var byteHash = hashAlgorithm.ComputeHash(buffer); + var hash = BitConverter.ToString(byteHash).Replace("-", string.Empty) + .ToLowerInvariant(); + return checkSumOptions.AsEnumerable().Any(p => + p.Equals(hash, StringComparison.InvariantCultureIgnoreCase)); + } + + /// + /// Parse the content of checksum file + /// + /// input file: see test for example + /// max number of SHA1 results + /// + internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 8) + { + var regexExifToolForWindowsName = new Regex("[a-z0-9]{40}", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + var results = regexExifToolForWindowsName.Matches(checksumsValue).Select(m => m.Value) + .ToArray(); + if ( results.Length < max ) return results; + + logger.LogError( + $"More than {max} checksums found, this is not expected, code stops now"); + return []; + } + + + internal static string GetUnixTarGzFromChecksum(string checksumsValue) + { + // (?<=SHA1\()Image-ExifTool-[\d\.]+\.zip + var regexExifToolForWindowsName = new Regex( + @"(?<=SHA1\()Image-ExifTool-[0-9\.]+\.tar.gz", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + return regexExifToolForWindowsName.Match(checksumsValue).Value; + } + + internal static string GetWindowsZipFromChecksum(string checksumsValue) + { + // (?<=SHA1\()exiftool-[\d\.]+\.zip + var regexExifToolForWindowsName = new Regex(@"(?<=SHA1\()exiftool-[0-9\.]+\.zip", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + return regexExifToolForWindowsName.Match(checksumsValue).Value; + } + +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownload.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownload.cs new file mode 100644 index 0000000000..0e52158ba3 --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownload.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using starsky.foundation.http.Interfaces; +using starsky.foundation.injection; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.storage.Interfaces; +using starsky.foundation.storage.Storage; +using starsky.foundation.writemeta.Helpers; +using starsky.foundation.writemeta.Interfaces; + +[assembly: InternalsVisibleTo("starskytest")] + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +[Service(typeof(IExifToolDownload), InjectionLifetime = InjectionLifetime.Singleton)] +[SuppressMessage("Usage", + "S1075:Refactor your code not to use hardcoded absolute paths or URIs", + Justification = "Source of files")] +[SuppressMessage("Usage", + "S4790:Make sure this weak hash algorithm is not used in a sensitive context here.", + Justification = "Safe")] +public sealed class ExifToolDownload : IExifToolDownload +{ + private readonly IHttpClientHelper _httpClientHelper; + private readonly AppSettings _appSettings; + private readonly IStorage _hostFileSystemStorage; + private readonly IWebLogger _logger; + private readonly ExifToolLocations _exifToolLocations; + private readonly ExifToolDownloadUnix _exifToolDownloadUnix; + + public ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, + IWebLogger logger) + { + _httpClientHelper = httpClientHelper; + _appSettings = appSettings; + _hostFileSystemStorage = new StorageHostFullPathFilesystem(logger); + _logger = logger; + _exifToolLocations = new ExifToolLocations(_appSettings); + _exifToolDownloadUnix = new ExifToolDownloadUnix(_hostFileSystemStorage, appSettings, httpClientHelper, logger); + _exifToolDownloadWindows = new ExifToolDownloadWindows(_hostFileSystemStorage, appSettings, httpClientHelper, logger); + + } + + internal ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, + IWebLogger logger, IStorage storage) + { + _httpClientHelper = httpClientHelper; + _appSettings = appSettings; + _hostFileSystemStorage = storage; + _logger = logger; + _exifToolLocations = new ExifToolLocations(_appSettings); + _exifToolDownloadUnix = new ExifToolDownloadUnix(_hostFileSystemStorage, appSettings, httpClientHelper, logger); + } + + /// + /// Auto Download Exiftool + /// + /// download windows version if true + /// check for min file size in bytes (Default = 30 bytes) + /// + public async Task DownloadExifTool(bool isWindows, int minimumSize = 30) + { + if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || _appSettings is + { AddSwaggerExport: true, AddSwaggerExportExitAfter: true } ) + { + var name = _appSettings.ExiftoolSkipDownloadOnStartup == true + ? "ExiftoolSkipDownloadOnStartup" + : "AddSwaggerExport and AddSwaggerExportExitAfter"; + _logger.LogInformation($"[DownloadExifTool] Skipped due true of {name} setting"); + return false; + } + + new CreateFolderIfNotExists(_logger, _appSettings) + .CreateDirectoryDependenciesTempFolderIfNotExists(); + + if ( isWindows && + ( !_hostFileSystemStorage.ExistFile( + _exifToolLocations.ExeExifToolWindowsFullFilePath()) || + _hostFileSystemStorage.Info(_exifToolLocations.ExeExifToolWindowsFullFilePath()) + .Size <= + minimumSize ) ) + { + return await StartDownloadForWindows(); + } + + if ( !isWindows && + ( !_hostFileSystemStorage.ExistFile(_exifToolLocations + .ExeExifToolUnixFullFilePath()) || + _hostFileSystemStorage.Info(_exifToolLocations.ExeExifToolUnixFullFilePath()).Size <= + minimumSize ) ) + { + return await _exifToolDownloadUnix.StartDownloadForUnix(); + } + + if ( _appSettings.IsVerbose() ) + { + _logger.LogInformation( + $"[DownloadExifTool] {_exifToolLocations.ExeExifToolFullFilePath(isWindows)}"); + } + + // When running deploy scripts rights might reset (only for unix) + if ( isWindows ) return true; + + return await _exifToolDownloadUnix.RunChmodOnExifToolUnixExe(); + } + + + + +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadUnix.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadUnix.cs new file mode 100644 index 0000000000..1e9564cf4b --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadUnix.cs @@ -0,0 +1,141 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Medallion.Shell; +using starsky.foundation.http.Interfaces; +using starsky.foundation.http.Services; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.storage.ArchiveFormats; +using starsky.foundation.storage.Interfaces; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class ExifToolDownloadUnix +{ + private readonly IStorage _hostFileSystemStorage; + private readonly AppSettings _appSettings; + private readonly IHttpClientHelper _httpClientHelper; + private readonly IWebLogger _logger; + private readonly ExifToolLocations _exifToolLocations; + private readonly CheckSums _checkSha1; + + public ExifToolDownloadUnix(IStorage hostFileSystemStorage, AppSettings appSettings, + IHttpClientHelper httpClientHelper, IWebLogger logger) + { + _hostFileSystemStorage = hostFileSystemStorage; + _appSettings = appSettings; + _httpClientHelper = httpClientHelper; + _logger = logger; + _exifToolLocations = new ExifToolLocations(_appSettings); + _checkSha1 = new CheckSums(_hostFileSystemStorage, httpClientHelper, logger); + } + + internal async Task StartDownloadForUnix() + { + var checksums = await DownloadCheckSums(); + if ( checksums == null ) return false; + var matchExifToolForUnixName = GetUnixTarGzFromChecksum(checksums.Value.Value); + return await DownloadForUnix(matchExifToolForUnixName, + GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); + } + + + internal async Task DownloadForUnix(string matchExifToolForUnixName, + IEnumerable getChecksumsFromTextFile, bool downloadFromMirror = false) + { + if ( _hostFileSystemStorage.ExistFile(_exifToolLocations.ExeExifToolUnixFullFilePath()) ) + { + return true; + } + + var tarGzArchiveFullFilePath = + Path.Combine(_appSettings.TempFolder, "exiftool.tar.gz"); + + var url = $"{ExifToolLocations.ExiftoolDownloadBasePath}{matchExifToolForUnixName}"; + if ( downloadFromMirror ) + url = $"{ExifToolLocations.ExiftoolDownloadBasePathMirror}{matchExifToolForUnixName}"; + + var unixDownloaded = await _httpClientHelper.Download(url, tarGzArchiveFullFilePath); + if ( !unixDownloaded ) + { + throw new HttpRequestException( + $"file is not downloaded {matchExifToolForUnixName}"); + } + + if ( !_checkSha1.CheckSha1(tarGzArchiveFullFilePath, getChecksumsFromTextFile) ) + { + throw new HttpRequestException( + $"checksum for {tarGzArchiveFullFilePath} is not valid"); + } + + await new TarBal(_hostFileSystemStorage).ExtractTarGz( + _hostFileSystemStorage.ReadStream(tarGzArchiveFullFilePath), + _appSettings.TempFolder, CancellationToken.None); + + var imageExifToolVersionFolder = _hostFileSystemStorage + .GetDirectories(_appSettings.TempFolder) + .FirstOrDefault(p => + p.StartsWith(Path.Combine(_appSettings.TempFolder, "Image-ExifTool-"))); + if ( imageExifToolVersionFolder != null ) + { + var exifToolUnixFolderFullFilePathTempFolder = + Path.Combine(_appSettings.TempFolder, "exiftool-unix"); + + if ( _hostFileSystemStorage.ExistFolder(exifToolUnixFolderFullFilePathTempFolder) ) + { + _hostFileSystemStorage.FolderDelete( + exifToolUnixFolderFullFilePathTempFolder); + } + + _hostFileSystemStorage.FolderMove(imageExifToolVersionFolder, + exifToolUnixFolderFullFilePathTempFolder); + + var exifToolUnixFolderFullFilePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix"); + + _hostFileSystemStorage.FileCopy(imageExifToolVersionFolder, + exifToolUnixFolderFullFilePathTempFolder); + } + else + { + _logger.LogError($"[DownloadForUnix] ExifTool folder does not exists"); + return false; + } + + // remove tar.gz file afterwards + _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); + + var exifToolExePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix", "exiftool"); + _logger.LogInformation( + $"[DownloadForUnix] ExifTool is just downloaded: {exifToolExePath} for {_appSettings.ApplicationType}"); + return await RunChmodOnExifToolUnixExe(); + } + + internal async Task RunChmodOnExifToolUnixExe() + { + // need to check again + // when not exist + if ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) return false; + if ( _appSettings.IsWindows ) return true; + + if ( !_hostFileSystemStorage.ExistFile("/bin/chmod") ) + { + _logger.LogError("[RunChmodOnExifToolUnixExe] WARNING: /bin/chmod does not exist"); + return true; + } + + // command.run does not care about the $PATH + var result = await Command.Run("/bin/chmod", "0755", ExeExifToolUnixFullFilePath()) + .Task; + if ( result.Success ) return true; + + _logger.LogError( + $"command failed with exit code {result.ExitCode}: {result.StandardError}"); + return false; + } +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadWindows.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadWindows.cs new file mode 100644 index 0000000000..87747d7fac --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadWindows.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using starsky.foundation.http.Interfaces; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.storage.ArchiveFormats; +using starsky.foundation.storage.Interfaces; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class ExifToolDownloadWindows(IStorage hostFileSystemStorage, AppSettings appSettings, + IHttpClientHelper httpClientHelper, IWebLogger logger) +{ + internal async Task StartDownloadForWindows() + { + var checksums = await DownloadCheckSums(); + if ( checksums == null ) return false; + + var matchExifToolForWindowsName = GetWindowsZipFromChecksum(checksums.Value.Value); + return await DownloadForWindows(matchExifToolForWindowsName, + GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); + } + + + private void MoveFileIfExist(string srcFullPath, string toFullPath) + { + if ( !_hostFileSystemStorage.ExistFile(srcFullPath) ) return; + _hostFileSystemStorage.FileMove(srcFullPath, toFullPath); + } + + internal async Task DownloadForWindows(string matchExifToolForWindowsName, + IEnumerable getChecksumsFromTextFile, bool downloadFromMirror = false) + { + if ( _hostFileSystemStorage.ExistFile( + ExeExifToolWindowsFullFilePath()) ) return true; + + var zipArchiveFullFilePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool.zip"); + var windowsExifToolFolder = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"); + + var url = $"{ExiftoolDownloadBasePath}{matchExifToolForWindowsName}"; + if ( downloadFromMirror ) + url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForWindowsName}"; + + var windowsDownloaded = await _httpClientHelper.Download(url, zipArchiveFullFilePath); + if ( !windowsDownloaded ) + { + throw new HttpRequestException( + $"file is not downloaded {matchExifToolForWindowsName}"); + } + + if ( !CheckSha1(zipArchiveFullFilePath, getChecksumsFromTextFile) ) + { + throw new HttpRequestException( + $"checksum for {zipArchiveFullFilePath} is not valid"); + } + + _hostFileSystemStorage.CreateDirectory(windowsExifToolFolder); + + new Zipper().ExtractZip(zipArchiveFullFilePath, windowsExifToolFolder); + MoveFileIfExist(Path.Combine(windowsExifToolFolder, "exiftool(-k).exe"), + Path.Combine(windowsExifToolFolder, "exiftool.exe")); + + _logger.LogInformation( + $"[DownloadForWindows] ExifTool downloaded: {ExeExifToolWindowsFullFilePath()}"); + return _hostFileSystemStorage.ExistFile(Path.Combine(windowsExifToolFolder, + "exiftool.exe")); + } +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolLocations.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolLocations.cs new file mode 100644 index 0000000000..1c5b6b2c30 --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolLocations.cs @@ -0,0 +1,39 @@ +using System.IO; +using starsky.foundation.platform.Models; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class ExifToolLocations(AppSettings appSettings) +{ + public string ExeExifToolFullFilePath(bool isWindows) + { + return isWindows ? ExeExifToolWindowsFullFilePath() : ExeExifToolUnixFullFilePath(); + } + + internal string ExeExifToolWindowsFullFilePath() + { + return Path.Combine(Path.Combine(appSettings.DependenciesFolder, "exiftool-windows"), + "exiftool.exe"); + } + + internal string ExeExifToolUnixFullFilePath() + { + var path = Path.Combine(appSettings.DependenciesFolder, + "exiftool-unix", + "exiftool"); + return path; + } + + internal const string Https = "https://"; + + internal const string CheckSumLocation = "exiftool.org/checksums.txt"; + + internal const string CheckSumLocationMirror = + "qdraw.nl/special/mirror/exiftool/checksums.txt"; + + internal const string + ExiftoolDownloadBasePath = "exiftool.org/"; // with slash at the end + + internal const string ExiftoolDownloadBasePathMirror = + "qdraw.nl/special/mirror/exiftool/"; // with slash at the end +} diff --git a/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs b/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs index f741ccfd64..7b946c502a 100644 --- a/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs +++ b/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs @@ -14,6 +14,7 @@ using starsky.foundation.storage.Interfaces; using starsky.foundation.storage.Storage; using starsky.foundation.writemeta.Services; +using starsky.foundation.writemeta.Services.ExifToolDownloader; using starskytest.FakeCreateAn; using starskytest.FakeMocks;