diff --git a/starsky/build/Build.cs b/starsky/build/Build.cs index d545276db5..fc5b7f1f24 100644 --- a/starsky/build/Build.cs +++ b/starsky/build/Build.cs @@ -184,7 +184,7 @@ List GetRuntimesWithoutGeneric() /// /// Link to GeoCli.csproj /// - const string GeoCliCsproj = "starskygeocli/starskygeocli.csproj"; + const string starskyDependenciesCliCsproj = "starskydependenciescli/starskydependenciescli.csproj"; /// /// Npm and node are required for preflight checks and building frontend code @@ -318,7 +318,8 @@ void ShowSettingsInfo() Configuration); DotnetTestHelper.TestNetCoreGenericCommand(Configuration, IsUnitTestDisabled()); - DotnetGenericHelper.DownloadDependencies(Configuration, GeoCliCsproj, + DotnetGenericHelper.DownloadDependencies(Configuration, starskyDependenciesCliCsproj, + GetRuntimesWithoutGeneric(), NoDependencies, GenericRuntimeName); MergeCoverageFiles.Merge(IsUnitTestDisabled()); SonarQube.SonarEnd(IsUnitTestDisabled(), NoSonar); @@ -333,7 +334,8 @@ void ShowSettingsInfo() .Executes(() => { DotnetGenericHelper.DownloadDependencies(Configuration, - "starskygeocli/starskygeocli.csproj", NoDependencies, + "starskydependenciescli/starskydependenciescli.csproj", + GetRuntimesWithoutGeneric(), NoDependencies, GenericRuntimeName); DotnetRuntimeSpecificHelper.CopyDependenciesFiles(NoDependencies, GenericRuntimeName, GetRuntimesWithoutGeneric()); diff --git a/starsky/build/helpers/DotnetGenericHelper.cs b/starsky/build/helpers/DotnetGenericHelper.cs index 0f4b9a36d8..489dbe60d6 100644 --- a/starsky/build/helpers/DotnetGenericHelper.cs +++ b/starsky/build/helpers/DotnetGenericHelper.cs @@ -1,5 +1,5 @@ using System; -using System.Diagnostics.CodeAnalysis; +using System.Collections.Generic; using System.IO; using build; using Nuke.Common.ProjectModel; @@ -10,9 +10,6 @@ namespace helpers { - [SuppressMessage("Sonar", - "S6664: Reduce the number of Information logging calls within this code block", - Justification = "Not production code.")] public static class DotnetGenericHelper { /// @@ -47,11 +44,12 @@ public static void BuildNetCoreGenericCommand(Solution solution, /// Download Exiftool and geo deps /// /// is Release - /// geo.csproj file + /// dependenciesCli.csproj file + /// which version are downloaded /// skip this step if true (external deps) /// genericNetcoreFolder public static void DownloadDependencies(Configuration configuration, - string geoCliCsproj, bool noDependencies, + string dependenciesCliCsproj, List getRuntimesWithoutGeneric, bool noDependencies, string genericNetcoreFolder) { if ( noDependencies ) @@ -60,6 +58,12 @@ public static void DownloadDependencies(Configuration configuration, return; } + if ( getRuntimesWithoutGeneric.Count == 0 ) + { + Log.Information("skip deps build due generic build"); + return; + } + var genericDepsFullPath = Path.Combine(BasePath(), genericNetcoreFolder, "dependencies"); Log.Information("genericDepsFullPath: {GenericDepsFullPath}", genericDepsFullPath); @@ -69,16 +73,16 @@ public static void DownloadDependencies(Configuration configuration, Environment.SetEnvironmentVariable("app__DependenciesFolder", genericDepsFullPath); Log.Information("Next: DownloadDependencies"); Log.Information("Run: {Path}", Path.Combine( - WorkingDirectory.GetSolutionParentFolder(), geoCliCsproj) + WorkingDirectory.GetSolutionParentFolder(), dependenciesCliCsproj) ); DotNetRun(p => p .SetConfiguration(configuration) .EnableNoRestore() .EnableNoBuild() - .SetApplicationArguments("--runtime linux-x64,win-x64") + .SetApplicationArguments($"--runtime {string.Join(',', getRuntimesWithoutGeneric)}") .SetProjectFile(Path.Combine(WorkingDirectory.GetSolutionParentFolder(), - geoCliCsproj))); + dependenciesCliCsproj))); } catch ( Exception exception ) { diff --git a/starsky/starsky.feature.externaldependencies/ExternalDependenciesService.cs b/starsky/starsky.feature.externaldependencies/ExternalDependenciesService.cs new file mode 100644 index 0000000000..b4edcd19cd --- /dev/null +++ b/starsky/starsky.feature.externaldependencies/ExternalDependenciesService.cs @@ -0,0 +1,60 @@ +using System.Runtime.InteropServices; +using starsky.feature.externaldependencies.Interfaces; +using starsky.feature.geolookup.Interfaces; +using starsky.foundation.database.Data; +using starsky.foundation.database.Helpers; +using starsky.foundation.injection; +using starsky.foundation.platform.Helpers; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.writemeta.Interfaces; + +namespace starsky.feature.externaldependencies; + +[Service(typeof(IExternalDependenciesService), InjectionLifetime = InjectionLifetime.Scoped)] +public class ExternalDependenciesService : IExternalDependenciesService +{ + private readonly IExifToolDownload _exifToolDownload; + private readonly ApplicationDbContext _dbContext; + private readonly IWebLogger _logger; + private readonly AppSettings _appSettings; + private readonly IGeoFileDownload _geoFileDownload; + + public ExternalDependenciesService(IExifToolDownload exifToolDownload, + ApplicationDbContext dbContext, IWebLogger logger, AppSettings appSettings, + IGeoFileDownload geoFileDownload) + { + _exifToolDownload = exifToolDownload; + _dbContext = dbContext; + _logger = logger; + _appSettings = appSettings; + _geoFileDownload = geoFileDownload; + } + + public async Task SetupAsync(string[] args) + { + await SetupAsync(ArgsHelper.GetRuntime(args)); + } + + public async Task SetupAsync(List<(OSPlatform?, Architecture?)> currentPlatforms) + { + await RunMigrations.Run(_dbContext, _logger, _appSettings); + + + if ( currentPlatforms.Count == 0 ) + { + currentPlatforms = + [ + ( PlatformParser.GetCurrentOsPlatform(), + PlatformParser.GetCurrentArchitecture() ) + ]; + } + + foreach ( var (osPlatform, _) in currentPlatforms ) + { + await _exifToolDownload.DownloadExifTool(osPlatform == OSPlatform.Windows); + } + + await _geoFileDownload.DownloadAsync(); + } +} diff --git a/starsky/starsky.feature.externaldependencies/Interfaces/IExternalDependenciesService.cs b/starsky/starsky.feature.externaldependencies/Interfaces/IExternalDependenciesService.cs new file mode 100644 index 0000000000..da6308a8c4 --- /dev/null +++ b/starsky/starsky.feature.externaldependencies/Interfaces/IExternalDependenciesService.cs @@ -0,0 +1,6 @@ +namespace starsky.feature.externaldependencies.Interfaces; + +public interface IExternalDependenciesService +{ + Task SetupAsync(string[] args); +} diff --git a/starsky/starsky.feature.externaldependencies/starsky.feature.externaldependencies.csproj b/starsky/starsky.feature.externaldependencies/starsky.feature.externaldependencies.csproj new file mode 100644 index 0000000000..3ede59eb51 --- /dev/null +++ b/starsky/starsky.feature.externaldependencies/starsky.feature.externaldependencies.csproj @@ -0,0 +1,16 @@ + + + + net8.0 + {5783f81c-618c-4971-984e-0e405ab1b609} + enable + enable + starsky.feature.externaldependencies + + + + + + + + diff --git a/starsky/starsky.feature.geolookup/Services/GeoFileDownload.cs b/starsky/starsky.feature.geolookup/Services/GeoFileDownload.cs index b1622a255a..9a7b0f2600 100644 --- a/starsky/starsky.feature.geolookup/Services/GeoFileDownload.cs +++ b/starsky/starsky.feature.geolookup/Services/GeoFileDownload.cs @@ -39,7 +39,7 @@ public async Task DownloadAsync() RemoveFailedDownload(); CreateDependenciesFolder(); const string https = "https://"; - const string admin1codesasciiTxt = "admin1CodesASCII.txt"; + const string admin1CodesasciiTxt = "admin1CodesASCII.txt"; if ( !_hostStorage.ExistFile( Path.Combine(_appSettings.DependenciesFolder, CountryName + ".txt")) ) @@ -59,18 +59,18 @@ await _httpClientHelper.Download(https + MirrorUrl + CountryName + ".zip", } if ( !_hostStorage.ExistFile( - Path.Combine(_appSettings.DependenciesFolder, admin1codesasciiTxt)) ) + Path.Combine(_appSettings.DependenciesFolder, admin1CodesasciiTxt)) ) { // code for the second administrative division, // a county in the US, see file admin2Codes.txt; varchar(80) var outputFile = Path.Combine(_appSettings.DependenciesFolder, - admin1codesasciiTxt); + admin1CodesasciiTxt); var baseResult = await _httpClientHelper.Download(https + - BaseUrl + admin1codesasciiTxt, outputFile); + BaseUrl + admin1CodesasciiTxt, outputFile); if ( !baseResult ) { await _httpClientHelper.Download(https + - MirrorUrl + admin1codesasciiTxt, outputFile); + MirrorUrl + admin1CodesasciiTxt, outputFile); } } } diff --git a/starsky/starsky.feature.packagetelemetry/Services/PackageTelemetry.cs b/starsky/starsky.feature.packagetelemetry/Services/PackageTelemetry.cs index db76e87d42..95e896a70e 100644 --- a/starsky/starsky.feature.packagetelemetry/Services/PackageTelemetry.cs +++ b/starsky/starsky.feature.packagetelemetry/Services/PackageTelemetry.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -17,9 +16,9 @@ using starsky.foundation.platform.Models; [assembly: InternalsVisibleTo("starskytest")] + namespace starsky.feature.packagetelemetry.Services { - [Service(typeof(IPackageTelemetry), InjectionLifetime = InjectionLifetime.Scoped)] public class PackageTelemetry : IPackageTelemetry { @@ -29,7 +28,8 @@ public class PackageTelemetry : IPackageTelemetry private readonly IQuery _query; private readonly IDeviceIdService _deviceIdService; - public PackageTelemetry(IHttpClientHelper httpClientHelper, AppSettings appSettings, IWebLogger logger, IQuery query, IDeviceIdService deviceIdService) + public PackageTelemetry(IHttpClientHelper httpClientHelper, AppSettings appSettings, + IWebLogger logger, IQuery query, IDeviceIdService deviceIdService) { _httpClientHelper = httpClientHelper; _appSettings = appSettings; @@ -38,49 +38,46 @@ public PackageTelemetry(IHttpClientHelper httpClientHelper, AppSettings appSetti _deviceIdService = deviceIdService; } - internal const string PackageTelemetryUrl = "qdraw.nl/special/starsky/telemetry/index.php"; + internal const string PackageTelemetryUrl = "qdraw.nl/special/starsky/telemetry/v2.php"; internal static object? GetPropValue(object? src, string propName) { return src?.GetType().GetProperty(propName)?.GetValue(src, null); } - internal static OSPlatform? GetCurrentOsPlatform() - { - OSPlatform? currentPlatform = null; - foreach ( var platform in new List{OSPlatform.Linux, - OSPlatform.Windows, OSPlatform.OSX, - OSPlatform.FreeBSD}.Where(RuntimeInformation.IsOSPlatform) ) - { - currentPlatform = platform; - } - - return currentPlatform; - } - - internal List> GetSystemData(OSPlatform? currentPlatform = null, string? deviceId = null) + internal List> GetSystemData( + OSPlatform? currentPlatform = null, string? deviceId = null) { - currentPlatform ??= GetCurrentOsPlatform(); + currentPlatform ??= PlatformParser.GetCurrentOsPlatform(); var dockerContainer = currentPlatform == OSPlatform.Linux && - Environment.GetEnvironmentVariable( - "DOTNET_RUNNING_IN_CONTAINER") == "true"; + Environment.GetEnvironmentVariable( + "DOTNET_RUNNING_IN_CONTAINER") == "true"; var data = new List> { - new KeyValuePair("UTCTime", DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)), + new KeyValuePair("UTCTime", + DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)), new KeyValuePair("AppVersion", _appSettings.AppVersion), - new KeyValuePair("NetVersion", RuntimeInformation.FrameworkDescription), - new KeyValuePair("OSArchitecture", RuntimeInformation.OSArchitecture.ToString()), - new KeyValuePair("ProcessArchitecture", RuntimeInformation.ProcessArchitecture.ToString()), - new KeyValuePair("OSVersion", Environment.OSVersion.Version.ToString()), - new KeyValuePair("OSDescriptionLong", RuntimeInformation.OSDescription.Replace(";", " ")), + new KeyValuePair("NetVersion", + RuntimeInformation.FrameworkDescription), + new KeyValuePair("OSArchitecture", + RuntimeInformation.OSArchitecture.ToString()), + new KeyValuePair("ProcessArchitecture", + RuntimeInformation.ProcessArchitecture.ToString()), + new KeyValuePair("OSVersion", + Environment.OSVersion.Version.ToString()), + new KeyValuePair("OSDescriptionLong", + RuntimeInformation.OSDescription.Replace(";", " ")), new KeyValuePair("OSPlatform", currentPlatform.ToString()!), new KeyValuePair("DockerContainer", dockerContainer.ToString()), - new KeyValuePair("CurrentCulture", CultureInfo.CurrentCulture.ThreeLetterISOLanguageName), - new KeyValuePair("AspNetCoreEnvironment", Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Not set"), + new KeyValuePair("CurrentCulture", + CultureInfo.CurrentCulture.ThreeLetterISOLanguageName), + new KeyValuePair("AspNetCoreEnvironment", + Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Not set"), new KeyValuePair("WebsiteName", GetEncryptedSiteName()), new KeyValuePair("DeviceId", deviceId ?? "Not set"), + new KeyValuePair("RuntimeIdentifier", RuntimeInformation.RuntimeIdentifier), }; return data; } @@ -89,11 +86,14 @@ private static string GetEncryptedSiteName() { var siteName = Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME"); - return string.IsNullOrEmpty(siteName) ? "Not set" : Sha256.ComputeSha256(siteName); // used in Azure web apps + return string.IsNullOrEmpty(siteName) + ? "Not set" + : Sha256.ComputeSha256(siteName); // used in Azure web apps } - internal async Task>> AddDatabaseData(List> data) + internal async Task>> AddDatabaseData( + List> data) { var fileIndexItemTotalCount = -1; var fileIndexItemDirectoryCount = -1; @@ -112,22 +112,27 @@ internal async Task>> AddDatabaseData(List> { - new KeyValuePair("FileIndexItemTotalCount",fileIndexItemTotalCount.ToString()), - new KeyValuePair("FileIndexItemDirectoryCount",fileIndexItemDirectoryCount.ToString()), - new KeyValuePair("FileIndexItemCount",fileIndexItemCount.ToString()) + new KeyValuePair("FileIndexItemTotalCount", + fileIndexItemTotalCount.ToString()), + new KeyValuePair("FileIndexItemDirectoryCount", + fileIndexItemDirectoryCount.ToString()), + new KeyValuePair("FileIndexItemCount", + fileIndexItemCount.ToString()) }); return data; } - internal List> AddAppSettingsData(List> data) + internal List> AddAppSettingsData( + List> data) { var type = typeof(AppSettings); var properties = type.GetProperties(); // ReSharper disable once LoopCanBeConvertedToQuery foreach ( var property in properties ) { - var someAttribute = Array.Find(Attribute.GetCustomAttributes(property), x => x is PackageTelemetryAttribute); + var someAttribute = Array.Find(Attribute.GetCustomAttributes(property), + x => x is PackageTelemetryAttribute); if ( someAttribute == null ) { continue; @@ -143,13 +148,16 @@ internal List> AddAppSettingsData(List) || - propValue?.GetType() == typeof(Dictionary>) ) + propValue?.GetType() == + typeof(Dictionary>) ) { value = ParseContent(propValue); } - data.Add(new KeyValuePair("AppSettings" + property.Name, value ?? "null")); + data.Add(new KeyValuePair("AppSettings" + property.Name, + value ?? "null")); } + return data; } @@ -160,7 +168,8 @@ internal static string ParseContent(object propValue) private async Task PostData(HttpContent formUrlEncodedContent) { - return ( await _httpClientHelper.PostString("https://" + PackageTelemetryUrl, formUrlEncodedContent, + return ( await _httpClientHelper.PostString("https://" + PackageTelemetryUrl, + formUrlEncodedContent, _appSettings.EnablePackageTelemetryDebug == true) ).Key; } @@ -171,7 +180,7 @@ private async Task PostData(HttpContent formUrlEncodedContent) return null; } - var currentOsPlatform = GetCurrentOsPlatform(); + var currentOsPlatform = PlatformParser.GetCurrentOsPlatform(); var deviceId = await _deviceIdService.DeviceId(currentOsPlatform); var telemetryDataItems = GetSystemData(currentOsPlatform, deviceId); @@ -194,4 +203,3 @@ private async Task PostData(HttpContent formUrlEncodedContent) } } } - diff --git a/starsky/starsky.foundation.http/Services/HttpClientHelper.cs b/starsky/starsky.foundation.http/Services/HttpClientHelper.cs index 8292f10cf0..1c80632de5 100644 --- a/starsky/starsky.foundation.http/Services/HttpClientHelper.cs +++ b/starsky/starsky.foundation.http/Services/HttpClientHelper.cs @@ -10,6 +10,7 @@ using starsky.foundation.injection; using starsky.foundation.platform.Helpers; using starsky.foundation.platform.Interfaces; +using starsky.foundation.storage.Helpers; using starsky.foundation.storage.Interfaces; using starsky.foundation.storage.Storage; @@ -100,12 +101,13 @@ public async Task> PostString(string sourceHttpUrl, // // allow whitelist and https only if ( !_allowedDomains.Contains(sourceUri.Host) || sourceUri.Scheme != "https" ) return new KeyValuePair(false, string.Empty); - + try { using ( HttpResponseMessage response = await _httpProvider.PostAsync(sourceHttpUrl, httpContent) ) using ( Stream streamToReadFrom = await response.Content.ReadAsStreamAsync() ) { + var reader = new StreamReader(streamToReadFrom, Encoding.UTF8); var result = await reader.ReadToEndAsync(); return new KeyValuePair(response.StatusCode == HttpStatusCode.OK, result); diff --git a/starsky/starsky.foundation.platform/Helpers/ArgsHelper.cs b/starsky/starsky.foundation.platform/Helpers/ArgsHelper.cs index 030ff415fb..5d974568dc 100644 --- a/starsky/starsky.foundation.platform/Helpers/ArgsHelper.cs +++ b/starsky/starsky.foundation.platform/Helpers/ArgsHelper.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; using System.Text.RegularExpressions; using starsky.foundation.platform.Interfaces; using starsky.foundation.platform.Models; @@ -504,7 +505,7 @@ private void AppSpecificHelp() /// private void ShowVersions() { - var version = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription; + var version = RuntimeInformation.FrameworkDescription; _console.WriteLine($".NET Version - {version}"); _console.WriteLine($"Starsky Version - {_appSettings.AppVersion} " + "- build at: " + @@ -636,6 +637,32 @@ public static ConsoleOutputMode GetConsoleOutputMode(IReadOnlyList args) return outputMode; } + /// + /// Get input runtime + /// + /// arg list + /// path + public static List<(OSPlatform?, Architecture?)> GetRuntime(IReadOnlyList args) + { + var outputModeList = new List<(OSPlatform?, Architecture?)>(); + for ( var arg = 0; arg < args.Count; arg++ ) + { + if ( args[arg]? + .Equals("--runtime", + StringComparison.CurrentCultureIgnoreCase) != true || + arg + 1 == args.Count ) + { + continue; + } + + var runtimeItem = args[arg + 1]; + var parsedRuntimeList = PlatformParser.RuntimeIdentifier(runtimeItem); + outputModeList.AddRange(parsedRuntimeList); + } + + return outputModeList; + } + /// /// Get the user input from -n or --name /// diff --git a/starsky/starsky.foundation.platform/Helpers/PlatformParser.cs b/starsky/starsky.foundation.platform/Helpers/PlatformParser.cs new file mode 100644 index 0000000000..be979cd8ee --- /dev/null +++ b/starsky/starsky.foundation.platform/Helpers/PlatformParser.cs @@ -0,0 +1,76 @@ +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace starsky.foundation.platform.Helpers; + +public static class PlatformParser +{ + public static OSPlatform? GetCurrentOsPlatform() + { + OSPlatform? currentPlatform = null; + foreach ( var platform in new List + { + OSPlatform.Linux, OSPlatform.Windows, OSPlatform.OSX, OSPlatform.FreeBSD + }.Where(RuntimeInformation.IsOSPlatform) ) + { + currentPlatform = platform; + } + + return currentPlatform; + } + + public static List<(OSPlatform?, Architecture?)> RuntimeIdentifier(string? runtimeIdentifiers) + { + var result = new List<(OSPlatform?, Architecture?)>(); + if ( runtimeIdentifiers == null ) + { + return []; + } + + var runtimes = runtimeIdentifiers.Split(",").Where(x => !string.IsNullOrEmpty(x)).ToList(); + foreach ( var runtime in runtimes ) + { + var singleRuntimeIdentifier = SingleRuntimeIdentifier(runtime); + if ( singleRuntimeIdentifier is { Item1: not null, Item2: not null } ) + { + result.Add(singleRuntimeIdentifier); + } + } + + return result; + } + + private static (OSPlatform?, Architecture?) SingleRuntimeIdentifier(string? runtimeIdentifier) + { + if ( runtimeIdentifier == null ) + { + return ( null, null ); + } + + switch ( runtimeIdentifier ) + { + case "win-x64": + return ( OSPlatform.Windows, Architecture.X64 ); + case "win-arm64": + return ( OSPlatform.Windows, Architecture.Arm64 ); + case "linux-arm": + return ( OSPlatform.Linux, Architecture.Arm ); + case "linux-arm64": + return ( OSPlatform.Linux, Architecture.Arm64 ); + case "linux-x64": + return ( OSPlatform.Linux, Architecture.X64 ); + case "osx-x64": + return ( OSPlatform.OSX, Architecture.X64 ); + case "osx-arm64": + return ( OSPlatform.OSX, Architecture.Arm64 ); + default: + return ( null, null ); + } + } + + public static Architecture GetCurrentArchitecture() + { + return Architecture.Wasm; + } +} diff --git a/starsky/starsky.foundation.storage/Interfaces/IStorage.cs b/starsky/starsky.foundation.storage/Interfaces/IStorage.cs index cfca155725..f4c4c85066 100644 --- a/starsky/starsky.foundation.storage/Interfaces/IStorage.cs +++ b/starsky/starsky.foundation.storage/Interfaces/IStorage.cs @@ -12,6 +12,7 @@ public interface IStorage bool ExistFolder(string path); FolderOrFileModel.FolderOrFileTypeList IsFolderOrFile(string path); void FolderMove(string fromPath, string toPath); + void FolderCopy(string fromPath, string toPath); bool FileMove(string fromPath, string toPath); void FileCopy(string fromPath, string toPath); bool FileDelete(string path); diff --git a/starsky/starsky.foundation.storage/Storage/StorageHostFullPathFilesystem.cs b/starsky/starsky.foundation.storage/Storage/StorageHostFullPathFilesystem.cs index ad92bf4f00..ad5d79fa4a 100644 --- a/starsky/starsky.foundation.storage/Storage/StorageHostFullPathFilesystem.cs +++ b/starsky/starsky.foundation.storage/Storage/StorageHostFullPathFilesystem.cs @@ -320,6 +320,39 @@ public void FolderMove(string fromPath, string toPath) Directory.Move(fromPath, toPath); } + public void FolderCopy(string fromPath, string toPath) + { + var dirs = new Stack(); + dirs.Push(fromPath); + + while (dirs.Count > 0) + { + var currentDir = dirs.Pop(); + var destinationSubDirectory = currentDir.Replace(fromPath, toPath); + + if (!Directory.Exists(destinationSubDirectory)) + { + Directory.CreateDirectory(destinationSubDirectory); + } + + var files = Directory.GetFiles(currentDir); + + foreach (var file in files) + { + var fileName = Path.GetFileName(file); + var destinationFile = Path.Combine(destinationSubDirectory, fileName); + File.Copy(file, destinationFile); + } + + var subdirectories = Directory.GetDirectories(currentDir); + + foreach (var subDirectory in subdirectories) + { + dirs.Push(subDirectory); + } + } + } + /// /// Move file on real filesystem /// diff --git a/starsky/starsky.foundation.storage/Storage/StorageSubPathFilesystem.cs b/starsky/starsky.foundation.storage/Storage/StorageSubPathFilesystem.cs index 695bc37f91..c282542464 100644 --- a/starsky/starsky.foundation.storage/Storage/StorageSubPathFilesystem.cs +++ b/starsky/starsky.foundation.storage/Storage/StorageSubPathFilesystem.cs @@ -101,6 +101,14 @@ public void FolderMove(string fromPath, string toPath) toFileFullPath); } + public void FolderCopy(string fromPath, string toPath) + { + var inputFileFullPath = _appSettings.DatabasePathToFilePath(fromPath); + var toFileFullPath = _appSettings.DatabasePathToFilePath(toPath); + new StorageHostFullPathFilesystem(_logger).FolderCopy(inputFileFullPath, + toFileFullPath); + } + /// /// Move a file /// diff --git a/starsky/starsky.foundation.storage/Storage/StorageThumbnailFilesystem.cs b/starsky/starsky.foundation.storage/Storage/StorageThumbnailFilesystem.cs index 27fac215a1..1e3bff458d 100644 --- a/starsky/starsky.foundation.storage/Storage/StorageThumbnailFilesystem.cs +++ b/starsky/starsky.foundation.storage/Storage/StorageThumbnailFilesystem.cs @@ -69,6 +69,11 @@ public void FolderMove(string fromPath, string toPath) throw new System.NotImplementedException(); } + public void FolderCopy(string fromPath, string toPath) + { + throw new NotImplementedException(); + } + public bool FileMove(string fromPath, string toPath) { var oldThumbPath = CombinePath(fromPath); diff --git a/starsky/starsky.foundation.writemeta/Helpers/CreateFolderIfNotExists.cs b/starsky/starsky.foundation.writemeta/Helpers/CreateFolderIfNotExists.cs new file mode 100644 index 0000000000..b8f767e316 --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Helpers/CreateFolderIfNotExists.cs @@ -0,0 +1,51 @@ +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.storage.Storage; + +namespace starsky.foundation.writemeta.Helpers; + +public class CreateFolderIfNotExists +{ + private readonly StorageHostFullPathFilesystem _hostFileSystemStorage; + private readonly IWebLogger _logger; + private readonly AppSettings _appSettings; + + public CreateFolderIfNotExists(IWebLogger logger, AppSettings appSettings) + { + _hostFileSystemStorage = new StorageHostFullPathFilesystem(logger); + _logger = logger; + _appSettings = appSettings; + } + + internal void CreateDirectoryDependenciesTempFolderIfNotExists() + { + CreateDirectoryDependenciesFolderIfNotExists(); + CreateDirectoryTempFolderIfNotExists(); + } + + private void CreateDirectoryDependenciesFolderIfNotExists() + { + if ( _hostFileSystemStorage.ExistFolder(_appSettings + .DependenciesFolder) ) + { + return; + } + + _logger.LogInformation("[DownloadExifTool] Create Directory: " + + _appSettings.DependenciesFolder); + _hostFileSystemStorage.CreateDirectory(_appSettings.DependenciesFolder); + } + + private void CreateDirectoryTempFolderIfNotExists() + { + if ( _hostFileSystemStorage.ExistFolder(_appSettings + .TempFolder) ) + { + return; + } + + _logger.LogInformation("[DownloadExifTool] Create Directory: " + + _appSettings.TempFolder); + _hostFileSystemStorage.CreateDirectory(_appSettings.TempFolder); + } +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs deleted file mode 100644 index dc268ebbb3..0000000000 --- a/starsky/starsky.foundation.writemeta/Services/ExifToolDownload.cs +++ /dev/null @@ -1,324 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Medallion.Shell; -using starsky.foundation.http.Interfaces; -using starsky.foundation.injection; -using starsky.foundation.platform.Interfaces; -using starsky.foundation.platform.Models; -using starsky.foundation.storage.ArchiveFormats; -using starsky.foundation.storage.Interfaces; -using starsky.foundation.storage.Storage; -using starsky.foundation.writemeta.Interfaces; - -[assembly: InternalsVisibleTo("starskytest")] -namespace starsky.foundation.writemeta.Services -{ - [Service(typeof(IExifToolDownload), InjectionLifetime = InjectionLifetime.Singleton)] - [SuppressMessage("Usage", "S1075:Refactor your code not to use hardcoded absolute paths or URIs", Justification = "Source of files")] - [SuppressMessage("Usage", "S4790:Make sure this weak hash algorithm is not used in a sensitive context here.", Justification = "Safe")] - public sealed class ExifToolDownload : IExifToolDownload - { - private readonly IHttpClientHelper _httpClientHelper; - private readonly AppSettings _appSettings; - private readonly IStorage _hostFileSystemStorage; - private readonly IWebLogger _logger; - - private const string CheckSumLocation = "https://exiftool.org/checksums.txt"; - private const string CheckSumLocationMirror = "https://qdraw.nl/special/mirror/exiftool/checksums.txt"; - private const string ExiftoolDownloadBasePath = "https://exiftool.org/"; // with slash at the end - private const string ExiftoolDownloadBasePathMirror = "https://qdraw.nl/special/mirror/exiftool/"; // with slash at the end - - public ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, IWebLogger logger) - { - _httpClientHelper = httpClientHelper; - _appSettings = appSettings; - _hostFileSystemStorage = new StorageHostFullPathFilesystem(logger); - _logger = logger; - } - - internal ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, IWebLogger logger, IStorage storage) - { - _httpClientHelper = httpClientHelper; - _appSettings = appSettings; - _hostFileSystemStorage = storage; - _logger = logger; - } - - /// - /// Auto Download Exiftool - /// - /// download windows version if true - /// check for min file size in bytes (Default = 30 bytes) - /// - public async Task DownloadExifTool(bool isWindows, int minimumSize = 30) - { - if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || ( - _appSettings.AddSwaggerExport == true && _appSettings.AddSwaggerExportExitAfter == true ) ) - { - var name = _appSettings.ExiftoolSkipDownloadOnStartup == true - ? "ExiftoolSkipDownloadOnStartup" - : "AddSwaggerExport and AddSwaggerExportExitAfter"; - _logger.LogInformation($"[DownloadExifTool] Skipped due true of {name} setting"); - return false; - } - - CreateDirectoryDependenciesFolderIfNotExists(); - - if ( isWindows && - ( !_hostFileSystemStorage.ExistFile(ExeExifToolWindowsFullFilePath()) || - _hostFileSystemStorage.Info(ExeExifToolWindowsFullFilePath()).Size <= minimumSize ) ) - { - return await StartDownloadForWindows(); - } - - if ( !isWindows && - ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) || - _hostFileSystemStorage.Info(ExeExifToolUnixFullFilePath()).Size <= minimumSize ) ) - { - return await StartDownloadForUnix(); - } - - if ( _appSettings.IsVerbose() ) - { - var debugPath = isWindows ? ExeExifToolWindowsFullFilePath() - : ExeExifToolUnixFullFilePath(); - _logger.LogInformation($"[DownloadExifTool] {debugPath}"); - } - - // When running deploy scripts rights might reset (only for unix) - if ( isWindows ) return true; - - return await RunChmodOnExifToolUnixExe(); - } - - private void CreateDirectoryDependenciesFolderIfNotExists() - { - if ( _hostFileSystemStorage.ExistFolder(_appSettings - .DependenciesFolder) ) return; - _logger.LogInformation("[DownloadExifTool] Create Directory: " + _appSettings.DependenciesFolder); - _hostFileSystemStorage.CreateDirectory(_appSettings.DependenciesFolder); - } - - internal async Task?> DownloadCheckSums() - { - var checksums = await _httpClientHelper.ReadString(CheckSumLocation); - if ( checksums.Key ) - { - return checksums; - } - _logger.LogError($"Checksum loading failed {CheckSumLocation}, next retry from mirror ~ error > " + checksums.Value); - - checksums = await _httpClientHelper.ReadString(CheckSumLocationMirror); - if ( checksums.Key ) return new KeyValuePair(false, checksums.Value); - - _logger.LogError($"Checksum loading failed {CheckSumLocationMirror}" + - $", next stop; please connect to internet and restart app ~ error > " + checksums.Value); - return null; - } - - internal async Task StartDownloadForUnix() - { - var checksums = await DownloadCheckSums(); - if ( checksums == null ) return false; - var matchExifToolForUnixName = GetUnixTarGzFromChecksum(checksums.Value.Value); - return await DownloadForUnix(matchExifToolForUnixName, - GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); - } - - internal static string GetUnixTarGzFromChecksum(string checksumsValue) - { - // (?<=SHA1\()Image-ExifTool-[\d\.]+\.zip - var regexExifToolForWindowsName = new Regex(@"(?<=SHA1\()Image-ExifTool-[0-9\.]+\.tar.gz", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - return regexExifToolForWindowsName.Match(checksumsValue).Value; - } - - private string ExeExifToolUnixFullFilePath() - { - var path = Path.Combine(_appSettings.DependenciesFolder, - "exiftool-unix", - "exiftool"); - return path; - } - - internal async Task DownloadForUnix(string matchExifToolForUnixName, - string[] getChecksumsFromTextFile, bool downloadFromMirror = false) - { - - if ( _hostFileSystemStorage.ExistFile( - ExeExifToolUnixFullFilePath()) ) return true; - - var tarGzArchiveFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool.tar.gz"); - - var url = $"{ExiftoolDownloadBasePath}{matchExifToolForUnixName}"; - if ( downloadFromMirror ) url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForUnixName}"; - - var unixDownloaded = await _httpClientHelper.Download(url, tarGzArchiveFullFilePath); - if ( !unixDownloaded ) - { - throw new HttpRequestException($"file is not downloaded {matchExifToolForUnixName}"); - } - if ( !CheckSha1(tarGzArchiveFullFilePath, getChecksumsFromTextFile) ) - { - throw new HttpRequestException($"checksum for {tarGzArchiveFullFilePath} is not valid"); - } - - await new TarBal(_hostFileSystemStorage).ExtractTarGz( - _hostFileSystemStorage.ReadStream(tarGzArchiveFullFilePath), _appSettings.DependenciesFolder, CancellationToken.None); - - var imageExifToolVersionFolder = _hostFileSystemStorage.GetDirectories(_appSettings.DependenciesFolder) - .FirstOrDefault(p => p.StartsWith(Path.Combine(_appSettings.DependenciesFolder, "Image-ExifTool-"))); - if ( imageExifToolVersionFolder != null ) - { - var exifToolUnixFolderFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix"); - if ( _hostFileSystemStorage.ExistFolder(exifToolUnixFolderFullFilePath) ) - { - _hostFileSystemStorage.FolderDelete( - exifToolUnixFolderFullFilePath); - } - _hostFileSystemStorage.FolderMove(imageExifToolVersionFolder, exifToolUnixFolderFullFilePath); - } - else - { - _logger.LogError($"[DownloadForUnix] ExifTool folder does not exists"); - return false; - } - - // remove tar.gz file afterwards - _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); - - var exifToolExePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix", "exiftool"); - _logger.LogInformation($"[DownloadForUnix] ExifTool is just downloaded: {exifToolExePath} for {_appSettings.ApplicationType}"); - return await RunChmodOnExifToolUnixExe(); - } - - internal async Task RunChmodOnExifToolUnixExe() - { - // need to check again - // when not exist - if ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) return false; - if ( _appSettings.IsWindows ) return true; - - if ( !_hostFileSystemStorage.ExistFile("/bin/chmod") ) - { - _logger.LogError("[RunChmodOnExifToolUnixExe] WARNING: /bin/chmod does not exist"); - return true; - } - - // command.run does not care about the $PATH - var result = await Command.Run("/bin/chmod", "0755", ExeExifToolUnixFullFilePath()).Task; - if ( result.Success ) return true; - - _logger.LogError($"command failed with exit code {result.ExitCode}: {result.StandardError}"); - return false; - } - - internal async Task StartDownloadForWindows() - { - var checksums = await DownloadCheckSums(); - if ( checksums == null ) return false; - - var matchExifToolForWindowsName = GetWindowsZipFromChecksum(checksums.Value.Value); - return await DownloadForWindows(matchExifToolForWindowsName, - GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); - } - - internal static string GetWindowsZipFromChecksum(string checksumsValue) - { - // (?<=SHA1\()exiftool-[\d\.]+\.zip - var regexExifToolForWindowsName = new Regex(@"(?<=SHA1\()exiftool-[0-9\.]+\.zip", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - return regexExifToolForWindowsName.Match(checksumsValue).Value; - } - - /// - /// Parse the content of checksum file - /// - /// input file: see test for example - /// max number of SHA1 results - /// - internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 8) - { - var regexExifToolForWindowsName = new Regex("[a-z0-9]{40}", - RegexOptions.None, TimeSpan.FromMilliseconds(100)); - var results = regexExifToolForWindowsName.Matches(checksumsValue). - Select(m => m.Value). - ToArray(); - if ( results.Length < max ) return results; - - _logger.LogError($"More than {max} checksums found, this is not expected, code stops now"); - return Array.Empty(); - } - - /// - /// Check if SHA1 hash is valid - /// Instead of SHA1CryptoServiceProvider, we use SHA1.Create - /// - /// path of exiftool.exe - /// list of sha1 hashes - /// - internal bool CheckSha1(string fullFilePath, IEnumerable checkSumOptions) - { - using var buffer = _hostFileSystemStorage.ReadStream(fullFilePath); - using var hashAlgorithm = SHA1.Create(); - - var byteHash = hashAlgorithm.ComputeHash(buffer); - var hash = BitConverter.ToString(byteHash).Replace("-", string.Empty).ToLowerInvariant(); - return checkSumOptions.AsEnumerable().Any(p => p.Equals(hash, StringComparison.InvariantCultureIgnoreCase)); - } - - private string ExeExifToolWindowsFullFilePath() - { - return Path.Combine(Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"), "exiftool.exe"); - } - - internal async Task DownloadForWindows(string matchExifToolForWindowsName, - string[] getChecksumsFromTextFile, bool downloadFromMirror = false) - { - if ( _hostFileSystemStorage.ExistFile( - ExeExifToolWindowsFullFilePath()) ) return true; - - var zipArchiveFullFilePath = Path.Combine(_appSettings.DependenciesFolder, "exiftool.zip"); - var windowsExifToolFolder = Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"); - - var url = $"{ExiftoolDownloadBasePath}{matchExifToolForWindowsName}"; - if ( downloadFromMirror ) url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForWindowsName}"; - - var windowsDownloaded = await _httpClientHelper.Download(url, zipArchiveFullFilePath); - if ( !windowsDownloaded ) - { - throw new HttpRequestException($"file is not downloaded {matchExifToolForWindowsName}"); - } - - if ( !CheckSha1(zipArchiveFullFilePath, getChecksumsFromTextFile) ) - { - throw new HttpRequestException($"checksum for {zipArchiveFullFilePath} is not valid"); - } - - _hostFileSystemStorage.CreateDirectory(windowsExifToolFolder); - - new Zipper().ExtractZip(zipArchiveFullFilePath, windowsExifToolFolder); - MoveFileIfExist(Path.Combine(windowsExifToolFolder, "exiftool(-k).exe"), - Path.Combine(windowsExifToolFolder, "exiftool.exe")); - - _logger.LogInformation($"[DownloadForWindows] ExifTool downloaded: {ExeExifToolWindowsFullFilePath()}"); - return _hostFileSystemStorage.ExistFile(Path.Combine(windowsExifToolFolder, - "exiftool.exe")); - } - - private void MoveFileIfExist(string srcFullPath, string toFullPath) - { - if ( !_hostFileSystemStorage.ExistFile(srcFullPath) ) return; - _hostFileSystemStorage.FileMove(srcFullPath, toFullPath); - } - } -} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs index a50862ed73..dea77dee14 100644 --- a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloadBackgroundService.cs @@ -6,6 +6,7 @@ using starsky.foundation.injection; using starsky.foundation.platform.Interfaces; using starsky.foundation.platform.Models; +using starsky.foundation.writemeta.Services.ExifToolDownloader; namespace starsky.foundation.writemeta.Services { diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/CheckSha1.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/CheckSha1.cs new file mode 100644 index 0000000000..05590b35cf --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/CheckSha1.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using starsky.foundation.http.Interfaces; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.storage.Interfaces; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class CheckSums(IStorage storage, IHttpClientHelper httpClientHelper, IWebLogger logger) +{ + internal async Task?> DownloadCheckSums(bool isMirror = false) + { + var url = !isMirror + ? ExifToolLocations.CheckSumLocation + : ExifToolLocations.CheckSumLocationMirror; + + var checksums = await httpClientHelper.ReadString(url); + if ( checksums.Key ) + { + return checksums; + } + + return null; + } + + /// + /// Check if SHA1 hash is valid + /// Instead of SHA1CryptoServiceProvider, we use SHA1.Create + /// + /// path of exiftool.exe + /// list of sha1 hashes + /// + internal bool CheckSha1(string fullFilePath, IEnumerable checkSumOptions) + { + using var buffer = storage.ReadStream(fullFilePath); + using var hashAlgorithm = SHA1.Create(); + + var byteHash = hashAlgorithm.ComputeHash(buffer); + var hash = BitConverter.ToString(byteHash).Replace("-", string.Empty) + .ToLowerInvariant(); + return checkSumOptions.AsEnumerable().Any(p => + p.Equals(hash, StringComparison.InvariantCultureIgnoreCase)); + } + + /// + /// Parse the content of checksum file + /// + /// input file: see test for example + /// max number of SHA1 results + /// + internal string[] GetChecksumsFromTextFile(string checksumsValue, int max = 8) + { + var regexExifToolForWindowsName = new Regex("[a-z0-9]{40}", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + var results = regexExifToolForWindowsName.Matches(checksumsValue).Select(m => m.Value) + .ToArray(); + if ( results.Length < max ) return results; + + logger.LogError( + $"More than {max} checksums found, this is not expected, code stops now"); + return []; + } + + + internal static string GetUnixTarGzFromChecksum(string checksumsValue) + { + // (?<=SHA1\()Image-ExifTool-[\d\.]+\.zip + var regexExifToolForWindowsName = new Regex( + @"(?<=SHA1\()Image-ExifTool-[0-9\.]+\.tar.gz", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + return regexExifToolForWindowsName.Match(checksumsValue).Value; + } + + internal static string GetWindowsZipFromChecksum(string checksumsValue) + { + // (?<=SHA1\()exiftool-[\d\.]+\.zip + var regexExifToolForWindowsName = new Regex(@"(?<=SHA1\()exiftool-[0-9\.]+\.zip", + RegexOptions.None, TimeSpan.FromMilliseconds(100)); + return regexExifToolForWindowsName.Match(checksumsValue).Value; + } + +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownload.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownload.cs new file mode 100644 index 0000000000..0e52158ba3 --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownload.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using starsky.foundation.http.Interfaces; +using starsky.foundation.injection; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.storage.Interfaces; +using starsky.foundation.storage.Storage; +using starsky.foundation.writemeta.Helpers; +using starsky.foundation.writemeta.Interfaces; + +[assembly: InternalsVisibleTo("starskytest")] + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +[Service(typeof(IExifToolDownload), InjectionLifetime = InjectionLifetime.Singleton)] +[SuppressMessage("Usage", + "S1075:Refactor your code not to use hardcoded absolute paths or URIs", + Justification = "Source of files")] +[SuppressMessage("Usage", + "S4790:Make sure this weak hash algorithm is not used in a sensitive context here.", + Justification = "Safe")] +public sealed class ExifToolDownload : IExifToolDownload +{ + private readonly IHttpClientHelper _httpClientHelper; + private readonly AppSettings _appSettings; + private readonly IStorage _hostFileSystemStorage; + private readonly IWebLogger _logger; + private readonly ExifToolLocations _exifToolLocations; + private readonly ExifToolDownloadUnix _exifToolDownloadUnix; + + public ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, + IWebLogger logger) + { + _httpClientHelper = httpClientHelper; + _appSettings = appSettings; + _hostFileSystemStorage = new StorageHostFullPathFilesystem(logger); + _logger = logger; + _exifToolLocations = new ExifToolLocations(_appSettings); + _exifToolDownloadUnix = new ExifToolDownloadUnix(_hostFileSystemStorage, appSettings, httpClientHelper, logger); + _exifToolDownloadWindows = new ExifToolDownloadWindows(_hostFileSystemStorage, appSettings, httpClientHelper, logger); + + } + + internal ExifToolDownload(IHttpClientHelper httpClientHelper, AppSettings appSettings, + IWebLogger logger, IStorage storage) + { + _httpClientHelper = httpClientHelper; + _appSettings = appSettings; + _hostFileSystemStorage = storage; + _logger = logger; + _exifToolLocations = new ExifToolLocations(_appSettings); + _exifToolDownloadUnix = new ExifToolDownloadUnix(_hostFileSystemStorage, appSettings, httpClientHelper, logger); + } + + /// + /// Auto Download Exiftool + /// + /// download windows version if true + /// check for min file size in bytes (Default = 30 bytes) + /// + public async Task DownloadExifTool(bool isWindows, int minimumSize = 30) + { + if ( _appSettings.ExiftoolSkipDownloadOnStartup == true || _appSettings is + { AddSwaggerExport: true, AddSwaggerExportExitAfter: true } ) + { + var name = _appSettings.ExiftoolSkipDownloadOnStartup == true + ? "ExiftoolSkipDownloadOnStartup" + : "AddSwaggerExport and AddSwaggerExportExitAfter"; + _logger.LogInformation($"[DownloadExifTool] Skipped due true of {name} setting"); + return false; + } + + new CreateFolderIfNotExists(_logger, _appSettings) + .CreateDirectoryDependenciesTempFolderIfNotExists(); + + if ( isWindows && + ( !_hostFileSystemStorage.ExistFile( + _exifToolLocations.ExeExifToolWindowsFullFilePath()) || + _hostFileSystemStorage.Info(_exifToolLocations.ExeExifToolWindowsFullFilePath()) + .Size <= + minimumSize ) ) + { + return await StartDownloadForWindows(); + } + + if ( !isWindows && + ( !_hostFileSystemStorage.ExistFile(_exifToolLocations + .ExeExifToolUnixFullFilePath()) || + _hostFileSystemStorage.Info(_exifToolLocations.ExeExifToolUnixFullFilePath()).Size <= + minimumSize ) ) + { + return await _exifToolDownloadUnix.StartDownloadForUnix(); + } + + if ( _appSettings.IsVerbose() ) + { + _logger.LogInformation( + $"[DownloadExifTool] {_exifToolLocations.ExeExifToolFullFilePath(isWindows)}"); + } + + // When running deploy scripts rights might reset (only for unix) + if ( isWindows ) return true; + + return await _exifToolDownloadUnix.RunChmodOnExifToolUnixExe(); + } + + + + +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadUnix.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadUnix.cs new file mode 100644 index 0000000000..1e9564cf4b --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadUnix.cs @@ -0,0 +1,141 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Medallion.Shell; +using starsky.foundation.http.Interfaces; +using starsky.foundation.http.Services; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.storage.ArchiveFormats; +using starsky.foundation.storage.Interfaces; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class ExifToolDownloadUnix +{ + private readonly IStorage _hostFileSystemStorage; + private readonly AppSettings _appSettings; + private readonly IHttpClientHelper _httpClientHelper; + private readonly IWebLogger _logger; + private readonly ExifToolLocations _exifToolLocations; + private readonly CheckSums _checkSha1; + + public ExifToolDownloadUnix(IStorage hostFileSystemStorage, AppSettings appSettings, + IHttpClientHelper httpClientHelper, IWebLogger logger) + { + _hostFileSystemStorage = hostFileSystemStorage; + _appSettings = appSettings; + _httpClientHelper = httpClientHelper; + _logger = logger; + _exifToolLocations = new ExifToolLocations(_appSettings); + _checkSha1 = new CheckSums(_hostFileSystemStorage, httpClientHelper, logger); + } + + internal async Task StartDownloadForUnix() + { + var checksums = await DownloadCheckSums(); + if ( checksums == null ) return false; + var matchExifToolForUnixName = GetUnixTarGzFromChecksum(checksums.Value.Value); + return await DownloadForUnix(matchExifToolForUnixName, + GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); + } + + + internal async Task DownloadForUnix(string matchExifToolForUnixName, + IEnumerable getChecksumsFromTextFile, bool downloadFromMirror = false) + { + if ( _hostFileSystemStorage.ExistFile(_exifToolLocations.ExeExifToolUnixFullFilePath()) ) + { + return true; + } + + var tarGzArchiveFullFilePath = + Path.Combine(_appSettings.TempFolder, "exiftool.tar.gz"); + + var url = $"{ExifToolLocations.ExiftoolDownloadBasePath}{matchExifToolForUnixName}"; + if ( downloadFromMirror ) + url = $"{ExifToolLocations.ExiftoolDownloadBasePathMirror}{matchExifToolForUnixName}"; + + var unixDownloaded = await _httpClientHelper.Download(url, tarGzArchiveFullFilePath); + if ( !unixDownloaded ) + { + throw new HttpRequestException( + $"file is not downloaded {matchExifToolForUnixName}"); + } + + if ( !_checkSha1.CheckSha1(tarGzArchiveFullFilePath, getChecksumsFromTextFile) ) + { + throw new HttpRequestException( + $"checksum for {tarGzArchiveFullFilePath} is not valid"); + } + + await new TarBal(_hostFileSystemStorage).ExtractTarGz( + _hostFileSystemStorage.ReadStream(tarGzArchiveFullFilePath), + _appSettings.TempFolder, CancellationToken.None); + + var imageExifToolVersionFolder = _hostFileSystemStorage + .GetDirectories(_appSettings.TempFolder) + .FirstOrDefault(p => + p.StartsWith(Path.Combine(_appSettings.TempFolder, "Image-ExifTool-"))); + if ( imageExifToolVersionFolder != null ) + { + var exifToolUnixFolderFullFilePathTempFolder = + Path.Combine(_appSettings.TempFolder, "exiftool-unix"); + + if ( _hostFileSystemStorage.ExistFolder(exifToolUnixFolderFullFilePathTempFolder) ) + { + _hostFileSystemStorage.FolderDelete( + exifToolUnixFolderFullFilePathTempFolder); + } + + _hostFileSystemStorage.FolderMove(imageExifToolVersionFolder, + exifToolUnixFolderFullFilePathTempFolder); + + var exifToolUnixFolderFullFilePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix"); + + _hostFileSystemStorage.FileCopy(imageExifToolVersionFolder, + exifToolUnixFolderFullFilePathTempFolder); + } + else + { + _logger.LogError($"[DownloadForUnix] ExifTool folder does not exists"); + return false; + } + + // remove tar.gz file afterwards + _hostFileSystemStorage.FileDelete(tarGzArchiveFullFilePath); + + var exifToolExePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-unix", "exiftool"); + _logger.LogInformation( + $"[DownloadForUnix] ExifTool is just downloaded: {exifToolExePath} for {_appSettings.ApplicationType}"); + return await RunChmodOnExifToolUnixExe(); + } + + internal async Task RunChmodOnExifToolUnixExe() + { + // need to check again + // when not exist + if ( !_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()) ) return false; + if ( _appSettings.IsWindows ) return true; + + if ( !_hostFileSystemStorage.ExistFile("/bin/chmod") ) + { + _logger.LogError("[RunChmodOnExifToolUnixExe] WARNING: /bin/chmod does not exist"); + return true; + } + + // command.run does not care about the $PATH + var result = await Command.Run("/bin/chmod", "0755", ExeExifToolUnixFullFilePath()) + .Task; + if ( result.Success ) return true; + + _logger.LogError( + $"command failed with exit code {result.ExitCode}: {result.StandardError}"); + return false; + } +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadWindows.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadWindows.cs new file mode 100644 index 0000000000..87747d7fac --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolDownloadWindows.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using starsky.foundation.http.Interfaces; +using starsky.foundation.platform.Interfaces; +using starsky.foundation.platform.Models; +using starsky.foundation.storage.ArchiveFormats; +using starsky.foundation.storage.Interfaces; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class ExifToolDownloadWindows(IStorage hostFileSystemStorage, AppSettings appSettings, + IHttpClientHelper httpClientHelper, IWebLogger logger) +{ + internal async Task StartDownloadForWindows() + { + var checksums = await DownloadCheckSums(); + if ( checksums == null ) return false; + + var matchExifToolForWindowsName = GetWindowsZipFromChecksum(checksums.Value.Value); + return await DownloadForWindows(matchExifToolForWindowsName, + GetChecksumsFromTextFile(checksums.Value.Value), !checksums.Value.Key); + } + + + private void MoveFileIfExist(string srcFullPath, string toFullPath) + { + if ( !_hostFileSystemStorage.ExistFile(srcFullPath) ) return; + _hostFileSystemStorage.FileMove(srcFullPath, toFullPath); + } + + internal async Task DownloadForWindows(string matchExifToolForWindowsName, + IEnumerable getChecksumsFromTextFile, bool downloadFromMirror = false) + { + if ( _hostFileSystemStorage.ExistFile( + ExeExifToolWindowsFullFilePath()) ) return true; + + var zipArchiveFullFilePath = + Path.Combine(_appSettings.DependenciesFolder, "exiftool.zip"); + var windowsExifToolFolder = + Path.Combine(_appSettings.DependenciesFolder, "exiftool-windows"); + + var url = $"{ExiftoolDownloadBasePath}{matchExifToolForWindowsName}"; + if ( downloadFromMirror ) + url = $"{ExiftoolDownloadBasePathMirror}{matchExifToolForWindowsName}"; + + var windowsDownloaded = await _httpClientHelper.Download(url, zipArchiveFullFilePath); + if ( !windowsDownloaded ) + { + throw new HttpRequestException( + $"file is not downloaded {matchExifToolForWindowsName}"); + } + + if ( !CheckSha1(zipArchiveFullFilePath, getChecksumsFromTextFile) ) + { + throw new HttpRequestException( + $"checksum for {zipArchiveFullFilePath} is not valid"); + } + + _hostFileSystemStorage.CreateDirectory(windowsExifToolFolder); + + new Zipper().ExtractZip(zipArchiveFullFilePath, windowsExifToolFolder); + MoveFileIfExist(Path.Combine(windowsExifToolFolder, "exiftool(-k).exe"), + Path.Combine(windowsExifToolFolder, "exiftool.exe")); + + _logger.LogInformation( + $"[DownloadForWindows] ExifTool downloaded: {ExeExifToolWindowsFullFilePath()}"); + return _hostFileSystemStorage.ExistFile(Path.Combine(windowsExifToolFolder, + "exiftool.exe")); + } +} diff --git a/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolLocations.cs b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolLocations.cs new file mode 100644 index 0000000000..1c5b6b2c30 --- /dev/null +++ b/starsky/starsky.foundation.writemeta/Services/ExifToolDownloader/ExifToolLocations.cs @@ -0,0 +1,39 @@ +using System.IO; +using starsky.foundation.platform.Models; + +namespace starsky.foundation.writemeta.Services.ExifToolDownloader; + +public class ExifToolLocations(AppSettings appSettings) +{ + public string ExeExifToolFullFilePath(bool isWindows) + { + return isWindows ? ExeExifToolWindowsFullFilePath() : ExeExifToolUnixFullFilePath(); + } + + internal string ExeExifToolWindowsFullFilePath() + { + return Path.Combine(Path.Combine(appSettings.DependenciesFolder, "exiftool-windows"), + "exiftool.exe"); + } + + internal string ExeExifToolUnixFullFilePath() + { + var path = Path.Combine(appSettings.DependenciesFolder, + "exiftool-unix", + "exiftool"); + return path; + } + + internal const string Https = "https://"; + + internal const string CheckSumLocation = "exiftool.org/checksums.txt"; + + internal const string CheckSumLocationMirror = + "qdraw.nl/special/mirror/exiftool/checksums.txt"; + + internal const string + ExiftoolDownloadBasePath = "exiftool.org/"; // with slash at the end + + internal const string ExiftoolDownloadBasePathMirror = + "qdraw.nl/special/mirror/exiftool/"; // with slash at the end +} diff --git a/starsky/starsky.sln b/starsky/starsky.sln index 729c69552f..db74985e2e 100644 --- a/starsky/starsky.sln +++ b/starsky/starsky.sln @@ -166,6 +166,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "starsky.feature.settings", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "starsky.feature.desktop", "starsky.feature.desktop\starsky.feature.desktop.csproj", "{B88C2815-D154-4C6D-AE37-2E150AEBF73D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "starskydependenciescli", "starskydependenciescli\starskydependenciescli.csproj", "{CDFE2199-CB93-49E4-AF8A-98A3239F1D3F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "starsky.feature.externaldependencies", "starsky.feature.externaldependencies\starsky.feature.externaldependencies.csproj", "{C34BB48B-0C65-4897-80A9-87CB1BDD5F48}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -368,6 +372,14 @@ Global {B88C2815-D154-4C6D-AE37-2E150AEBF73D}.Debug|Any CPU.Build.0 = Debug|Any CPU {B88C2815-D154-4C6D-AE37-2E150AEBF73D}.Release|Any CPU.ActiveCfg = Release|Any CPU {B88C2815-D154-4C6D-AE37-2E150AEBF73D}.Release|Any CPU.Build.0 = Release|Any CPU + {CDFE2199-CB93-49E4-AF8A-98A3239F1D3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDFE2199-CB93-49E4-AF8A-98A3239F1D3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDFE2199-CB93-49E4-AF8A-98A3239F1D3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDFE2199-CB93-49E4-AF8A-98A3239F1D3F}.Release|Any CPU.Build.0 = Release|Any CPU + {C34BB48B-0C65-4897-80A9-87CB1BDD5F48}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C34BB48B-0C65-4897-80A9-87CB1BDD5F48}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C34BB48B-0C65-4897-80A9-87CB1BDD5F48}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C34BB48B-0C65-4897-80A9-87CB1BDD5F48}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -428,5 +440,7 @@ Global {A62C129C-5D0C-4A0A-B5AA-261E041FF55D} = {4B9276C3-651E-48D3-B3A7-3F4D74F3D01A} {F2C4C9DE-22A1-4B34-AC1D-0F08353E0742} = {4B9276C3-651E-48D3-B3A7-3F4D74F3D01A} {B88C2815-D154-4C6D-AE37-2E150AEBF73D} = {4B9276C3-651E-48D3-B3A7-3F4D74F3D01A} + {CDFE2199-CB93-49E4-AF8A-98A3239F1D3F} = {B974FC20-C3EE-4EB0-AF25-F0D8DA2C28D7} + {C34BB48B-0C65-4897-80A9-87CB1BDD5F48} = {4B9276C3-651E-48D3-B3A7-3F4D74F3D01A} EndGlobalSection EndGlobal diff --git a/starsky/starskydependenciescli/Program.cs b/starsky/starskydependenciescli/Program.cs new file mode 100644 index 0000000000..c8ef0dc340 --- /dev/null +++ b/starsky/starskydependenciescli/Program.cs @@ -0,0 +1,49 @@ +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using starsky.feature.externaldependencies; +using starsky.feature.externaldependencies.Interfaces; +using starsky.foundation.database.Helpers; +using starsky.foundation.http.Interfaces; +using starsky.foundation.http.Services; +using starsky.foundation.injection; +using starsky.foundation.platform.Helpers; +using starsky.foundation.platform.Models; +using starsky.foundation.webtelemetry.Extensions; +using starsky.foundation.webtelemetry.Helpers; + +namespace starskyDependenciesCli +{ + public static class Program + { + public static async Task Main(string[] args) + { + // Use args in application + new ArgsHelper().SetEnvironmentByArgs(args); + + var services = new ServiceCollection(); + + // Setup AppSettings + services = await SetupAppSettings.FirstStepToAddSingleton(services); + + // Inject services + RegisterDependencies.Configure(services); + var serviceProvider = services.BuildServiceProvider(); + var appSettings = serviceProvider.GetRequiredService(); + + services.AddOpenTelemetryMonitoring(appSettings); + services.AddTelemetryLogging(appSettings); + + new SetupDatabaseTypes(appSettings, services).BuilderDb(); + + services.AddScoped(); + services.AddScoped(); + + serviceProvider = services.BuildServiceProvider(); + + var dependenciesService = + serviceProvider.GetRequiredService(); + + await dependenciesService.SetupAsync(args); + } + } +} diff --git a/starsky/starskydependenciescli/Properties/default-init-launchSettings.json b/starsky/starskydependenciescli/Properties/default-init-launchSettings.json new file mode 100644 index 0000000000..bec23311cc --- /dev/null +++ b/starsky/starskydependenciescli/Properties/default-init-launchSettings.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "starskydependenciescli": { + "commandName": "Project", + "commandLineArgs": "-h", + "environmentVariables": { + } + } + } +} diff --git a/starsky/starskydependenciescli/starskydependenciescli.csproj b/starsky/starskydependenciescli/starskydependenciescli.csproj new file mode 100644 index 0000000000..d8d90559e0 --- /dev/null +++ b/starsky/starskydependenciescli/starskydependenciescli.csproj @@ -0,0 +1,32 @@ + + + + Exe + net8.0 + + {04d4b161-7dad-4258-8edc-f0a77e39dc3f} + 8.0.3 + starskydependenciescli + 0.6.0 + enable + disable + + + + true + + + + + + + + + + + + + + + + diff --git a/starsky/starskytest/FakeMocks/FakeExifToolDownload.cs b/starsky/starskytest/FakeMocks/FakeExifToolDownload.cs index ea5360ec1f..df52df19a1 100644 --- a/starsky/starskytest/FakeMocks/FakeExifToolDownload.cs +++ b/starsky/starskytest/FakeMocks/FakeExifToolDownload.cs @@ -7,9 +7,10 @@ namespace starskytest.FakeMocks public class FakeExifToolDownload : IExifToolDownload { public List Called { get; set; } = new List(); + public Task DownloadExifTool(bool isWindows, int minimumSize = 30) { - Called.Add(true); + Called.Add(isWindows); return Task.FromResult(true); } } diff --git a/starsky/starskytest/FakeMocks/FakeIStorage.cs b/starsky/starskytest/FakeMocks/FakeIStorage.cs index 398d5f77ad..55bec3f6f5 100644 --- a/starsky/starskytest/FakeMocks/FakeIStorage.cs +++ b/starsky/starskytest/FakeMocks/FakeIStorage.cs @@ -130,6 +130,11 @@ public void FolderMove(string fromPath, string toPath) _outputSubPathFolders[indexOfFolders] = toPath; } + public void FolderCopy(string fromPath, string toPath) + { + throw new NotImplementedException(); + } + public bool FileMove(string fromPath, string toPath) { var existOldFile = ExistFile(fromPath); diff --git a/starsky/starskytest/starsky.feature.externaldependencies/ExternalDependenciesServiceTest.cs b/starsky/starskytest/starsky.feature.externaldependencies/ExternalDependenciesServiceTest.cs new file mode 100644 index 0000000000..b24955f6a8 --- /dev/null +++ b/starsky/starskytest/starsky.feature.externaldependencies/ExternalDependenciesServiceTest.cs @@ -0,0 +1,89 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using starsky.feature.externaldependencies; +using starsky.foundation.database.Data; +using starsky.foundation.platform.Models; +using starskytest.FakeMocks; + +namespace starskytest.starsky.feature.externaldependencies; + +[TestClass] +public class ExternalDependenciesServiceTest +{ + private readonly ApplicationDbContext _dbContext; + + private static IServiceScopeFactory CreateNewScope() + { + var services = new ServiceCollection(); + services.AddDbContext(options => + options.UseInMemoryDatabase(nameof(ExternalDependenciesServiceTest))); + var serviceProvider = services.BuildServiceProvider(); + return serviceProvider.GetRequiredService(); + } + + public ExternalDependenciesServiceTest() + { + var serviceScope = CreateNewScope(); + var scope = serviceScope.CreateScope(); + _dbContext = scope.ServiceProvider.GetRequiredService(); + } + + [TestMethod] + public async Task ExternalDependenciesServiceTest_ExifTool_Default() + { + var fakeExifToolDownload = new FakeExifToolDownload(); + var appSettings = new AppSettings + { + DatabaseType = AppSettings.DatabaseTypeList.InMemoryDatabase + }; + var externalDependenciesService = new ExternalDependenciesService(fakeExifToolDownload, + _dbContext, new FakeIWebLogger(), appSettings, new FakeIGeoFileDownload()); + await externalDependenciesService.SetupAsync(new List().ToArray()); + + Assert.AreEqual(new AppSettings().IsWindows, fakeExifToolDownload.Called[0]); + } + + [DataTestMethod] + [DataRow("osx-arm64", false)] + [DataRow("linux-x64", false)] + [DataRow("win-x64", true)] + public async Task ExternalDependenciesServiceTest_ExifTool_Single_DataTestMethod(string input, + bool expected) + { + var fakeExifToolDownload = new FakeExifToolDownload(); + var appSettings = new AppSettings + { + DatabaseType = AppSettings.DatabaseTypeList.InMemoryDatabase + }; + var externalDependenciesService = new ExternalDependenciesService(fakeExifToolDownload, + _dbContext, new FakeIWebLogger(), appSettings, new FakeIGeoFileDownload()); + + await externalDependenciesService.SetupAsync(["--runtime", input]); + + Assert.AreEqual(expected, fakeExifToolDownload.Called[0]); + } + + [DataTestMethod] + [DataRow("osx-arm64,osx-x64", false, false)] + [DataRow("linux-x64,osx-x64", false, false)] + [DataRow("win-x64,osx-x64", true, false)] + public async Task ExternalDependenciesServiceTest_ExifTool_Multiple_DataTestMethod(string input, + bool expected1, bool expected2) + { + var appSettings = new AppSettings + { + DatabaseType = AppSettings.DatabaseTypeList.InMemoryDatabase + }; + var fakeExifToolDownload = new FakeExifToolDownload(); + var externalDependenciesService = new ExternalDependenciesService(fakeExifToolDownload, + _dbContext, new FakeIWebLogger(), appSettings, new FakeIGeoFileDownload()); + + await externalDependenciesService.SetupAsync(["--runtime", input]); + + Assert.AreEqual(expected1, fakeExifToolDownload.Called[0]); + Assert.AreEqual(expected2, fakeExifToolDownload.Called[1]); + } +} diff --git a/starsky/starskytest/starsky.feature.import/Services/ImportCliTest.cs b/starsky/starskytest/starsky.feature.import/Services/ImportCliTest.cs index d4bd569c1a..ae3c2e53a6 100644 --- a/starsky/starskytest/starsky.feature.import/Services/ImportCliTest.cs +++ b/starsky/starskytest/starsky.feature.import/Services/ImportCliTest.cs @@ -22,7 +22,7 @@ public async Task ImporterCli_CheckIfExifToolIsCalled() new FakeIImport(new FakeSelectorStorage()), new AppSettings { TempFolder = "/___not___found_" }, fakeConsole, new FakeIWebLogger(), fakeExifToolDownload) - .Importer(new List().ToArray()); + .Importer([]); Assert.IsTrue(fakeExifToolDownload.Called.Count != 0); } diff --git a/starsky/starskytest/starsky.feature.packagetelemetry/Helpers/PackageTelemetryTest.cs b/starsky/starskytest/starsky.feature.packagetelemetry/Helpers/PackageTelemetryTest.cs index 53525e69e0..8123add981 100644 --- a/starsky/starskytest/starsky.feature.packagetelemetry/Helpers/PackageTelemetryTest.cs +++ b/starsky/starskytest/starsky.feature.packagetelemetry/Helpers/PackageTelemetryTest.cs @@ -17,19 +17,6 @@ namespace starskytest.starsky.feature.packagetelemetry.Helpers [TestClass] public sealed class PackageTelemetryTest { - [TestMethod] - public void GetCurrentOsPlatformTest() - { - var content = PackageTelemetry.GetCurrentOsPlatform(); - Assert.IsNotNull(content); - - var allOsPlatforms = new List - { - OSPlatform.Linux, OSPlatform.Windows, OSPlatform.OSX, OSPlatform.FreeBSD - }; - Assert.IsTrue(allOsPlatforms.Contains(content.Value)); - } - [TestMethod] public void GetSystemDataTest() { @@ -55,6 +42,10 @@ public void GetSystemDataTest() Assert.IsTrue(systemData.Exists(p => p.Key == "DockerContainer")); Assert.IsTrue(systemData.Exists(p => p.Key == "CurrentCulture")); Assert.IsTrue(systemData.Exists(p => p.Key == "AspNetCoreEnvironment")); + Assert.IsTrue(systemData.Exists(p => p.Key == "RuntimeIdentifier")); + Assert.AreEqual(systemData.Find(p => p.Key == "RuntimeIdentifier").Value, + RuntimeInformation.RuntimeIdentifier); + Assert.IsTrue(systemData.Exists(p => p.Key == "DeviceId")); } [TestMethod] diff --git a/starsky/starskytest/starsky.foundation.database/DataProtection/SqlXmlRepositoryTest.cs b/starsky/starskytest/starsky.foundation.database/DataProtection/SqlXmlRepositoryTest.cs index 9ea09b7bde..248a15872d 100644 --- a/starsky/starskytest/starsky.foundation.database/DataProtection/SqlXmlRepositoryTest.cs +++ b/starsky/starskytest/starsky.foundation.database/DataProtection/SqlXmlRepositoryTest.cs @@ -22,7 +22,7 @@ public class SqlXmlRepositoryTest private readonly SqlXmlRepository _repository; private readonly ApplicationDbContext _dbContext; - private IServiceScopeFactory CreateNewScope() + private static IServiceScopeFactory CreateNewScope() { var services = new ServiceCollection(); services.AddDbContext(options => options.UseInMemoryDatabase(nameof(SqlXmlRepositoryTest))); diff --git a/starsky/starskytest/starsky.foundation.platform/Helpers/ArgsHelperTest.cs b/starsky/starskytest/starsky.foundation.platform/Helpers/ArgsHelperTest.cs index f65ccaddb7..ffee2b88a5 100644 --- a/starsky/starskytest/starsky.foundation.platform/Helpers/ArgsHelperTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Helpers/ArgsHelperTest.cs @@ -629,5 +629,29 @@ public void Name() var args = new List { "-n", "test" }.ToArray(); Assert.AreEqual("test", ArgsHelper.GetName(args)); } + + [DataTestMethod] + [DataRow("osx-arm64", "OSX")] + [DataRow("osx-x64", "OSX")] + [DataRow("linux-x64", "LINUX")] + [DataRow("linux-arm", "LINUX")] + [DataRow("linux-arm64", "LINUX")] + [DataRow("win-x64", "WINDOWS")] + [DataRow("win-arm64", "WINDOWS")] + public void GetRuntimeTest(string input, string expected) + { + var args = new List { "--runtime", input }.ToArray(); + Assert.AreEqual(expected, ArgsHelper.GetRuntime(args)[0].Item1.ToString()); + } + + [DataTestMethod] + [DataRow("test")] + [DataRow(null)] + [DataRow("win-x86")] + public void GetRuntimeTestNoContent(string input) + { + var args = new List { "--runtime", input }.ToArray(); + Assert.AreEqual(0, ArgsHelper.GetRuntime(args).Count); + } } } diff --git a/starsky/starskytest/starsky.foundation.platform/Helpers/PlatformParserTest.cs b/starsky/starskytest/starsky.foundation.platform/Helpers/PlatformParserTest.cs new file mode 100644 index 0000000000..b0696f06f2 --- /dev/null +++ b/starsky/starskytest/starsky.foundation.platform/Helpers/PlatformParserTest.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; +using System.Runtime.InteropServices; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using starsky.foundation.platform.Helpers; + +namespace starskytest.starsky.foundation.platform.Helpers; + +[TestClass] +public class PlatformParserTest +{ + [TestMethod] + public void GetCurrentOsPlatformTest() + { + var content = PlatformParser.GetCurrentOsPlatform(); + Assert.IsNotNull(content); + + var allOsPlatforms = new List + { + OSPlatform.Linux, OSPlatform.Windows, OSPlatform.OSX, OSPlatform.FreeBSD + }; + Assert.IsTrue(allOsPlatforms.Contains(content.Value)); + } + + [DataTestMethod] + [DataRow("osx-arm64", "OSX", "Arm64")] + [DataRow("osx-x64", "OSX", "X64")] + [DataRow("linux-x64", "LINUX", "X64")] + [DataRow("linux-arm", "LINUX", "Arm")] + [DataRow("linux-arm64", "LINUX", "Arm64")] + [DataRow("win-x64", "WINDOWS", "X64")] + [DataRow("win-arm64", "WINDOWS", "Arm64")] + public void RuntimeIdentifierTest(string input, string expectedOs, string expectedArch) + { + var result = PlatformParser.RuntimeIdentifier(input); + Assert.AreEqual(expectedOs, result[0].Item1.ToString()); + Assert.AreEqual(expectedArch, result[0].Item2.ToString()); + } + + [DataTestMethod] + [DataRow("test")] + [DataRow(null)] + [DataRow("win-x86")] + public void RuntimeIdentifierTestNoContent(string input) + { + var result = PlatformParser.RuntimeIdentifier(input); + Assert.AreEqual(0, result.Count); + } +} diff --git a/starsky/starskytest/starsky.foundation.storage/Storage/StorageHostFullPathFilesystemTest.cs b/starsky/starskytest/starsky.foundation.storage/Storage/StorageHostFullPathFilesystemTest.cs index d5f7c31867..c4164519ac 100644 --- a/starsky/starskytest/starsky.foundation.storage/Storage/StorageHostFullPathFilesystemTest.cs +++ b/starsky/starskytest/starsky.foundation.storage/Storage/StorageHostFullPathFilesystemTest.cs @@ -273,6 +273,68 @@ public void IsFileReady_HostService() Assert.IsFalse(hostStorage.ExistFile(filePathNewFile)); } + + [TestMethod] + public void MoveFolder() + { + var hostStorage = new StorageHostFullPathFilesystem(new FakeIWebLogger()); + var beforeDirectoryFullPath = Path.Combine(new CreateAnImage().BasePath, "28934283492349_before"); + var afterDirectoryFullPath = Path.Combine(new CreateAnImage().BasePath, "28934283492349_after"); + hostStorage.FolderDelete(afterDirectoryFullPath); + + hostStorage.CreateDirectory(beforeDirectoryFullPath); + hostStorage.FolderMove(beforeDirectoryFullPath, afterDirectoryFullPath); + + Assert.IsFalse(hostStorage.ExistFolder(beforeDirectoryFullPath)); + Assert.IsTrue(hostStorage.ExistFolder(afterDirectoryFullPath)); + + hostStorage.FolderDelete(beforeDirectoryFullPath); + hostStorage.FolderDelete(afterDirectoryFullPath); + } + + [TestMethod] + public void CopyFolder_FlatFolder() + { + var hostStorage = new StorageHostFullPathFilesystem(new FakeIWebLogger()); + var beforeDirectoryFullPath = Path.Combine(new CreateAnImage().BasePath, "453984583495_before"); + var afterDirectoryFullPath = Path.Combine(new CreateAnImage().BasePath, "453984583495_after"); + hostStorage.FolderDelete(afterDirectoryFullPath); + + hostStorage.CreateDirectory(beforeDirectoryFullPath); + hostStorage.FolderCopy(beforeDirectoryFullPath, afterDirectoryFullPath); + + Assert.IsTrue(hostStorage.ExistFolder(beforeDirectoryFullPath)); + Assert.IsTrue(hostStorage.ExistFolder(afterDirectoryFullPath)); + + hostStorage.FolderDelete(beforeDirectoryFullPath); + hostStorage.FolderDelete(afterDirectoryFullPath); + } + + [TestMethod] + public async Task CopyFolder_ChildItems() + { + var hostStorage = new StorageHostFullPathFilesystem(new FakeIWebLogger()); + var beforeDirectoryFullPath = Path.Combine(new CreateAnImage().BasePath, "9057678234_before"); + var childFullPath = Path.Combine(beforeDirectoryFullPath, "child"); + var childFullPathFile = Path.Combine(childFullPath, "child.test"); + + var afterDirectoryFullPath = Path.Combine(new CreateAnImage().BasePath, "9057678234_after"); + hostStorage.FolderDelete(afterDirectoryFullPath); + + hostStorage.CreateDirectory(beforeDirectoryFullPath); + hostStorage.CreateDirectory(childFullPath); + await hostStorage.WriteStreamAsync(new MemoryStream(new byte[1]), childFullPathFile); + + hostStorage.FolderCopy(beforeDirectoryFullPath, afterDirectoryFullPath); + + Assert.IsTrue(hostStorage.ExistFolder(beforeDirectoryFullPath)); + Assert.IsTrue(hostStorage.ExistFolder(afterDirectoryFullPath)); + Assert.IsTrue(hostStorage.ExistFile(childFullPathFile.Replace("_before", "_after"))); + Assert.IsTrue(hostStorage.ExistFile(childFullPathFile)); + + hostStorage.FolderDelete(beforeDirectoryFullPath); + hostStorage.FolderDelete(afterDirectoryFullPath); + } [TestMethod] public void GetDirectoryRecursive_NotFound() diff --git a/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs b/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs index f741ccfd64..7b946c502a 100644 --- a/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs +++ b/starsky/starskytest/starsky.foundation.writemeta/Helpers/ExifToolDownloadTest.cs @@ -14,6 +14,7 @@ using starsky.foundation.storage.Interfaces; using starsky.foundation.storage.Storage; using starsky.foundation.writemeta.Services; +using starsky.foundation.writemeta.Services.ExifToolDownloader; using starskytest.FakeCreateAn; using starskytest.FakeMocks; diff --git a/starsky/starskytest/starskytest.csproj b/starsky/starskytest/starskytest.csproj index 0a684b8cdd..dc0e6727a6 100644 --- a/starsky/starskytest/starskytest.csproj +++ b/starsky/starskytest/starskytest.csproj @@ -44,6 +44,7 @@ +