diff --git a/history.md b/history.md index bf0ff67731..9b3fe042f4 100644 --- a/history.md +++ b/history.md @@ -47,6 +47,8 @@ Semantic Versioning 2.0.0 is from version 0.1.6+ - [x] (Fixed) _Docs_ Demo site is not working (PR #1486) - [x] (Fixed) _Back-end_ GetFileNameRegex refactor to avoid timeouts (PR #1515) - [x] (Changed) Back-end Upgrade to .NET 8 - SDK 8.0.204 (Runtime: 8.0.4) (PR #1541) +- [x] (Fixed) _Back-end_ Unhandled exception DbUpdateException (PR #1558 Issue #1489) +- [x] (Fixed) _Back-end_ Regex timeout IsExtensionForce (PR #1542 Issue #1537) ## version 0.6.0 - 2024-03-15 {#v0.6.0} diff --git a/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs b/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs index 7d130f5381..350ca29c0e 100644 --- a/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs +++ b/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using MySqlConnector; using starsky.foundation.database.Data; using starsky.foundation.database.Interfaces; using starsky.foundation.database.Models; @@ -22,23 +23,28 @@ public class ThumbnailQuery : IThumbnailQuery private readonly IServiceScopeFactory? _scopeFactory; private readonly IWebLogger _logger; - public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeFactory, IWebLogger logger) + public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeFactory, + IWebLogger logger) { _context = context; _scopeFactory = scopeFactory; _logger = logger; } - public Task?> AddThumbnailRangeAsync(List thumbnailItems) + public Task?> AddThumbnailRangeAsync( + List thumbnailItems) { if ( thumbnailItems.Exists(p => string.IsNullOrEmpty(p.FileHash)) ) { - throw new ArgumentNullException(nameof(thumbnailItems), "[AddThumbnailRangeAsync] FileHash is null or empty"); + throw new ArgumentNullException(nameof(thumbnailItems), + "[AddThumbnailRangeAsync] FileHash is null or empty"); } + return AddThumbnailRangeInternalRetryDisposedAsync(thumbnailItems); } - private async Task?> AddThumbnailRangeInternalRetryDisposedAsync(List thumbnailItems) + private async Task?> AddThumbnailRangeInternalRetryDisposedAsync( + List thumbnailItems) { try { @@ -48,11 +54,12 @@ public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeF catch ( InvalidOperationException ) { if ( _scopeFactory == null ) throw; - return await AddThumbnailRangeInternalAsync(new InjectServiceScope(_scopeFactory).Context(), thumbnailItems); + return await AddThumbnailRangeInternalAsync( + new InjectServiceScope(_scopeFactory).Context(), thumbnailItems); } } - private static async Task?> AddThumbnailRangeInternalAsync( + private async Task?> AddThumbnailRangeInternalAsync( ApplicationDbContext dbContext, IReadOnlyCollection thumbnailItems) { @@ -63,9 +70,10 @@ public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeF var updateThumbnailNewItemsList = new List(); foreach ( var item in thumbnailItems - .Where(p => p.FileHash != null).DistinctBy(p => p.FileHash) ) + .Where(p => p.FileHash != null).DistinctBy(p => p.FileHash) ) { - updateThumbnailNewItemsList.Add(new ThumbnailItem(item.FileHash!, item.TinyMeta, item.Small, item.Large, item.ExtraLarge, item.Reasons)); + updateThumbnailNewItemsList.Add(new ThumbnailItem(item.FileHash!, item.TinyMeta, + item.Small, item.Large, item.ExtraLarge, item.Reasons)); } var (newThumbnailItems, @@ -100,7 +108,7 @@ public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeF return allResults; } - private static async Task SaveChangesDuplicate(DbContext dbContext) + private async Task SaveChangesDuplicate(DbContext dbContext) { try { @@ -110,11 +118,15 @@ private static async Task SaveChangesDuplicate(DbContext dbContext) { // Skip if Duplicate entry // MySqlConnector.MySqlException (0x80004005): Duplicate entry for key 'PRIMARY' - // https://github.com/qdraw/starsky/issues/1248 - if ( !exception.Message.Contains("Duplicate") ) + // https://github.com/qdraw/starsky/issues/1248 https://github.com/qdraw/starsky/issues/1489 + if ( exception is MySqlException { ErrorCode: MySqlErrorCode.DuplicateKey } ) { - throw; + return; } + + _logger.LogError($"[SaveChangesDuplicate] T:{exception.GetType()} M:{exception.Message} I: {exception.InnerException}"); + + throw; } } @@ -128,7 +140,8 @@ public async Task> Get(string? fileHash = null) catch ( InvalidOperationException ) { if ( _scopeFactory == null ) throw; - return await GetInternalAsync(new InjectServiceScope(_scopeFactory).Context(), fileHash); + return await GetInternalAsync(new InjectServiceScope(_scopeFactory).Context(), + fileHash); } } @@ -136,10 +149,12 @@ private static async Task> GetInternalAsync( ApplicationDbContext context, string? fileHash = null) { - return fileHash == null ? await context - .Thumbnails.ToListAsync() : await context - .Thumbnails.Where(p => p.FileHash == fileHash) - .ToListAsync(); + return fileHash == null + ? await context + .Thumbnails.ToListAsync() + : await context + .Thumbnails.Where(p => p.FileHash == fileHash) + .ToListAsync(); } public async Task RemoveThumbnailsAsync(List deletedFileHashes) @@ -157,7 +172,8 @@ public async Task RemoveThumbnailsAsync(List deletedFileHashes) catch ( InvalidOperationException ) { if ( _scopeFactory == null ) throw; - await RemoveThumbnailsInternalAsync(new InjectServiceScope(_scopeFactory).Context(), deletedFileHashes); + await RemoveThumbnailsInternalAsync(new InjectServiceScope(_scopeFactory).Context(), + deletedFileHashes); } } @@ -172,10 +188,12 @@ internal static async Task RemoveThumbnailsInternalAsync( foreach ( var fileNamesInChunk in deletedFileHashes.ChunkyEnumerable(100) ) { - var thumbnailItems = await context.Thumbnails.Where(p => fileNamesInChunk.Contains(p.FileHash)).ToListAsync(); + var thumbnailItems = await context.Thumbnails + .Where(p => fileNamesInChunk.Contains(p.FileHash)).ToListAsync(); context.Thumbnails.RemoveRange(thumbnailItems); await context.SaveChangesAsync(); } + return true; } @@ -189,11 +207,13 @@ public async Task RenameAsync(string beforeFileHash, string newFileHash) catch ( InvalidOperationException ) { if ( _scopeFactory == null ) throw; - return await RenameInternalAsync(new InjectServiceScope(_scopeFactory).Context(), beforeFileHash, newFileHash); + return await RenameInternalAsync(new InjectServiceScope(_scopeFactory).Context(), + beforeFileHash, newFileHash); } catch ( DbUpdateConcurrencyException concurrencyException ) { - _logger.LogInformation("[ThumbnailQuery] try to fix DbUpdateConcurrencyException", concurrencyException); + _logger.LogInformation("[ThumbnailQuery] try to fix DbUpdateConcurrencyException", + concurrencyException); SolveConcurrency.SolveConcurrencyExceptionLoop(concurrencyException.Entries); try { @@ -201,14 +221,17 @@ public async Task RenameAsync(string beforeFileHash, string newFileHash) } catch ( DbUpdateConcurrencyException e ) { - _logger.LogInformation(e, "[ThumbnailQuery] save failed after DbUpdateConcurrencyException"); + _logger.LogInformation(e, + "[ThumbnailQuery] save failed after DbUpdateConcurrencyException"); return false; } + return true; } } - private static async Task RenameInternalAsync(ApplicationDbContext dbContext, string beforeFileHash, string newFileHash) + private static async Task RenameInternalAsync(ApplicationDbContext dbContext, + string beforeFileHash, string newFileHash) { var beforeOrNewItems = await dbContext.Thumbnails.Where(p => p.FileHash == beforeFileHash || p.FileHash == newFileHash).ToListAsync(); @@ -226,7 +249,8 @@ private static async Task RenameInternalAsync(ApplicationDbContext dbConte } await dbContext.Thumbnails.AddRangeAsync(new ThumbnailItem(newFileHash, - beforeItem.TinyMeta, beforeItem.Small, beforeItem.Large, beforeItem.ExtraLarge, beforeItem.Reasons)); + beforeItem.TinyMeta, beforeItem.Small, beforeItem.Large, beforeItem.ExtraLarge, + beforeItem.Reasons)); await dbContext.SaveChangesAsync(); @@ -236,8 +260,9 @@ await dbContext.Thumbnails.AddRangeAsync(new ThumbnailItem(newFileHash, public async Task> UnprocessedGeneratedThumbnails() { return await _context.Thumbnails.Where(p => ( p.ExtraLarge == null - || p.Large == null || p.Small == null ) - && !string.IsNullOrEmpty(p.FileHash)).ToListAsync(); + || p.Large == null || p.Small == null ) + && !string.IsNullOrEmpty(p.FileHash)) + .ToListAsync(); } public async Task UpdateAsync(ThumbnailItem item) @@ -254,7 +279,8 @@ public async Task UpdateAsync(ThumbnailItem item) } } - internal static async Task UpdateInternalAsync(ApplicationDbContext dbContext, ThumbnailItem item) + internal static async Task UpdateInternalAsync(ApplicationDbContext dbContext, + ThumbnailItem item) { dbContext.Thumbnails.Update(item); await dbContext.SaveChangesAsync(); @@ -280,8 +306,9 @@ internal static async Task UpdateInternalAsync(ApplicationDbContext dbCont .Contains(p.FileHash)).ToListAsync(); var alreadyExistingThumbnails = dbThumbnailItems.Select(p => p.FileHash).Distinct(); - var newThumbnailItems = nonNullItems.Where(p => !alreadyExistingThumbnails. - Contains(p!.FileHash)).Cast().DistinctBy(p => p.FileHash).ToList(); + var newThumbnailItems = nonNullItems + .Where(p => !alreadyExistingThumbnails.Contains(p!.FileHash)).Cast() + .DistinctBy(p => p.FileHash).ToList(); var alreadyExistingThumbnailItems = nonNullItems .Where(p => alreadyExistingThumbnails.Contains(p!.FileHash)) @@ -293,7 +320,8 @@ internal static async Task UpdateInternalAsync(ApplicationDbContext dbCont // merge two items together foreach ( var item in dbThumbnailItems ) { - var indexOfAlreadyExists = alreadyExistingThumbnailItems.FindIndex(p => p.FileHash == item.FileHash); + var indexOfAlreadyExists = + alreadyExistingThumbnailItems.FindIndex(p => p.FileHash == item.FileHash); if ( indexOfAlreadyExists == -1 ) continue; var alreadyExists = alreadyExistingThumbnailItems[indexOfAlreadyExists]; context.Attach(item).State = EntityState.Detached; @@ -306,23 +334,24 @@ internal static async Task UpdateInternalAsync(ApplicationDbContext dbCont if ( !alreadyExists.Reasons!.Contains(item.Reasons!) ) { - var reasons = new StringBuilder(alreadyExistingThumbnailItems[indexOfAlreadyExists].Reasons); + var reasons = + new StringBuilder(alreadyExistingThumbnailItems[indexOfAlreadyExists].Reasons); reasons.Append($",{item.Reasons}"); alreadyExistingThumbnailItems[indexOfAlreadyExists].Reasons = reasons.ToString(); } if ( item.TinyMeta == alreadyExists.TinyMeta && - item.Large == alreadyExists.Large && - item.Small == alreadyExists.Small && - item.ExtraLarge == alreadyExists.ExtraLarge ) + item.Large == alreadyExists.Large && + item.Small == alreadyExists.Small && + item.ExtraLarge == alreadyExists.ExtraLarge ) { equalThumbnailItems.Add(alreadyExists); continue; } + updateThumbnailItems.Add(alreadyExists); } - return (newThumbnailItems, updateThumbnailItems, equalThumbnailItems); + return ( newThumbnailItems, updateThumbnailItems, equalThumbnailItems ); } - } diff --git a/starsky/starsky.foundation.platform/Helpers/ExtensionRolesHelper.cs b/starsky/starsky.foundation.platform/Helpers/ExtensionRolesHelper.cs index e65a253f58..8df8f18f3a 100644 --- a/starsky/starsky.foundation.platform/Helpers/ExtensionRolesHelper.cs +++ b/starsky/starsky.foundation.platform/Helpers/ExtensionRolesHelper.cs @@ -8,7 +8,7 @@ namespace starsky.foundation.platform.Helpers { - public static class ExtensionRolesHelper + public static partial class ExtensionRolesHelper { /// /// Xmp sidecar file @@ -88,12 +88,8 @@ private static readonly Dictionary> public static ImageFormat MapFileTypesToExtension(string filename) { if ( string.IsNullOrEmpty(filename) ) return ImageFormat.unknown; - - // without escaped values: - // \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$ - var matchCollection = new Regex("\\.([0-9a-z]+)(?=[?#])|(\\.)(?:[\\w]+)$", - RegexOptions.None, TimeSpan.FromMilliseconds(100) - ).Matches(filename); + + var matchCollection = FileExtensionRegex().Matches(filename); if ( matchCollection.Count == 0 ) { return ImageFormat.unknown; @@ -280,11 +276,8 @@ public static bool IsExtensionSidecar(string? filename) private static bool IsExtensionForce(string? filename, List checkThisList) { if ( string.IsNullOrEmpty(filename) ) return false; - - // without escaped values: - // \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$ - var matchCollection = new Regex("\\.([0-9a-z]+)(?=[?#])|(\\.)(?:[\\w]+)$", - RegexOptions.IgnoreCase, TimeSpan.FromMilliseconds(200)).Matches(filename); + + var matchCollection = FileExtensionRegex().Matches(filename); if ( matchCollection.Count == 0 ) return false; foreach ( var matchValue in matchCollection.Select(p => p.Value) ) @@ -316,10 +309,7 @@ public static string ReplaceExtensionWithXmp(string? filename) return string.Empty; } - // without escaped values: - // \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$ - var matchCollection = new Regex("\\.([0-9a-z]+)(?=[?#])|(\\.)(?:[\\w]+)$", - RegexOptions.None, TimeSpan.FromMilliseconds(100)).Matches(filename); + var matchCollection = FileExtensionRegex().Matches(filename); if ( matchCollection.Count == 0 ) return string.Empty; foreach ( Match match in matchCollection ) @@ -339,6 +329,18 @@ public static string ReplaceExtensionWithXmp(string? filename) return string.Empty; } + + /// + /// Check for file extensions + /// without escaped values: + /// \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$ + /// + /// Regex object + [GeneratedRegex( + @"\.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$", + RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, + matchTimeoutMilliseconds: 500)] + private static partial Regex FileExtensionRegex(); /// /// ImageFormat based on first bytes diff --git a/starsky/starskytest/starsky.foundation.database/Thumbnails/ThumbnailQueryErrorTest.cs b/starsky/starskytest/starsky.foundation.database/Thumbnails/ThumbnailQueryErrorTest.cs index a268defb29..9912ddf5bb 100644 --- a/starsky/starskytest/starsky.foundation.database/Thumbnails/ThumbnailQueryErrorTest.cs +++ b/starsky/starskytest/starsky.foundation.database/Thumbnails/ThumbnailQueryErrorTest.cs @@ -29,42 +29,42 @@ public void SetOriginalValue(IProperty property, object? value) { throw new System.NotImplementedException(); } - + public void SetPropertyModified(IProperty property) { throw new System.NotImplementedException(); } - + public bool IsModified(IProperty property) { throw new System.NotImplementedException(); } - + public bool HasTemporaryValue(IProperty property) { throw new System.NotImplementedException(); } - + public bool IsStoreGenerated(IProperty property) { throw new System.NotImplementedException(); } - + public object GetCurrentValue(IPropertyBase propertyBase) { throw new System.NotImplementedException(); } - + public TProperty GetCurrentValue(IPropertyBase propertyBase) { throw new System.NotImplementedException(); } - + public object GetOriginalValue(IPropertyBase propertyBase) { throw new System.NotImplementedException(); } - + public TProperty GetOriginalValue(IProperty property) { throw new System.NotImplementedException(); @@ -75,7 +75,7 @@ public void SetStoreGeneratedValue(IProperty property, object? value, { throw new NotImplementedException(); } - + public EntityEntry ToEntityEntry() { IsCalledDbUpdateConcurrency = true; @@ -103,12 +103,13 @@ public bool IsConceptualNull(IProperty property) // ReSharper disable once UnassignedGetOnlyAutoProperty public IEntityType EntityType { get; } public EntityState EntityState { get; set; } + // ReSharper disable once UnassignedGetOnlyAutoProperty public IUpdateEntry SharedIdentityEntry { get; } #pragma warning restore 8618 } - + private class AppDbContextConcurrencyException : ApplicationDbContext { [SuppressMessage("ReSharper", "VirtualMemberCallInConstructor")] @@ -134,17 +135,18 @@ public override Task SaveChangesAsync(CancellationToken cancellationToken = if ( Count <= MinCount ) { throw new DbUpdateConcurrencyException("t", - new List{new UpdateEntryUpdateConcurrency()}); + new List { new UpdateEntryUpdateConcurrency() }); } + return Task.FromResult(Count); - } - + } + public override Task AddRangeAsync(params object[] entities) { return Task.CompletedTask; - } + } } - + [TestMethod] public async Task ThumbnailQuery_ConcurrencyException() { @@ -152,16 +154,15 @@ public async Task ThumbnailQuery_ConcurrencyException() var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: "MovieListDatabase") .Options; - - var fakeQuery = new ThumbnailQuery(new AppDbContextConcurrencyException(options) - { - MinCount = 1 - },null!,new FakeIWebLogger()); - await fakeQuery.RenameAsync("1","1"); - + + var fakeQuery = + new ThumbnailQuery(new AppDbContextConcurrencyException(options) { MinCount = 1 }, + null!, new FakeIWebLogger()); + await fakeQuery.RenameAsync("1", "1"); + Assert.IsTrue(IsCalledDbUpdateConcurrency); } - + [TestMethod] public async Task ThumbnailQuery_DoubleConcurrencyException() { @@ -169,16 +170,15 @@ public async Task ThumbnailQuery_DoubleConcurrencyException() var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: "MovieListDatabase") .Options; - - var fakeQuery = new ThumbnailQuery(new AppDbContextConcurrencyException(options) - { - MinCount = 2 - },null!,new FakeIWebLogger()); - await fakeQuery.RenameAsync("1","2"); - + + var fakeQuery = + new ThumbnailQuery(new AppDbContextConcurrencyException(options) { MinCount = 2 }, + null!, new FakeIWebLogger()); + await fakeQuery.RenameAsync("1", "2"); + Assert.IsTrue(IsCalledDbUpdateConcurrency); } - + [TestMethod] public async Task ThumbnailQuery_3ConcurrencyException() { @@ -186,16 +186,14 @@ public async Task ThumbnailQuery_3ConcurrencyException() var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: "MovieListDatabase") .Options; - - var fakeQuery = new ThumbnailQuery(new AppDbContextConcurrencyException(options) - { - MinCount = 3 - },null!,new FakeIWebLogger()); - await fakeQuery.RenameAsync("1","2"); - + + var fakeQuery = + new ThumbnailQuery(new AppDbContextConcurrencyException(options) { MinCount = 3 }, + null!, new FakeIWebLogger()); + await fakeQuery.RenameAsync("1", "2"); + Assert.IsTrue(IsCalledDbUpdateConcurrency); } - private static bool IsCalledMySqlSaveDbExceptionContext { get; set; } @@ -203,20 +201,24 @@ public async Task ThumbnailQuery_3ConcurrencyException() private class MySqlSaveDbExceptionContext : ApplicationDbContext { private readonly string _error; + private readonly MySqlErrorCode _key; - public MySqlSaveDbExceptionContext(DbContextOptions options, string error) : base(options) + public MySqlSaveDbExceptionContext(DbContextOptions options, string error, + MySqlErrorCode key) : base(options) { _error = error; + _key = key; } - + public override Task SaveChangesAsync(CancellationToken cancellationToken = default) { IsCalledMySqlSaveDbExceptionContext = true; - throw CreateMySqlException(_error); + throw CreateMySqlException(_error, _key); } - - [SuppressMessage("Usage", "S6602:\"Find\" method should be used instead of the \"FirstOrDefault\" extension")] - private static MySqlException CreateMySqlException(string message) + + [SuppressMessage("Usage", + "S6602:\"Find\" method should be used instead of the \"FirstOrDefault\" extension")] + private static MySqlException CreateMySqlException(string message, MySqlErrorCode key) { // MySqlErrorCode errorCode, string? sqlState, string message, Exception? innerException @@ -224,21 +226,19 @@ private static MySqlException CreateMySqlException(string message) typeof(MySqlException).GetConstructors( BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.InvokeMethod); - var ctor = ctorLIst.FirstOrDefault(p => - p.ToString() == "Void .ctor(MySqlConnector.MySqlErrorCode, System.String, System.String, System.Exception)" ); - + var ctor = ctorLIst.FirstOrDefault(p => + p.ToString() == + "Void .ctor(MySqlConnector.MySqlErrorCode, System.String, System.String, System.Exception)"); + var instance = - ( MySqlException ) ctor?.Invoke(new object[] + ( MySqlException )ctor?.Invoke(new object[] { - MySqlErrorCode.AccessDenied, - "test", - message, - new Exception() + key, "test", message, new Exception() })!; return instance; } } - + [TestMethod] public async Task AddThumbnailRangeAsync_ShouldCatchPrimaryKeyHit() { @@ -246,18 +246,19 @@ public async Task AddThumbnailRangeAsync_ShouldCatchPrimaryKeyHit() var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: "MovieListDatabase") .Options; - - var fakeQuery = new ThumbnailQuery(new MySqlSaveDbExceptionContext(options,"Duplicate entry '1' for key 'PRIMARY'"), - null!,new FakeIWebLogger()); - + + var fakeQuery = new ThumbnailQuery( + new MySqlSaveDbExceptionContext(options, "Duplicate entry '1' for key 'PRIMARY'", MySqlErrorCode.DuplicateKey), + null!, new FakeIWebLogger()); + await fakeQuery.AddThumbnailRangeAsync(new List { new ThumbnailResultDataTransferModel("t") }); - + Assert.IsTrue(IsCalledMySqlSaveDbExceptionContext); } - + [TestMethod] [ExpectedException(typeof(MySqlException))] public async Task AddThumbnailRangeAsync_SomethingElseShould_ExpectedException() @@ -266,10 +267,11 @@ public async Task AddThumbnailRangeAsync_SomethingElseShould_ExpectedException() var options = new DbContextOptionsBuilder() .UseInMemoryDatabase(databaseName: "MovieListDatabase") .Options; - - var fakeQuery = new ThumbnailQuery(new MySqlSaveDbExceptionContext(options,"Something else"), - null!,new FakeIWebLogger()); - + + var fakeQuery = new ThumbnailQuery( + new MySqlSaveDbExceptionContext(options, "Something else", MySqlErrorCode.AbortingConnection), + null!, new FakeIWebLogger()); + await fakeQuery.AddThumbnailRangeAsync(new List { new ThumbnailResultDataTransferModel("t")