Skip to content

Commit

Permalink
Merge pull request #1558 from qdraw/feature/202404-1537-regex-timeout
Browse files Browse the repository at this point in the history 
#1537 add regex timeout & change exception handeling
  • Loading branch information
@qdraw
qdraw authored Apr 24, 2024
2 parents e16a4e5 + 993f022 commit c6ae6d4
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 116 deletions.
2 changes: 2 additions & 0 deletions history.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ Semantic Versioning 2.0.0 is from version 0.1.6+
- [x] (Fixed) _Docs_ Demo site is not working (PR #1486)
- [x] (Fixed) _Back-end_ GetFileNameRegex refactor to avoid timeouts (PR #1515)
- [x] (Changed) Back-end Upgrade to .NET 8 - SDK 8.0.204 (Runtime: 8.0.4) (PR #1541)
- [x] (Fixed) _Back-end_ Unhandled exception DbUpdateException (PR #1558 Issue #1489)
- [x] (Fixed) _Back-end_ Regex timeout IsExtensionForce (PR #1542 Issue #1537)

## version 0.6.0 - 2024-03-15 {#v0.6.0}

Expand Down
101 changes: 65 additions & 36 deletions starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using MySqlConnector;
using starsky.foundation.database.Data;
using starsky.foundation.database.Interfaces;
using starsky.foundation.database.Models;
Expand All @@ -22,23 +23,28 @@ public class ThumbnailQuery : IThumbnailQuery
private readonly IServiceScopeFactory? _scopeFactory;
private readonly IWebLogger _logger;

public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeFactory, IWebLogger logger)
public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeFactory,
IWebLogger logger)
{
_context = context;
_scopeFactory = scopeFactory;
_logger = logger;
}

public Task<List<ThumbnailItem>?> AddThumbnailRangeAsync(List<ThumbnailResultDataTransferModel> thumbnailItems)
public Task<List<ThumbnailItem>?> AddThumbnailRangeAsync(
List<ThumbnailResultDataTransferModel> thumbnailItems)
{
if ( thumbnailItems.Exists(p => string.IsNullOrEmpty(p.FileHash)) )
{
throw new ArgumentNullException(nameof(thumbnailItems), "[AddThumbnailRangeAsync] FileHash is null or empty");
throw new ArgumentNullException(nameof(thumbnailItems),
"[AddThumbnailRangeAsync] FileHash is null or empty");
}

return AddThumbnailRangeInternalRetryDisposedAsync(thumbnailItems);
}

private async Task<List<ThumbnailItem>?> AddThumbnailRangeInternalRetryDisposedAsync(List<ThumbnailResultDataTransferModel> thumbnailItems)
private async Task<List<ThumbnailItem>?> AddThumbnailRangeInternalRetryDisposedAsync(
List<ThumbnailResultDataTransferModel> thumbnailItems)
{
try
{
Expand All @@ -48,11 +54,12 @@ public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeF
catch ( InvalidOperationException )
{
if ( _scopeFactory == null ) throw;
return await AddThumbnailRangeInternalAsync(new InjectServiceScope(_scopeFactory).Context(), thumbnailItems);
return await AddThumbnailRangeInternalAsync(
new InjectServiceScope(_scopeFactory).Context(), thumbnailItems);
}
}

private static async Task<List<ThumbnailItem>?> AddThumbnailRangeInternalAsync(
private async Task<List<ThumbnailItem>?> AddThumbnailRangeInternalAsync(
ApplicationDbContext dbContext,
IReadOnlyCollection<ThumbnailResultDataTransferModel> thumbnailItems)
{
Expand All @@ -63,9 +70,10 @@ public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeF

var updateThumbnailNewItemsList = new List<ThumbnailItem>();
foreach ( var item in thumbnailItems
.Where(p => p.FileHash != null).DistinctBy(p => p.FileHash) )
.Where(p => p.FileHash != null).DistinctBy(p => p.FileHash) )
{
updateThumbnailNewItemsList.Add(new ThumbnailItem(item.FileHash!, item.TinyMeta, item.Small, item.Large, item.ExtraLarge, item.Reasons));
updateThumbnailNewItemsList.Add(new ThumbnailItem(item.FileHash!, item.TinyMeta,
item.Small, item.Large, item.ExtraLarge, item.Reasons));
}

var (newThumbnailItems,
Expand Down Expand Up @@ -100,7 +108,7 @@ public ThumbnailQuery(ApplicationDbContext context, IServiceScopeFactory? scopeF
return allResults;
}

private static async Task SaveChangesDuplicate(DbContext dbContext)
private async Task SaveChangesDuplicate(DbContext dbContext)
{
try
{
Expand All @@ -110,11 +118,15 @@ private static async Task SaveChangesDuplicate(DbContext dbContext)
{
// Skip if Duplicate entry
// MySqlConnector.MySqlException (0x80004005): Duplicate entry for key 'PRIMARY'
// https://github.com/qdraw/starsky/issues/1248
if ( !exception.Message.Contains("Duplicate") )
// https://github.com/qdraw/starsky/issues/1248 https://github.com/qdraw/starsky/issues/1489
if ( exception is MySqlException { ErrorCode: MySqlErrorCode.DuplicateKey } )
{
throw;
return;
}

_logger.LogError($"[SaveChangesDuplicate] T:{exception.GetType()} M:{exception.Message} I: {exception.InnerException}");

throw;
}
}

Expand All @@ -128,18 +140,21 @@ public async Task<List<ThumbnailItem>> Get(string? fileHash = null)
catch ( InvalidOperationException )
{
if ( _scopeFactory == null ) throw;
return await GetInternalAsync(new InjectServiceScope(_scopeFactory).Context(), fileHash);
return await GetInternalAsync(new InjectServiceScope(_scopeFactory).Context(),
fileHash);
}
}

private static async Task<List<ThumbnailItem>> GetInternalAsync(
ApplicationDbContext context,
string? fileHash = null)
{
return fileHash == null ? await context
.Thumbnails.ToListAsync() : await context
.Thumbnails.Where(p => p.FileHash == fileHash)
.ToListAsync();
return fileHash == null
? await context
.Thumbnails.ToListAsync()
: await context
.Thumbnails.Where(p => p.FileHash == fileHash)
.ToListAsync();
}

public async Task RemoveThumbnailsAsync(List<string> deletedFileHashes)
Expand All @@ -157,7 +172,8 @@ public async Task RemoveThumbnailsAsync(List<string> deletedFileHashes)
catch ( InvalidOperationException )
{
if ( _scopeFactory == null ) throw;
await RemoveThumbnailsInternalAsync(new InjectServiceScope(_scopeFactory).Context(), deletedFileHashes);
await RemoveThumbnailsInternalAsync(new InjectServiceScope(_scopeFactory).Context(),
deletedFileHashes);
}
}

Expand All @@ -172,10 +188,12 @@ internal static async Task<bool> RemoveThumbnailsInternalAsync(

foreach ( var fileNamesInChunk in deletedFileHashes.ChunkyEnumerable(100) )
{
var thumbnailItems = await context.Thumbnails.Where(p => fileNamesInChunk.Contains(p.FileHash)).ToListAsync();
var thumbnailItems = await context.Thumbnails
.Where(p => fileNamesInChunk.Contains(p.FileHash)).ToListAsync();
context.Thumbnails.RemoveRange(thumbnailItems);
await context.SaveChangesAsync();
}

return true;
}

Expand All @@ -189,26 +207,31 @@ public async Task<bool> RenameAsync(string beforeFileHash, string newFileHash)
catch ( InvalidOperationException )
{
if ( _scopeFactory == null ) throw;
return await RenameInternalAsync(new InjectServiceScope(_scopeFactory).Context(), beforeFileHash, newFileHash);
return await RenameInternalAsync(new InjectServiceScope(_scopeFactory).Context(),
beforeFileHash, newFileHash);
}
catch ( DbUpdateConcurrencyException concurrencyException )
{
_logger.LogInformation("[ThumbnailQuery] try to fix DbUpdateConcurrencyException", concurrencyException);
_logger.LogInformation("[ThumbnailQuery] try to fix DbUpdateConcurrencyException",
concurrencyException);
SolveConcurrency.SolveConcurrencyExceptionLoop(concurrencyException.Entries);
try
{
await _context.SaveChangesAsync();
}
catch ( DbUpdateConcurrencyException e )
{
_logger.LogInformation(e, "[ThumbnailQuery] save failed after DbUpdateConcurrencyException");
_logger.LogInformation(e,
"[ThumbnailQuery] save failed after DbUpdateConcurrencyException");
return false;
}

return true;
}
}

private static async Task<bool> RenameInternalAsync(ApplicationDbContext dbContext, string beforeFileHash, string newFileHash)
private static async Task<bool> RenameInternalAsync(ApplicationDbContext dbContext,
string beforeFileHash, string newFileHash)
{
var beforeOrNewItems = await dbContext.Thumbnails.Where(p =>
p.FileHash == beforeFileHash || p.FileHash == newFileHash).ToListAsync();
Expand All @@ -226,7 +249,8 @@ private static async Task<bool> RenameInternalAsync(ApplicationDbContext dbConte
}

await dbContext.Thumbnails.AddRangeAsync(new ThumbnailItem(newFileHash,
beforeItem.TinyMeta, beforeItem.Small, beforeItem.Large, beforeItem.ExtraLarge, beforeItem.Reasons));
beforeItem.TinyMeta, beforeItem.Small, beforeItem.Large, beforeItem.ExtraLarge,
beforeItem.Reasons));

await dbContext.SaveChangesAsync();

Expand All @@ -236,8 +260,9 @@ await dbContext.Thumbnails.AddRangeAsync(new ThumbnailItem(newFileHash,
public async Task<List<ThumbnailItem>> UnprocessedGeneratedThumbnails()
{
return await _context.Thumbnails.Where(p => ( p.ExtraLarge == null
|| p.Large == null || p.Small == null )
&& !string.IsNullOrEmpty(p.FileHash)).ToListAsync();
|| p.Large == null || p.Small == null )
&& !string.IsNullOrEmpty(p.FileHash))
.ToListAsync();
}

public async Task<bool> UpdateAsync(ThumbnailItem item)
Expand All @@ -254,7 +279,8 @@ public async Task<bool> UpdateAsync(ThumbnailItem item)
}
}

internal static async Task<bool> UpdateInternalAsync(ApplicationDbContext dbContext, ThumbnailItem item)
internal static async Task<bool> UpdateInternalAsync(ApplicationDbContext dbContext,
ThumbnailItem item)
{
dbContext.Thumbnails.Update(item);
await dbContext.SaveChangesAsync();
Expand All @@ -280,8 +306,9 @@ internal static async Task<bool> UpdateInternalAsync(ApplicationDbContext dbCont
.Contains(p.FileHash)).ToListAsync();
var alreadyExistingThumbnails = dbThumbnailItems.Select(p => p.FileHash).Distinct();

var newThumbnailItems = nonNullItems.Where(p => !alreadyExistingThumbnails.
Contains(p!.FileHash)).Cast<ThumbnailItem>().DistinctBy(p => p.FileHash).ToList();
var newThumbnailItems = nonNullItems
.Where(p => !alreadyExistingThumbnails.Contains(p!.FileHash)).Cast<ThumbnailItem>()
.DistinctBy(p => p.FileHash).ToList();

var alreadyExistingThumbnailItems = nonNullItems
.Where(p => alreadyExistingThumbnails.Contains(p!.FileHash))
Expand All @@ -293,7 +320,8 @@ internal static async Task<bool> UpdateInternalAsync(ApplicationDbContext dbCont
// merge two items together
foreach ( var item in dbThumbnailItems )
{
var indexOfAlreadyExists = alreadyExistingThumbnailItems.FindIndex(p => p.FileHash == item.FileHash);
var indexOfAlreadyExists =
alreadyExistingThumbnailItems.FindIndex(p => p.FileHash == item.FileHash);
if ( indexOfAlreadyExists == -1 ) continue;
var alreadyExists = alreadyExistingThumbnailItems[indexOfAlreadyExists];
context.Attach(item).State = EntityState.Detached;
Expand All @@ -306,23 +334,24 @@ internal static async Task<bool> UpdateInternalAsync(ApplicationDbContext dbCont

if ( !alreadyExists.Reasons!.Contains(item.Reasons!) )
{
var reasons = new StringBuilder(alreadyExistingThumbnailItems[indexOfAlreadyExists].Reasons);
var reasons =
new StringBuilder(alreadyExistingThumbnailItems[indexOfAlreadyExists].Reasons);
reasons.Append($",{item.Reasons}");
alreadyExistingThumbnailItems[indexOfAlreadyExists].Reasons = reasons.ToString();
}

if ( item.TinyMeta == alreadyExists.TinyMeta &&
item.Large == alreadyExists.Large &&
item.Small == alreadyExists.Small &&
item.ExtraLarge == alreadyExists.ExtraLarge )
item.Large == alreadyExists.Large &&
item.Small == alreadyExists.Small &&
item.ExtraLarge == alreadyExists.ExtraLarge )
{
equalThumbnailItems.Add(alreadyExists);
continue;
}

updateThumbnailItems.Add(alreadyExists);
}

return (newThumbnailItems, updateThumbnailItems, equalThumbnailItems);
return ( newThumbnailItems, updateThumbnailItems, equalThumbnailItems );
}

}
34 changes: 18 additions & 16 deletions starsky/starsky.foundation.platform/Helpers/ExtensionRolesHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

namespace starsky.foundation.platform.Helpers
{
public static class ExtensionRolesHelper
public static partial class ExtensionRolesHelper
{
/// <summary>
/// Xmp sidecar file
Expand Down Expand Up @@ -88,12 +88,8 @@ private static readonly Dictionary<ImageFormat, List<string>>
public static ImageFormat MapFileTypesToExtension(string filename)
{
if ( string.IsNullOrEmpty(filename) ) return ImageFormat.unknown;

// without escaped values:
// \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$
var matchCollection = new Regex("\\.([0-9a-z]+)(?=[?#])|(\\.)(?:[\\w]+)$",
RegexOptions.None, TimeSpan.FromMilliseconds(100)
).Matches(filename);

var matchCollection = FileExtensionRegex().Matches(filename);
if ( matchCollection.Count == 0 )
{
return ImageFormat.unknown;
Expand Down Expand Up @@ -280,11 +276,8 @@ public static bool IsExtensionSidecar(string? filename)
private static bool IsExtensionForce(string? filename, List<string> checkThisList)
{
if ( string.IsNullOrEmpty(filename) ) return false;

// without escaped values:
// \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$
var matchCollection = new Regex("\\.([0-9a-z]+)(?=[?#])|(\\.)(?:[\\w]+)$",
RegexOptions.IgnoreCase, TimeSpan.FromMilliseconds(200)).Matches(filename);

var matchCollection = FileExtensionRegex().Matches(filename);

if ( matchCollection.Count == 0 ) return false;
foreach ( var matchValue in matchCollection.Select(p => p.Value) )
Expand Down Expand Up @@ -316,10 +309,7 @@ public static string ReplaceExtensionWithXmp(string? filename)
return string.Empty;
}

// without escaped values:
// \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$
var matchCollection = new Regex("\\.([0-9a-z]+)(?=[?#])|(\\.)(?:[\\w]+)$",
RegexOptions.None, TimeSpan.FromMilliseconds(100)).Matches(filename);
var matchCollection = FileExtensionRegex().Matches(filename);

if ( matchCollection.Count == 0 ) return string.Empty;
foreach ( Match match in matchCollection )
Expand All @@ -339,6 +329,18 @@ public static string ReplaceExtensionWithXmp(string? filename)

return string.Empty;
}

/// <summary>
/// Check for file extensions
/// without escaped values:
/// \.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$
/// </summary>
/// <returns>Regex object</returns>
[GeneratedRegex(
@"\.([0-9a-z]+)(?=[?#])|(\.)(?:[\w]+)$",
RegexOptions.CultureInvariant | RegexOptions.IgnoreCase,
matchTimeoutMilliseconds: 500)]
private static partial Regex FileExtensionRegex();

/// <summary>
/// ImageFormat based on first bytes
Expand Down
Loading

0 comments on commit c6ae6d4

Please sign in to comment.