From 8d413639c52c6cc930d8acd3e242aaad33da13f6 Mon Sep 17 00:00:00 2001 From: n0099 Date: Fri, 17 May 2024 23:32:57 +0800 Subject: [PATCH] * rename class `PostContent` to `BasePostContent` @ crawler * fix or suppress all violations of Roslyn analyzer rules @ c# --- .../PublishProfiles/FolderProfile.pubxml | 2 +- .../{PostContent.cs => BasePostContent.cs} | 2 +- .../src/Db/Post/PostContent/ReplyContent.cs | 2 +- .../Db/Post/PostContent/SubReplyContent.cs | 2 +- c#/crawler/src/Db/User.cs | 2 +- .../Tieba/Crawl/Parser/Post/ReplyParser.cs | 1 + .../src/Tieba/Crawl/Saver/Post/ReplySaver.cs | 52 ++++++------ .../Tieba/Crawl/Saver/Post/SubReplySaver.cs | 30 +++---- .../src/Tieba/Crawl/Saver/Post/ThreadSaver.cs | 56 +++++++------ .../Crawl/Saver/ReplyContentImageSaver.cs | 8 +- .../Tieba/Crawl/Saver/SaverWithRevision.cs | 16 ++-- c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs | 84 ++++++++++--------- .../PushAllPostContentsIntoSonicWorker.cs | 7 +- .../PublishProfiles/FolderProfile.pubxml | 2 +- .../src/ImageBatchConsumingWorker.cs | 5 +- c#/shared/src/Db/TbmDbContext.cs | 3 + c#/shared/{ => src}/SharedHelper.cs | 8 +- 17 files changed, 152 insertions(+), 130 deletions(-) rename c#/crawler/src/Db/Post/PostContent/{PostContent.cs => BasePostContent.cs} (71%) rename c#/shared/{ => src}/SharedHelper.cs (100%) diff --git a/c#/crawler/Properties/PublishProfiles/FolderProfile.pubxml b/c#/crawler/Properties/PublishProfiles/FolderProfile.pubxml index a085d5d3..060114e4 100644 --- a/c#/crawler/Properties/PublishProfiles/FolderProfile.pubxml +++ b/c#/crawler/Properties/PublishProfiles/FolderProfile.pubxml @@ -1,6 +1,6 @@ diff --git a/c#/crawler/src/Db/Post/PostContent/PostContent.cs b/c#/crawler/src/Db/Post/PostContent/BasePostContent.cs similarity index 71% rename from c#/crawler/src/Db/Post/PostContent/PostContent.cs rename to c#/crawler/src/Db/Post/PostContent/BasePostContent.cs index d37cc45b..ae624f59 100644 --- a/c#/crawler/src/Db/Post/PostContent/PostContent.cs +++ b/c#/crawler/src/Db/Post/PostContent/BasePostContent.cs @@ -1,7 +1,7 @@ // ReSharper disable PropertyCanBeMadeInitOnly.Global namespace tbm.Crawler.Db.Post.PostContent; -public abstract class PostContent : RowVersionedEntity +public abstract class BasePostContent : RowVersionedEntity { public byte[]? ProtoBufBytes { get; set; } } diff --git a/c#/crawler/src/Db/Post/PostContent/ReplyContent.cs b/c#/crawler/src/Db/Post/PostContent/ReplyContent.cs index 7660e604..f698966b 100644 --- a/c#/crawler/src/Db/Post/PostContent/ReplyContent.cs +++ b/c#/crawler/src/Db/Post/PostContent/ReplyContent.cs @@ -1,7 +1,7 @@ // ReSharper disable PropertyCanBeMadeInitOnly.Global namespace tbm.Crawler.Db.Post.PostContent; -public class ReplyContent : PostContent +public class ReplyContent : BasePostContent { [Key] public ulong Pid { get; set; } } diff --git a/c#/crawler/src/Db/Post/PostContent/SubReplyContent.cs b/c#/crawler/src/Db/Post/PostContent/SubReplyContent.cs index c027beea..65e12da9 100644 --- a/c#/crawler/src/Db/Post/PostContent/SubReplyContent.cs +++ b/c#/crawler/src/Db/Post/PostContent/SubReplyContent.cs @@ -1,7 +1,7 @@ // ReSharper disable PropertyCanBeMadeInitOnly.Global namespace tbm.Crawler.Db.Post.PostContent; -public class SubReplyContent : PostContent +public class SubReplyContent : BasePostContent { [Key] public ulong Spid { get; set; } } diff --git a/c#/crawler/src/Db/User.cs b/c#/crawler/src/Db/User.cs index 593912f9..4ce32946 100644 --- a/c#/crawler/src/Db/User.cs +++ b/c#/crawler/src/Db/User.cs @@ -28,7 +28,7 @@ public override bool Equals(User? x, User? y) => x == y || ( public override int GetHashCode(User obj) { - var hash = new HashCode(); + var hash = default(HashCode); hash.Add(obj.Uid); hash.Add(obj.Name); hash.Add(obj.DisplayName); diff --git a/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs index 9b1ea5f7..42c1a62d 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs @@ -54,6 +54,7 @@ protected override ReplyPost Convert(Reply inPost) o.Pid, c.OriginSrc, SharedHelper.UnescapedJsonSerialize(c)); } } + // AuthorId rarely respond with 0, Author should always be null with no guarantee o.AuthorUid = inPost.AuthorId.NullIfZero() ?? inPost.Author?.Uid ?? 0; diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs index f9f48bb4..45ee4b0c 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs @@ -13,25 +13,6 @@ public class ReplySaver( { public delegate ReplySaver New(ConcurrentDictionary posts); - protected override bool FieldUpdateIgnorance - (string propName, object? oldValue, object? newValue) => propName switch - { // possible randomly respond with null - nameof(ReplyPost.SignatureId) when newValue is null && oldValue is not null => true, - _ => false - }; - - public override bool UserFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch - { // FansNickname in reply response will always be null - nameof(User.FansNickname) when newValue is null && oldValue is not null => true, - _ => false - }; - - public override bool UserFieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => propName switch - { // user icon will be null after UserParser.ResetUsersIcon() get invoked - nameof(User.Icon) when newValue is not null && oldValue is null => true, - _ => false - }; - protected override Dictionary AddRevisionDelegatesKeyBySplitEntityType { get; } = new() { @@ -52,13 +33,16 @@ protected override Dictionary } }; - [SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")] - protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch - { - nameof(ReplyPost.IsFold) => 1 << 2, - nameof(ReplyPost.DisagreeCount) => 1 << 4, - nameof(ReplyPost.Geolocation) => 1 << 5, - _ => 0 + public override bool UserFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch + { // FansNickname in reply response will always be null + nameof(User.FansNickname) when newValue is null && oldValue is not null => true, + _ => false + }; + + public override bool UserFieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => propName switch + { // user icon will be null after UserParser.ResetUsersIcon() get invoked + nameof(User.Icon) when newValue is not null && oldValue is null => true, + _ => false }; public override SaverChangeSet Save(CrawlerDbContext db) @@ -73,4 +57,20 @@ public override SaverChangeSet Save(CrawlerDbContext db) return changeSet; } + + protected override bool FieldUpdateIgnorance + (string propName, object? oldValue, object? newValue) => propName switch + { // possible randomly respond with null + nameof(ReplyPost.SignatureId) when newValue is null && oldValue is not null => true, + _ => false + }; + + [SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")] + protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch + { + nameof(ReplyPost.IsFold) => 1 << 2, + nameof(ReplyPost.DisagreeCount) => 1 << 4, + nameof(ReplyPost.Geolocation) => 1 << 5, + _ => 0 + }; } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs index 66033ef5..b87d556c 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs @@ -11,20 +11,6 @@ public class SubReplySaver( { public delegate SubReplySaver New(ConcurrentDictionary posts); - public override bool UserFieldUpdateIgnorance - (string propName, object? oldValue, object? newValue) => propName switch - { // always ignore updates on iconinfo due to some rare user will show some extra icons - // compare to reply response in the response of sub reply - nameof(User.Icon) => true, - - // FansNickname in sub reply response will always be null - nameof(User.FansNickname) when newValue is null && oldValue is not null => true, - - // DisplayName in users embedded in sub replies from response will be the legacy nickname - nameof(User.DisplayName) => true, - _ => false - }; - protected override Dictionary AddRevisionDelegatesKeyBySplitEntityType { get; } = new() { @@ -40,7 +26,19 @@ protected override Dictionary } }; - protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => 0; + public override bool UserFieldUpdateIgnorance + (string propName, object? oldValue, object? newValue) => propName switch + { // always ignore updates on iconinfo due to some rare user will show some extra icons + // compare to reply response in the response of sub reply + nameof(User.Icon) => true, + + // FansNickname in sub reply response will always be null + nameof(User.FansNickname) when newValue is null && oldValue is not null => true, + + // DisplayName in users embedded in sub replies from response will be the legacy nickname + nameof(User.DisplayName) => true, + _ => false + }; public override SaverChangeSet Save(CrawlerDbContext db) { @@ -51,4 +49,6 @@ public override SaverChangeSet Save(CrawlerDbContext db) return changeSet; } + + protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => 0; } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs index 8575afc9..8ac56330 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs @@ -11,15 +11,43 @@ public class ThreadSaver( { public delegate ThreadSaver New(ConcurrentDictionary posts); + protected override Dictionary + AddRevisionDelegatesKeyBySplitEntityType { get; } = new() + { + { + typeof(ThreadRevision.SplitViewCount), (db, revisions) => + db.Set() + .AddRange(revisions.OfType()) + } + }; + + public override bool UserFieldUpdateIgnorance + (string propName, object? oldValue, object? newValue) => propName switch + { // Icon.SpriteInfo will be an empty array and the icon url is a smaller one + // so we should mark it as null temporarily + // note this will cause we can't record when did a user update its iconinfo to null + // since these null values have been ignored in ReplySaver and SubReplySaver + nameof(User.Icon) => true, + _ => false + }; + + public override SaverChangeSet Save(CrawlerDbContext db) => + Save(db, th => th.Tid, + th => new ThreadRevision {TakenAt = th.UpdatedAt ?? th.CreatedAt, Tid = th.Tid}, + PredicateBuilder.New(th => Posts.Keys.Contains(th.Tid))); + protected override bool FieldUpdateIgnorance (string propName, object? oldValue, object? newValue) => propName switch { // will be updated by ThreadLateCrawler and ThreadLateCrawlFacade nameof(ThreadPost.AuthorPhoneType) => true, + // prevent overwrite existing value of field liker_id which is saved by legacy crawler // and Zan itself is deprecated by tieba, so it shouldn't get updated nameof(ThreadPost.Zan) => true, + // possible randomly respond with null nameof(ThreadPost.Geolocation) when newValue is null => true, + // empty string means the author had not written a title // its value generated from the first reply within response of reply crawler // will be later set by ReplyCrawlFacade.SaveParentThreadTitle() @@ -30,8 +58,10 @@ when newValue is "" // due to the thread is a multi forum topic thread // thus its title can be varied within the forum and within the thread || (newValue is not "" && oldValue is not "") => true, + // possible randomly respond with 0.NullIfZero() nameof(ThreadPost.DisagreeCount) when newValue is null && oldValue is not null => true, + // when the latest reply post is deleted and there's no new reply after delete // this field but not LatestReplyPostedAt will be null nameof(ThreadPost.LatestReplierUid) when newValue is null => true, @@ -42,31 +72,12 @@ protected override bool FieldRevisionIgnorance (string propName, object? oldValue, object? newValue) => propName switch { // empty string from response has been updated by ReplyCrawlFacade.OnPostParse() nameof(ThreadPost.Title) when oldValue is "" => true, + // null values will be later set by tieba client 6.0.2 response at ThreadParser.ParseInternal() nameof(ThreadPost.LatestReplierUid) when oldValue is null => true, _ => false }; - public override bool UserFieldUpdateIgnorance - (string propName, object? oldValue, object? newValue) => propName switch - { // Icon.SpriteInfo will be an empty array and the icon url is a smaller one - // so we should mark it as null temporarily - // note this will cause we can't record when did a user update its iconinfo to null - // since these null values have been ignored in ReplySaver and SubReplySaver - nameof(User.Icon) => true, - _ => false - }; - - protected override Dictionary - AddRevisionDelegatesKeyBySplitEntityType { get; } = new() - { - { - typeof(ThreadRevision.SplitViewCount), (db, revisions) => - db.Set() - .AddRange(revisions.OfType()) - } - }; - [SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")] protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch { @@ -81,9 +92,4 @@ protected override Dictionary nameof(ThreadPost.Geolocation) => 1 << 10, _ => 0 }; - - public override SaverChangeSet Save(CrawlerDbContext db) => - Save(db, th => th.Tid, - th => new ThreadRevision {TakenAt = th.UpdatedAt ?? th.CreatedAt, Tid = th.Tid}, - PredicateBuilder.New(th => Posts.Keys.Contains(th.Tid))); } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/ReplyContentImageSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/ReplyContentImageSaver.cs index 4abb96e4..a9b58062 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/ReplyContentImageSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/ReplyContentImageSaver.cs @@ -36,7 +36,11 @@ where imagesKeyByUrlFilename.Keys.Contains(e.UrlFilename) throw new InvalidOperationException(); alreadyLocked.ForEach(urlFilename => { - lock (LocksKeyByUrlFilename[urlFilename]) { } + lock (LocksKeyByUrlFilename[urlFilename]) +#pragma warning disable S108 // Either remove or fill this block of code. + { + } +#pragma warning restore S108 // Either remove or fill this block of code. }); existingImages = existingImages .Concat(( @@ -69,6 +73,8 @@ on existing.UrlFilename equals newInContent.UrlFilename if (newlyLocked.Any(urlFilename => !LocksKeyByUrlFilename.TryRemove(urlFilename, out _))) throw new InvalidOperationException(); +#pragma warning disable IDISP007 // Don't dispose injected locks.Dispose(); +#pragma warning restore IDISP007 // Don't dispose injected } } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs b/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs index 4941eaca..c90576e9 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs @@ -2,8 +2,8 @@ namespace tbm.Crawler.Tieba.Crawl.Saver; -public abstract class SaverWithRevision - (ILogger> logger) +public abstract partial class SaverWithRevision( + ILogger> logger) : IRevisionProperties where TBaseRevision : BaseRevisionWithSplitting { @@ -11,15 +11,17 @@ public abstract class SaverWithRevision protected abstract IReadOnlyDictionary AddRevisionDelegatesKeyBySplitEntityType { get; } protected abstract NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName); + protected virtual bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry) => false; + protected virtual bool FieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => false; + protected virtual bool FieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => false; private static bool GlobalFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch { // possible rarely respond with the protoBuf default value 0 nameof(BasePost.AuthorUid) when newValue is 0L && oldValue is not null => true, _ => false }; - protected virtual bool FieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => false; - protected virtual bool FieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => false; - protected virtual bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry) => false; - +} +public abstract partial class SaverWithRevision +{ protected void SaveEntitiesWithRevision( CrawlerDbContext db, Func revisionFactory, @@ -69,7 +71,7 @@ bool IsTimestampingFieldName(string name) => name is nameof(BasePost.LastSeenAt) continue; // skip following revision check } if (FieldRevisionIgnorance( - pName, p.OriginalValue, p.CurrentValue) + pName, p.OriginalValue, p.CurrentValue) || (entityIsUser && userFieldRevisionIgnorance!( pName, p.OriginalValue, p.CurrentValue))) continue; diff --git a/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs index 4ba8bed5..eadf5c4d 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs @@ -4,10 +4,52 @@ namespace tbm.Crawler.Tieba.Crawl.Saver; public partial class UserSaver { + protected override Dictionary + AddRevisionDelegatesKeyBySplitEntityType { get; } = new() + { + { + typeof(UserRevision.SplitDisplayName), (db, revisions) => + db.Set() + .AddRange(revisions.OfType()) + }, + { + typeof(UserRevision.SplitPortraitUpdatedAt), (db, revisions) => + db.Set() + .AddRange(revisions.OfType()) + }, + { + typeof(UserRevision.SplitIpGeolocation), (db, revisions) => + db.Set() + .AddRange(revisions.OfType()) + } + }; + + protected override bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry) + { + // ThreadCrawlFacade.ParseLatestRepliers() will save users with empty string as portrait + // they may soon be updated by (sub) reply crawler after it find out the latest reply + // so we should ignore its revision update for all fields + // ignore entire record is not possible via IFieldChangeIgnorance.GlobalFieldChangeIgnorance.Revision() + // since it can only determine one field at the time + if (propName != nameof(User.Portrait) || propEntry.OriginalValue is not "") return false; + + // invokes OriginalValues.ToObject() to get a new instance + // since entityInTracking is reference to the changed one + var user = (User)entityEntry.OriginalValues.ToObject(); + + // create another user instance with only fields of latest replier filled + var latestReplier = User.CreateLatestReplier(user.Uid, user.Name, user.DisplayName); + + // if they are same by fields values, the original one is the latest replier + // that previously generated by ParseLatestRepliers() + return User.EqualityComparer.Instance.Equals(user, latestReplier); + } + protected override bool FieldUpdateIgnorance (string propName, object? oldValue, object? newValue) => propName switch { // possible randomly respond with null nameof(User.IpGeolocation) when newValue is null => true, + // possible clock drift across multiple response from tieba api // they should sync their servers with NTP /* following sql can track these drift @@ -33,47 +75,6 @@ protected override bool FieldRevisionIgnorance _ => false }; - protected override bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry) - { - // ThreadCrawlFacade.ParseLatestRepliers() will save users with empty string as portrait - // they may soon be updated by (sub) reply crawler after it find out the latest reply - // so we should ignore its revision update for all fields - // ignore entire record is not possible via IFieldChangeIgnorance.GlobalFieldChangeIgnorance.Revision() - // since it can only determine one field at the time - if (propName != nameof(User.Portrait) || propEntry.OriginalValue is not "") return false; - - // invokes OriginalValues.ToObject() to get a new instance - // since entityInTracking is reference to the changed one - var user = (User)entityEntry.OriginalValues.ToObject(); - - // create another user instance with only fields of latest replier filled - var latestReplier = User.CreateLatestReplier(user.Uid, user.Name, user.DisplayName); - - // if they are same by fields values, the original one is the latest replier - // that previously generated by ParseLatestRepliers() - return User.EqualityComparer.Instance.Equals(user, latestReplier); - } - - protected override Dictionary - AddRevisionDelegatesKeyBySplitEntityType { get; } = new() - { - { - typeof(UserRevision.SplitDisplayName), (db, revisions) => - db.Set() - .AddRange(revisions.OfType()) - }, - { - typeof(UserRevision.SplitPortraitUpdatedAt), (db, revisions) => - db.Set() - .AddRange(revisions.OfType()) - }, - { - typeof(UserRevision.SplitIpGeolocation), (db, revisions) => - db.Set() - .AddRange(revisions.OfType()) - } - }; - [SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")] protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch { @@ -122,5 +123,6 @@ where newlyLocked.Contains(user.Uid) public IEnumerable AcquireUidLocksForSave(IEnumerable usersId) => locks.AcquireLocks(usersId.ToList()); + [SuppressMessage("IDisposableAnalyzers.Correctness", "IDISP007:Don't dispose injected")] public void OnPostSave() => locks.Dispose(); } diff --git a/c#/crawler/src/Worker/PushAllPostContentsIntoSonicWorker.cs b/c#/crawler/src/Worker/PushAllPostContentsIntoSonicWorker.cs index 81dc1c74..276ef139 100644 --- a/c#/crawler/src/Worker/PushAllPostContentsIntoSonicWorker.cs +++ b/c#/crawler/src/Worker/PushAllPostContentsIntoSonicWorker.cs @@ -18,14 +18,15 @@ protected override async Task DoWork(CancellationToken stoppingToken) await using var dbDefaultFactory = dbContextDefaultFactory(); var db = dbDefaultFactory.Value(); #pragma warning disable IDISP004 // Don't ignore created IDisposable - var forumPostCountsTuples = db.Database.GetDbConnection() + var forumPostCountsTuples = (await db.Database.GetDbConnection() #pragma warning restore IDISP004 // Don't ignore created IDisposable - .Query<(Fid Fid, int ReplyCount, int SubReplyCount)>( + .QueryAsync<(Fid Fid, int ReplyCount, int SubReplyCount)>( string.Join(" UNION ALL ", (from f in db.Forums.AsNoTracking() select f.Fid) .AsEnumerable().Select(fid => $"SELECT '{fid}'," + $"COALESCE((SELECT id FROM \"tbmc_f{fid}_reply\" ORDER BY id DESC LIMIT 1), 0)," - + $"COALESCE((SELECT id FROM \"tbmc_f{fid}_subReply\" ORDER BY id DESC LIMIT 1), 0)"))) + + $"COALESCE((SELECT id FROM \"tbmc_f{fid}_subReply\" ORDER BY id DESC LIMIT 1), 0)")), + stoppingToken)) .ToList(); var forumCount = forumPostCountsTuples.Count * 2; // reply and sub reply var totalPostCount = forumPostCountsTuples.Sum(t => t.ReplyCount) diff --git a/c#/imagePipeline/Properties/PublishProfiles/FolderProfile.pubxml b/c#/imagePipeline/Properties/PublishProfiles/FolderProfile.pubxml index a085d5d3..060114e4 100644 --- a/c#/imagePipeline/Properties/PublishProfiles/FolderProfile.pubxml +++ b/c#/imagePipeline/Properties/PublishProfiles/FolderProfile.pubxml @@ -1,6 +1,6 @@ diff --git a/c#/imagePipeline/src/ImageBatchConsumingWorker.cs b/c#/imagePipeline/src/ImageBatchConsumingWorker.cs index 6ef513bd..4e000833 100644 --- a/c#/imagePipeline/src/ImageBatchConsumingWorker.cs +++ b/c#/imagePipeline/src/ImageBatchConsumingWorker.cs @@ -197,8 +197,9 @@ private async Task ConsumeOcrConsumer( IQueryable forumScripts, CancellationToken stoppingToken = default) { - var scriptGroupings = forumScripts.AsNoTracking() - .GroupBy(e => e.Fid, e => e.Script).ToList(); + var scriptGroupings = await forumScripts.AsNoTracking() + .GroupBy(e => e.Fid, e => e.Script) + .ToListAsync(stoppingToken); var scripts = scriptGroupings.SelectMany(i => i).Distinct().ToList(); var recognizedTextLinesKeyByScript = new Dictionary>(scripts.Count); scripts.ForEach(script => recognizedTextLinesKeyByScript[script] = []); diff --git a/c#/shared/src/Db/TbmDbContext.cs b/c#/shared/src/Db/TbmDbContext.cs index 752ac182..8c8d1b81 100644 --- a/c#/shared/src/Db/TbmDbContext.cs +++ b/c#/shared/src/Db/TbmDbContext.cs @@ -97,6 +97,7 @@ public class TbmDbContext(ILogger? _dataSourceSingleton; // ReSharper disable once UnusedAutoPropertyAccessor.Global @@ -140,6 +141,8 @@ protected void OnModelCreatingWithFid(ModelBuilder b, uint fid) => protected virtual void OnConfiguringNpgsql(NpgsqlDbContextOptionsBuilder builder) { } protected virtual void OnBuildingNpgsqlDataSource(NpgsqlDataSourceBuilder builder) { } + + [SuppressMessage("Critical Code Smell", "S2696:Instance members should not write to \"static\" fields")] private Lazy GetNpgsqlDataSource(string? connectionString) => _dataSourceSingleton ??= new(() => { diff --git a/c#/shared/SharedHelper.cs b/c#/shared/src/SharedHelper.cs similarity index 100% rename from c#/shared/SharedHelper.cs rename to c#/shared/src/SharedHelper.cs index 81d8d569..497b4982 100644 --- a/c#/shared/SharedHelper.cs +++ b/c#/shared/src/SharedHelper.cs @@ -9,12 +9,12 @@ namespace tbm.Shared; public static class SharedHelper #pragma warning restore AV1708 // Type name contains term that should be avoided { - public static void GetNowTimestamp(out UInt32 now) => now = GetNowTimestamp(); - [SuppressMessage("Maintainability", "AV1551:Method overload should call another overload")] - public static UInt32 GetNowTimestamp() => (UInt32)DateTimeOffset.Now.ToUnixTimeSeconds(); - private static readonly JsonSerializerOptions UnescapedSerializeOptions = new() {Encoder = JavaScriptEncoder.Create(UnicodeRanges.All)}; public static string UnescapedJsonSerialize(TValue value) => JsonSerializer.Serialize(value, UnescapedSerializeOptions); + + public static void GetNowTimestamp(out UInt32 now) => now = GetNowTimestamp(); + [SuppressMessage("Maintainability", "AV1551:Method overload should call another overload")] + public static UInt32 GetNowTimestamp() => (UInt32)DateTimeOffset.Now.ToUnixTimeSeconds(); }