diff --git a/c#/crawler/src/Tieba/Crawl/Saver/AuthorRevisionSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/AuthorRevisionSaver.cs index 584ecf1e..d1599c16 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/AuthorRevisionSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/AuthorRevisionSaver.cs @@ -55,7 +55,6 @@ private static void SaveAuthorRevisions( .Select(latestRevisionProjectionFactory) .AsCte() // https://stackoverflow.com/questions/49854322/usage-of-for-update-in-window-function-postgres#comment86726589_49854322 .Where(e => e.Rank == 1) - .AsPostgreSQL().ForNoKeyUpdateHint() .ToLinqToDB().AsEnumerable() .Join(posts, e => e.Uid, p => p.AuthorUid, (e, p) => ( diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs index 45ee4b0c..d44ba4ea 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs @@ -1,5 +1,3 @@ -using PredicateBuilder = LinqKit.PredicateBuilder; - namespace tbm.Crawler.Tieba.Crawl.Saver.Post; public class ReplySaver( @@ -49,7 +47,7 @@ public override SaverChangeSet Save(CrawlerDbContext db) { var changeSet = Save(db, r => r.Pid, r => new ReplyRevision {TakenAt = r.UpdatedAt ?? r.CreatedAt, Pid = r.Pid}, - PredicateBuilder.New(r => Posts.Keys.Contains(r.Pid))); + LinqKit.PredicateBuilder.New(r => Posts.Keys.Contains(r.Pid))); replyContentImageSaver.Save(db, changeSet.NewlyAdded); PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke; diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs index b87d556c..65909144 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs @@ -1,5 +1,3 @@ -using LinqKit; - namespace tbm.Crawler.Tieba.Crawl.Saver.Post; public class SubReplySaver( @@ -44,7 +42,7 @@ public override SaverChangeSet Save(CrawlerDbContext db) { var changeSet = Save(db, sr => sr.Spid, sr => new SubReplyRevision {TakenAt = sr.UpdatedAt ?? sr.CreatedAt, Spid = sr.Spid}, - PredicateBuilder.New(sr => Posts.Keys.Contains(sr.Spid))); + LinqKit.PredicateBuilder.New(sr => Posts.Keys.Contains(sr.Spid))); PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke; return changeSet; diff --git a/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs b/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs index c90576e9..d545475a 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs @@ -25,21 +25,21 @@ public abstract partial class SaverWithRevision protected void SaveEntitiesWithRevision( CrawlerDbContext db, Func revisionFactory, - ILookup existingOrNewLookup, + ILookup isExistingEntityLookup, Func existingSelector, UserSaver.FieldChangeIgnorance? userFieldUpdateIgnorance = null, UserSaver.FieldChangeIgnorance? userFieldRevisionIgnorance = null) - where TEntity : class + where TEntity : RowVersionedEntity where TRevision : BaseRevisionWithSplitting { - db.Set().AddRange(existingOrNewLookup[false]); // newly added - var newRevisions = existingOrNewLookup[true].Select(newEntity => + db.Set().AddRange(isExistingEntityLookup[false]); // newly added + var newRevisions = isExistingEntityLookup[true].Select(newEntity => { var entityInTracking = existingSelector(newEntity); var entityEntry = db.Entry(entityInTracking); - // this will mutate existingEntity which is referenced by entry - entityEntry.CurrentValues.SetValues(newEntity); + entityEntry.CurrentValues.SetValues(newEntity); // mutate existingEntity that referenced by entry + entityEntry.Property(e => e.Version).IsModified = false; // newEntity.Version will always be default 0 bool IsTimestampingFieldName(string name) => name is nameof(BasePost.LastSeenAt) or nameof(TimestampedEntity.CreatedAt) or nameof(TimestampedEntity.UpdatedAt); diff --git a/c#/crawler/src/Worker/RetryCrawlWorker.cs b/c#/crawler/src/Worker/RetryCrawlWorker.cs index 6ba72f28..329d4012 100644 --- a/c#/crawler/src/Worker/RetryCrawlWorker.cs +++ b/c#/crawler/src/Worker/RetryCrawlWorker.cs @@ -2,7 +2,7 @@ namespace tbm.Crawler.Worker; public class RetryCrawlWorker( ILogger logger, - IIndex registeredLocksLookup, + IIndex registeredLocksKeyByType, CrawlPost crawlPost, Func> dbContextDefaultFactory, Func> threadLateCrawlFacadeFactory, @@ -16,7 +16,7 @@ protected override async Task DoWork(CancellationToken stoppingToken) foreach (var lockType in Enum.GetValues()) { if (stoppingToken.IsCancellationRequested) return; - var failed = registeredLocksLookup[lockType].RetryAllFailed(); + var failed = registeredLocksKeyByType[lockType].RetryAllFailed(); if (failed.Count == 0) continue; // skip current lock type if there's nothing needs to retry if (lockType == CrawlerLocks.Type.ThreadLate) {