+ method AddSplitRevisions()

+ abstract method `RevisionEntityIdSelector()` & `IsRevisionEntityIdEqualsExpression()`
* rename abstract prop `AddRevisionDelegatesKeyBySplitEntityType` to `AddSplitRevisionsDelegatesKeyByEntityType`
* rename delegate `AddRevisionDelegate` to `AddSplitRevisionsDelegate` and its param `revision` to `revisions`
+ type param `TEntityId`
@ SaverWithRevision.cs

+ type param `TPostId` @ PostSaver.cs
+ field `DuplicateIndex` @ BaseRevisionWithSplitting.cs
* add field `BaseRevisionWithSplitting.DuplicateIndex` to primary key columns @ `CrawlerDbContext.OnModelCreating()`
* move nested class `tbm.Crawler.Db.Revision.Splitting.RevisionWithSplitting.ModelBuilder.ReplaceParameterTypeVisitor` to `tbm.Crawler`
@ c#/crawler

c#/crawler/src/Db/CrawlerDbContext.cs

@@ -78,25 +78,25 @@ protected override void OnModelCreating(ModelBuilder b)
         b.Entity<SubReplyContent>().ToTable($"tbmc_f{Fid}_subReply_content");
 
         _ = new RevisionWithSplitting<BaseThreadRevision>
-                .ModelBuilder(b, "tbmcr_thread", e => new {e.Tid, e.TakenAt})
+                .ModelBuilder(b, "tbmcr_thread", e => new {e.Tid, e.TakenAt, e.DuplicateIndex})
             .HasBaseTable<ThreadRevision>()
             .SplitToTable<SplitViewCount>("viewCount");
 
         _ = new RevisionWithSplitting<BaseReplyRevision>
-                .ModelBuilder(b, "tbmcr_reply", e => new {e.Pid, e.TakenAt})
+                .ModelBuilder(b, "tbmcr_reply", e => new {e.Pid, e.TakenAt, e.DuplicateIndex})
             .HasBaseTable<ReplyRevision>()
             .SplitToTable<ReplyRevision.SplitAgreeCount>("agreeCount")
             .SplitToTable<SplitSubReplyCount>("subReplyCount")
             .SplitToTable<SplitFloor>("floor");
 
         _ = new RevisionWithSplitting<BaseSubReplyRevision>
-                .ModelBuilder(b, "tbmcr_subReply", e => new {e.Spid, e.TakenAt})
+                .ModelBuilder(b, "tbmcr_subReply", e => new {e.Spid, e.TakenAt, e.DuplicateIndex})
             .HasBaseTable<SubReplyRevision>()
             .SplitToTable<SubReplyRevision.SplitAgreeCount>("agreeCount")
             .SplitToTable<SplitDisagreeCount>("disagreeCount");
 
         _ = new RevisionWithSplitting<BaseUserRevision>
-                .ModelBuilder(b, "tbmcr_user", e => new {e.Uid, e.TakenAt})
+                .ModelBuilder(b, "tbmcr_user", e => new {e.Uid, e.TakenAt, e.DuplicateIndex})
             .HasBaseTable<UserRevision>()
             .SplitToTable<SplitIpGeolocation>("ipGeolocation")
             .SplitToTable<SplitPortraitUpdatedAt>("portraitUpdatedAt")

c#/crawler/src/Db/Revision/Splitting/BaseRevisionWithSplitting.cs

@@ -5,6 +5,7 @@ namespace tbm.Crawler.Db.Revision.Splitting;
 public abstract class BaseRevisionWithSplitting : RowVersionedEntity
 {
     public uint TakenAt { get; set; }
+    public ushort? DuplicateIndex { get; set; }
     public ushort? NullFieldsBitMask { get; set; }
     public virtual bool IsAllFieldsIsNullExceptSplit() => throw new NotSupportedException();
 }

c#/crawler/src/Db/Revision/Splitting/RevisionWithSplitting.cs

@@ -1,5 +1,3 @@
-using System.Collections.ObjectModel;
-
 namespace tbm.Crawler.Db.Revision.Splitting;
 
 public abstract class RevisionWithSplitting<TBaseRevision> : BaseRevisionWithSplitting
@@ -50,26 +48,5 @@ public ModelBuilder SplitToTable<TRevisionWithSplitting>(string tableNameSuffix)
                     visitor.Visit(keySelector));
             return this;
         }
-
-        /// <see>https://stackoverflow.com/questions/38316519/replace-parameter-type-in-lambda-expression/38345590#38345590</see>
-        private sealed class ReplaceParameterTypeVisitor<TSource, TTarget> : ExpressionVisitor
-        {
-            private ReadOnlyCollection<ParameterExpression>? _parameters;
-
-            protected override Expression VisitParameter(ParameterExpression node) =>
-                _parameters?.FirstOrDefault(p => p.Name == node.Name) ??
-                (node.Type == typeof(TSource) ? Expression.Parameter(typeof(TTarget), node.Name) : node);
-
-            protected override Expression VisitLambda<T>(Expression<T> node)
-            {
-                _parameters = VisitAndConvert(node.Parameters, nameof(VisitLambda));
-                return Expression.Lambda(Visit(node.Body), _parameters);
-            }
-
-            protected override Expression VisitMember(MemberExpression node) =>
-                node.Member.DeclaringType == typeof(TSource)
-                    ? Expression.Property(Visit(node.Expression)!, node.Member.Name)
-                    : base.VisitMember(node);
-        }
     }
 }

c#/crawler/src/EntryPoint.cs

@@ -37,7 +37,7 @@ protected override void ConfigureContainer(HostBuilderContext context, Container
         builder.RegisterImplementsOfBaseTypes(typeof(EntryPoint).Assembly,
         [
             typeof(ICrawler<,>), typeof(ICrawlFacade<>),
-            typeof(IPostParser<,>), typeof(SaverWithRevision<>)
+            typeof(IPostParser<,>), typeof(SaverWithRevision<,>)
         ]);
         builder.RegisterType<CrawlerDbContext>();
         builder.RegisterType<ClientRequester>();

c#/crawler/src/GlobalUsings.cs

@@ -62,6 +62,7 @@
 global using PostId = System.UInt64;
 global using Tid = System.UInt64;
 global using Pid = System.UInt64;
+global using Spid = System.UInt64;
 global using Time = System.UInt32;
 global using Page = System.UInt32;
 global using FailureCount = System.UInt16;

c#/crawler/src/ReplaceParameterTypeVisitor.cs

@@ -0,0 +1,24 @@
+using System.Collections.ObjectModel;
+
+namespace tbm.Crawler;
+
+/// <see>https://stackoverflow.com/questions/38316519/replace-parameter-type-in-lambda-expression/38345590#38345590</see>
+public class ReplaceParameterTypeVisitor<TSource, TTarget> : ExpressionVisitor
+{
+    private ReadOnlyCollection<ParameterExpression>? _parameters;
+
+    protected override Expression VisitParameter(ParameterExpression node) =>
+        _parameters?.FirstOrDefault(p => p.Name == node.Name) ??
+        (node.Type == typeof(TSource) ? Expression.Parameter(typeof(TTarget), node.Name) : node);
+
+    protected override Expression VisitLambda<T>(Expression<T> node)
+    {
+        _parameters = VisitAndConvert(node.Parameters, nameof(VisitLambda));
+        return Expression.Lambda(Visit(node.Body), _parameters);
+    }
+
+    protected override Expression VisitMember(MemberExpression node) =>
+        node.Member.DeclaringType == typeof(TSource)
+            ? Expression.Property(Visit(node.Expression)!, node.Member.Name)
+            : base.VisitMember(node);
+}

c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs

@@ -2,12 +2,12 @@
 
 namespace tbm.Crawler.Tieba.Crawl.Saver.Post;
 
-public abstract class PostSaver<TPost, TBaseRevision>(
-    ILogger<PostSaver<TPost, TBaseRevision>> logger,
+public abstract class PostSaver<TPost, TBaseRevision, TPostId>(
+    ILogger<PostSaver<TPost, TBaseRevision, TPostId>> logger,
     ConcurrentDictionary<PostId, TPost> posts,
     AuthorRevisionSaver.New authorRevisionSaverFactory,
     PostType currentPostType)
-    : SaverWithRevision<TBaseRevision>(logger), IPostSaver<TPost>
+    : SaverWithRevision<TBaseRevision, TPostId>(logger), IPostSaver<TPost>
     where TPost : BasePost
     where TBaseRevision : BaseRevisionWithSplitting
 {

c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs

@@ -6,30 +6,25 @@ public class ReplySaver(
     ReplyContentImageSaver replyContentImageSaver,
     ReplySignatureSaver replySignatureSaver,
     AuthorRevisionSaver.New authorRevisionSaverFactory)
-    : PostSaver<ReplyPost, BaseReplyRevision>(
+    : PostSaver<ReplyPost, BaseReplyRevision, Pid>(
         logger, posts, authorRevisionSaverFactory, PostType.Reply)
 {
     public delegate ReplySaver New(ConcurrentDictionary<PostId, ReplyPost> posts);
 
-    protected override Dictionary<Type, AddRevisionDelegate>
-        AddRevisionDelegatesKeyBySplitEntityType { get; } = new()
-    {
-        {
-            typeof(ReplyRevision.SplitFloor), (db, revisions) =>
-                db.Set<ReplyRevision.SplitFloor>()
-                    .AddRange(revisions.OfType<ReplyRevision.SplitFloor>())
-        },
-        {
-            typeof(ReplyRevision.SplitSubReplyCount), (db, revisions) =>
-                db.Set<ReplyRevision.SplitSubReplyCount>()
-                    .AddRange(revisions.OfType<ReplyRevision.SplitSubReplyCount>())
-        },
+    private Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>? _addSplitRevisionsDelegatesKeyByEntityType;
+    protected override Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>
+        AddSplitRevisionsDelegatesKeyByEntityType =>
+        _addSplitRevisionsDelegatesKeyByEntityType ??= new(() => new()
         {
-            typeof(ReplyRevision.SplitAgreeCount), (db, revisions) =>
-                db.Set<ReplyRevision.SplitAgreeCount>()
-                    .AddRange(revisions.OfType<ReplyRevision.SplitAgreeCount>())
-        }
-    };
+            {typeof(ReplyRevision.SplitFloor), AddSplitRevisions<ReplyRevision.SplitFloor>},
+            {typeof(ReplyRevision.SplitSubReplyCount), AddSplitRevisions<ReplyRevision.SplitSubReplyCount>},
+            {typeof(ReplyRevision.SplitAgreeCount), AddSplitRevisions<ReplyRevision.SplitAgreeCount>}
+        });
+
+    protected override Pid RevisionEntityIdSelector(BaseReplyRevision entity) => entity.Pid;
+    protected override Expression<Func<BaseReplyRevision, bool>>
+        IsRevisionEntityIdEqualsExpression(BaseReplyRevision newRevision) =>
+        existingRevision => existingRevision.Pid == newRevision.Pid;
 
     public override bool UserFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch
     { // FansNickname in reply response will always be null

c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs

@@ -4,25 +4,24 @@ public class SubReplySaver(
     ILogger<SubReplySaver> logger,
     ConcurrentDictionary<PostId, SubReplyPost> posts,
     AuthorRevisionSaver.New authorRevisionSaverFactory)
-    : PostSaver<SubReplyPost, BaseSubReplyRevision>(
+    : PostSaver<SubReplyPost, BaseSubReplyRevision, Spid>(
         logger, posts, authorRevisionSaverFactory, PostType.SubReply)
 {
     public delegate SubReplySaver New(ConcurrentDictionary<PostId, SubReplyPost> posts);
 
-    protected override Dictionary<Type, AddRevisionDelegate>
-        AddRevisionDelegatesKeyBySplitEntityType { get; } = new()
-    {
-        {
-            typeof(SubReplyRevision.SplitAgreeCount), (db, revisions) =>
-                db.Set<SubReplyRevision.SplitAgreeCount>()
-                    .AddRange(revisions.OfType<SubReplyRevision.SplitAgreeCount>())
-        },
+    private Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>? _addSplitRevisionsDelegatesKeyByEntityType;
+    protected override Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>
+        AddSplitRevisionsDelegatesKeyByEntityType =>
+        _addSplitRevisionsDelegatesKeyByEntityType ??= new(() => new()
         {
-            typeof(SubReplyRevision.SplitDisagreeCount), (db, revisions) =>
-                db.Set<SubReplyRevision.SplitDisagreeCount>()
-                    .AddRange(revisions.OfType<SubReplyRevision.SplitDisagreeCount>())
-        }
-    };
+            {typeof(SubReplyRevision.SplitAgreeCount), AddSplitRevisions<SubReplyRevision.SplitAgreeCount>},
+            {typeof(SubReplyRevision.SplitDisagreeCount), AddSplitRevisions<SubReplyRevision.SplitDisagreeCount>},
+        });
+
+    protected override Spid RevisionEntityIdSelector(BaseSubReplyRevision entity) => entity.Spid;
+    protected override Expression<Func<BaseSubReplyRevision, bool>>
+        IsRevisionEntityIdEqualsExpression(BaseSubReplyRevision newRevision) =>
+        existingRevision => existingRevision.Spid == newRevision.Spid;
 
     public override bool UserFieldUpdateIgnorance
         (string propName, object? oldValue, object? newValue) => propName switch

c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs

@@ -6,20 +6,23 @@ public class ThreadSaver(
     ILogger<ThreadSaver> logger,
     ConcurrentDictionary<Tid, ThreadPost> posts,
     AuthorRevisionSaver.New authorRevisionSaverFactory)
-    : PostSaver<ThreadPost, BaseThreadRevision>(
+    : PostSaver<ThreadPost, BaseThreadRevision, Tid>(
         logger, posts, authorRevisionSaverFactory, PostType.Thread)
 {
     public delegate ThreadSaver New(ConcurrentDictionary<Tid, ThreadPost> posts);
 
-    protected override Dictionary<Type, AddRevisionDelegate>
-        AddRevisionDelegatesKeyBySplitEntityType { get; } = new()
-    {
+    private Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>? _addSplitRevisionsDelegatesKeyByEntityType;
+    protected override Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>
+        AddSplitRevisionsDelegatesKeyByEntityType =>
+        _addSplitRevisionsDelegatesKeyByEntityType ??= new(() => new()
         {
-            typeof(ThreadRevision.SplitViewCount), (db, revisions) =>
-                db.Set<ThreadRevision.SplitViewCount>()
-                    .AddRange(revisions.OfType<ThreadRevision.SplitViewCount>())
-        }
-    };
+            {typeof(ThreadRevision.SplitViewCount), AddSplitRevisions<ThreadRevision.SplitViewCount>}
+        });
+
+    protected override Tid RevisionEntityIdSelector(BaseThreadRevision entity) => entity.Tid;
+    protected override Expression<Func<BaseThreadRevision, bool>>
+        IsRevisionEntityIdEqualsExpression(BaseThreadRevision newRevision) =>
+        existingRevision => existingRevision.Tid == newRevision.Tid;
 
     public override bool UserFieldUpdateIgnorance
         (string propName, object? oldValue, object? newValue) => propName switch

c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs

@@ -2,15 +2,21 @@
 
 namespace tbm.Crawler.Tieba.Crawl.Saver;
 
-public abstract partial class SaverWithRevision<TBaseRevision>(
-    ILogger<SaverWithRevision<TBaseRevision>> logger)
+public abstract partial class SaverWithRevision<TBaseRevision, TEntityId>(
+    ILogger<SaverWithRevision<TBaseRevision, TEntityId>> logger)
     : IRevisionProperties
     where TBaseRevision : BaseRevisionWithSplitting
 {
-    protected delegate void AddRevisionDelegate(CrawlerDbContext db, IEnumerable<TBaseRevision> revision);
-    protected abstract IReadOnlyDictionary<Type, AddRevisionDelegate> AddRevisionDelegatesKeyBySplitEntityType { get; }
+    protected delegate void AddSplitRevisionsDelegate(CrawlerDbContext db, IEnumerable<TBaseRevision> revisions);
+    protected abstract Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>
+        AddSplitRevisionsDelegatesKeyByEntityType { get; }
+
     protected abstract NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName);
 
+    protected abstract TEntityId RevisionEntityIdSelector(TBaseRevision entity);
+    protected abstract Expression<Func<TBaseRevision, bool>>
+        IsRevisionEntityIdEqualsExpression(TBaseRevision newRevision);
+
     protected virtual bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry) => false;
     protected virtual bool FieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => false;
     protected virtual bool FieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => false;
@@ -19,8 +25,34 @@ public abstract partial class SaverWithRevision<TBaseRevision>(
         nameof(BasePost.AuthorUid) when newValue is 0L && oldValue is not null => true,
         _ => false
     };
+
+    protected void AddSplitRevisions<TRevision>(CrawlerDbContext db, IEnumerable<TBaseRevision> revisions)
+        where TRevision : TBaseRevision
+    {
+        var newRevisions = revisions.OfType<TRevision>().ToList();
+        var dbSet = db.Set<TRevision>();
+        var visitor = new ReplaceParameterTypeVisitor<TBaseRevision, TRevision>();
+        var existingRevisions = dbSet
+            .Where(newRevisions.Aggregate(
+
+                // https://github.com/npgsql/npgsql/issues/4437
+                // https://github.com/dotnet/efcore/issues/32092
+                LinqKit.PredicateBuilder.New<TRevision>(),
+                (predicate, newRevision) => predicate.Or(LinqKit.PredicateBuilder
+                    .New<TRevision>(existingRevision => existingRevision.TakenAt == newRevision.TakenAt)
+                    .And((Expression<Func<TRevision, bool>>)
+                        visitor.Visit(IsRevisionEntityIdEqualsExpression(newRevision))))))
+            .ToList();
+        (from existingRevision in existingRevisions
+                join newRevision in newRevisions
+                    on RevisionEntityIdSelector(existingRevision) equals RevisionEntityIdSelector(newRevision)
+                select (existingRevision, newRevision))
+            .ForEach(t =>
+                t.newRevision.DuplicateIndex = t.existingRevision.DuplicateIndex ?? 0 + 1);
+        dbSet.AddRange(newRevisions);
+    }
 }
-public abstract partial class SaverWithRevision<TBaseRevision>
+public abstract partial class SaverWithRevision<TBaseRevision, TEntityId>
 {
     protected void SaveEntitiesWithRevision<TEntity, TRevision>(
         CrawlerDbContext db,
@@ -120,6 +152,6 @@ bool IsTimestampingFieldName(string name) => name is nameof(BasePost.LastSeenAt)
         newRevisions.OfType<RevisionWithSplitting<TBaseRevision>>()
             .SelectMany(rev => rev.SplitEntities)
             .GroupBy(pair => pair.Key, pair => pair.Value)
-            .ForEach(g => AddRevisionDelegatesKeyBySplitEntityType[g.Key](db, g));
+            .ForEach(g => AddSplitRevisionsDelegatesKeyByEntityType.Value[g.Key](db, g));
     }
 }

c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs

@@ -4,25 +4,20 @@ namespace tbm.Crawler.Tieba.Crawl.Saver;
 
 public partial class UserSaver
 {
-    protected override Dictionary<Type, AddRevisionDelegate>
-        AddRevisionDelegatesKeyBySplitEntityType { get; } = new()
-    {
-        {
-            typeof(UserRevision.SplitDisplayName), (db, revisions) =>
-                db.Set<UserRevision.SplitDisplayName>()
-                    .AddRange(revisions.OfType<UserRevision.SplitDisplayName>())
-        },
-        {
-            typeof(UserRevision.SplitPortraitUpdatedAt), (db, revisions) =>
-                db.Set<UserRevision.SplitPortraitUpdatedAt>()
-                    .AddRange(revisions.OfType<UserRevision.SplitPortraitUpdatedAt>())
-        },
+    private Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>? _addSplitRevisionsDelegatesKeyByEntityType;
+    protected override Lazy<Dictionary<Type, AddSplitRevisionsDelegate>>
+        AddSplitRevisionsDelegatesKeyByEntityType =>
+        _addSplitRevisionsDelegatesKeyByEntityType ??= new(() => new()
         {
-            typeof(UserRevision.SplitIpGeolocation), (db, revisions) =>
-                db.Set<UserRevision.SplitIpGeolocation>()
-                    .AddRange(revisions.OfType<UserRevision.SplitIpGeolocation>())
-        }
-    };
+            {typeof(UserRevision.SplitDisplayName), AddSplitRevisions<UserRevision.SplitDisplayName>},
+            {typeof(UserRevision.SplitPortraitUpdatedAt), AddSplitRevisions<UserRevision.SplitPortraitUpdatedAt>},
+            {typeof(UserRevision.SplitIpGeolocation), AddSplitRevisions<UserRevision.SplitIpGeolocation>}
+        });
+
+    protected override Uid RevisionEntityIdSelector(BaseUserRevision entity) => entity.Uid;
+    protected override Expression<Func<BaseUserRevision, bool>>
+        IsRevisionEntityIdEqualsExpression(BaseUserRevision newRevision) =>
+        existingRevision => existingRevision.Uid == newRevision.Uid;
 
     protected override bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry)
     {
@@ -87,7 +82,7 @@ protected override bool FieldRevisionIgnorance
 public partial class UserSaver(
     ILogger<UserSaver> logger, SaverLocks<Uid> locks,
     IDictionary<Uid, User> users)
-    : SaverWithRevision<BaseUserRevision>(logger)
+    : SaverWithRevision<BaseUserRevision, Uid>(logger)
 {
     public delegate UserSaver New(IDictionary<Uid, User> users);
     public delegate bool FieldChangeIgnorance(string propName, object? oldValue, object? newValue);