diff --git a/c#/GlobalSuppressions.cs b/c#/GlobalSuppressions.cs index a7ae8735..bfd97d2f 100644 --- a/c#/GlobalSuppressions.cs +++ b/c#/GlobalSuppressions.cs @@ -88,3 +88,4 @@ [assembly: SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1009:Closing parenthesis should be spaced correctly")] [assembly: SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1012:Opening braces should be spaced correctly")] [assembly: SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1013:Closing braces should be spaced correctly")] +[assembly: SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1015:Closing generic brackets should be spaced correctly", Justification = "https://github.com/DotNetAnalyzers/StyleCopAnalyzers/issues/3856")] diff --git a/c#/crawler/src/Db/Revision/Splitting/RevisionWithSplitting.cs b/c#/crawler/src/Db/Revision/Splitting/RevisionWithSplitting.cs index 4bd00a2d..158553f4 100644 --- a/c#/crawler/src/Db/Revision/Splitting/RevisionWithSplitting.cs +++ b/c#/crawler/src/Db/Revision/Splitting/RevisionWithSplitting.cs @@ -32,8 +32,7 @@ public ModelBuilder HasBaseTable() where TRevision : TBaseRevision { var visitor = new ReplaceParameterTypeVisitor(); _ = builder.Entity().ToTable(baseTableName) - .HasKey((Expression>) - visitor.Visit(keySelector)); + .HasKey((Expression>)visitor.Visit(keySelector)); return this; } @@ -44,8 +43,7 @@ public ModelBuilder SplitToTable(string tableNameSuffix) _ = builder.Entity() .Ignore(e => e.NullFieldsBitMask) .ToTable($"{baseTableName}_{tableNameSuffix}") - .HasKey((Expression>) - visitor.Visit(keySelector)); + .HasKey((Expression>)visitor.Visit(keySelector)); return this; } } diff --git a/c#/crawler/src/ReplaceParameterTypeVisitor.cs b/c#/crawler/src/ReplaceParameterTypeVisitor.cs index 93fb11f9..e386f044 100644 --- a/c#/crawler/src/ReplaceParameterTypeVisitor.cs +++ b/c#/crawler/src/ReplaceParameterTypeVisitor.cs @@ -4,7 +4,9 @@ namespace tbm.Crawler; /// https://stackoverflow.com/questions/38316519/replace-parameter-type-in-lambda-expression/38345590#38345590 +#pragma warning disable SA1618 // Generic type parameters should be documented public class ReplaceParameterTypeVisitor : ExpressionVisitor +#pragma warning restore SA1618 // Generic type parameters should be documented { private ReadOnlyCollection? _parameters; diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs index 093c00ee..0e1e636d 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs @@ -9,9 +9,10 @@ public class ReplySaver( : PostSaver( logger, posts, authorRevisionSaverFactory, PostType.Reply) { + private Lazy>? _addSplitRevisionsDelegatesKeyByEntityType; + public delegate ReplySaver New(ConcurrentDictionary posts); - private Lazy>? _addSplitRevisionsDelegatesKeyByEntityType; protected override Lazy> AddSplitRevisionsDelegatesKeyByEntityType => _addSplitRevisionsDelegatesKeyByEntityType ??= new(() => new() @@ -21,17 +22,11 @@ protected override Lazy> {typeof(ReplyRevision.SplitAgreeCount), AddSplitRevisions} }); - protected override Pid RevisionEntityIdSelector(BaseReplyRevision entity) => entity.Pid; - protected override Expression> - IsRevisionEntityIdEqualsExpression(BaseReplyRevision newRevision) => - existingRevision => existingRevision.Pid == newRevision.Pid; - public override bool UserFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch { // FansNickname in reply response will always be null nameof(User.FansNickname) when newValue is null && oldValue is not null => true, _ => false }; - public override bool UserFieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => propName switch { // user icon will be null after UserParser.ResetUsersIcon() get invoked nameof(User.Icon) when newValue is not null && oldValue is null => true, @@ -51,6 +46,11 @@ public override SaverChangeSet Save(CrawlerDbContext db) return changeSet; } + protected override Pid RevisionEntityIdSelector(BaseReplyRevision entity) => entity.Pid; + protected override Expression> + IsRevisionEntityIdEqualsExpression(BaseReplyRevision newRevision) => + existingRevision => existingRevision.Pid == newRevision.Pid; + protected override bool FieldUpdateIgnorance (string propName, object? oldValue, object? newValue) => propName switch { // possible randomly respond with null diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs index 2a18dc08..ad237e9b 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs @@ -7,9 +7,10 @@ public class SubReplySaver( : PostSaver( logger, posts, authorRevisionSaverFactory, PostType.SubReply) { + private Lazy>? _addSplitRevisionsDelegatesKeyByEntityType; + public delegate SubReplySaver New(ConcurrentDictionary posts); - private Lazy>? _addSplitRevisionsDelegatesKeyByEntityType; protected override Lazy> AddSplitRevisionsDelegatesKeyByEntityType => _addSplitRevisionsDelegatesKeyByEntityType ??= new(() => new() @@ -18,11 +19,6 @@ protected override Lazy> {typeof(SubReplyRevision.SplitDisagreeCount), AddSplitRevisions}, }); - protected override Spid RevisionEntityIdSelector(BaseSubReplyRevision entity) => entity.Spid; - protected override Expression> - IsRevisionEntityIdEqualsExpression(BaseSubReplyRevision newRevision) => - existingRevision => existingRevision.Spid == newRevision.Spid; - public override bool UserFieldUpdateIgnorance (string propName, object? oldValue, object? newValue) => propName switch { // always ignore updates on iconinfo due to some rare user will show some extra icons @@ -47,5 +43,10 @@ public override SaverChangeSet Save(CrawlerDbContext db) return changeSet; } + protected override Spid RevisionEntityIdSelector(BaseSubReplyRevision entity) => entity.Spid; + protected override Expression> + IsRevisionEntityIdEqualsExpression(BaseSubReplyRevision newRevision) => + existingRevision => existingRevision.Spid == newRevision.Spid; + protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => 0; } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs index e1e05e8e..c6773cb6 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs @@ -9,9 +9,10 @@ public class ThreadSaver( : PostSaver( logger, posts, authorRevisionSaverFactory, PostType.Thread) { + private Lazy>? _addSplitRevisionsDelegatesKeyByEntityType; + public delegate ThreadSaver New(ConcurrentDictionary posts); - private Lazy>? _addSplitRevisionsDelegatesKeyByEntityType; protected override Lazy> AddSplitRevisionsDelegatesKeyByEntityType => _addSplitRevisionsDelegatesKeyByEntityType ??= new(() => new() @@ -19,11 +20,6 @@ protected override Lazy> {typeof(ThreadRevision.SplitViewCount), AddSplitRevisions} }); - protected override Tid RevisionEntityIdSelector(BaseThreadRevision entity) => entity.Tid; - protected override Expression> - IsRevisionEntityIdEqualsExpression(BaseThreadRevision newRevision) => - existingRevision => existingRevision.Tid == newRevision.Tid; - public override bool UserFieldUpdateIgnorance (string propName, object? oldValue, object? newValue) => propName switch { // Icon.SpriteInfo will be an empty array and the icon url is a smaller one @@ -39,6 +35,11 @@ public override SaverChangeSet Save(CrawlerDbContext db) => th => new ThreadRevision {TakenAt = th.UpdatedAt ?? th.CreatedAt, Tid = th.Tid}, PredicateBuilder.New(th => Posts.Keys.Contains(th.Tid))); + protected override Tid RevisionEntityIdSelector(BaseThreadRevision entity) => entity.Tid; + protected override Expression> + IsRevisionEntityIdEqualsExpression(BaseThreadRevision newRevision) => + existingRevision => existingRevision.Tid == newRevision.Tid; + protected override bool FieldUpdateIgnorance (string propName, object? oldValue, object? newValue) => propName switch { // will be updated by ThreadLateCrawler and ThreadLateCrawlFacade diff --git a/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs b/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs index e78179a3..2086010b 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs @@ -11,37 +11,22 @@ public abstract partial class SaverWithRevision( protected abstract Lazy> AddSplitRevisionsDelegatesKeyByEntityType { get; } - protected abstract NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName); - - protected abstract TEntityId RevisionEntityIdSelector(TBaseRevision entity); - protected abstract Expression> - IsRevisionEntityIdEqualsExpression(TBaseRevision newRevision); - - protected virtual bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry) => false; - protected virtual bool FieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => false; - protected virtual bool FieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => false; - private static bool GlobalFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch - { // possible rarely respond with the protoBuf default value 0 - nameof(BasePost.AuthorUid) when newValue is 0L && oldValue is not null => true, - _ => false - }; - protected void AddSplitRevisions(CrawlerDbContext db, IEnumerable revisions) where TRevision : TBaseRevision { var newRevisions = revisions.OfType().ToList(); var dbSet = db.Set(); var visitor = new ReplaceParameterTypeVisitor(); + + // https://github.com/npgsql/npgsql/issues/4437 + // https://github.com/dotnet/efcore/issues/32092 var existingRevisions = dbSet .Where(newRevisions.Aggregate( - - // https://github.com/npgsql/npgsql/issues/4437 - // https://github.com/dotnet/efcore/issues/32092 LinqKit.PredicateBuilder.New(), (predicate, newRevision) => predicate.Or(LinqKit.PredicateBuilder .New(existingRevision => existingRevision.TakenAt == newRevision.TakenAt) - .And((Expression>) - visitor.Visit(IsRevisionEntityIdEqualsExpression(newRevision)))))) + .And((Expression>)visitor + .Visit(IsRevisionEntityIdEqualsExpression(newRevision)))))) .ToList(); (from existingRevision in existingRevisions join newRevision in newRevisions @@ -51,6 +36,21 @@ on RevisionEntityIdSelector(existingRevision) equals RevisionEntityIdSelector(ne t.newRevision.DuplicateIndex = (ushort)(t.existingRevision.DuplicateIndex + 1)); dbSet.AddRange(newRevisions); } + + protected abstract NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName); + + protected abstract TEntityId RevisionEntityIdSelector(TBaseRevision entity); + protected abstract Expression> + IsRevisionEntityIdEqualsExpression(TBaseRevision newRevision); + + protected virtual bool ShouldIgnoreEntityRevision(string propName, PropertyEntry propEntry, EntityEntry entityEntry) => false; + protected virtual bool FieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => false; + protected virtual bool FieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => false; + private static bool GlobalFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch + { // possible rarely respond with the protoBuf default value 0 + nameof(BasePost.AuthorUid) when newValue is 0L && oldValue is not null => true, + _ => false + }; } public abstract partial class SaverWithRevision {