Skip to content

Commit

Permalink
* split abstract class CommonInSavers<> into two abstract classes `…
Browse files Browse the repository at this point in the history
…BaseSaver<>` & `SaverWithRevision<>` to fix `AV1000: Type '' contains the word 'and', which suggests it has multiple purposes`

* split abstract class `StaticCommonInSavers` into interface `IFieldChangeIgnorance` & `IRevisionProperties` to fix `AV1000`
@ c#/crawler
  • Loading branch information
n0099 committed Mar 30, 2024
1 parent d08413c commit 54ae1be
Show file tree
Hide file tree
Showing 10 changed files with 51 additions and 38 deletions.
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public class ThreadCrawlFacade(
protected override void BeforeCommitSaveHook(CrawlerDbContext db, UserSaver userSaver)
{ // BeforeCommitSaveHook() should get invoked after UserSaver.Save() by the base.SaveCrawled()
// so only latest repliers that not exists in parsed users are being inserted
// note this will bypass user revision detection since not invoking CommonInSavers.SavePostsOrUsers() but directly DbContext.AddRange()
// note this will bypass user revision detection since not invoking BaseSaver.SavePostsOrUsers() but directly DbContext.AddRange()

// users has already been added into DbContext and tracking
var existingUsersId = db.ChangeTracker.Entries<User>().Select(ee => ee.Entity.Uid);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,13 @@

namespace tbm.Crawler.Tieba.Crawl.Saver;

public abstract class CommonInSavers<TBaseRevision>(ILogger<CommonInSavers<TBaseRevision>> logger)
: StaticCommonInSavers
public abstract class BaseSaver<TBaseRevision>(ILogger<BaseSaver<TBaseRevision>> logger)
: SaverWithRevision<TBaseRevision>, IFieldChangeIgnorance
where TBaseRevision : class, IRevision
{
protected delegate void RevisionUpsertDelegate(CrawlerDbContext db, IEnumerable<TBaseRevision> revision);

protected virtual IDictionary<Type, RevisionUpsertDelegate>
RevisionUpsertDelegatesKeyBySplitEntityType => throw new NotSupportedException();

protected void SavePostsOrUsers<TPostOrUser, TRevision>(
CrawlerDbContext db,
FieldChangeIgnoranceDelegates userFieldChangeIgnorance,
IFieldChangeIgnorance.FieldChangeIgnoranceDelegates userFieldChangeIgnorance,
Func<TPostOrUser, TRevision> revisionFactory,
ILookup<bool, TPostOrUser> existingOrNewLookup,
Func<TPostOrUser, TPostOrUser> existingSelector)
Expand Down Expand Up @@ -48,21 +43,21 @@ bool IsTimestampingFieldName(string name) => name is nameof(IPost.LastSeenAt)
var pName = p.Metadata.Name;
if (!p.IsModified || IsTimestampingFieldName(pName)) continue;

if (GlobalFieldChangeIgnorance.Update(whichPostType, pName, p.OriginalValue, p.CurrentValue)
if (IFieldChangeIgnorance.GlobalFieldChangeIgnorance.Update(whichPostType, pName, p.OriginalValue, p.CurrentValue)
|| (entryIsUser && userFieldChangeIgnorance.Update(
whichPostType, pName, p.OriginalValue, p.CurrentValue)))
{
p.IsModified = false;
continue; // skip following revision check
}
if (GlobalFieldChangeIgnorance.Revision(whichPostType, pName, p.OriginalValue, p.CurrentValue)
if (IFieldChangeIgnorance.GlobalFieldChangeIgnorance.Revision(whichPostType, pName, p.OriginalValue, p.CurrentValue)
|| (entryIsUser && userFieldChangeIgnorance.Revision(
whichPostType, pName, p.OriginalValue, p.CurrentValue)))
continue;

if (IsLatestReplierUser(pName, p, entry)) return null;

if (!RevisionPropertiesCache[typeof(TRevision)].TryGetValue(pName, out var revisionProp))
if (!IRevisionProperties.Cache[typeof(TRevision)].TryGetValue(pName, out var revisionProp))
{
object? ToHexWhenByteArray(object? value) =>
value is byte[] bytes ? $"0x{Convert.ToHexString(bytes).ToLowerInvariant()}" : value;
Expand Down Expand Up @@ -109,14 +104,12 @@ bool IsTimestampingFieldName(string name) => name is nameof(IPost.LastSeenAt)
.ForEach(g => RevisionUpsertDelegatesKeyBySplitEntityType[g.Key](db, g));
}

protected virtual NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => throw new NotSupportedException();

private static bool IsLatestReplierUser(string pName, PropertyEntry p, EntityEntry entry)
{
// ThreadCrawlFacade.ParseLatestRepliers() will save users with empty string as portrait
// they will soon be updated by (sub) reply crawler after it find out the latest reply
// so we should ignore its revision update for all fields
// ignore entire record is not possible via GlobalFieldChangeIgnorance.Revision()
// ignore entire record is not possible via IFieldChangeIgnorance.GlobalFieldChangeIgnorance.Revision()
// since it can only determine one field at the time
if (pName != nameof(User.Portrait) || p.OriginalValue is not "") return false;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
namespace tbm.Crawler.Tieba.Crawl.Saver;

public abstract class StaticCommonInSavers
public partial interface IFieldChangeIgnorance
{
public delegate bool FieldChangeIgnoranceDelegate(
Type whichPostType, string propName, object? oldValue, object? newValue);

// static field in this non-generic class will be shared across all reified generic derived classes
protected static IDictionary<Type, IDictionary<string, PropertyInfo>> RevisionPropertiesCache { get; } = GetPropsKeyByType(
[typeof(ThreadRevision), typeof(ReplyRevision), typeof(SubReplyRevision), typeof(UserRevision)]);

public record FieldChangeIgnoranceDelegates(
FieldChangeIgnoranceDelegate Update,
FieldChangeIgnoranceDelegate Revision);
}
public partial interface IFieldChangeIgnorance
{
protected static FieldChangeIgnoranceDelegates GlobalFieldChangeIgnorance { get; } = new(
Update: (whichPostType, propName, oldValue, newValue) =>
{
Expand Down Expand Up @@ -90,13 +92,4 @@ when newValue is ""
}
return false;
});

[SuppressMessage("Performance", "CA1859:Use concrete types when possible for improved performance")]
private static IDictionary<Type, IDictionary<string, PropertyInfo>> GetPropsKeyByType(IEnumerable<Type> types) =>
types.ToDictionary(type => type, type =>
(IDictionary<string, PropertyInfo>)type.GetProperties().ToDictionary(prop => prop.Name));

public record FieldChangeIgnoranceDelegates(
FieldChangeIgnoranceDelegate Update,
FieldChangeIgnoranceDelegate Revision);
}
11 changes: 11 additions & 0 deletions c#/crawler/src/Tieba/Crawl/Saver/IRevisionProperties.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
namespace tbm.Crawler.Tieba.Crawl.Saver;

public interface IRevisionProperties
{
protected static IDictionary<Type, IDictionary<string, PropertyInfo>> Cache { get; } = GetPropsKeyByType(
[typeof(ThreadRevision), typeof(ReplyRevision), typeof(SubReplyRevision), typeof(UserRevision)]);

private static IDictionary<Type, IDictionary<string, PropertyInfo>> GetPropsKeyByType(IEnumerable<Type> types) =>
types.ToDictionary(type => type, type =>
(IDictionary<string, PropertyInfo>)type.GetProperties().ToDictionary(prop => prop.Name));
}
8 changes: 4 additions & 4 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/BasePostSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ public abstract class BasePostSaver<TPost, TBaseRevision>(
ConcurrentDictionary<PostId, TPost> posts,
AuthorRevisionSaver.New authorRevisionSaverFactory,
string postType)
: CommonInSavers<TBaseRevision>(logger)
: BaseSaver<TBaseRevision>(logger)
where TPost : class, IPost
where TBaseRevision : class, IRevision
{
protected delegate void PostSaveEventHandler();
[SuppressMessage("Design", "MA0046:Use EventHandler<T> to declare events")]
protected event PostSaveEventHandler PostSaveEvent = () => { };

public virtual FieldChangeIgnoranceDelegates UserFieldChangeIgnorance =>
throw new NotSupportedException();
public virtual IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
UserFieldChangeIgnorance => throw new NotSupportedException();
public string PostType { get; } = postType;
protected ConcurrentDictionary<PostId, TPost> Posts { get; } = posts;
protected AuthorRevisionSaver AuthorRevisionSaver { get; } = authorRevisionSaverFactory(postType);
Expand All @@ -35,7 +35,7 @@ protected SaverChangeSet<TPost> Save<TRevision>(

var existingPostsKeyById = dbSet.Where(existingPostPredicate).ToDictionary(postIdSelector);

// deep copy before entities get mutated by CommonInSavers.SavePostsOrUsers()
// deep copy before entities get mutated by BaseSaver.SavePostsOrUsers()
var existingBeforeMerge = existingPostsKeyById.Select(pair => (TPost)pair.Value.Clone()).ToList();

SavePostsOrUsers(db, UserFieldChangeIgnorance, revisionFactory,
Expand Down
3 changes: 2 additions & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ public partial class ReplySaver(
{
public delegate ReplySaver New(ConcurrentDictionary<PostId, ReplyPost> posts);

public override FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; } = new(
public override IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
UserFieldChangeIgnorance { get; } = new(
Update: (_, propName, oldValue, newValue) => propName switch
{ // FansNickname in reply response will always be null
nameof(User.FansNickname) when oldValue is not null && newValue is null => true,
Expand Down
3 changes: 2 additions & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ public class SubReplySaver(
{
public delegate SubReplySaver New(ConcurrentDictionary<PostId, SubReplyPost> posts);

public override FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; } = new(
public override IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
UserFieldChangeIgnorance { get; } = new(
Update: (_, propName, oldValue, newValue) => propName switch
{ // always ignore updates on iconinfo due to some rare user will show some extra icons
// compare to reply response in the response of sub reply
Expand Down
3 changes: 2 additions & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ public class ThreadSaver(
{
public delegate ThreadSaver New(ConcurrentDictionary<Tid, ThreadPost> posts);

public override FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; } = new(
public override IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
UserFieldChangeIgnorance { get; } = new(
Update: (_, propName, _, _) => propName switch
{ // Icon.SpriteInfo will be an empty array and the icon url is a smaller one
// so we should mark it as null temporarily
Expand Down
13 changes: 13 additions & 0 deletions c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
namespace tbm.Crawler.Tieba.Crawl.Saver;

public abstract class SaverWithRevision<TBaseRevision> : IRevisionProperties
where TBaseRevision : class, IRevision
{
protected delegate void RevisionUpsertDelegate(CrawlerDbContext db, IEnumerable<TBaseRevision> revision);

protected virtual IDictionary<Type, RevisionUpsertDelegate> RevisionUpsertDelegatesKeyBySplitEntityType =>
throw new NotSupportedException();

protected virtual NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) =>
throw new NotSupportedException();
}
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ protected override Dictionary<Type, RevisionUpsertDelegate>
};
}
public partial class UserSaver(ILogger<UserSaver> logger, ConcurrentDictionary<Uid, User> users)
: CommonInSavers<BaseUserRevision>(logger)
: BaseSaver<BaseUserRevision>(logger)
{
private static readonly HashSet<Uid> UserIdLocks = [];
private readonly List<Uid> _savedUsersId = [];
Expand All @@ -42,7 +42,7 @@ public partial class UserSaver(ILogger<UserSaver> logger, ConcurrentDictionary<U
public void Save(
CrawlerDbContext db,
string postType,
FieldChangeIgnoranceDelegates userFieldChangeIgnorance)
IFieldChangeIgnorance.FieldChangeIgnoranceDelegates userFieldChangeIgnorance)
{
if (users.IsEmpty) return;
lock (UserIdLocks)
Expand Down

0 comments on commit 54ae1be

Please sign in to comment.