Skip to content

Commit

Permalink
* move static field GlobalLocks to primary ctor param `globalLocked…
Browse files Browse the repository at this point in the history
…` to fix different instance sharing the same type param `TKey` will also share this static field

* rename field `_localLocks` to `_localLocked`
* rename method `AcquireLocks()` to `Acquire()`
@ SaverLocks.cs

+ static field `GlobalLocked*` & field `_*saverLocks` for primary ctor param `saverLocksFactory` @ class `UserSaver`, `ReplySignatureSaver` & `AuthorRevisionSaver`
@ c#/crawler
  • Loading branch information
n0099 committed Jun 3, 2024
1 parent 686d845 commit c949474
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 22 deletions.
12 changes: 8 additions & 4 deletions c#/crawler/src/Tieba/Crawl/Saver/AuthorRevisionSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,20 @@ namespace tbm.Crawler.Tieba.Crawl.Saver;
// this prevents inserting multiple entities with similar time and other fields with the same values
public class AuthorRevisionSaver(
ILogger<AuthorRevisionSaver> logger,
SaverLocks<(Fid Fid, Uid Uid)> authorExpGradeLocks,
SaverLocks<(Fid Fid, Uid Uid)>.New saverLocksFactory,
PostType triggeredByPostType)
{
public delegate AuthorRevisionSaver New(PostType triggeredByPostType);

private static readonly HashSet<(Fid Fid, Uid Uid)> GlobalLockedAuthorExpGradeKeys = [];
private readonly Lazy<SaverLocks<(Fid Fid, Uid Uid)>> _authorExpGradeLocksSaverLocks =
new(() => saverLocksFactory(GlobalLockedAuthorExpGradeKeys));

public Action SaveAuthorExpGradeRevisions<TPostWithAuthorExpGrade>
(CrawlerDbContext db, IReadOnlyCollection<TPostWithAuthorExpGrade> posts)
where TPostWithAuthorExpGrade : PostWithAuthorExpGrade
{
Save(db, posts, authorExpGradeLocks,
Save(db, posts, _authorExpGradeLocksSaverLocks.Value,
db.AuthorExpGradeRevisions,
p => p.AuthorExpGrade,
(a, b) => a != b,
Expand All @@ -32,7 +36,7 @@ public Action SaveAuthorExpGradeRevisions<TPostWithAuthorExpGrade>
TriggeredBy = triggeredByPostType,
AuthorExpGrade = t.Value
});
return authorExpGradeLocks.Dispose;
return _authorExpGradeLocksSaverLocks.Value.Dispose;
}

private void Save<TPost, TRevision, TValue>(
Expand Down Expand Up @@ -83,7 +87,7 @@ private void Save<TPost, TRevision, TValue>(
.Select(revisionFactory)
.ToDictionary(revision => (revision.Fid, revision.Uid), revision => revision);
db.Set<TRevision>().AddRange(newRevisions
.IntersectByKey(locks.AcquireLocks(newRevisions.Keys))
.IntersectByKey(locks.Acquire(newRevisions.Keys))
.Values());
}

Expand Down
10 changes: 7 additions & 3 deletions c#/crawler/src/Tieba/Crawl/Saver/ReplySignatureSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,12 @@ namespace tbm.Crawler.Tieba.Crawl.Saver;

public class ReplySignatureSaver(
ILogger<ReplySignatureSaver> logger,
SaverLocks<ReplySignatureSaver.UniqueSignature> locks)
SaverLocks<ReplySignatureSaver.UniqueSignature>.New saverLocksFactory)
{
private static readonly HashSet<UniqueSignature> GlobalLockedSignatures = [];
private readonly Lazy<SaverLocks<UniqueSignature>> _saverLocks =
new(() => saverLocksFactory(GlobalLockedSignatures));

public Action Save(CrawlerDbContext db, IEnumerable<ReplyPost> replies)
{
SharedHelper.GetNowTimestamp(out var now);
Expand Down Expand Up @@ -51,11 +55,11 @@ join newInReply in signatures on existing.SignatureId equals newInReply.Signatur

var newSignatures = signatures
.ExceptBy(existingSignatures.Select(s => s.SignatureId), s => s.SignatureId).ToList();
var newlyLocked = locks.AcquireLocks(
var newlyLocked = _saverLocks.Value.Acquire(
newSignatures.Select(s => new UniqueSignature(s.SignatureId, s.XxHash3)).ToList());
db.ReplySignatures.AddRange(
newSignatures.IntersectBy(newlyLocked, s => new(s.SignatureId, s.XxHash3)));
return locks.Dispose;
return _saverLocks.Value.Dispose;
}

[SuppressMessage("Class Design", "AV1000:Type name contains the word 'and', which suggests it has multiple purposes")]
Expand Down
21 changes: 11 additions & 10 deletions c#/crawler/src/Tieba/Crawl/Saver/SaverLocks.cs
Original file line number Diff line number Diff line change
@@ -1,26 +1,27 @@
namespace tbm.Crawler.Tieba.Crawl.Saver;

public sealed class SaverLocks<TKey> : IDisposable
public sealed class SaverLocks<TKey>(ISet<TKey> globalLocked) : IDisposable
{
private static readonly HashSet<TKey> GlobalLocks = [];
private readonly List<TKey> _localLocks = [];
public delegate SaverLocks<TKey> New(ISet<TKey> globalLocked);

private readonly List<TKey> _localLocked = [];

public void Dispose()
{
lock (GlobalLocks) GlobalLocks.ExceptWith(_localLocks);
lock (_localLocks) _localLocks.Clear();
lock (globalLocked) globalLocked.ExceptWith(_localLocked);
lock (_localLocked) _localLocked.Clear();
}

public IReadOnlyCollection<TKey> AcquireLocks(IReadOnlyCollection<TKey> pendingLocking)
public IReadOnlyCollection<TKey> Acquire(IReadOnlyCollection<TKey> pendingLocking)
{
if (pendingLocking.Count == 0) return [];
var newlyLocked = new List<TKey>(pendingLocking.Count);
lock (GlobalLocks)
lock (globalLocked)
{
newlyLocked.AddRange(pendingLocking.Except(GlobalLocks));
GlobalLocks.UnionWith(newlyLocked);
newlyLocked.AddRange(pendingLocking.Except(globalLocked));
globalLocked.UnionWith(newlyLocked);
}
lock (_localLocks) _localLocks.AddRange(newlyLocked);
lock (_localLocked) _localLocked.AddRange(newlyLocked);
return newlyLocked;
}
}
14 changes: 9 additions & 5 deletions c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,26 @@
namespace tbm.Crawler.Tieba.Crawl.Saver;

public partial class UserSaver(
ILogger<UserSaver> logger, SaverLocks<Uid> locks,
ILogger<UserSaver> logger,
SaverLocks<Uid>.New saverLocksFactory,
IDictionary<Uid, User> users)
: SaverWithRevision<BaseUserRevision, Uid>(logger)
{
public delegate UserSaver New(IDictionary<Uid, User> users);
public delegate bool FieldChangeIgnorance(string propName, object? oldValue, object? newValue);

private static readonly HashSet<Uid> GlobalLockedUid = [];
private readonly Lazy<SaverLocks<Uid>> _saverLocks =
new(() => saverLocksFactory(GlobalLockedUid));

public void Save(
CrawlerDbContext db,
PostType postType,
FieldChangeIgnorance userFieldUpdateIgnorance,
FieldChangeIgnorance userFieldRevisionIgnorance)
{
if (users.Count == 0) return;
var newlyLocked = locks.AcquireLocks(users.Keys().ToList());
var newlyLocked = _saverLocks.Value.Acquire(users.Keys().ToList());

// existingUsers may have new revisions to insert so excluding already locked users
// to prevent inserting duplicate revision
Expand All @@ -39,10 +44,9 @@ where newlyLocked.Contains(user.Uid)
}

public IEnumerable<Uid> AcquireUidLocksForSave(IEnumerable<Uid> usersId) =>
locks.AcquireLocks(usersId.ToList());
_saverLocks.Value.Acquire(usersId.ToList());

[SuppressMessage("IDisposableAnalyzers.Correctness", "IDISP007:Don't dispose injected")]
public void OnPostSave() => locks.Dispose();
public void OnPostSave() => _saverLocks.Value.Dispose();
}
public partial class UserSaver
{
Expand Down

0 comments on commit c949474

Please sign in to comment.