Skip to content

Commit

Permalink
* reorder hooks being invoked to partial revert previous commit 97d426a
Browse files Browse the repository at this point in the history
… @ `CrawlFacade.SaveCrawled()`

* replace event `PostSaveEvent` to auto prop `PostSaveHooks`
* rename delegate `PostSaveEventHandler` to `PostSaveHook`
@ PostSaver.cs
* rename method `OnPostSaveEvent()` to `TriggerPostSave()` @ IPostSaver.cs
@ crawler

+ primary ctor param `ILogger<TbmDbContext> logger` and method `LogDbUpdateConcurrencyException()` @ TbmDbContext
@ shared
@ c#
  • Loading branch information
n0099 committed May 11, 2024
1 parent fd9ac3c commit 0f36b03
Show file tree
Hide file tree
Showing 8 changed files with 31 additions and 26 deletions.
7 changes: 4 additions & 3 deletions c#/crawler/src/Db/CrawlerDbContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@

namespace tbm.Crawler.Db;

public class CrawlerDbContext(Fid fid) : TbmDbContext<CrawlerDbContext.ModelCacheKeyFactory>
public class CrawlerDbContext(ILogger<CrawlerDbContext> logger, Fid fid)
: TbmDbContext<CrawlerDbContext.ModelCacheKeyFactory>(logger)
{
private static Lazy<NpgsqlDataSource>? _dataSourceSingleton;

public CrawlerDbContext() : this(fid: 0) { }
public delegate CrawlerDbContext NewDefault();
public CrawlerDbContext(ILogger<CrawlerDbContext> logger) : this(logger, fid: 0) { }
public delegate CrawlerDbContext NewDefault(ILogger<CrawlerDbContext> logger);
public delegate CrawlerDbContext New(Fid fid);

public Fid Fid { get; } = fid;
Expand Down
16 changes: 7 additions & 9 deletions c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ public virtual void Dispose()
[SuppressMessage("Major Bug", "S1751:Loops with at most one iteration should be refactored")]
public SaverChangeSet<TPost>? SaveCrawled(CancellationToken stoppingToken = default)
{
var db = DbContextFactory(Fid);
using var transaction = db.Database.BeginTransaction(IsolationLevel.ReadCommitted);

while (true)
{
using var db = DbContextFactory(Fid); // dispose after each loop when retrying
using var transaction = db.Database.BeginTransaction(IsolationLevel.ReadCommitted);

var postSaver = postSaverFactory(Posts);
var savedPosts = Posts.IsEmpty ? null : postSaver.Save(db);

Expand All @@ -57,21 +57,19 @@ public virtual void Dispose()
{
db.TimestampingEntities();
_ = db.SaveChanges();
if (savedPosts != null) PostCommitSaveHook(savedPosts, stoppingToken);
postSaver.OnPostSaveEvent();
transaction.Commit();
if (savedPosts != null) PostCommitSaveHook(savedPosts, stoppingToken);
return savedPosts;
}
catch (DbUpdateConcurrencyException e)
{
Logger.LogError(e, "DbUpdateConcurrencyException: {}",
e.Entries.GroupBy(ee => ee.Entity.GetType())
.ToDictionary(g => g.Key, g => g.Count()));
db.LogDbUpdateConcurrencyException(e);
}
finally
{
postSaver.TriggerPostSave();
userSaver.PostSaveHook();
}
return savedPosts;
}
}

Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/Post/IPostSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ public interface IPostSaver<TPost> where TPost : BasePost
{
public IFieldChangeIgnorance.FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; }
public PostType CurrentPostType { get; }
public void OnPostSaveEvent();
public void TriggerPostSave();
public SaverChangeSet<TPost> Save(CrawlerDbContext db);
}
8 changes: 3 additions & 5 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,16 @@ public abstract class PostSaver<TPost, TBaseRevision>(
where TPost : BasePost
where TBaseRevision : BaseRevisionWithSplitting
{
protected delegate void PostSaveEventHandler();
[SuppressMessage("Design", "MA0046:Use EventHandler<T> to declare events")]
protected event PostSaveEventHandler PostSaveEvent = () => { };
protected delegate void PostSaveHook();
protected PostSaveHook PostSaveHooks { get; set; } = () => { };

public virtual IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
UserFieldChangeIgnorance => throw new NotSupportedException();
public PostType CurrentPostType { get; } = currentPostType;
protected ConcurrentDictionary<PostId, TPost> Posts { get; } = posts;
protected AuthorRevisionSaver AuthorRevisionSaver { get; } = authorRevisionSaverFactory(currentPostType);

[SuppressMessage("Misc", "AV1225:Method that raises an event should be protected virtual and be named 'On' followed by event name")]
public void OnPostSaveEvent() => PostSaveEvent();
public void TriggerPostSave() => PostSaveHooks();
public abstract SaverChangeSet<TPost> Save(CrawlerDbContext db);

protected SaverChangeSet<TPost> Save<TRevision>(
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)
db.ReplyContents.AddRange(changeSet.NewlyAdded
.Select(r => new ReplyContent {Pid = r.Pid, ProtoBufBytes = r.Content}));
SaveReplyContentImages(db, changeSet.NewlyAdded);
PostSaveEvent += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
PostSaveEvent += SaveReplySignatures(db, changeSet.AllAfter).Invoke;
PostSaveHooks += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
PostSaveHooks += SaveReplySignatures(db, changeSet.AllAfter).Invoke;

return changeSet;
}
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)

db.SubReplyContents.AddRange(changeSet.NewlyAdded.Select(sr =>
new SubReplyContent {Spid = sr.Spid, ProtoBufBytes = sr.Content}));
PostSaveEvent += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
PostSaveHooks += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;

return changeSet;
}
Expand Down
6 changes: 3 additions & 3 deletions c#/imagePipeline/src/Db/ImagePipelineDbContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

namespace tbm.ImagePipeline.Db;

public class ImagePipelineDbContext(Fid fid, string script)
public class ImagePipelineDbContext(ILogger<ImagePipelineDbContext> logger, Fid fid, string script)
: TbmDbContext<ImagePipelineDbContext.ModelCacheKeyFactory>
{
public ImagePipelineDbContext() : this(fid: 0, script: "") { }
public delegate ImagePipelineDbContext NewDefault();
public ImagePipelineDbContext(ILogger<ImagePipelineDbContext> logger) : this(logger, fid: 0, script: "") { }
public delegate ImagePipelineDbContext NewDefault(ILogger<ImagePipelineDbContext> logger);
public delegate ImagePipelineDbContext New(Fid fid, string script);

public DbSet<ForumScript> ForumScripts => Set<ForumScript>();
Expand Down
12 changes: 10 additions & 2 deletions c#/shared/src/Db/TbmDbContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,13 @@

namespace tbm.Shared.Db;

public abstract class TbmDbContext : DbContext
public abstract class TbmDbContext(ILogger<TbmDbContext> logger) : DbContext
{
public void LogDbUpdateConcurrencyException(DbUpdateConcurrencyException e) =>
logger.LogError(e, "DbUpdateConcurrencyException: {}",
e.Entries.GroupBy(ee => ee.Entity.GetType())
.ToDictionary(g => g.Key, g => g.Count()));

public int SaveChangesForUpdate()
{
while (true)
Expand All @@ -21,6 +26,7 @@ public int SaveChangesForUpdate()
}
catch (DbUpdateConcurrencyException e)
{
LogDbUpdateConcurrencyException(e);
foreach (var entry in e.Entries)
{
var existing = entry.GetDatabaseValues();
Expand All @@ -41,6 +47,7 @@ public async Task<int> SaveChangesForUpdateAsync(CancellationToken stoppingToken
}
catch (DbUpdateConcurrencyException e)
{
LogDbUpdateConcurrencyException(e);
foreach (var entry in e.Entries)
{
var existing = await entry.GetDatabaseValuesAsync(stoppingToken);
Expand Down Expand Up @@ -84,7 +91,8 @@ private sealed class NoSavePointTransaction(IRelationalConnection connection,
}
}
}
public class TbmDbContext<TModelCacheKeyFactory> : TbmDbContext
public class TbmDbContext<TModelCacheKeyFactory>(ILogger<TbmDbContext<TModelCacheKeyFactory>> logger)
: TbmDbContext(logger)
where TModelCacheKeyFactory : class, IModelCacheKeyFactory
{
// ReSharper disable once UnusedAutoPropertyAccessor.Global
Expand Down

0 comments on commit 0f36b03

Please sign in to comment.