Skip to content

Commit

Permalink
* rename variables whose type inherits from BaseCrawlFacade from `c…
Browse files Browse the repository at this point in the history
…rawler` to `facade` @ `crawler.CrawlPost`, `ArchiveCrawlWorker` & `RetryCrawlWorker`

* fix `Assembly.GetExecutingAssembly()` will always returning the one with `tbm.Shared` by adding param to inject it from caller @ `shared.ExtensionMethods.RegisterImplementsOfBaseTypes()`
* suppress Roslyn analyzer rule `MA0076:Do not use implicit culture-sensitive ToString in interpolated strings` due to https://stackoverflow.com/questions/8492449/is-int32-tostring-culture-specific @ GlobalSuppressions.cs
@ c#
  • Loading branch information
n0099 committed Mar 30, 2024
1 parent 54ae1be commit 07b6acc
Show file tree
Hide file tree
Showing 11 changed files with 38 additions and 32 deletions.
1 change: 1 addition & 0 deletions c#/GlobalSuppressions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
[assembly: SuppressMessage("Design", "CC0120:Your Switch maybe include default clause")]
[assembly: SuppressMessage("Design", "MA0048:File name must match type name")]
[assembly: SuppressMessage("Design", "MA0051:Method is too long")]
[assembly: SuppressMessage("Design", "MA0076:Do not use implicit culture-sensitive ToString in interpolated strings", Justification = "https://stackoverflow.com/questions/8492449/is-int32-tostring-culture-specific")]
[assembly: SuppressMessage("Documentation", "AV2305:Missing XML comment for internally visible type, member or parameter")]
[assembly: SuppressMessage("Framework", "AV2220:Simple query should be replaced by extension method call")]
[assembly: SuppressMessage("Maintainability", "AV1500:Member or local function contains too many statements")]
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/EntryPoint.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ protected override void ConfigureServices(HostBuilderContext context, IServiceCo
[SuppressMessage("Style", "IDE0058:Expression value is never used")]
protected override void ConfigureContainer(HostBuilderContext context, ContainerBuilder builder)
{
builder.RegisterImplementsOfBaseTypes(
builder.RegisterImplementsOfBaseTypes(typeof(EntryPoint).Assembly,
[
typeof(BaseCrawler<,>), typeof(BaseCrawlFacade<,,,>),
typeof(BaseParser<,>), typeof(CommonInSavers<>)
typeof(BasePostParser<,>), typeof(BaseSaver<>)
]);
builder.RegisterType<CrawlerDbContext>();
builder.RegisterType<ClientRequester>();
Expand Down
16 changes: 8 additions & 8 deletions c#/crawler/src/Tieba/Crawl/CrawlPost.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ public async Task<SavedThreadsList> CrawlThreads
{
crawlingPage++;
await using var facadeFactory = threadCrawlFacadeFactory();
var crawler = facadeFactory.Value(fid, forumName);
var currentPageChangeSet = (await crawler.CrawlPageRange(
var facade = facadeFactory.Value(fid, forumName);
var currentPageChangeSet = (await facade.CrawlPageRange(
crawlingPage, crawlingPage, stoppingToken)).SaveCrawled(stoppingToken);
if (currentPageChangeSet != null)
{
Expand All @@ -60,8 +60,8 @@ await Task.WhenAll(savedThreads.Select(async threads =>
if (stoppingToken.IsCancellationRequested) return;
var failureCountsKeyByTid = threads.NewlyAdded
.ToDictionary(th => th.Tid, _ => (FailureCount)0);
await using var threadLateFactory = threadLateCrawlFacadeFactory();
await threadLateFactory.Value(fid).CrawlThenSave(failureCountsKeyByTid, stoppingToken);
await using var threadLateFacade = threadLateCrawlFacadeFactory();
await threadLateFacade.Value(fid).CrawlThenSave(failureCountsKeyByTid, stoppingToken);
}));

return savedThreads;
Expand Down Expand Up @@ -89,10 +89,10 @@ await Task.WhenAll(shouldCrawlParentPosts.Select(async tid =>
{
if (stoppingToken.IsCancellationRequested) return;
await using var facadeFactory = replyCrawlFacadeFactory();
var crawler = facadeFactory.Value(fid, tid).AddExceptionHandler(
var facade = facadeFactory.Value(fid, tid).AddExceptionHandler(
SaveThreadMissingFirstReply(fid, tid, savedThreads).Invoke);
savedRepliesKeyByTid.SetIfNotNull(tid,
(await crawler.CrawlPageRange(1, stoppingToken: stoppingToken)).SaveCrawled(stoppingToken));
(await facade.CrawlPageRange(1, stoppingToken: stoppingToken)).SaveCrawled(stoppingToken));
}));
return savedRepliesKeyByTid;
}
Expand Down Expand Up @@ -121,8 +121,8 @@ await Task.WhenAll(shouldCrawlParentPosts.Select(async t =>
if (stoppingToken.IsCancellationRequested) return;
var (tid, pid) = t;
await using var facadeFactory = subReplyCrawlFacadeFactory();
var crawler = facadeFactory.Value(fid, tid, pid);
_ = (await crawler.CrawlPageRange(1, stoppingToken: stoppingToken))
var facade = facadeFactory.Value(fid, tid, pid);
_ = (await facade.CrawlPageRange(1, stoppingToken: stoppingToken))
.SaveCrawled(stoppingToken);
}));
}
Expand Down
2 changes: 2 additions & 0 deletions c#/crawler/src/Tieba/Crawl/Saver/BaseSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
namespace tbm.Crawler.Tieba.Crawl.Saver;

public abstract class BaseSaver<TBaseRevision>(ILogger<BaseSaver<TBaseRevision>> logger)
#pragma warning disable S1939 // Inheritance list should not be redundant
: SaverWithRevision<TBaseRevision>, IFieldChangeIgnorance
#pragma warning restore S1939 // Inheritance list should not be redundant
where TBaseRevision : class, IRevision
{
protected void SavePostsOrUsers<TPostOrUser, TRevision>(
Expand Down
1 change: 1 addition & 0 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/BasePostSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public virtual IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
protected ConcurrentDictionary<PostId, TPost> Posts { get; } = posts;
protected AuthorRevisionSaver AuthorRevisionSaver { get; } = authorRevisionSaverFactory(postType);

[SuppressMessage("Misc", "AV1225:Method that raises an event should be protected virtual and be named 'On' followed by event name")]
public void OnPostSaveEvent() => PostSaveEvent();
public abstract SaverChangeSet<TPost> Save(CrawlerDbContext db);

Expand Down
12 changes: 6 additions & 6 deletions c#/crawler/src/Worker/ArchiveCrawlWorker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -118,17 +118,17 @@ private async Task<SaverChangeSet<ThreadPost>?> CrawlThreads
(Page page, string forumName, Fid fid, CancellationToken stoppingToken = default)
{
await using var facadeFactory = threadArchiveCrawlFacadeFactory();
var crawler = facadeFactory.Value(fid, forumName);
var savedThreads = (await crawler.CrawlPageRange(
var facade = facadeFactory.Value(fid, forumName);
var savedThreads = (await facade.CrawlPageRange(
page, page, stoppingToken)).SaveCrawled(stoppingToken);

// ReSharper disable once InvertIf
if (savedThreads != null)
{
var failureCountsKeyByTid = savedThreads.NewlyAdded
.ToDictionary(th => th.Tid, _ => (FailureCount)0);
await using var threadLate = threadLateCrawlFacadeFactory();
await threadLate.Value(fid).CrawlThenSave(failureCountsKeyByTid, stoppingToken);
await using var threadLateFacade = threadLateCrawlFacadeFactory();
await threadLateFacade.Value(fid).CrawlThenSave(failureCountsKeyByTid, stoppingToken);
}
return savedThreads;
}
Expand All @@ -147,9 +147,9 @@ await Task.WhenAll(savedThreads.AllAfter.Select(th => th.Tid).Distinct().Select(
{
if (stoppingToken.IsCancellationRequested) return;
await using var facadeFactory = replyCrawlFacadeFactory();
var crawler = facadeFactory.Value(fid, tid);
var facade = facadeFactory.Value(fid, tid);
savedRepliesKeyByTid.SetIfNotNull(tid,
(await crawler.CrawlPageRange(1, stoppingToken: stoppingToken)).SaveCrawled(stoppingToken));
(await facade.CrawlPageRange(1, stoppingToken: stoppingToken)).SaveCrawled(stoppingToken));
}));
return savedRepliesKeyByTid;
}
Expand Down
22 changes: 11 additions & 11 deletions c#/crawler/src/Worker/RetryCrawlWorker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ private async Task RetryThreadLate(
IReadOnlyDictionary<CrawlerLocks.LockId, IReadOnlyDictionary<Page, FailureCount>> failureCountWithPagesKeyByLockId,
CancellationToken stoppingToken = default)
{
await using var threadLate = threadLateCrawlFacadeFactory();
await using var threadLateFacade = threadLateCrawlFacadeFactory();
foreach (var tidGroupByFid in failureCountWithPagesKeyByLockId
.Keys.GroupBy(lockId => lockId.Fid, lockId => lockId.Tid))
{
Expand All @@ -71,7 +71,7 @@ FailureCount FailureCountSelector(Tid tid) =>
.Cast<Tid>().ToDictionary(tid => tid, FailureCountSelector);
logger.LogTrace("Retrying previous failed thread late crawl with fid={}, threadsId={}",
fid, Helper.UnescapedJsonSerialize(tidGroupByFid));
await threadLate.Value(fid).CrawlThenSave(failureCountsKeyByTid, stoppingToken);
await threadLateFacade.Value(fid).CrawlThenSave(failureCountsKeyByTid, stoppingToken);
}
}

Expand All @@ -95,9 +95,9 @@ from f in dbFactory.Value().Forums.AsNoTracking()

logger.LogTrace("Retrying previous failed {} pages in thread crawl for fid={}, forumName={}",
failureCount, fid, forumName);
await using var crawlerFactory = threadCrawlFacadeFactory();
var crawler = crawlerFactory.Value(fid, forumName);
var savedThreads = await crawler.RetryThenSave(pages, failureCountSelector, stoppingToken);
await using var facadeFactory = threadCrawlFacadeFactory();
var facade = facadeFactory.Value(fid, forumName);
var savedThreads = await facade.RetryThenSave(pages, failureCountSelector, stoppingToken);
if (savedThreads == null) return;
var savedReplies = await crawlPost.CrawlReplies
([savedThreads], fid, stoppingToken);
Expand All @@ -113,9 +113,9 @@ private async Task RetryReply(
{
logger.LogTrace("Retrying previous failed {} pages reply crawl for fid={}, tid={}",
failureCount, fid, tid);
await using var crawlerFactory = replyCrawlFacadeFactory();
var crawler = crawlerFactory.Value(fid, tid);
var savedReplies = await crawler.RetryThenSave(pages, failureCountSelector, stoppingToken);
await using var facadeFactory = replyCrawlFacadeFactory();
var facade = facadeFactory.Value(fid, tid);
var savedReplies = await facade.RetryThenSave(pages, failureCountSelector, stoppingToken);
if (savedReplies == null) return;
var savedRepliesKeyByTid = new Dictionary<PostId, SaverChangeSet<ReplyPost>> {{tid, savedReplies}};
await crawlPost.CrawlSubReplies(savedRepliesKeyByTid, fid, stoppingToken);
Expand All @@ -130,8 +130,8 @@ private async Task RetrySubReply(
{
logger.LogTrace("Retrying previous failed {} pages sub reply crawl for fid={}, tid={}, pid={}",
failureCount, fid, tid, pid);
await using var crawlerFactory = subReplyCrawlFacadeFactory();
var crawler = crawlerFactory.Value(fid, tid, pid);
_ = await crawler.RetryThenSave(pages, failureCountSelector, stoppingToken);
await using var facadeFactory = subReplyCrawlFacadeFactory();
var facade = facadeFactory.Value(fid, tid, pid);
_ = await facade.RetryThenSave(pages, failureCountSelector, stoppingToken);
}
}
3 changes: 2 additions & 1 deletion c#/imagePipeline/src/Consumer/MetadataConsumer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ private static partial class ExifDateTimeTagValuesParser
?? ParseAsUnixTimestamp(exifDateTime)
?? throw new ArgumentException(
$"Failed to parse provided EXIF date time \"{exifDateTime}\""
+ $" with fractional seconds {fractionalSeconds.ToString(CultureInfo.InvariantCulture)}.");
+ $" with fractional seconds {fractionalSeconds}.");
return fractionalSeconds == 0 ? ret : ret with
{
DateTime = ret.DateTime.AddSeconds(fractionalSeconds / Math.Pow(10, CountDigits(fractionalSeconds)))
Expand Down Expand Up @@ -376,6 +376,7 @@ private static partial class ExifDateTimeTagValuesParser
}
: null;

[SuppressMessage("Performance", "CA1852:Seal internal types")]
public record DateTimeAndOffset(DateTime DateTime, string? Offset);
}
}
2 changes: 1 addition & 1 deletion c#/imagePipeline/src/EntryPoint.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ protected override void ConfigureServices(HostBuilderContext context, IServiceCo
[SuppressMessage("Style", "IDE0058:Expression value is never used")]
protected override void ConfigureContainer(HostBuilderContext context, ContainerBuilder builder)
{
builder.RegisterImplementsOfBaseTypes([typeof(IConsumer<>)]);
builder.RegisterImplementsOfBaseTypes(typeof(EntryPoint).Assembly, [typeof(IConsumer<>)]);
builder.RegisterType<ImagePipelineDbContext>();
builder.RegisterType<JointRecognizer>();
builder.RegisterType<ImageRequester>();
Expand Down
2 changes: 1 addition & 1 deletion c#/imagePipeline/src/ImageBatchConsumingWorker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ ImageKeyWithMatrix DecodeFrame(ImageFrame<Rgb24> frame, int frameIndex)
#pragma warning restore IDISP001 // Dispose created
return frameMat.Empty()
? throw new InvalidOperationException(
$"Failed to decode frame {frameIndex.ToString(CultureInfo.InvariantCulture)} of image {imageId}.")
$"Failed to decode frame {frameIndex} of image {imageId}.")
: new(imageId, (uint)frameIndex, frameMat);
}

Expand Down
5 changes: 3 additions & 2 deletions c#/shared/src/ExtensionMethods.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ public static void AddRange<T>(this IList<T> list, IEnumerable<T> items)
}
public static partial class ExtensionMethods
{
public static void RegisterImplementsOfBaseTypes(this ContainerBuilder builder, IEnumerable<Type> baseTypes) =>
builder.RegisterAssemblyTypes(Assembly.GetExecutingAssembly())
public static void RegisterImplementsOfBaseTypes
(this ContainerBuilder builder, Assembly assembly, IEnumerable<Type> baseTypes) =>
builder.RegisterAssemblyTypes(assembly)
.Where(type => baseTypes.Any(baseType => baseType.IsSubTypeOfRawGeneric(type)))
.AsSelf();

Expand Down

0 comments on commit 07b6acc

Please sign in to comment.