Skip to content

Commit

Permalink
* split into two partial classes @ ReplySaver.cs
Browse files Browse the repository at this point in the history
@ crawler

* reorder methods
* expand list with newline
@ c#
  • Loading branch information
n0099 committed May 15, 2024
1 parent a0f48f3 commit 7d929b0
Show file tree
Hide file tree
Showing 6 changed files with 33 additions and 28 deletions.
13 changes: 6 additions & 7 deletions c#/crawler/src/Tieba/Crawl/Crawler/ThreadCrawler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,10 @@ protected override IEnumerable<Request> GetRequestsForPage(Page page, Cancellati
];
}

protected ThreadRequest.Types.Data GetRequestDataForClientVersion602(Page page) =>
new()
{
Kw = forumName,
Pn = (int)page,
Rn = 30
};
protected ThreadRequest.Types.Data GetRequestDataForClientVersion602(Page page) => new()
{
Kw = forumName,
Pn = (int)page,
Rn = 30
};
}
8 changes: 7 additions & 1 deletion c#/crawler/src/Tieba/Crawl/Parser/UserParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,13 @@ public void Parse(IEnumerable<TbClient.User> inUsers) =>
var (portrait, portraitUpdatedAt) = ExtractPortrait(el.Portrait);
if (uid < 0) // historical anonymous user
{
return new() {Uid = uid, Name = el.NameShow, Portrait = portrait, PortraitUpdatedAt = portraitUpdatedAt};
return new()
{
Uid = uid,
Name = el.NameShow,
Portrait = portrait,
PortraitUpdatedAt = portraitUpdatedAt
};
}

// will be an empty string when the user hasn't set a username for their baidu account yet
Expand Down
24 changes: 13 additions & 11 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace tbm.Crawler.Tieba.Crawl.Saver;

public class ReplySaver(
public partial class ReplySaver(
ILogger<ReplySaver> logger,
ConcurrentDictionary<PostId, ReplyPost> posts,
ReplySignatureSaver replySignatureSaver,
Expand Down Expand Up @@ -51,6 +51,15 @@ protected override Dictionary<Type, AddRevisionDelegate>
}
};

[SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")]
protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch
{
nameof(ReplyPost.IsFold) => 1 << 2,
nameof(ReplyPost.DisagreeCount) => 1 << 4,
nameof(ReplyPost.Geolocation) => 1 << 5,
_ => 0
};

public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)
{
var changeSet = Save(db, r => r.Pid,
Expand All @@ -65,16 +74,9 @@ public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)

return changeSet;
}

[SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")]
protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch
{
nameof(ReplyPost.IsFold) => 1 << 2,
nameof(ReplyPost.DisagreeCount) => 1 << 4,
nameof(ReplyPost.Geolocation) => 1 << 5,
_ => 0
};

}
public partial class ReplySaver
{
private static void SaveReplyContentImages(CrawlerDbContext db, IEnumerable<ReplyPost> replies)
{
var pidAndImageList = (
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ protected override Dictionary<Type, AddRevisionDelegate>
}
};

protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => 0;

public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)
{
var changeSet = Save(db, sr => sr.Spid,
Expand All @@ -52,6 +54,4 @@ public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)

return changeSet;
}

protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => 0;
}
10 changes: 5 additions & 5 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,6 @@ protected override Dictionary<Type, AddRevisionDelegate>
}
};

public override SaverChangeSet<ThreadPost> Save(CrawlerDbContext db) =>
Save(db, th => th.Tid,
th => new ThreadRevision {TakenAt = th.UpdatedAt ?? th.CreatedAt, Tid = th.Tid},
PredicateBuilder.New<ThreadPost>(th => Posts.Keys.Contains(th.Tid)));

[SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")]
protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch
{
Expand All @@ -86,4 +81,9 @@ public override SaverChangeSet<ThreadPost> Save(CrawlerDbContext db) =>
nameof(ThreadPost.Geolocation) => 1 << 10,
_ => 0
};

public override SaverChangeSet<ThreadPost> Save(CrawlerDbContext db) =>
Save(db, th => th.Tid,
th => new ThreadRevision {TakenAt = th.UpdatedAt ?? th.CreatedAt, Tid = th.Tid},
PredicateBuilder.New<ThreadPost>(th => Posts.Keys.Contains(th.Tid)));
}
2 changes: 0 additions & 2 deletions c#/shared/src/Db/TbmDbContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,7 @@ protected void OnModelCreatingWithFid(ModelBuilder b, uint fid) =>
b.Entity<ReplyContentImage>().ToTable($"tbmc_f{fid}_reply_content_image");

protected virtual void OnConfiguringNpgsql(NpgsqlDbContextOptionsBuilder builder) { }

protected virtual void OnBuildingNpgsqlDataSource(NpgsqlDataSourceBuilder builder) { }

private Lazy<NpgsqlDataSource> GetNpgsqlDataSource(string? connectionString) =>
_dataSourceSingleton ??= new(() =>

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (crawler)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (crawler)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (imagePipeline)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (imagePipeline)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (shared)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (shared)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)
{
Expand Down

0 comments on commit 7d929b0

Please sign in to comment.