Skip to content

Commit

Permalink
* now will dispose instances of ByteArrayContent & `MultipartFormDa…
Browse files Browse the repository at this point in the history
…taContent` that inherits `HttpContent` @ `ClientRequester.PostProtoBuf()`

* implement `IDisposable` pattern to dispose `System.Timers.Timer` @ ClientRequesterTcs.cs & WithLogTrace.cs

* fix `AV1250: Method returns the result of a call to 'Concat', which uses deferred execution` @ `JointRecognizer.RecognizeMatrices()`
* suppress some violations of Roslyn analyzer rules
* suppress `MA0001` & `MA0002` @ GlobalSuppressions.cs
@ c#
  • Loading branch information
n0099 committed Mar 26, 2024
1 parent c0378a7 commit ddf98c9
Show file tree
Hide file tree
Showing 8 changed files with 37 additions and 12 deletions.
2 changes: 2 additions & 0 deletions c#/GlobalSuppressions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
[assembly: SuppressMessage("Style", "MA0007:Add a comma after the last value")]
[assembly: SuppressMessage("Style", "VSTHRD200:Use \"Async\" suffix for async methods")]
[assembly: SuppressMessage("Usage", "CC0057:Unused parameters")]
[assembly: SuppressMessage("Usage", "MA0001:StringComparison is missing")]
[assembly: SuppressMessage("Usage", "MA0002:IEqualityComparer<string> or IComparer<string> is missing", Justification = "https://stackoverflow.com/questions/56478995/default-stringcomparer-used-by-dictionarystring-t")]
[assembly: SuppressMessage("Usage", "MA0004:Use Task.ConfigureAwait")]
[assembly: SuppressMessage("Usage", "MA0006:Use String.Equals instead of equality operator")]
[assembly: SuppressMessage("Usage", "MA0015:Specify the parameter name in ArgumentException")]
Expand Down
1 change: 1 addition & 0 deletions c#/crawler/src/Helper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public static byte[]? SerializedProtoBufWrapperOrNullIfEmpty<T>
contents == null ? null : new() {Value = {contents}};

public static void GetNowTimestamp(out Time now) => now = GetNowTimestamp();
[SuppressMessage("Maintainability", "AV1551:Method overload should call another overload")]
public static Time GetNowTimestamp() => (Time)DateTimeOffset.Now.ToUnixTimeSeconds();
}
public abstract partial class Helper
Expand Down
11 changes: 9 additions & 2 deletions c#/crawler/src/Tieba/ClientRequester.cs
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ private async Task<HttpResponseMessage> PostJson(
acc += pair.Key + '=' + pair.Value;
return acc;
}) + "tiebaclient!!!";
#pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms
var signMd5 = BitConverter.ToString(MD5.HashData(Encoding.UTF8.GetBytes(sign))).Replace("-", "");
#pragma warning restore CA5351 // Do Not Use Broken Cryptographic Algorithms
postData.Add(KeyValuePair.Create("sign", signMd5));

return await Post(async http =>
Expand All @@ -123,15 +125,20 @@ private async Task<HttpResponseMessage> PostProtoBuf<TRequest>(
commonParamSetter(requestParam, new() {ClientVersion = clientVersion, ClientType = 2});

// https://github.com/dotnet/runtime/issues/22996 http://test.greenbytes.de/tech/tc2231
var protoBufFile = new ByteArrayContent(requestParam.ToByteArray());
using var protoBufFile = new ByteArrayContent(requestParam.ToByteArray());
protoBufFile.Headers.Add("Content-Disposition", "form-data; name=\"data\"; filename=\"file\"");
var content = new MultipartFormDataContent {protoBufFile};
#pragma warning disable IDE0028 // Simplify collection initialization
using var content = new MultipartFormDataContent();
#pragma warning restore IDE0028 // Simplify collection initialization
content.Add(protoBufFile);

// https://stackoverflow.com/questions/30926645/httpcontent-boundary-double-quotes
var boundary = content.Headers.ContentType?.Parameters.First(header => header.Name == "boundary");
if (boundary != null) boundary.Value = boundary.Value?.Replace("\"", "");

#pragma warning disable CC0008 // Use object initializer
using var request = new HttpRequestMessage(HttpMethod.Post, url);
#pragma warning restore CC0008 // Use object initializer
request.Content = content;
_ = request.Headers.UserAgent.TryParseAdd($"bdtb for Android {clientVersion}");
request.Headers.Add("x_bd_data_type", "protobuf");
Expand Down
8 changes: 7 additions & 1 deletion c#/crawler/src/Tieba/ClientRequesterTcs.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace tbm.Crawler.Tieba;

public class ClientRequesterTcs : WithLogTrace
public sealed class ClientRequesterTcs : WithLogTrace
{
private readonly ILogger<ClientRequesterTcs> _logger;
private readonly IConfigurationSection _config;
Expand Down Expand Up @@ -41,6 +41,12 @@ private double MaxRps
}
}

public override void Dispose()
{
_timer.Dispose();
base.Dispose();
}

public void Increase() => MaxRps = Math.Min(
_config.GetValue("LimitRps:1", 1000),
MaxRps + _config.GetValue("DeltaRps:0", 0.01));
Expand Down
1 change: 1 addition & 0 deletions c#/crawler/src/Tieba/Crawl/Saver/BaseSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public abstract class BaseSaver<TPost, TBaseRevision>(
where TBaseRevision : class, IRevision
{
protected delegate void PostSaveEventHandler();
[SuppressMessage("Design", "MA0046:Use EventHandler<T> to declare events")]
protected event PostSaveEventHandler PostSaveEvent = () => { };

public virtual FieldChangeIgnoranceDelegates UserFieldChangeIgnorance =>
Expand Down
18 changes: 13 additions & 5 deletions c#/crawler/src/WithLogTrace.cs
Original file line number Diff line number Diff line change
@@ -1,23 +1,31 @@
namespace tbm.Crawler;

public abstract class WithLogTrace
#pragma warning disable S3881 // "IDisposable" should be implemented correctly
public abstract class WithLogTrace : IDisposable
#pragma warning restore S3881 // "IDisposable" should be implemented correctly
{
private readonly IConfigurationSection _config;
private readonly Timer _timerLogTrace = new() {Enabled = true};
private readonly Timer _timer = new() {Enabled = true};

protected WithLogTrace(IConfiguration config, string section)
{
_config = config.GetSection(section).GetSection("LogTrace");
_timerLogTrace.Interval = _config.GetValue("LogIntervalMs", 1000);
_timerLogTrace.Elapsed += (_, _) => LogTrace();
_timer.Interval = _config.GetValue("LogIntervalMs", 1000);
_timer.Elapsed += (_, _) => LogTrace();
}

public virtual void Dispose()
{
GC.SuppressFinalize(this);
_timer.Dispose();
}

protected abstract void LogTrace();

protected bool ShouldLogTrace()
{
if (!_config.GetValue("Enabled", false)) return false;
_timerLogTrace.Interval = _config.GetValue("LogIntervalMs", 1000);
_timer.Interval = _config.GetValue("LogIntervalMs", 1000);
return true;
}
}
3 changes: 1 addition & 2 deletions c#/imagePipeline/src/Consumer/OcrConsumer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ protected override IEnumerable<ImageId> ConsumeInternal(
return KeyValuePair.Create(new ImageKey(i.ImageId, i.FrameIndex), mat);
}).ToList();
var recognizedEithers = _recognizer
.RecognizeMatrices(matricesKeyByImageKey.Rights().ToDictionary(), stoppingToken)
.ToList();
.RecognizeMatrices(matricesKeyByImageKey.Rights().ToDictionary(), stoppingToken);
var recognizedFailedImagesId = recognizedEithers.Lefts().ToList();
var recognizedResults = recognizedEithers

Expand Down
5 changes: 3 additions & 2 deletions c#/imagePipeline/src/Ocr/JointRecognizer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public class JointRecognizer(
public async Task InitializePaddleOcr(CancellationToken stoppingToken = default) =>
await _paddleOcrRecognizerAndDetector.Initialize(stoppingToken);

public IEnumerable<Either<ImageId, IRecognitionResult>> RecognizeMatrices
public List<Either<ImageId, IRecognitionResult>> RecognizeMatrices

Check failure on line 37 in c#/imagePipeline/src/Ocr/JointRecognizer.cs

View workflow job for this annotation

GitHub Actions / build (imagePipeline)

Return type in signature for 'JointRecognizer.RecognizeMatrices(Dictionary<ImageKey, Mat>, CancellationToken)' should be an interface to an unchangeable collection (https://github.com/dennisdoomen/CSharpGuidelines/blob/5.6.0/_rules/1130.md)

Check failure on line 37 in c#/imagePipeline/src/Ocr/JointRecognizer.cs

View workflow job for this annotation

GitHub Actions / build (imagePipeline)

Prefer using collection abstraction instead of implementation (https://github.com/meziantou/Meziantou.Analyzer/blob/main/docs/Rules/MA0016.md)
(Dictionary<ImageKey, Mat> matricesKeyByImageKey, CancellationToken stoppingToken = default)

Check failure on line 38 in c#/imagePipeline/src/Ocr/JointRecognizer.cs

View workflow job for this annotation

GitHub Actions / build (imagePipeline)

Prefer using collection abstraction instead of implementation (https://github.com/meziantou/Meziantou.Analyzer/blob/main/docs/Rules/MA0016.md)
{
var recognizedEithersViaPaddleOcr = _paddleOcrRecognizerAndDetector
Expand Down Expand Up @@ -63,7 +63,8 @@ public IEnumerable<Either<ImageId, IRecognitionResult>> RecognizeMatrices
.Lefts()
.Concat(detectedEithers.Lefts())
.Concat(recognizedEithersViaTesseract.Lefts())
.Select(Either<ImageId, IRecognitionResult>.Left));
.Select(Either<ImageId, IRecognitionResult>.Left))
.ToList();
}

public Dictionary<ImageKey, string> GetRecognizedTextLines
Expand Down

0 comments on commit ddf98c9

Please sign in to comment.