From 8bd09ff93ab612f41c535713f9cc5e6cee81b5a7 Mon Sep 17 00:00:00 2001
From: devinleighsmith <devinleighsmith@gmail.com>
Date: Wed, 22 Jan 2025 17:26:16 -0800
Subject: [PATCH 1/3] psp-9893 mayan document queue corrections.
---
.../api/Services/DocumentQueueService.cs | 22 +++-----
source/backend/scheduler/Startup.cs | 16 ++++--
.../api/Services/DocumentQueueServiceTest.cs | 53 ++++++++++++++++---
3 files changed, 66 insertions(+), 25 deletions(-)
diff --git a/source/backend/api/Services/DocumentQueueService.cs b/source/backend/api/Services/DocumentQueueService.cs
index b4389ed017..03d2d3263e 100644
--- a/source/backend/api/Services/DocumentQueueService.cs
+++ b/source/backend/api/Services/DocumentQueueService.cs
@@ -212,14 +212,6 @@ public async Task<PimsDocumentQueue> Upload(PimsDocumentQueue documentQueue)
}
databaseDocumentQueue.DocProcessStartDt = DateTime.UtcNow;
- // if the document queued for upload is already in an error state, update the retries.
- if (databaseDocumentQueue.DocumentQueueStatusTypeCode == DocumentQueueStatusTypes.PIMS_ERROR.ToString() || databaseDocumentQueue.DocumentQueueStatusTypeCode == DocumentQueueStatusTypes.MAYAN_ERROR.ToString())
- {
- this.Logger.LogDebug("Document Queue {documentQueueId}, previously errored, retrying", documentQueue.DocumentQueueId);
- databaseDocumentQueue.DocProcessRetries += 1;
- databaseDocumentQueue.DocProcessEndDt = null;
- }
-
bool isValid = ValidateQueuedDocument(databaseDocumentQueue, documentQueue);
if (!isValid)
{
@@ -229,6 +221,14 @@ public async Task<PimsDocumentQueue> Upload(PimsDocumentQueue documentQueue)
return databaseDocumentQueue;
}
+ // if the document queued for upload is already in an error state, update the retries.
+ if (databaseDocumentQueue.DocumentQueueStatusTypeCode == DocumentQueueStatusTypes.PIMS_ERROR.ToString() || databaseDocumentQueue.DocumentQueueStatusTypeCode == DocumentQueueStatusTypes.MAYAN_ERROR.ToString())
+ {
+ this.Logger.LogDebug("Document Queue {documentQueueId}, previously errored, retrying", documentQueue.DocumentQueueId);
+ databaseDocumentQueue.DocProcessRetries = ++databaseDocumentQueue.DocProcessRetries ?? 1;
+ databaseDocumentQueue.DocProcessEndDt = null;
+ }
+
PimsDocument relatedDocument = null;
relatedDocument = _documentRepository.TryGetDocumentRelationships(databaseDocumentQueue.DocumentId.Value);
if (relatedDocument?.DocumentTypeId == null)
@@ -238,12 +238,6 @@ public async Task<PimsDocumentQueue> Upload(PimsDocumentQueue documentQueue)
UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.PIMS_ERROR);
return databaseDocumentQueue;
}
- else if (relatedDocument?.MayanId != null && relatedDocument?.MayanId > 0)
- {
- this.Logger.LogInformation("Queued document {documentQueueId} already has a mayan id {mayanid}, no further processing required.", databaseDocumentQueue.DocumentQueueId, relatedDocument.MayanId);
- UpdateDocumentQueueStatus(databaseDocumentQueue, DocumentQueueStatusTypes.SUCCESS);
- return databaseDocumentQueue; // The document poll job should pick this up and fix the document queue status.
- }
try
{
diff --git a/source/backend/scheduler/Startup.cs b/source/backend/scheduler/Startup.cs
index 74899aab00..698b516cf3 100644
--- a/source/backend/scheduler/Startup.cs
+++ b/source/backend/scheduler/Startup.cs
@@ -245,7 +245,12 @@ public void ConfigureServices(IServiceCollection services)
{
options.UseSimpleAssemblyNameTypeSerializer().UseRecommendedSerializerSettings().UseRedisStorage(redisConnection).UseSerilogLogProvider().UseConsole();
});
- services.AddHangfireServer();
+
+ BackgroundJobServerOptions hangfireOptions = Hangfire.Extensions.Configuration.ConfigurationExtensions.GetHangfireBackgroundJobServerOptions(this.Configuration);
+ services.AddHangfireServer(options =>
+ {
+ options.Queues = hangfireOptions.Queues;
+ });
services.AddHangfireConsoleExtensions();
services.Configure<ForwardedHeadersOptions>(options =>
@@ -341,10 +346,13 @@ private static void ConfigureSecureHeaders(IApplicationBuilder app)
private void ScheduleHangfireJobs(IServiceProvider services)
{
+ BackgroundJobServerOptions hangfireOptions = Hangfire.Extensions.Configuration.ConfigurationExtensions.GetHangfireBackgroundJobServerOptions(this.Configuration);
+
// provide default definition of all jobs.
- RecurringJob.AddOrUpdate<IDocumentQueueService>(nameof(DocumentQueueService.UploadQueuedDocuments), x => x.UploadQueuedDocuments(), Cron.Minutely);
- RecurringJob.AddOrUpdate<IDocumentQueueService>(nameof(DocumentQueueService.RetryQueuedDocuments), x => x.RetryQueuedDocuments(), "0 0 * * *");
- RecurringJob.AddOrUpdate<IDocumentQueueService>(nameof(DocumentQueueService.QueryProcessingDocuments), x => x.QueryProcessingDocuments(), Cron.Minutely);
+
+ RecurringJob.AddOrUpdate<IDocumentQueueService>(nameof(DocumentQueueService.UploadQueuedDocuments), hangfireOptions.Queues.FirstOrDefault() ?? "default", x => x.UploadQueuedDocuments(), Cron.Minutely);
+ RecurringJob.AddOrUpdate<IDocumentQueueService>(nameof(DocumentQueueService.RetryQueuedDocuments), hangfireOptions.Queues.FirstOrDefault() ?? "default", x => x.RetryQueuedDocuments(), "0 0 * * *");
+ RecurringJob.AddOrUpdate<IDocumentQueueService>(nameof(DocumentQueueService.QueryProcessingDocuments), hangfireOptions.Queues.FirstOrDefault() ?? "default", x => x.QueryProcessingDocuments(), Cron.Minutely);
// override scheduled jobs with configuration.
JobScheduleOptions jobOptions = this.Configuration.GetSection("JobOptions").Get<JobScheduleOptions>();
diff --git a/source/backend/tests/unit/api/Services/DocumentQueueServiceTest.cs b/source/backend/tests/unit/api/Services/DocumentQueueServiceTest.cs
index 0c7d48579c..a6704c8c2a 100644
--- a/source/backend/tests/unit/api/Services/DocumentQueueServiceTest.cs
+++ b/source/backend/tests/unit/api/Services/DocumentQueueServiceTest.cs
@@ -544,24 +544,63 @@ public async Task Upload_ValidationFails_Retries()
[Fact]
public async Task Upload_RelatedDocument_MayanId()
{
- var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin);
// Arrange
- var documentQueue = new PimsDocumentQueue { DocumentQueueId = 1, DocumentId = 1, Document = new byte[] { 1, 2, 3 } };
+ var service = CreateDocumentQueueServiceWithPermissions(Permissions.SystemAdmin);
+ var documentQueue = new PimsDocumentQueue
+ {
+ DocumentQueueId = 1,
+ DocumentId = 1,
+ DocumentQueueStatusTypeCode = DocumentQueueStatusTypes.PENDING.ToString(),
+ Document = new byte[] { 1, 2, 3 },
+ DocProcessRetries = 0,
+ };
+
+ var relatedDocument = new PimsDocument
+ {
+ DocumentId = 1,
+ DocumentTypeId = 1,
+ FileName = "test.pdf",
+ DocumentStatusTypeCode = "STATUS",
+ MayanId = 1
+ };
+
+ var documentType = new PimsDocumentTyp
+ {
+ DocumentTypeId = 1,
+ MayanId = 1
+ };
+
+ var documentUploadResponse = new DocumentUploadResponse
+ {
+ DocumentExternalResponse = new ExternalResponse<DocumentDetailModel>
+ {
+ Status = ExternalResponseStatus.Success,
+ Payload = new DocumentDetailModel
+ {
+ }
+ },
+ MetadataExternalResponse = new List<ExternalResponse<DocumentMetadataModel>>()
+ };
var documentRepositoryMock = this._helper.GetService<Mock<IDocumentRepository>>();
var documentQueueRepositoryMock = this._helper.GetService<Mock<IDocumentQueueRepository>>();
var documentServiceMock = this._helper.GetService<Mock<IDocumentService>>();
+ var documentTypeRepositoryMock = this._helper.GetService<Mock<IDocumentTypeRepository>>();
documentQueueRepositoryMock.Setup(x => x.TryGetById(It.IsAny<long>())).Returns(documentQueue);
- documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny<long>())).Returns(new PimsDocument() { DocumentTypeId = 1, MayanId = 1 });
+ documentRepositoryMock.Setup(x => x.TryGetDocumentRelationships(It.IsAny<long>())).Returns(relatedDocument);
+ documentTypeRepositoryMock.Setup(x => x.GetById(It.IsAny<long>())).Returns(documentType);
+ documentServiceMock.Setup(x => x.UploadDocumentAsync(It.IsAny<DocumentUploadRequest>(), true)).ReturnsAsync(documentUploadResponse);
// Act
- var result = await service.Upload(documentQueue);
+ var result = await service.Upload(documentQueue);
// Assert
- result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.SUCCESS.ToString());
- documentQueueRepositoryMock.Verify(x => x.TryGetById(It.IsAny<long>()), Times.Once);
- documentRepositoryMock.Verify(x => x.TryGetDocumentRelationships(It.IsAny<long>()), Times.Once);
+ result.Should().NotBeNull();
+ result.DocumentQueueStatusTypeCode.Should().Be(DocumentQueueStatusTypes.PROCESSING.ToString());
+ documentQueueRepositoryMock.Verify(x => x.Update(It.IsAny<PimsDocumentQueue>(), It.IsAny<bool>()), Times.AtLeastOnce);
+ documentQueueRepositoryMock.Verify(x => x.CommitTransaction(), Times.AtLeastOnce);
+ documentServiceMock.Verify(x => x.UploadDocumentAsync(It.IsAny<DocumentUploadRequest>(), true), Times.Once);
}
[Fact]
From f2df7fc73c010853cf58b88b1aabefb75f1be01d Mon Sep 17 00:00:00 2001
From: devinleighsmith <devinleighsmith@gmail.com>
Date: Wed, 29 Jan 2025 12:43:17 -0800
Subject: [PATCH 2/3] hotfix version bump.
---
source/backend/api/Pims.Api.csproj | 4 ++--
source/frontend/package.json | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/source/backend/api/Pims.Api.csproj b/source/backend/api/Pims.Api.csproj
index f09d8d6653..a5788ba515 100644
--- a/source/backend/api/Pims.Api.csproj
+++ b/source/backend/api/Pims.Api.csproj
@@ -2,8 +2,8 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<UserSecretsId>0ef6255f-9ea0-49ec-8c65-c172304b4926</UserSecretsId>
- <Version>5.7.2-96.22</Version>
- <AssemblyVersion>5.7.2.96</AssemblyVersion>
+ <Version>5.7.3-96.22</Version>
+ <AssemblyVersion>5.7.3.96</AssemblyVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<ProjectGuid>{16BC0468-78F6-4C91-87DA-7403C919E646}</ProjectGuid>
<TargetFramework>net8.0</TargetFramework>
diff --git a/source/frontend/package.json b/source/frontend/package.json
index 4932d87988..ac7cb8502c 100644
--- a/source/frontend/package.json
+++ b/source/frontend/package.json
@@ -1,6 +1,6 @@
{
"name": "frontend",
- "version": "5.7.2-96.22",
+ "version": "5.7.3-96.22",
"private": true,
"dependencies": {
"@bcgov/bc-sans": "1.0.1",
From 88d046e97ca6af94e26dc07867b5b1b46ac4dbf5 Mon Sep 17 00:00:00 2001
From: devinleighsmith <devinleighsmith@gmail.com>
Date: Wed, 29 Jan 2025 15:24:25 -0800
Subject: [PATCH 3/3] psp-9666 hotfix: correct loading of partial property ids.
---
.../src/components/maps/leaflet/Layers/util.test.tsx | 5 +++++
source/frontend/src/hooks/layer-api/layerUtils.ts | 3 ++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/source/frontend/src/components/maps/leaflet/Layers/util.test.tsx b/source/frontend/src/components/maps/leaflet/Layers/util.test.tsx
index dc357301ca..012648e7ad 100644
--- a/source/frontend/src/components/maps/leaflet/Layers/util.test.tsx
+++ b/source/frontend/src/components/maps/leaflet/Layers/util.test.tsx
@@ -205,5 +205,10 @@ describe('mapUtils tests', () => {
const cql = toCqlFilterValue({ PID: '12345678', PIN: '54321' }, { forceExactMatch: true });
expect(cql).toBe("PID = '12345678' AND PIN='54321'");
});
+
+ it('if property id is specified, exact search should always be used', () => {
+ const cql = toCqlFilterValue({ PROPERTY_ID: '1' });
+ expect(cql).toBe("PROPERTY_ID = '1'");
+ });
});
});
diff --git a/source/frontend/src/hooks/layer-api/layerUtils.ts b/source/frontend/src/hooks/layer-api/layerUtils.ts
index c54a9fa604..c19baaa751 100644
--- a/source/frontend/src/hooks/layer-api/layerUtils.ts
+++ b/source/frontend/src/hooks/layer-api/layerUtils.ts
@@ -28,7 +28,8 @@ export const toCqlFilterValue = (object: Record<string, string>, flags?: IWfsCql
} else if (
((key === 'PID' || key === 'PID_PADDED' || key === 'PID_NUMBER') &&
object[key]?.length === 9) ||
- flags?.forceExactMatch
+ flags?.forceExactMatch ||
+ key === 'PROPERTY_ID'
) {
cql.push(`${key} = '${object[key]}'`);
} else {