Feat/improve logs (#113)

* add some logs for dashboards * fix statistics logs --------- Co-authored-by: Antoine Jeanneney <[email protected]>
Cour-de-cassation · Jul 3, 2024 · b3943b0 · b3943b0
1 parent af65476
commit b3943b0
Show file tree

Hide file tree

Showing 6 changed files with 229 additions and 6 deletions.
diff --git a/packages/generic/backend/src/lib/annotator/buildAnnotator.ts b/packages/generic/backend/src/lib/annotator/buildAnnotator.ts
@@ -133,7 +133,13 @@ function buildAnnotator(
             operationName: 'annotateDocumentsWithoutAnnotations',
             msg: `Error while annotating document ${formatDocumentInfos(
               currentDocumentToAnnotate,
-            )}. Setting the document to its previous status...`,
+            )}.`,
+            data: {
+              decision: {
+                sourceId: currentDocumentToAnnotate.documentNumber,
+                sourceName: currentDocumentToAnnotate.source,
+              },
+            },
           });
           await documentService.updateDocumentStatus(
             currentDocumentToAnnotate._id,
@@ -200,7 +206,13 @@ function buildAnnotator(
     } = await annotatorConfig.fetchAnnotationOfDocument(settings, document);
     logger.log({
       operationName: 'annotateDocument',
-      msg: 'NLP annotation succeeded',
+      msg: 'NLP annotation succesfully fetched',
+      data: {
+        decision: {
+          sourceId: document.documentNumber,
+          sourceName: document.source,
+        },
+      },
     });
     documentService.updateDocumentNlpVersions(documentId, version);
 
@@ -216,6 +228,12 @@ function buildAnnotator(
     logger.log({
       operationName: 'annotateDocument',
       msg: 'NLP treatment created in DB',
+      data: {
+        decision: {
+          sourceId: document.documentNumber,
+          sourceName: document.source,
+        },
+      },
     });
 
     if (document.decisionMetadata.motivationOccultation === true) {
@@ -227,6 +245,12 @@ function buildAnnotator(
         logger.log({
           operationName: 'annotateDocument',
           msg: 'Annotate motivation zone because motivationOccultation is true',
+          data: {
+            decision: {
+              sourceId: document.documentNumber,
+              sourceName: document.source,
+            },
+          },
         });
 
         createMotivationOccultationTreatment(
@@ -242,6 +266,12 @@ function buildAnnotator(
           msg: `Annotate decision motivation zone impossible because motication zone was not found for document ${formatDocumentInfos(
             document,
           )}`,
+          data: {
+            decision: {
+              sourceId: document.documentNumber,
+              sourceName: document.source,
+            },
+          },
         });
       }
     }
@@ -251,6 +281,12 @@ function buildAnnotator(
     logger.log({
       operationName: 'annotateDocument',
       msg: 'Annotation report created in DB',
+      data: {
+        decision: {
+          sourceId: document.documentNumber,
+          sourceName: document.source,
+        },
+      },
     });
 
     if (
@@ -260,6 +296,12 @@ function buildAnnotator(
       logger.log({
         operationName: 'annotateDocument',
         msg: `additionalTermsParsingFailed found, updating with value ${additionalTermsParsingFailed}`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
       });
       await documentService.updateDocumentAdditionalTermsParsingFailed(
         documentId,
@@ -272,6 +314,12 @@ function buildAnnotator(
       logger.log({
         operationName: 'annotateDocument',
         msg: `categoriesToUnAnnotate found, adding '${newCategoriesToUnAnnotate}' to categoriesToOmit if not already in`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
       });
       newCategoriesToOmit = Array.from(
         new Set(newCategoriesToOmit.concat(newCategoriesToUnAnnotate)),
@@ -282,6 +330,12 @@ function buildAnnotator(
       logger.log({
         operationName: 'annotateDocument',
         msg: `categoriesToAnnotate found, removing '${newCategoriesToAnnotate}' from categoriesToOmit if present`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
       });
       newCategoriesToOmit = newCategoriesToOmit.filter(
         (category) => !newCategoriesToAnnotate.includes(category),
@@ -292,6 +346,12 @@ function buildAnnotator(
       logger.log({
         operationName: 'annotateDocument',
         msg: `updating categoriesToOmit in database`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
       });
       documentService.updateDocumentCategoriesToOmit(
         documentId,
@@ -304,6 +364,12 @@ function buildAnnotator(
         operationName: 'annotateDocument',
         msg:
           'Additionals terms to annotate or to unannotate found, adding to document...',
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
       });
 
       await documentService.updateDocumentComputedAdditionalTerms(
@@ -352,6 +418,12 @@ function buildAnnotator(
     logger.log({
       operationName: 'annotateDocument',
       msg: `Annotation done for document ${formatDocumentInfos(document)}`,
+      data: {
+        decision: {
+          sourceId: document.documentNumber,
+          sourceName: document.source,
+        },
+      },
     });
   }
 

diff --git a/packages/generic/backend/src/lib/connector/buildConnector.ts b/packages/generic/backend/src/lib/connector/buildConnector.ts
@@ -30,8 +30,8 @@ function buildConnector(connectorConfig: connectorConfigType) {
     threshold: number;
   }) {
     logger.log({
-      operationName: 'annotateDocument',
-      msg: `importChainedDocumentsFromSder: ${threshold}`,
+      operationName: 'importChainedDocumentsFromSder',
+      msg: `Import chained documents from sder with ${threshold} threshold`,
     });
 
     const documentRepository = buildDocumentRepository();
@@ -813,12 +813,59 @@ async function insertDocuments(documents: documentType[]) {
   const documentRepository = buildDocumentRepository();
 
   for await (const document of documents) {
-    await documentRepository.insert(document);
+    try {
+      await documentRepository.insert(document);
+      logger.log({
+        operationName: 'documentInsertion',
+        msg: `Document ${document.source}:${document.documentNumber} has been inserted in database imported by ${document.importer}`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
+      });
+    } catch {
+      logger.error({
+        operationName: 'documentInsertion',
+        msg: `Failed to import ${document.source}:${document.documentNumber} document`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
+      });
+    }
   }
 }
 
 function insertDocument(document: documentType) {
   const documentRepository = buildDocumentRepository();
 
-  return documentRepository.insert(document);
+  try {
+    const insertedDocument = documentRepository.insert(document);
+    logger.log({
+      operationName: 'documentInsertion',
+      msg: `Document ${document.source}:${document.documentNumber} has been inserted in database imported by ${document.importer}`,
+      data: {
+        decision: {
+          sourceId: document.documentNumber,
+          sourceName: document.source,
+        },
+      },
+    });
+    return insertedDocument;
+  } catch {
+    logger.error({
+      operationName: 'documentInsertion',
+      msg: `Failed to import ${document.source}:${document.documentNumber} document`,
+      data: {
+        decision: {
+          sourceId: document.documentNumber,
+          sourceName: document.source,
+        },
+      },
+    });
+  }
 }
diff --git a/packages/generic/backend/src/lib/exporter/buildExporter.ts b/packages/generic/backend/src/lib/exporter/buildExporter.ts
@@ -199,6 +199,16 @@ function buildExporter(
         pseudonymizationText: anonymizer.anonymizeDocument(document).text,
         labelTreatments: treatmentModule.lib.concat(treatments, version),
       });
+      logger.log({
+        operationName: 'exportDocument',
+        msg: `Document ${document.source}:${document.documentNumber} has been exported`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+        },
+      });
 
       await statisticService.saveStatisticsOfDocument(document, settings);
 

diff --git a/...ages/generic/backend/src/modules/document/service/documentService/updateDocumentStatus.ts b/...ages/generic/backend/src/modules/document/service/documentService/updateDocumentStatus.ts
@@ -3,6 +3,7 @@ import { documentType, idModule } from '@label/core';
 import { assignationService } from '../../../assignation';
 import { buildDocumentRepository } from '../../repository';
 import { resetDocument } from './resetDocument';
+import { logger } from '../../../../utils';
 
 export { updateDocumentStatus };
 
@@ -27,5 +28,15 @@ async function updateDocumentStatus(
   } else if (status === 'loaded') {
     await resetDocument(_id);
   }
+  logger.log({
+    operationName: 'updateDocumentStatus',
+    msg: `Document ${updatedDocument.source}:${updatedDocument.documentNumber} status updated to ${status}`,
+    data: {
+      decision: {
+        sourceId: updatedDocument.documentNumber,
+        sourceName: updatedDocument.source,
+      },
+    },
+  });
   return updatedDocument;
 }
diff --git a/packages/generic/backend/src/modules/problemReport/service/problemReportService.ts b/packages/generic/backend/src/modules/problemReport/service/problemReportService.ts
@@ -8,6 +8,7 @@ import {
 import { documentService } from '../../document';
 import { userService } from '../../user';
 import { buildProblemReportRepository } from '../repository';
+import { logger } from '../../../utils';
 
 export { problemReportService };
 
@@ -35,6 +36,28 @@ const problemReportService = {
         type: problemType,
       }),
     );
+    const documents = await documentService.fetchAllDocumentsByIds([
+      documentId,
+    ]);
+    const users = await userService.fetchUsersByIds([userId]);
+
+    const document = documents[idModule.lib.convertToString(documentId)];
+    const user = users[idModule.lib.convertToString(userId)];
+
+    if (document && user) {
+      logger.log({
+        operationName: 'createProblemReport',
+        msg: `Problem report created on document ${document.source}:${document.documentNumber} by ${user.name}`,
+        data: {
+          decision: {
+            sourceId: document.documentNumber,
+            sourceName: document.source,
+          },
+          userId: userId,
+          userName: user.name,
+        },
+      });
+    }
   },
 
   async deleteProblemReportById(problemReportId: problemReportType['_id']) {

diff --git a/packages/generic/backend/src/modules/statistic/service/saveStatisticsOfDocument.ts b/packages/generic/backend/src/modules/statistic/service/saveStatisticsOfDocument.ts
@@ -1,12 +1,16 @@
 import {
   documentType,
+  idModule,
   settingsType,
   statisticsCreator,
+  timeOperator,
   treatmentModule,
 } from '@label/core';
 import { assignationService } from '../../assignation';
 import { treatmentService } from '../../treatment';
 import { buildStatisticRepository } from '../repository';
+import { logger } from '../../../utils';
+import { userService } from '../../../modules/user';
 
 export { saveStatisticsOfDocument };
 
@@ -38,5 +42,61 @@ async function saveStatisticsOfDocument(
     settings,
   });
 
+  if (humanTreatments && humanTreatments.length > 0) {
+    const userIds = humanTreatments.map((humanTreatment) =>
+      idModule.lib.buildId(humanTreatment.userId),
+    );
+    const users = await userService.fetchUsersByIds(userIds);
+
+    for (const humanTreatment of humanTreatments) {
+      const user = users[idModule.lib.convertToString(humanTreatment.userId)];
+
+      if (user) {
+        logger.log({
+          operationName: 'documentStatistics',
+          msg: `Human treatment for document ${document.source}:${
+            document.documentNumber
+          } : ${
+            user.name
+          } treat the document in ${timeOperator.convertDurationToReadableDuration(
+            humanTreatment.treatment.duration,
+          )} on ${timeOperator.convertTimestampToReadableDate(
+            humanTreatment.treatment.lastUpdateDate,
+            true,
+          )}`,
+          data: {
+            decision: {
+              sourceId: document.documentNumber,
+              sourceName: document.source,
+            },
+            treatmentDuration: humanTreatment.treatment.duration,
+            lastUpdateDate: humanTreatment.treatment.lastUpdateDate,
+            userName: user.name,
+          },
+        });
+      }
+    }
+  }
+
+  logger.log({
+    operationName: 'documentStatistics',
+    msg: `Create statistics for document ${document.source}:${document.documentNumber}`,
+    data: {
+      decision: {
+        sourceId: statistic.documentNumber,
+        sourceName: statistic.source,
+      },
+      statistics: {
+        route: statistic.route,
+        wordsCount: statistic.wordsCount,
+        surAnnotationsCount: statistic.surAnnotationsCount,
+        subAnnotationsSensitiveCount: statistic.subAnnotationsSensitiveCount,
+        subAnnotationsNonSensitiveCount:
+          statistic.subAnnotationsNonSensitiveCount,
+        linkedEntitiesCount: statistic.linkedEntitiesCount,
+      },
+    },
+  });
+
   await statisticRepository.insert(statistic);
 }