PIN-4193 BKE - Metric report generator: consolidated in unique excel …

…file (#165)

metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/Main.scala

@@ -7,13 +7,16 @@ import it.pagopa.interop.commons.logging._
 import it.pagopa.interop.commons.mail.{InteropMailer, MailAttachment, TextMail}
 import it.pagopa.interop.commons.utils.CORRELATION_ID_HEADER
 import it.pagopa.interop.metricsreportgenerator.util._
+import spoiwo.model.Workbook
+import spoiwo.natures.xlsx.Model2XlsxConversions._
 
 import java.util.UUID
 import java.util.concurrent.{ExecutorService, Executors}
 import scala.concurrent.ExecutionContext.global
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Future}
 import scala.util.Failure
+import java.io.ByteArrayOutputStream
 
 object Main extends App {
 
@@ -37,14 +40,14 @@ object Main extends App {
     tokensJob <- Future(new TokensJobs(s3))
   } yield (es, blockingEC, fm, s3, rm, jobs, tokensJob, config)
 
-  def sendMail(config: Configuration): List[MailAttachment] => Future[Unit] = ats => {
+  def sendMail(config: Configuration, attachments: Seq[MailAttachment]): Future[Unit] = {
     val mail: TextMail =
       TextMail(
         UUID.randomUUID(),
         config.recipients,
         s"Report ${config.environment}",
         s"Data report of ${config.environment}",
-        ats
+        attachments
       )
     InteropMailer.from(config.mailer).send(mail)
   }
@@ -56,22 +59,37 @@ object Main extends App {
     blockingEC: ExecutionContextExecutor
   ): Future[Unit] = {
     val env: String = config.environment
-
-    val agreementsJobResult: Future[MailAttachment] = jobs.getAgreementRecord
-      .map(_.getBytes)
-      .flatMap(bs => s3.saveAgreementsReport(bs).map(_ => asAttachment(s"agreements-${env}.csv", bs)))
-
-    val activeDescriptorsJobResult: Future[MailAttachment] = jobs.getDescriptorsRecord
-      .map(_.getBytes)
-      .flatMap(bs => s3.saveActiveDescriptorsReport(bs).map(_ => asAttachment(s"active-descriptors-${env}.csv", bs)))
-
-    val tokensJobResult: Future[MailAttachment] = tokensJob.getTokensData
-      .map(_.getBytes)
-      .flatMap(bs => s3.saveTokensReport(bs).map(_ => asAttachment(s"tokens-${env}.csv", bs)))
-
-    Future
-      .sequence(List(agreementsJobResult, tokensJobResult, activeDescriptorsJobResult))
-      .flatMap(sendMail(config))
+    for {
+
+      agreements  <- jobs.getAgreements
+      purposes    <- jobs.getPurposes
+      descriptors <- jobs.getActiveDescriptors
+
+      agreementRecord = jobs.getAgreementRecord(agreements, purposes).getBytes
+      _ <- s3.saveAgreementsReport(agreementRecord)
+
+      descriptorRecord <- jobs.getDescriptorsRecord(descriptors).map(_.getBytes)
+      _                <- s3.saveActiveDescriptorsReport(descriptorRecord)
+
+      tokenReport <- tokensJob.getTokenReport()
+      tokenCsv = tokenReport.renderCsv
+      _ <- s3.saveTokensReport(tokenCsv.getBytes)
+
+      agreementsSheet = jobs.getAgreementSheet(agreements, purposes)
+      descriptorsSheet <- jobs.getDescriptorsSheet(descriptors)
+      tokenSheet = tokenReport.renderSheet
+      _ <- sendMail(
+        config,
+        Seq(
+          asAttachment(
+            s"report-${env}.xlsx",
+            Workbook(agreementsSheet, descriptorsSheet, tokenSheet)
+              .writeToOutputStream(new ByteArrayOutputStream())
+              .toByteArray()
+          )
+        )
+      )
+    } yield ()
   }
 
   def job(implicit ec: ExecutionContext): Future[Unit] = resources.flatMap {

metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/Jobs.scala

@@ -5,7 +5,15 @@ import cats.syntax.all._
 import com.typesafe.scalalogging.LoggerTakingImplicit
 import it.pagopa.interop.commons.cqrs.service.ReadModelService
 import it.pagopa.interop.commons.logging._
-import it.pagopa.interop.metricsreportgenerator.util.models.MetricDescriptor
+import it.pagopa.interop.metricsreportgenerator.util.models.{
+  SheetStyle,
+  Agreement,
+  Purpose,
+  Descriptor,
+  MetricDescriptor
+}
+
+import spoiwo.model._
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -14,36 +22,82 @@ class Jobs(config: Configuration, readModel: ReadModelService)(implicit
   context: ContextFieldsToLog
 ) {
 
-  def getAgreementRecord(implicit ec: ExecutionContext): Future[String] = {
+  def getAgreements(implicit ec: ExecutionContext): Future[List[Agreement]] = ReadModelQueries
+    .getAllActiveAgreements(config.collections, readModel)
+    .map(_.toList)
+
+  def getPurposes(implicit ec: ExecutionContext): Future[List[Purpose]] =
+    ReadModelQueries.getAllPurposes(config.collections, readModel).map(_.toList)
+
+  def getActiveDescriptors(implicit ec: ExecutionContext): Future[List[Descriptor]] = ReadModelQueries
+    .getAllDescriptors(config.collections, readModel)
+    .map(_.filter(_.isActive).toList)
+
+  def getAgreementRecord(agreements: List[Agreement], purposes: List[Purpose]): String = {
     logger.info("Gathering Agreements Information")
+    val header = "eserviceId,eservice,producerId,producer,consumerId,consumer,agreementId,state,purposes,purposeIds"
+    (header :: agreements.map { a =>
+      val (purposeIds, purposeNames): (Seq[String], Seq[String]) = purposes
+        .filter(p => p.consumerId == a.consumerId && p.eserviceId == a.eserviceId)
+        .map(p => (p.purposeId, p.name))
+        .separate
+      List(
+        a.eserviceId,
+        a.eservice,
+        a.producerId,
+        a.producer,
+        a.consumerId,
+        a.consumer,
+        a.agreementId,
+        a.state,
+        purposeNames.mkString("§"),
+        purposeIds.mkString("§")
+      ).map(s => s"\"$s\"").mkString(",")
+    }).mkString("\n")
+  }
+
+  def getAgreementSheet(agreements: List[Agreement], purposes: List[Purpose]): Sheet = {
+    logger.info("Gathering Agreements Information")
+    val headerList = List(
+      "EserviceId",
+      "Eservice",
+      "Producer",
+      "ProducerId",
+      "Consumer",
+      "ConsumerId",
+      "AgreementId",
+      "Purposes",
+      "PurposeIds"
+    )
+    val headerRow  =
+      Row(style = SheetStyle.headerStyle).withCellValues(headerList)
+
+    val columns = (0 until (headerList.size)).toList
+      .map(index => Column(index = index, style = CellStyle(font = Font(bold = true)), autoSized = true))
 
-    ReadModelQueries
-      .getAllActiveAgreements(config.collections, readModel)
-      .zip(ReadModelQueries.getAllPurposes(config.collections, readModel))
-      .map { case (agreements, purposes) =>
-        val header = "eserviceId,eservice,producerId,producer,consumerId,consumer,agreementId,state,purposes,purposeIds"
-        (header :: agreements.toList.map { a =>
-          val (purposeIds, purposeNames): (Seq[String], Seq[String]) = purposes
-            .filter(p => p.consumerId == a.consumerId && p.eserviceId == a.eserviceId)
-            .map(p => (p.purposeId, p.name))
-            .separate
-          List(
-            a.eserviceId,
-            a.eservice,
-            a.producerId,
-            a.producer,
-            a.consumerId,
-            a.consumer,
-            a.agreementId,
-            a.state,
-            purposeNames.mkString("§"),
-            purposeIds.mkString("§")
-          ).map(s => s"\"$s\"").mkString(",")
-        }).mkString("\n")
-      }
+    val rows = agreements.map { a =>
+      val (purposeIds, purposeNames): (Seq[String], Seq[String]) = purposes
+        .filter(p => p.consumerId == a.consumerId && p.eserviceId == a.eserviceId)
+        .map(p => (p.purposeId, p.name))
+        .separate
+
+      Row(style = SheetStyle.rowStyle).withCellValues(
+        a.eserviceId,
+        a.eservice,
+        a.producer,
+        a.producerId,
+        a.consumer,
+        a.consumerId,
+        a.agreementId,
+        purposeNames.mkString(", "),
+        purposeIds.mkString(", ")
+      )
+    }
+
+    Sheet(name = "Agreements", rows = headerRow :: rows, columns = columns)
   }
 
-  def getDescriptorsRecord(implicit ec: ExecutionContext): Future[String] = {
+  def getDescriptorsRecord(descriptors: List[Descriptor])(implicit ec: ExecutionContext): Future[String] = {
     logger.info("Gathering Descriptors Information")
 
     val header: String = "name,createdAt,producerId,producer,descriptorId,state,fingerprint,tokenDuration"
@@ -54,11 +108,37 @@ class Jobs(config: Configuration, readModel: ReadModelService)(implicit
 
     val asCsv: List[MetricDescriptor] => List[String] = asCsvRows.andThen(addHeader)
 
-    ReadModelQueries
-      .getAllDescriptors(config.collections, readModel)
-      .map(_.filter(_.isActive).toList)
-      .flatMap(xs => Future.traverse(xs)(_.toMetric))
+    descriptors
+      .traverse(_.toMetric)
       .map(asCsv)
       .map(_.mkString("\n"))
   }
+
+  def getDescriptorsSheet(descriptors: List[Descriptor])(implicit ec: ExecutionContext): Future[Sheet] = {
+    logger.info("Gathering Descriptors Information")
+
+    val headerList = List("Name", "CreatedAt", "ProducerId", "Producer", "DescriptorId", "State", "Fingerprint")
+    val headerRow  =
+      Row(style = SheetStyle.headerStyle).withCellValues(headerList)
+
+    val columns = (0 until (headerList.size)).toList
+      .map(index => Column(index = index, style = CellStyle(font = Font(bold = true)), autoSized = true))
+
+    for {
+      metricDescriptors <- descriptors.traverse(_.toMetric)
+      rows = metricDescriptors
+        .map(descriptor =>
+          Row(style = SheetStyle.rowStyle).withCellValues(
+            descriptor.name,
+            descriptor.createdAt,
+            descriptor.producerId,
+            descriptor.producer,
+            descriptor.descriptorId,
+            descriptor.state,
+            descriptor.fingerprint
+          )
+        )
+        .toList
+    } yield Sheet(name = "Descriptors", rows = headerRow :: rows, columns = columns)
+  }
 }

metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/TokensJobs.scala

@@ -44,7 +44,8 @@ class TokensJobs(s3: S3)(implicit
       .flatMap(_.toFuture)
   }
 
-  def getTokensData: Future[String] = {
+  def getTokenReport(): Future[Report] = {
+
     logger.info("Gathering tokens information")
 
     // * The data on S3 is skewed, meaning that a specific date-folder (i.e.
@@ -74,7 +75,5 @@ class TokensJobs(s3: S3)(implicit
             .map(_.flatten)
             .flatMap(updateReport(afterThan, beforeThan)(prunedReport))
       }
-      .map(_.render)
   }
-
 }

metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/models.scala

@@ -10,6 +10,9 @@ import spray.json._
 import java.time.{Instant, LocalDate, ZoneId}
 import scala.concurrent.Future
 import scala.util._
+import spoiwo.model._
+import spoiwo.model.enums._
+import scala.collection.immutable.ListMap
 
 final case class Agreement(
   activationDate: Option[String],
@@ -81,6 +84,39 @@ final case class MetricDescriptor(
   tokenDuration: Int
 )
 
+object SheetStyle {
+
+  val headerStyle =
+    CellStyle(
+      fillPattern = CellFill.Solid,
+      fillForegroundColor = Color.LightGreen,
+      font = Font(bold = true),
+      locked = true,
+      borders = CellBorders(
+        leftStyle = CellBorderStyle.Thin,
+        bottomStyle = CellBorderStyle.Thin,
+        rightStyle = CellBorderStyle.Thin,
+        topStyle = CellBorderStyle.Thin
+      ),
+      horizontalAlignment = CellHorizontalAlignment.Center,
+      verticalAlignment = CellVerticalAlignment.Center,
+      wrapText = true
+    )
+
+  val rowStyle =
+    CellStyle(
+      fillPattern = CellFill.Solid,
+      fillForegroundColor = Color.White,
+      font = Font(bold = false),
+      borders = CellBorders(
+        leftStyle = CellBorderStyle.Thin,
+        bottomStyle = CellBorderStyle.Thin,
+        rightStyle = CellBorderStyle.Thin,
+        topStyle = CellBorderStyle.Thin
+      )
+    )
+}
+
 final case class Report private (map: Map[Report.RecordValue, Int]) {
   def addIfInRange(after: Instant, before: Instant)(record: String): Try[Report] = Report
     .extractDataFromToken(record)
@@ -102,11 +138,21 @@ final case class Report private (map: Map[Report.RecordValue, Int]) {
 
   def allButLastDate: Report = Report(map.filterNot { case ((_, _, date), _) => date.isEqual(lastDate) })
 
-  def render: String = (Report.header :: map.map(Report.renderLine).toList.sorted).mkString("\n")
+  def renderCsv: String = (Report.headerCsv :: map.map(Report.renderLineCsv).toList.sorted).mkString("\n")
+
+  def renderSheet: Sheet = {
+    val headerRow =
+      Row(style = SheetStyle.headerStyle).withCellValues(Report.headerSheet)
+    val columns   = (0 until (Report.headerSheet.size)).toList
+      .map(index => Column(index = index, style = CellStyle(font = Font(bold = true)), autoSized = true))
+
+    val rows = ListMap(map.toSeq.sorted: _*).map(Report.renderLineSheet).toList
+    Sheet(name = "Tokens", rows = headerRow :: rows, columns = columns)
+  }
 }
 
 object Report {
-  type RecordValue    = (String, String, LocalDate)
+  type RecordValue    = Tuple3[String, String, LocalDate]
   type RawRecordValue = (String, String, Instant)
 
   val europeRome: ZoneId = ZoneId.of("Europe/Rome")
@@ -129,7 +175,8 @@ object Report {
     (aId, pId, time)
   }
 
-  private val header: String = "agreementId,purposeId,year,month,day,tokencount"
+  private val headerSheet: List[String] = List("agreementId", "purposeId", "year", "month", "day", "tokencount")
+  private val headerCsv: String         = headerSheet.mkString(",")
 
   private val row = raw""""([\w|-]{36})","([\w|-]{36})","(\d{4})","(\d*)","(\d*)","(\d*)"""".r
 
@@ -139,14 +186,23 @@ object Report {
     case _                                      => Failure(GenericError(s"Csv line hasn't the right format: $line"))
   }
 
-  private def renderLine(row: (RecordValue, Int)): String = row match {
+  private def renderLineCsv(row: (RecordValue, Int)): String = row match {
     case ((aId, pId, time), count) =>
       val year: String  = f"${time.getYear()}%04d"
       val month: String = f"${time.getMonthValue()}%02d"
       val day: String   = f"${time.getDayOfMonth()}%02d"
       s""""${aId}","${pId}","${year}","${month}","${day}","${count}""""
   }
 
+  private def renderLineSheet(row: (RecordValue, Int)): Row = row match {
+    case ((aId, pId, time), count) =>
+      val year: String  = f"${time.getYear()}%04d"
+      val month: String = f"${time.getMonthValue()}%02d"
+      val day: String   = f"${time.getDayOfMonth()}%02d"
+
+      Row(style = SheetStyle.rowStyle).withCellValues(aId, pId, year, month, day, count)
+  }
+
   def from(bytes: Array[Byte]): Try[Report] =
     new String(bytes).split("\n").toList.tail.traverse(parseCSVLine).map(_.toMap).map(new Report(_))