From c54b4a65097840877087de00b5586b1df3ae0543 Mon Sep 17 00:00:00 2001 From: Riccardo Torsoli <122275960+nttdata-rtorsoli@users.noreply.github.com> Date: Thu, 11 Jan 2024 11:16:06 +0100 Subject: [PATCH] PIN-4193 BKE - Metric report generator: consolidated in unique excel file (#165) --- .../interop/metricsreportgenerator/Main.scala | 54 ++++--- .../metricsreportgenerator/util/Jobs.scala | 142 ++++++++++++++---- .../util/TokensJobs.scala | 5 +- .../metricsreportgenerator/util/models.scala | 64 +++++++- project/Dependencies.scala | 5 + project/Versions.scala | 1 + 6 files changed, 215 insertions(+), 56 deletions(-) diff --git a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/Main.scala b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/Main.scala index 0cb6fe6e..445a27df 100644 --- a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/Main.scala +++ b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/Main.scala @@ -7,6 +7,8 @@ import it.pagopa.interop.commons.logging._ import it.pagopa.interop.commons.mail.{InteropMailer, MailAttachment, TextMail} import it.pagopa.interop.commons.utils.CORRELATION_ID_HEADER import it.pagopa.interop.metricsreportgenerator.util._ +import spoiwo.model.Workbook +import spoiwo.natures.xlsx.Model2XlsxConversions._ import java.util.UUID import java.util.concurrent.{ExecutorService, Executors} @@ -14,6 +16,7 @@ import scala.concurrent.ExecutionContext.global import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Future} import scala.util.Failure +import java.io.ByteArrayOutputStream object Main extends App { @@ -37,14 +40,14 @@ object Main extends App { tokensJob <- Future(new TokensJobs(s3)) } yield (es, blockingEC, fm, s3, rm, jobs, tokensJob, config) - def sendMail(config: Configuration): List[MailAttachment] => Future[Unit] = ats => { + def sendMail(config: Configuration, attachments: Seq[MailAttachment]): Future[Unit] = { val mail: TextMail = TextMail( UUID.randomUUID(), config.recipients, s"Report ${config.environment}", s"Data report of ${config.environment}", - ats + attachments ) InteropMailer.from(config.mailer).send(mail) } @@ -56,22 +59,37 @@ object Main extends App { blockingEC: ExecutionContextExecutor ): Future[Unit] = { val env: String = config.environment - - val agreementsJobResult: Future[MailAttachment] = jobs.getAgreementRecord - .map(_.getBytes) - .flatMap(bs => s3.saveAgreementsReport(bs).map(_ => asAttachment(s"agreements-${env}.csv", bs))) - - val activeDescriptorsJobResult: Future[MailAttachment] = jobs.getDescriptorsRecord - .map(_.getBytes) - .flatMap(bs => s3.saveActiveDescriptorsReport(bs).map(_ => asAttachment(s"active-descriptors-${env}.csv", bs))) - - val tokensJobResult: Future[MailAttachment] = tokensJob.getTokensData - .map(_.getBytes) - .flatMap(bs => s3.saveTokensReport(bs).map(_ => asAttachment(s"tokens-${env}.csv", bs))) - - Future - .sequence(List(agreementsJobResult, tokensJobResult, activeDescriptorsJobResult)) - .flatMap(sendMail(config)) + for { + + agreements <- jobs.getAgreements + purposes <- jobs.getPurposes + descriptors <- jobs.getActiveDescriptors + + agreementRecord = jobs.getAgreementRecord(agreements, purposes).getBytes + _ <- s3.saveAgreementsReport(agreementRecord) + + descriptorRecord <- jobs.getDescriptorsRecord(descriptors).map(_.getBytes) + _ <- s3.saveActiveDescriptorsReport(descriptorRecord) + + tokenReport <- tokensJob.getTokenReport() + tokenCsv = tokenReport.renderCsv + _ <- s3.saveTokensReport(tokenCsv.getBytes) + + agreementsSheet = jobs.getAgreementSheet(agreements, purposes) + descriptorsSheet <- jobs.getDescriptorsSheet(descriptors) + tokenSheet = tokenReport.renderSheet + _ <- sendMail( + config, + Seq( + asAttachment( + s"report-${env}.xlsx", + Workbook(agreementsSheet, descriptorsSheet, tokenSheet) + .writeToOutputStream(new ByteArrayOutputStream()) + .toByteArray() + ) + ) + ) + } yield () } def job(implicit ec: ExecutionContext): Future[Unit] = resources.flatMap { diff --git a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/Jobs.scala b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/Jobs.scala index dab9dca9..8996adec 100644 --- a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/Jobs.scala +++ b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/Jobs.scala @@ -5,7 +5,15 @@ import cats.syntax.all._ import com.typesafe.scalalogging.LoggerTakingImplicit import it.pagopa.interop.commons.cqrs.service.ReadModelService import it.pagopa.interop.commons.logging._ -import it.pagopa.interop.metricsreportgenerator.util.models.MetricDescriptor +import it.pagopa.interop.metricsreportgenerator.util.models.{ + SheetStyle, + Agreement, + Purpose, + Descriptor, + MetricDescriptor +} + +import spoiwo.model._ import scala.concurrent.{ExecutionContext, Future} @@ -14,36 +22,82 @@ class Jobs(config: Configuration, readModel: ReadModelService)(implicit context: ContextFieldsToLog ) { - def getAgreementRecord(implicit ec: ExecutionContext): Future[String] = { + def getAgreements(implicit ec: ExecutionContext): Future[List[Agreement]] = ReadModelQueries + .getAllActiveAgreements(config.collections, readModel) + .map(_.toList) + + def getPurposes(implicit ec: ExecutionContext): Future[List[Purpose]] = + ReadModelQueries.getAllPurposes(config.collections, readModel).map(_.toList) + + def getActiveDescriptors(implicit ec: ExecutionContext): Future[List[Descriptor]] = ReadModelQueries + .getAllDescriptors(config.collections, readModel) + .map(_.filter(_.isActive).toList) + + def getAgreementRecord(agreements: List[Agreement], purposes: List[Purpose]): String = { logger.info("Gathering Agreements Information") + val header = "eserviceId,eservice,producerId,producer,consumerId,consumer,agreementId,state,purposes,purposeIds" + (header :: agreements.map { a => + val (purposeIds, purposeNames): (Seq[String], Seq[String]) = purposes + .filter(p => p.consumerId == a.consumerId && p.eserviceId == a.eserviceId) + .map(p => (p.purposeId, p.name)) + .separate + List( + a.eserviceId, + a.eservice, + a.producerId, + a.producer, + a.consumerId, + a.consumer, + a.agreementId, + a.state, + purposeNames.mkString("§"), + purposeIds.mkString("§") + ).map(s => s"\"$s\"").mkString(",") + }).mkString("\n") + } + + def getAgreementSheet(agreements: List[Agreement], purposes: List[Purpose]): Sheet = { + logger.info("Gathering Agreements Information") + val headerList = List( + "EserviceId", + "Eservice", + "Producer", + "ProducerId", + "Consumer", + "ConsumerId", + "AgreementId", + "Purposes", + "PurposeIds" + ) + val headerRow = + Row(style = SheetStyle.headerStyle).withCellValues(headerList) + + val columns = (0 until (headerList.size)).toList + .map(index => Column(index = index, style = CellStyle(font = Font(bold = true)), autoSized = true)) - ReadModelQueries - .getAllActiveAgreements(config.collections, readModel) - .zip(ReadModelQueries.getAllPurposes(config.collections, readModel)) - .map { case (agreements, purposes) => - val header = "eserviceId,eservice,producerId,producer,consumerId,consumer,agreementId,state,purposes,purposeIds" - (header :: agreements.toList.map { a => - val (purposeIds, purposeNames): (Seq[String], Seq[String]) = purposes - .filter(p => p.consumerId == a.consumerId && p.eserviceId == a.eserviceId) - .map(p => (p.purposeId, p.name)) - .separate - List( - a.eserviceId, - a.eservice, - a.producerId, - a.producer, - a.consumerId, - a.consumer, - a.agreementId, - a.state, - purposeNames.mkString("§"), - purposeIds.mkString("§") - ).map(s => s"\"$s\"").mkString(",") - }).mkString("\n") - } + val rows = agreements.map { a => + val (purposeIds, purposeNames): (Seq[String], Seq[String]) = purposes + .filter(p => p.consumerId == a.consumerId && p.eserviceId == a.eserviceId) + .map(p => (p.purposeId, p.name)) + .separate + + Row(style = SheetStyle.rowStyle).withCellValues( + a.eserviceId, + a.eservice, + a.producer, + a.producerId, + a.consumer, + a.consumerId, + a.agreementId, + purposeNames.mkString(", "), + purposeIds.mkString(", ") + ) + } + + Sheet(name = "Agreements", rows = headerRow :: rows, columns = columns) } - def getDescriptorsRecord(implicit ec: ExecutionContext): Future[String] = { + def getDescriptorsRecord(descriptors: List[Descriptor])(implicit ec: ExecutionContext): Future[String] = { logger.info("Gathering Descriptors Information") val header: String = "name,createdAt,producerId,producer,descriptorId,state,fingerprint,tokenDuration" @@ -54,11 +108,37 @@ class Jobs(config: Configuration, readModel: ReadModelService)(implicit val asCsv: List[MetricDescriptor] => List[String] = asCsvRows.andThen(addHeader) - ReadModelQueries - .getAllDescriptors(config.collections, readModel) - .map(_.filter(_.isActive).toList) - .flatMap(xs => Future.traverse(xs)(_.toMetric)) + descriptors + .traverse(_.toMetric) .map(asCsv) .map(_.mkString("\n")) } + + def getDescriptorsSheet(descriptors: List[Descriptor])(implicit ec: ExecutionContext): Future[Sheet] = { + logger.info("Gathering Descriptors Information") + + val headerList = List("Name", "CreatedAt", "ProducerId", "Producer", "DescriptorId", "State", "Fingerprint") + val headerRow = + Row(style = SheetStyle.headerStyle).withCellValues(headerList) + + val columns = (0 until (headerList.size)).toList + .map(index => Column(index = index, style = CellStyle(font = Font(bold = true)), autoSized = true)) + + for { + metricDescriptors <- descriptors.traverse(_.toMetric) + rows = metricDescriptors + .map(descriptor => + Row(style = SheetStyle.rowStyle).withCellValues( + descriptor.name, + descriptor.createdAt, + descriptor.producerId, + descriptor.producer, + descriptor.descriptorId, + descriptor.state, + descriptor.fingerprint + ) + ) + .toList + } yield Sheet(name = "Descriptors", rows = headerRow :: rows, columns = columns) + } } diff --git a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/TokensJobs.scala b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/TokensJobs.scala index 199c4ea5..b5549cb0 100644 --- a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/TokensJobs.scala +++ b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/TokensJobs.scala @@ -44,7 +44,8 @@ class TokensJobs(s3: S3)(implicit .flatMap(_.toFuture) } - def getTokensData: Future[String] = { + def getTokenReport(): Future[Report] = { + logger.info("Gathering tokens information") // * The data on S3 is skewed, meaning that a specific date-folder (i.e. @@ -74,7 +75,5 @@ class TokensJobs(s3: S3)(implicit .map(_.flatten) .flatMap(updateReport(afterThan, beforeThan)(prunedReport)) } - .map(_.render) } - } diff --git a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/models.scala b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/models.scala index a1203424..46f2c206 100644 --- a/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/models.scala +++ b/metrics-report-generator/src/main/scala/it/pagopa/interop/metricsreportgenerator/util/models.scala @@ -10,6 +10,9 @@ import spray.json._ import java.time.{Instant, LocalDate, ZoneId} import scala.concurrent.Future import scala.util._ +import spoiwo.model._ +import spoiwo.model.enums._ +import scala.collection.immutable.ListMap final case class Agreement( activationDate: Option[String], @@ -81,6 +84,39 @@ final case class MetricDescriptor( tokenDuration: Int ) +object SheetStyle { + + val headerStyle = + CellStyle( + fillPattern = CellFill.Solid, + fillForegroundColor = Color.LightGreen, + font = Font(bold = true), + locked = true, + borders = CellBorders( + leftStyle = CellBorderStyle.Thin, + bottomStyle = CellBorderStyle.Thin, + rightStyle = CellBorderStyle.Thin, + topStyle = CellBorderStyle.Thin + ), + horizontalAlignment = CellHorizontalAlignment.Center, + verticalAlignment = CellVerticalAlignment.Center, + wrapText = true + ) + + val rowStyle = + CellStyle( + fillPattern = CellFill.Solid, + fillForegroundColor = Color.White, + font = Font(bold = false), + borders = CellBorders( + leftStyle = CellBorderStyle.Thin, + bottomStyle = CellBorderStyle.Thin, + rightStyle = CellBorderStyle.Thin, + topStyle = CellBorderStyle.Thin + ) + ) +} + final case class Report private (map: Map[Report.RecordValue, Int]) { def addIfInRange(after: Instant, before: Instant)(record: String): Try[Report] = Report .extractDataFromToken(record) @@ -102,11 +138,21 @@ final case class Report private (map: Map[Report.RecordValue, Int]) { def allButLastDate: Report = Report(map.filterNot { case ((_, _, date), _) => date.isEqual(lastDate) }) - def render: String = (Report.header :: map.map(Report.renderLine).toList.sorted).mkString("\n") + def renderCsv: String = (Report.headerCsv :: map.map(Report.renderLineCsv).toList.sorted).mkString("\n") + + def renderSheet: Sheet = { + val headerRow = + Row(style = SheetStyle.headerStyle).withCellValues(Report.headerSheet) + val columns = (0 until (Report.headerSheet.size)).toList + .map(index => Column(index = index, style = CellStyle(font = Font(bold = true)), autoSized = true)) + + val rows = ListMap(map.toSeq.sorted: _*).map(Report.renderLineSheet).toList + Sheet(name = "Tokens", rows = headerRow :: rows, columns = columns) + } } object Report { - type RecordValue = (String, String, LocalDate) + type RecordValue = Tuple3[String, String, LocalDate] type RawRecordValue = (String, String, Instant) val europeRome: ZoneId = ZoneId.of("Europe/Rome") @@ -129,7 +175,8 @@ object Report { (aId, pId, time) } - private val header: String = "agreementId,purposeId,year,month,day,tokencount" + private val headerSheet: List[String] = List("agreementId", "purposeId", "year", "month", "day", "tokencount") + private val headerCsv: String = headerSheet.mkString(",") private val row = raw""""([\w|-]{36})","([\w|-]{36})","(\d{4})","(\d*)","(\d*)","(\d*)"""".r @@ -139,7 +186,7 @@ object Report { case _ => Failure(GenericError(s"Csv line hasn't the right format: $line")) } - private def renderLine(row: (RecordValue, Int)): String = row match { + private def renderLineCsv(row: (RecordValue, Int)): String = row match { case ((aId, pId, time), count) => val year: String = f"${time.getYear()}%04d" val month: String = f"${time.getMonthValue()}%02d" @@ -147,6 +194,15 @@ object Report { s""""${aId}","${pId}","${year}","${month}","${day}","${count}"""" } + private def renderLineSheet(row: (RecordValue, Int)): Row = row match { + case ((aId, pId, time), count) => + val year: String = f"${time.getYear()}%04d" + val month: String = f"${time.getMonthValue()}%02d" + val day: String = f"${time.getDayOfMonth()}%02d" + + Row(style = SheetStyle.rowStyle).withCellValues(aId, pId, year, month, day, count) + } + def from(bytes: Array[Byte]): Try[Report] = new String(bytes).split("\n").toList.tail.traverse(parseCSVLine).map(_.toMap).map(new Report(_)) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 8e6a6ac3..11d60002 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -108,6 +108,10 @@ object Dependencies { lazy val parser = namespace %% "interop-commons-parser" % commonsVersion } + private[this] object spoiwo { + lazy val spoiwo = "com.norbitltd" %% "spoiwo" % spoiwoVersion + } + private[this] object scanamo { lazy val scanamo = "org.scanamo" %% "scanamo" % scanamoVersion lazy val testkit = "org.scanamo" %% "scanamo-testkit" % scanamoVersion @@ -197,6 +201,7 @@ object Dependencies { "javax.annotation" % "javax.annotation-api" % "1.3.2" % "compile", cats.core % Compile, "com.github.pureconfig" %% "pureconfig" % "0.17.2", + spoiwo.spoiwo % Compile, circe.core % Compile, circe.generic % Compile, logback.classic % Compile, diff --git a/project/Versions.scala b/project/Versions.scala index 7fb4eddd..7a937097 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -11,6 +11,7 @@ object Versions { lazy val sttpModelVersion = "1.5.5" lazy val scalaMockVersion = "5.2.0" lazy val scalatestVersion = "3.2.16" + lazy val spoiwoVersion = "2.2.1" } object PagopaVersions {