Merge pull request #1526 from betagouv/master

MEP [TRELLO-2117] Implements probes to detect anomalies (#1525)

app/config/TaskConfiguration.scala

@@ -11,7 +11,8 @@ case class TaskConfiguration(
     reportClosure: ReportClosureTaskConfiguration,
     reportReminders: ReportRemindersTaskConfiguration,
     inactiveAccounts: InactiveAccountsTaskConfiguration,
-    companyUpdate: CompanyUpdateTaskConfiguration
+    companyUpdate: CompanyUpdateTaskConfiguration,
+    probe: ProbeConfiguration
 )
 
 case class SubscriptionTaskConfiguration(startTime: LocalTime, startDay: DayOfWeek)
@@ -38,3 +39,5 @@ case class ReportRemindersTaskConfiguration(
     mailReminderDelay: Period // 7days.
 
 )
+
+case class ProbeConfiguration(active: Boolean)

app/loader/SignalConsoApplicationLoader.scala

@@ -64,6 +64,7 @@ import repositories.emailvalidation.EmailValidationRepositoryInterface
 import repositories.event.EventRepository
 import repositories.event.EventRepositoryInterface
 import repositories.barcode.BarcodeProductRepository
+import repositories.probe.ProbeRepository
 import repositories.rating.RatingRepository
 import repositories.rating.RatingRepositoryInterface
 import repositories.report.ReportRepository
@@ -96,6 +97,8 @@ import tasks.account.InactiveAccountTask
 import tasks.account.InactiveDgccrfAccountReminderTask
 import tasks.account.InactiveDgccrfAccountRemoveTask
 import tasks.company._
+import tasks.probe.LowRateLanceurDAlerteTask
+import tasks.probe.LowRateReponseConsoTask
 import tasks.report.ReportClosureTask
 import tasks.report.ReportNotificationTask
 import tasks.report.ReportRemindersTask
@@ -697,6 +700,30 @@ class SignalConsoComponents(
     new Exception("This is a test Alert, used to check that Sentry alert are still active on each new deployments.")
   )
 
+  // Probes
+  if (applicationConfiguration.task.probe.active) {
+    logger.debug("Probes are enabled")
+    val probeRepository = new ProbeRepository(dbConfig)
+    new LowRateReponseConsoTask(
+      actorSystem,
+      applicationConfiguration.task,
+      taskRepository,
+      probeRepository,
+      userRepository,
+      mailService
+    ).schedule()
+    new LowRateLanceurDAlerteTask(
+      actorSystem,
+      applicationConfiguration.task,
+      taskRepository,
+      probeRepository,
+      userRepository,
+      mailService
+    ).schedule()
+  } else {
+    logger.debug("Probes are disabled")
+  }
+
   // Routes
   lazy val router: Router =
     new _root_.router.Routes(

app/repositories/probe/ProbeRepository.scala

@@ -0,0 +1,33 @@
+package repositories.probe
+
+import repositories.PostgresProfile.api._
+import slick.basic.DatabaseConfig
+import slick.jdbc.JdbcProfile
+
+import scala.concurrent.Future
+import scala.concurrent.duration.FiniteDuration
+
+class ProbeRepository(dbConfig: DatabaseConfig[JdbcProfile]) {
+
+  import dbConfig._
+
+  def getReponseConsoRate(interval: FiniteDuration): Future[Option[Double]] = db.run(
+    sql"""
+          SELECT (CAST(SUM(CASE
+                                                          WHEN forward_to_reponseconso = true THEN 1
+                                                          ELSE 0
+    END) AS FLOAT) / count(*)) * 100 ratio FROM reports WHERE reports.creation_date < (now() - INTERVAL '${interval
+        .toString()}');
+        """.as[Double].headOption
+  )
+
+  def getLancerDalerteRate(interval: FiniteDuration): Future[Option[Double]] = db.run(
+    sql"""
+          SELECT (CAST(SUM(CASE
+                     WHEN status = 'LanceurAlerte' THEN 1
+                     ELSE 0
+    END) AS FLOAT) / count(*)) * 100 ratio FROM reports WHERE reports.creation_date < (now() - INTERVAL '${interval
+        .toString()}');
+       """.as[Double].headOption
+  )
+}

app/services/Email.scala

@@ -228,6 +228,15 @@ object Email {
     override val recipients: List[EmailAddress] = List(recipient)
   }
 
+  final case class ProbeTriggered(recipients: Seq[EmailAddress], probeName: String, rate: Double, issue: String)
+      extends AdminEmail {
+
+    override val subject: String = EmailSubjects.ADMIN_PROBE_TRIGGERED
+
+    override def getBody: (FrontRoute, EmailAddress) => String = (_, _) =>
+      views.html.mails.admin.probetriggered(probeName, rate, issue).toString()
+  }
+
   final case class ReportDeletionConfirmation(report: Report, maybeCompany: Option[Company], messagesApi: MessagesApi)
       extends ConsumerEmail {
     private val lang                                        = Lang(getLocaleOrDefault(report.lang))

app/tasks/probe/LowRateLanceurDAlerteTask.scala

@@ -0,0 +1,47 @@
+package tasks.probe
+
+import akka.actor.ActorSystem
+import config.TaskConfiguration
+import models.UserRole
+import play.api.Logger
+import repositories.probe.ProbeRepository
+import repositories.tasklock.TaskRepositoryInterface
+import repositories.user.UserRepositoryInterface
+import services.Email.ProbeTriggered
+import services.MailService
+import tasks.ScheduledTask
+import utils.Logs.RichLogger
+
+import java.time.LocalTime
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+import scala.concurrent.duration.DurationInt
+import scala.concurrent.duration.FiniteDuration
+
+class LowRateLanceurDAlerteTask(
+    actorSystem: ActorSystem,
+    taskConfiguration: TaskConfiguration,
+    taskRepository: TaskRepositoryInterface,
+    probeRepository: ProbeRepository,
+    userRepository: UserRepositoryInterface,
+    mailService: MailService
+)(implicit executionContext: ExecutionContext)
+    extends ScheduledTask(101, "low_rate_lanceur_dalerte", taskRepository, actorSystem, taskConfiguration) {
+
+  override val logger: Logger           = Logger(this.getClass)
+  override val startTime: LocalTime     = LocalTime.of(2, 0)
+  override val interval: FiniteDuration = 12.hours
+
+  override def runTask(): Future[Unit] = probeRepository.getLancerDalerteRate(interval).flatMap {
+    case Some(rate) if rate < 0.1d =>
+      logger.warnWithTitle("probe_triggered", s"Taux de signalements 'Lanceur d'alerte' faible : $rate%")
+      for {
+        users <- userRepository.listForRoles(Seq(UserRole.Admin))
+        _ <- mailService
+          .send(ProbeTriggered(users.map(_.email), "Taux de signalements 'Lanceur d'alerte' faible", rate, "bas"))
+      } yield ()
+    case rate =>
+      logger.debug(s"Taux de signalements correct: $rate%")
+      Future.unit
+  }
+}

app/tasks/probe/LowRateReponseConsoTask.scala

@@ -0,0 +1,47 @@
+package tasks.probe
+
+import akka.actor.ActorSystem
+import config.TaskConfiguration
+import models.UserRole
+import play.api.Logger
+import repositories.probe.ProbeRepository
+import repositories.tasklock.TaskRepositoryInterface
+import repositories.user.UserRepositoryInterface
+import services.Email.ProbeTriggered
+import services.MailService
+import tasks.ScheduledTask
+import utils.Logs.RichLogger
+
+import java.time.LocalTime
+import scala.concurrent.duration.DurationInt
+import scala.concurrent.duration.FiniteDuration
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+class LowRateReponseConsoTask(
+    actorSystem: ActorSystem,
+    taskConfiguration: TaskConfiguration,
+    taskRepository: TaskRepositoryInterface,
+    probeRepository: ProbeRepository,
+    userRepository: UserRepositoryInterface,
+    mailService: MailService
+)(implicit executionContext: ExecutionContext)
+    extends ScheduledTask(100, "low_rate_reponse_conso", taskRepository, actorSystem, taskConfiguration) {
+
+  override val logger: Logger           = Logger(this.getClass)
+  override val startTime: LocalTime     = LocalTime.of(2, 0)
+  override val interval: FiniteDuration = 12.hours
+
+  override def runTask(): Future[Unit] = probeRepository.getReponseConsoRate(interval).flatMap {
+    case Some(rate) if rate < 1.0d =>
+      logger.warnWithTitle("probe_triggered", s"Taux de signalements 'Réponse conso' faible : $rate%")
+      for {
+        users <- userRepository.listForRoles(Seq(UserRole.Admin))
+        _ <- mailService
+          .send(ProbeTriggered(users.map(_.email), "Taux de signalements 'Réponse conso' faible", rate, "bas"))
+      } yield ()
+    case rate =>
+      logger.debug(s"Taux de signalements correct: $rate%")
+      Future.unit
+  }
+}

app/utils/EmailSubjects.scala

@@ -7,6 +7,7 @@ object EmailSubjects {
   val COMPANY_ACCESS_INVITATION = (companyName: String) => s"Rejoignez l'entreprise ${companyName} sur SignalConso"
   val DGCCRF_ACCESS_LINK        = "Votre accès DGCCRF sur SignalConso"
   val ADMIN_ACCESS_LINK         = "Votre accès Administrateur sur SignalConso"
+  val ADMIN_PROBE_TRIGGERED     = "Sonde Signal conso déclenchée"
   val VALIDATE_EMAIL            = "Veuillez valider cette adresse email"
   val NEW_REPORT                = "Nouveau signalement"
   val REPORT_REOPENING          = "Réouverture signalement en attente de réponse"

app/views/mails/admin/probetriggered.scala.html

@@ -0,0 +1,15 @@
+@import java.net.URI
+@import utils.FrontRoute
+
+@(probeName: String, rate: Double, issue: String)
+
+@views.html.mails.layout("Sonde déclenchée") {
+  <p>
+      Cet email a été envoyée car la sonde @probeName a été déclenchée.
+  </p>
+
+  <p>Le taux détecté (@rate %) est anormalement @issue.
+  </p>
+
+  <p>Contactez un développeur pour investiguer.</p>
+}

conf/common/task.conf

@@ -36,4 +36,9 @@ task {
     etablissement-api-key = ${ETABLISSEMENT_API_KEY}
   }
 
+  probe {
+    active = true
+    active = ${?SIGNAL_CONSO_SCHEDULED_PROBES_ACTIVE}
+  }
+
 }

test/tasks/report/ReportRemindersTaskUnitSpec.scala

@@ -1,6 +1,7 @@
 package tasks.report
 
 import akka.actor.testkit.typed.scaladsl.ActorTestKit
+import config.ProbeConfiguration
 import config.ReportRemindersTaskConfiguration
 import config.TaskConfiguration
 import models.company.AccessLevel
@@ -49,7 +50,8 @@ class ReportRemindersTaskUnitSpec extends Specification with FutureMatchers {
       mailReminderDelay = Period.ofDays(7)
     ),
     inactiveAccounts = null,
-    companyUpdate = null
+    companyUpdate = null,
+    probe = ProbeConfiguration(false)
   )
 
   val testKit = ActorTestKit()