Merge pull request #1858 from betagouv/master

MEP

app/config/TaskConfiguration.scala

@@ -18,6 +18,7 @@ case class TaskConfiguration(
     probe: ProbeConfiguration,
     exportReportsToSFTP: ExportReportsToSFTPConfiguration,
     subcategoryLabels: SubcategoryLabelsTaskConfiguration,
+    siretExtraction: SiretExtractionConfiguration,
     sampleData: SampleDataConfiguration
 )
 
@@ -64,6 +65,11 @@ case class SubcategoryLabelsTaskConfiguration(
     interval: FiniteDuration
 )
 
+case class SiretExtractionConfiguration(
+    interval: FiniteDuration,
+    websiteCount: Int
+)
+
 case class ProbeConfiguration(active: Boolean)
 
 case class ExportReportsToSFTPConfiguration(filePath: String, startTime: LocalTime)

app/controllers/SiretExtractorController.scala

@@ -6,25 +6,38 @@ import models.UserRole
 import play.api.mvc.ControllerComponents
 import services.SiretExtractorService
 import authentication.actions.UserAction.WithRole
+import play.api.libs.json.Json
+import repositories.siretextraction.SiretExtractionRepositoryInterface
 
 import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
 
 class SiretExtractorController(
+    siretExtractionRepository: SiretExtractionRepositoryInterface,
     siretExtractorService: SiretExtractorService,
     authenticator: Authenticator[User],
     controllerComponents: ControllerComponents
 )(implicit val ec: ExecutionContext)
     extends BaseController(authenticator, controllerComponents) {
 
-  def extractSiret() = SecuredAction.andThen(WithRole(UserRole.Admins)).async { request =>
-    siretExtractorService
-      .extractSiret(request.body.asJson)
-      .map { response =>
-        response.body match {
-          case Left(body)  => Status(response.code.code)(body.getMessage)
-          case Right(body) => Status(response.code.code)(body)
-        }
-      }
+  def extractSiret() = SecuredAction.andThen(WithRole(UserRole.Admins)).async(parse.json) { request =>
+    val maybeWebsite = (request.body \ "website").asOpt[String]
+
+    maybeWebsite match {
+      case Some(website) =>
+        siretExtractorService
+          .extractSiret(website)
+          .flatMap { response =>
+            response.body match {
+              case Left(body) => Future.successful(Status(response.code.code)(body.getMessage))
+              case Right(body) =>
+                logger.debug(s"Saving siret extraction result (${body.status}) in DB before returning")
+                siretExtractionRepository.insertOrReplace(body).map(_ => Status(response.code.code)(Json.toJson(body)))
+            }
+          }
+      case None => Future.successful(BadRequest)
+    }
+
   }
 
 }

app/controllers/WebsiteController.scala

@@ -137,7 +137,7 @@ class WebsiteController(
           errors => Future.successful(BadRequest(JsError.toJson(errors))),
           company =>
             websitesOrchestrator
-              .updateCompany(websiteId, company, request.identity)
+              .updateCompany(websiteId, company, Some(request.identity))
               .map(websiteAndCompany => Ok(Json.toJson(websiteAndCompany)))
         )
     }

app/loader/SignalConsoApplicationLoader.scala

@@ -93,6 +93,7 @@ import repositories.reportmetadata.ReportMetadataRepository
 import repositories.reportmetadata.ReportMetadataRepositoryInterface
 import repositories.signalconsoreview.SignalConsoReviewRepository
 import repositories.signalconsoreview.SignalConsoReviewRepositoryInterface
+import repositories.siretextraction.SiretExtractionRepository
 import repositories.socialnetwork.SocialNetworkRepository
 import repositories.socialnetwork.SocialNetworkRepositoryInterface
 import repositories.subcategorylabel.SubcategoryLabelRepository
@@ -128,6 +129,7 @@ import tasks.report.ReportNotificationTask
 import tasks.report.ReportRemindersTask
 import tasks.report.SampleDataGenerationTask
 import tasks.subcategorylabel.SubcategoryLabelTask
+import tasks.website.SiretExtractionTask
 import utils.CustomIpFilter
 import utils.EmailAddress
 import utils.FrontRoute
@@ -255,6 +257,8 @@ class SignalConsoComponents(
 
   val albertClassificationRepository = new AlbertClassificationRepository(dbConfig)
 
+  val siretExtractionRepository = new SiretExtractionRepository(dbConfig)
+
   val crypter = new JcaCrypter(applicationConfiguration.crypter)
   val signer  = new JcaSigner(applicationConfiguration.signer)
 
@@ -320,6 +324,8 @@ class SignalConsoComponents(
     attachmentService
   )
 
+  val siretExtractorService = new SiretExtractorService(applicationConfiguration.siretExtractor)
+
   // Orchestrator
 
   val userOrchestrator = new UserOrchestrator(userRepository, eventRepository)
@@ -723,6 +729,16 @@ class SignalConsoComponents(
     taskRepository
   )
 
+  val siretExtractionTask = new SiretExtractionTask(
+    actorSystem,
+    taskConfiguration,
+    taskRepository,
+    siretExtractionRepository,
+    siretExtractorService,
+    websitesOrchestrator,
+    companyRepository
+  )
+
   // Controller
 
   val blacklistedEmailsController =
@@ -894,9 +910,13 @@ class SignalConsoComponents(
     controllerComponents
   )
 
-  val siretExtractorService = new SiretExtractorService(applicationConfiguration.siretExtractor)
   val siretExtractorController =
-    new SiretExtractorController(siretExtractorService, cookieAuthenticator, controllerComponents)
+    new SiretExtractorController(
+      siretExtractionRepository,
+      siretExtractorService,
+      cookieAuthenticator,
+      controllerComponents
+    )
 
   val importOrchestrator = new ImportOrchestrator(
     companyRepository,
@@ -982,6 +1002,7 @@ class SignalConsoComponents(
     subcategoryLabelTask.schedule()
     companyAlbertLabelTask.schedule()
     companyReportCountViewRefresherTask.schedule()
+    siretExtractionTask.schedule()
   }
 
   override def config: Config = ConfigFactory.load()

app/models/website/WebsiteCompanyReportCount.scala

@@ -5,6 +5,7 @@ import models.company.Company
 import models.investigation.InvestigationStatus
 import play.api.libs.json.Json
 import play.api.libs.json.Writes
+import tasks.website.ExtractionResultApi
 import utils.Country
 
 import java.time.OffsetDateTime
@@ -23,22 +24,26 @@ case class WebsiteCompanyReportCount(
     kind: String,
     company: Option[Company],
     investigationStatus: InvestigationStatus,
-    count: Int
+    count: Int,
+    siretExtraction: Option[ExtractionResultApi]
 )
 
 object WebsiteCompanyReportCount {
 
   implicit val WebsiteCompanyCountWrites: Writes[WebsiteCompanyReportCount] = Json.writes[WebsiteCompanyReportCount]
 
-  def toApi(countByWebsiteCompany: ((Website, Option[Company]), Int)): WebsiteCompanyReportCount = {
-    val ((website, maybeCompany), count) = countByWebsiteCompany
+  def toApi(
+      countByWebsiteCompany: (Website, Option[Company], Option[ExtractionResultApi], Int)
+  ): WebsiteCompanyReportCount = {
+    val (website, maybeCompany, maybeSiretExtraction, count) = countByWebsiteCompany
     website
       .into[WebsiteCompanyReportCount]
       .withFieldComputed(_.id, _.id)
       .withFieldConst(_.company, maybeCompany)
       .withFieldConst(_.companyCountry, website.companyCountry.map(Country.fromCode(_)))
       .withFieldConst(_.count, count)
       .withFieldConst(_.kind, IdentificationStatus.toKind(website.identificationStatus))
+      .withFieldConst(_.siretExtraction, maybeSiretExtraction)
       .transform
   }
 }

app/orchestrators/ImportOrchestrator.scala

@@ -131,8 +131,12 @@ class ImportOrchestrator(
                       companyId = Some(company.id),
                       isMarketplace = true
                     )
-                    updatedWebsite <- websitesOrchestrator.updateIdentification(websiteToUpdate, user)
-                    _ <- websitesOrchestrator.updatePreviousReportsAssociatedToWebsite(website.host, company, user.id)
+                    updatedWebsite <- websitesOrchestrator.updateIdentification(websiteToUpdate, Some(user))
+                    _ <- websitesOrchestrator.updatePreviousReportsAssociatedToWebsite(
+                      website.host,
+                      company,
+                      Some(user.id)
+                    )
                   } yield updatedWebsite
               }
             case Nil =>
@@ -145,7 +149,7 @@ class ImportOrchestrator(
               )
               for {
                 createdWebsite <- websiteRepository.create(website)
-                _ <- websitesOrchestrator.updatePreviousReportsAssociatedToWebsite(website.host, company, user.id)
+                _ <- websitesOrchestrator.updatePreviousReportsAssociatedToWebsite(website.host, company, Some(user.id))
               } yield createdWebsite
 
           }

app/orchestrators/ReportOrchestrator.scala

@@ -618,14 +618,14 @@ class ReportOrchestrator(
       updatedReport <- updateReportCompany(
         existingReport,
         reportCompany,
-        requestingUserId
+        Some(requestingUserId)
       )
     } yield updatedReport
 
   def updateReportCompanyForWebsite(
       existingReport: Report,
       reportCompany: ReportCompany,
-      adminUserId: UUID
+      adminUserId: Option[UUID]
   ) =
     if (isReportTooOld(existingReport)) {
       logger.debug(s"Report ${existingReport.id} is too old to be updated")
@@ -636,7 +636,7 @@ class ReportOrchestrator(
   private def updateReportCompany(
       existingReport: Report,
       reportCompany: ReportCompany,
-      adminUserId: UUID
+      adminUserId: Option[UUID]
   ): Future[Report] = {
     val updateDateTime = OffsetDateTime.now()
 
@@ -687,9 +687,9 @@ class ReportOrchestrator(
               UUID.randomUUID(),
               Some(updatedReport.id),
               Some(company.id),
-              Some(adminUserId),
+              adminUserId,
               updateDateTime,
-              Constants.EventType.ADMIN,
+              if (adminUserId.isDefined) Constants.EventType.ADMIN else Constants.EventType.SYSTEM,
               Constants.ActionEvent.REPORT_COMPANY_CHANGE,
               stringToDetailsJsValue(
                 s"Entreprise précédente : Siret ${existingReport.companySiret

app/orchestrators/WebsitesOrchestrator.scala

@@ -64,7 +64,7 @@ class WebsitesOrchestrator(
         identificationStatus = IdentificationStatus.Identified
       )
       createdWebsite <- repository.create(website)
-      _              <- updatePreviousReportsAssociatedToWebsite(website.host, createdCompany, user.id)
+      _              <- updatePreviousReportsAssociatedToWebsite(website.host, createdCompany, Some(user.id))
     } yield createdWebsite
 
   def searchByHost(host: String): Future[Seq[Country]] =
@@ -135,7 +135,7 @@ class WebsitesOrchestrator(
             .getOrElse(Future.successful(None))
           _ = logger.debug(s"Company Siret is ${maybeCompany.map(_.siret)}")
           _ <- maybeCompany
-            .map(company => updatePreviousReportsAssociatedToWebsite(website.host, company, user.id))
+            .map(company => updatePreviousReportsAssociatedToWebsite(website.host, company, Some(user.id)))
             .getOrElse(Future.unit)
         } yield ()
       } else Future.unit
@@ -152,7 +152,11 @@ class WebsitesOrchestrator(
     } yield website
   }
 
-  def updateCompany(websiteId: WebsiteId, companyToAssign: CompanyCreation, user: User): Future[WebsiteAndCompany] =
+  def updateCompany(
+      websiteId: WebsiteId,
+      companyToAssign: CompanyCreation,
+      user: Option[User]
+  ): Future[WebsiteAndCompany] =
     for {
       company <- {
         logger.debug(s"Updating website (id ${websiteId}) with company siret : ${companyToAssign.siret}")
@@ -171,7 +175,7 @@ class WebsitesOrchestrator(
         companyId = Some(company.id)
       )
       updatedWebsite <- updateIdentification(websiteToUpdate, user)
-      _              <- updatePreviousReportsAssociatedToWebsite(website.host, company, user.id)
+      _              <- updatePreviousReportsAssociatedToWebsite(website.host, company, user.map(_.id))
     } yield WebsiteAndCompany.toApi(updatedWebsite, Some(company))
 
   def updateCompanyCountry(websiteId: WebsiteId, companyCountry: String, user: User): Future[WebsiteAndCompany] = for {
@@ -188,17 +192,17 @@ class WebsitesOrchestrator(
       companyCountry = Some(companyCountry),
       companyId = None
     )
-    updatedWebsite <- updateIdentification(websiteToUpdate, user)
+    updatedWebsite <- updateIdentification(websiteToUpdate, Some(user))
   } yield WebsiteAndCompany.toApi(updatedWebsite, maybeCompany = None)
 
-  def updateIdentification(website: Website, user: User): Future[Website] = {
+  def updateIdentification(website: Website, user: Option[User]): Future[Website] = {
     logger.debug(s"Removing other websites with the same host : ${website.host}")
     for {
       _ <- repository
         .removeOtherNonIdentifiedWebsitesWithSameHost(website)
       _ = logger.debug(s"updating identification status when Admin is updating identification")
       websiteToUpdate =
-        if (UserRole.isAdmin(user.userRole)) website.copy(identificationStatus = Identified) else website
+        if (user.map(_.userRole).forall(UserRole.isAdmin)) website.copy(identificationStatus = Identified) else website
       _ = logger.debug(s"Website to update : ${websiteToUpdate}")
       updatedWebsite <- update(websiteToUpdate)
       _ = logger.debug(s"Website company country successfully updated")
@@ -264,7 +268,7 @@ class WebsitesOrchestrator(
   def updatePreviousReportsAssociatedToWebsite(
       websiteHost: String,
       company: Company,
-      userId: UUID
+      userId: Option[UUID]
   ): Future[Unit] = {
     val reportCompany = ReportCompany(
       name = company.name,

app/repositories/siretextraction/SiretExtractionRepository.scala

@@ -0,0 +1,48 @@
+package repositories.siretextraction
+
+import models.website.IdentificationStatus
+import models.website.Website
+import slick.basic.DatabaseConfig
+import slick.jdbc.JdbcProfile
+import repositories.PostgresProfile.api._
+import repositories.website.WebsiteColumnType._
+import repositories.website.WebsiteTable
+import tasks.website.ExtractionResultApi
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+trait SiretExtractionRepositoryInterface {
+  def getByHost(host: String): Future[Option[ExtractionResultApi]]
+  def listUnextractedWebsiteHosts(n: Int): Future[List[Website]]
+  def insertOrReplace(extraction: ExtractionResultApi): Future[Unit]
+}
+
+class SiretExtractionRepository(dbConfig: DatabaseConfig[JdbcProfile])(implicit ec: ExecutionContext)
+    extends SiretExtractionRepositoryInterface {
+
+  val table = SiretExtractionTable.table
+
+  import dbConfig._
+
+  override def getByHost(host: String): Future[Option[ExtractionResultApi]] = db.run(
+    table.filter(_.host === host).to[List].result.headOption
+  )
+
+  def listUnextractedWebsiteHosts(n: Int): Future[List[Website]] = db.run(
+    WebsiteTable.table
+      .joinLeft(table)
+      .on(_.host === _.host)
+      .filter(_._1.identificationStatus === (IdentificationStatus.NotIdentified: IdentificationStatus))
+      .filter(_._2.isEmpty)
+      .map(_._1)
+      .distinctOn(_.host)
+      .take(n)
+      .to[List]
+      .result
+  )
+
+  override def insertOrReplace(extraction: ExtractionResultApi): Future[Unit] = db.run(
+    table.insertOrUpdate(extraction).map(_ => ())
+  )
+}