From 545824e87e4ef167c5ece923bd2e3fa02b36edbb Mon Sep 17 00:00:00 2001 From: Charles Dufour Date: Thu, 8 Feb 2024 18:43:09 +0100 Subject: [PATCH 01/10] [TRELLO-2168] Implement a better 'lifecycle' for categories (#1539) * [TRELLO-2168] Implement a better 'lifecycle' for categories * [TRELLO-2168] Implement a better 'lifecycle' for categories --- app/controllers/ConstantController.scala | 17 ++++++- app/models/report/ReportCategory.scala | 59 +++++++++++++++--------- conf/routes | 1 + 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/app/controllers/ConstantController.scala b/app/controllers/ConstantController.scala index 22dcddfb3..77b501dbc 100644 --- a/app/controllers/ConstantController.scala +++ b/app/controllers/ConstantController.scala @@ -3,6 +3,7 @@ package controllers import authentication.Authenticator import models.User import models.report.ReportCategory +import models.report.ReportCategoryStatus import play.api.Logger import play.api.libs.json.Json import play.api.mvc.ControllerComponents @@ -21,7 +22,21 @@ class ConstantController(authenticator: Authenticator[User], controllerComponent } def getCategories = Action.async { - Future(Ok(Json.toJson(ReportCategory.values.filterNot(_.legacy)))) + Future(Ok(Json.toJson(ReportCategory.values.filter(_.status != ReportCategoryStatus.Legacy)))) + } + + def getCategoriesByStatus() = Action.async { + val legacy = ReportCategory.values.filter(_.status == ReportCategoryStatus.Legacy) + val closed = ReportCategory.values.filter(_.status == ReportCategoryStatus.Closed) + val inactive = ReportCategory.values.filter(_.status == ReportCategoryStatus.Inactive) + val active = ReportCategory.values.filter(_.status == ReportCategoryStatus.Active) + val json = Json.obj( + "active" -> active, + "inactive" -> inactive, + "legacy" -> legacy, + "closed" -> closed + ) + Future(Ok(json)) } } diff --git a/app/models/report/ReportCategory.scala b/app/models/report/ReportCategory.scala index eaf867623..454b3a333 100644 --- a/app/models/report/ReportCategory.scala +++ b/app/models/report/ReportCategory.scala @@ -4,31 +4,46 @@ import controllers.error.AppError.MalformedValue import enumeratum.EnumEntry import enumeratum.PlayEnum -sealed abstract class ReportCategory(val label: String, val legacy: Boolean = false) extends EnumEntry +sealed trait ReportCategoryStatus +object ReportCategoryStatus { + // Modified / splitted / migrated, should never come back + case object Legacy extends ReportCategoryStatus + // Old category, not available and no investigation, but still 'valid' + case object Closed extends ReportCategoryStatus + // Category not available in the website, but an investigation could still happen (still searchable) + case object Inactive extends ReportCategoryStatus + case object Active extends ReportCategoryStatus +} + +sealed abstract class ReportCategory(val label: String, val status: ReportCategoryStatus = ReportCategoryStatus.Active) + extends EnumEntry object ReportCategory extends PlayEnum[ReportCategory] { - case object RetraitRappelSpecifique extends ReportCategory("Retrait-Rappel spécifique") - case object Coronavirus extends ReportCategory("COVID-19 (coronavirus)") - case object CafeRestaurant extends ReportCategory("Café / Restaurant") - case object AchatMagasinLegacy extends ReportCategory("Achat / Magasin", legacy = true) - case object AchatMagasinInternet extends ReportCategory("Achat (Magasin ou Internet)", legacy = true) - case object AchatMagasin extends ReportCategory("Achat en Magasin") - case object AchatInternet extends ReportCategory("Achat sur internet") - case object ServicesAuxParticuliers extends ReportCategory("Services aux particuliers") - case object TelEauGazElec extends ReportCategory("Téléphonie / Eau-Gaz-Electricité", legacy = true) - case object EauGazElectricite extends ReportCategory("Eau / Gaz / Electricité") - case object TelephonieFaiMedias extends ReportCategory("Téléphonie / Fournisseur d'accès internet / médias") - case object BanqueAssuranceMutuelle extends ReportCategory("Banque / Assurance / Mutuelle") - case object IntoxicationAlimentaire extends ReportCategory("Intoxication alimentaire") - case object ProduitsObjets extends ReportCategory("Produits / Objets", legacy = true) - case object Internet extends ReportCategory("Internet (hors achats)") - case object TravauxRenovations extends ReportCategory("Travaux / Rénovation") - case object VoyageLoisirs extends ReportCategory("Voyage / Loisirs") - case object Immobilier extends ReportCategory("Immobilier") - case object Sante extends ReportCategory("Secteur de la santé") - case object VoitureVehicule extends ReportCategory("Voiture / Véhicule", legacy = true) - case object Animaux extends ReportCategory("Animaux") + case object RetraitRappelSpecifique + extends ReportCategory("Retrait-Rappel spécifique", status = ReportCategoryStatus.Closed) + case object Coronavirus extends ReportCategory("COVID-19 (coronavirus)", status = ReportCategoryStatus.Closed) + case object CafeRestaurant extends ReportCategory("Café / Restaurant") + case object AchatMagasinLegacy extends ReportCategory("Achat / Magasin", status = ReportCategoryStatus.Legacy) + case object AchatMagasinInternet + extends ReportCategory("Achat (Magasin ou Internet)", status = ReportCategoryStatus.Legacy) + case object AchatMagasin extends ReportCategory("Achat en Magasin") + case object AchatInternet extends ReportCategory("Achat sur internet") + case object ServicesAuxParticuliers extends ReportCategory("Services aux particuliers") + case object TelEauGazElec + extends ReportCategory("Téléphonie / Eau-Gaz-Electricité", status = ReportCategoryStatus.Legacy) + case object EauGazElectricite extends ReportCategory("Eau / Gaz / Electricité") + case object TelephonieFaiMedias extends ReportCategory("Téléphonie / Fournisseur d'accès internet / médias") + case object BanqueAssuranceMutuelle extends ReportCategory("Banque / Assurance / Mutuelle") + case object IntoxicationAlimentaire extends ReportCategory("Intoxication alimentaire") + case object ProduitsObjets extends ReportCategory("Produits / Objets", status = ReportCategoryStatus.Legacy) + case object Internet extends ReportCategory("Internet (hors achats)") + case object TravauxRenovations extends ReportCategory("Travaux / Rénovation") + case object VoyageLoisirs extends ReportCategory("Voyage / Loisirs") + case object Immobilier extends ReportCategory("Immobilier") + case object Sante extends ReportCategory("Secteur de la santé") + case object VoitureVehicule extends ReportCategory("Voiture / Véhicule", status = ReportCategoryStatus.Legacy) + case object Animaux extends ReportCategory("Animaux") case object DemarchesAdministratives extends ReportCategory("Démarches administratives") case object VoitureVehiculeVelo extends ReportCategory("Voiture / Véhicule / Vélo") case object DemarchageAbusif extends ReportCategory("Démarchage abusif") diff --git a/conf/routes b/conf/routes index 079a3d31d..895b6fc92 100644 --- a/conf/routes +++ b/conf/routes @@ -85,6 +85,7 @@ GET /api/async-files controll # Constants API GET /api/constants/countries controllers.ConstantController.getCountries() GET /api/constants/categories controllers.ConstantController.getCategories() +GET /api/constants/categoriesByStatus controllers.ConstantController.getCategoriesByStatus() # Mobile app specific API GET /api/mobileapp/requirements controllers.MobileAppController.getRequirements() From 91c9a029cb37c753aeb5b1d5bac04f1aa3668c7b Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Sun, 4 Feb 2024 13:18:45 +0100 Subject: [PATCH 02/10] TRELLO-2174 : CertifiedInfluencer api --- app/controllers/SocialNetworkController.scala | 28 +++++++++++++++++ app/controllers/package.scala | 8 +++++ app/loader/SignalConsoApplicationLoader.scala | 10 ++++++ .../socialnetwork/CertifiedInfluencer.scala | 13 ++++++++ .../socialmedia/InfluencerOrchestrator.scala | 18 +++++++++++ .../influencer/InfluencerRepository.scala | 31 +++++++++++++++++++ .../InfluencerRepositoryInterface.scala | 11 +++++++ .../influencer/InfluencerTable.scala | 20 ++++++++++++ build.sbt | 2 ++ .../default/V19__influencer_table.sql | 5 +++ conf/routes | 2 ++ 11 files changed, 148 insertions(+) create mode 100644 app/controllers/SocialNetworkController.scala create mode 100644 app/models/report/socialnetwork/CertifiedInfluencer.scala create mode 100644 app/orchestrators/socialmedia/InfluencerOrchestrator.scala create mode 100644 app/repositories/influencer/InfluencerRepository.scala create mode 100644 app/repositories/influencer/InfluencerRepositoryInterface.scala create mode 100644 app/repositories/influencer/InfluencerTable.scala create mode 100644 conf/db/migration/default/V19__influencer_table.sql diff --git a/app/controllers/SocialNetworkController.scala b/app/controllers/SocialNetworkController.scala new file mode 100644 index 000000000..2b7343ab4 --- /dev/null +++ b/app/controllers/SocialNetworkController.scala @@ -0,0 +1,28 @@ +package controllers + +import authentication.Authenticator +import models.User +import models.report.SocialNetworkSlug +import orchestrators.socialmedia.InfluencerOrchestrator +import play.api.Logger +import play.api.libs.json.Json +import play.api.mvc.ControllerComponents + +import scala.concurrent.ExecutionContext + +class SocialNetworkController( + influencerOrchestrator: InfluencerOrchestrator, + authenticator: Authenticator[User], + controllerComponents: ControllerComponents +)(implicit + val ec: ExecutionContext +) extends BaseController(authenticator, controllerComponents) { + val logger: Logger = Logger(this.getClass) + + def get(name: String, socialNetwork: SocialNetworkSlug) = Action.async { _ => + influencerOrchestrator + .get(name, socialNetwork) + .map(product => Ok(Json.toJson(product))) + } + +} diff --git a/app/controllers/package.scala b/app/controllers/package.scala index 61524e862..e62517b11 100644 --- a/app/controllers/package.scala +++ b/app/controllers/package.scala @@ -14,6 +14,7 @@ import models.PublicStat import models.extractUUID import models.report.ReportFileOrigin import models.report.ReportResponseType +import models.report.SocialNetworkSlug import models.report.delete.ReportAdminActionType import models.report.reportfile.ReportFileId import utils.DateUtils @@ -35,6 +36,13 @@ package object controllers { identificationStatus => identificationStatus.entryName ) + implicit val SocialNetworkSlugQueryStringBindable: QueryStringBindable[SocialNetworkSlug] = + QueryStringBindable.bindableString + .transform[SocialNetworkSlug]( + socialNetwork => SocialNetworkSlug.withName(socialNetwork), + socialNetwork => socialNetwork.entryName + ) + implicit val UUIDPathBindable: PathBindable[UUID] = PathBindable.bindableString .transform[UUID]( diff --git a/app/loader/SignalConsoApplicationLoader.scala b/app/loader/SignalConsoApplicationLoader.scala index f83f56fff..2339b5a7f 100644 --- a/app/loader/SignalConsoApplicationLoader.scala +++ b/app/loader/SignalConsoApplicationLoader.scala @@ -19,6 +19,8 @@ import com.typesafe.config.ConfigFactory import config._ import models.report.ArborescenceNode import orchestrators._ +import orchestrators.socialmedia.InfluencerOrchestrator +import orchestrators.socialmedia.SocialBladeClient import org.flywaydb.core.Flyway import play.api._ import play.api.db.slick.DbName @@ -64,6 +66,8 @@ import repositories.emailvalidation.EmailValidationRepositoryInterface import repositories.event.EventRepository import repositories.event.EventRepositoryInterface import repositories.barcode.BarcodeProductRepository +import repositories.influencer.InfluencerRepository +import repositories.influencer.InfluencerRepositoryInterface import repositories.probe.ProbeRepository import repositories.rating.RatingRepository import repositories.rating.RatingRepositoryInterface @@ -200,6 +204,7 @@ class SignalConsoComponents( def eventRepository: EventRepositoryInterface = new EventRepository(dbConfig) val ratingRepository: RatingRepositoryInterface = new RatingRepository(dbConfig) + val influencerRepository: InfluencerRepositoryInterface = new InfluencerRepository(dbConfig) def reportRepository: ReportRepositoryInterface = new ReportRepository(dbConfig) val reportMetadataRepository: ReportMetadataRepositoryInterface = new ReportMetadataRepository(dbConfig) val reportNotificationBlockedRepository: ReportNotificationBlockedRepositoryInterface = @@ -389,6 +394,8 @@ class SignalConsoComponents( messagesApi ) + val influencerOrchestrator = new InfluencerOrchestrator(influencerRepository) + val reportsExtractActor: typed.ActorRef[ReportsExtractActor.ReportsExtractCommand] = actorSystem.spawn( ReportsExtractActor.create( @@ -540,6 +547,8 @@ class SignalConsoComponents( controllerComponents ) + val socialNetworkController = + new SocialNetworkController(influencerOrchestrator, cookieAuthenticator, controllerComponents) val asyncFileController = new AsyncFileController(asyncFileRepository, s3Service, cookieAuthenticator, controllerComponents) @@ -741,6 +750,7 @@ class SignalConsoComponents( adminController, asyncFileController, constantController, + socialNetworkController, mobileAppController, authController, accountController, diff --git a/app/models/report/socialnetwork/CertifiedInfluencer.scala b/app/models/report/socialnetwork/CertifiedInfluencer.scala new file mode 100644 index 000000000..3fbfffbeb --- /dev/null +++ b/app/models/report/socialnetwork/CertifiedInfluencer.scala @@ -0,0 +1,13 @@ +package models.report.socialnetwork + +import models.report.SocialNetworkSlug +import play.api.libs.json.Json +import play.api.libs.json.OFormat + +import java.util.UUID + +case class CertifiedInfluencer(id: UUID, socialNetwork: SocialNetworkSlug, name: String) + +object CertifiedInfluencer { + implicit val format: OFormat[CertifiedInfluencer] = Json.format[CertifiedInfluencer] +} diff --git a/app/orchestrators/socialmedia/InfluencerOrchestrator.scala b/app/orchestrators/socialmedia/InfluencerOrchestrator.scala new file mode 100644 index 000000000..56f16d127 --- /dev/null +++ b/app/orchestrators/socialmedia/InfluencerOrchestrator.scala @@ -0,0 +1,18 @@ +package orchestrators.socialmedia + +import models.report.SocialNetworkSlug +import models.report.socialnetwork.CertifiedInfluencer +import repositories.influencer.InfluencerRepositoryInterface + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future + +class InfluencerOrchestrator( + influencerRepository: InfluencerRepositoryInterface +)(implicit + val executionContext: ExecutionContext +) { + + def get(name: String, socialNetwork: SocialNetworkSlug): Future[Seq[CertifiedInfluencer]] = + influencerRepository.get(name, socialNetwork) +} diff --git a/app/repositories/influencer/InfluencerRepository.scala b/app/repositories/influencer/InfluencerRepository.scala new file mode 100644 index 000000000..67967bbf8 --- /dev/null +++ b/app/repositories/influencer/InfluencerRepository.scala @@ -0,0 +1,31 @@ +package repositories.influencer + +import models.report.SocialNetworkSlug +import models.report.socialnetwork.CertifiedInfluencer +import play.api.Logger +import repositories.CRUDRepository +import repositories.PostgresProfile.api._ +import slick.basic.DatabaseConfig +import slick.jdbc.JdbcProfile +import slick.lifted.TableQuery + +import scala.concurrent.ExecutionContext +class InfluencerRepository(override val dbConfig: DatabaseConfig[JdbcProfile])(implicit + override val ec: ExecutionContext +) extends CRUDRepository[InfluencerTable, CertifiedInfluencer] + with InfluencerRepositoryInterface { + + val logger: Logger = Logger(this.getClass) + val table: TableQuery[InfluencerTable] = InfluencerTable.table + import dbConfig._ + + override def get(name: String, socialNetwork: SocialNetworkSlug) = db.run( + table + .filter { result => + (result.name <-> (name: String)).<(0.55) + } + .filter(_.socialNetwork === socialNetwork) + .result + ) + +} diff --git a/app/repositories/influencer/InfluencerRepositoryInterface.scala b/app/repositories/influencer/InfluencerRepositoryInterface.scala new file mode 100644 index 000000000..eb561ad76 --- /dev/null +++ b/app/repositories/influencer/InfluencerRepositoryInterface.scala @@ -0,0 +1,11 @@ +package repositories.influencer + +import models.report.SocialNetworkSlug +import models.report.socialnetwork.CertifiedInfluencer +import repositories.CRUDRepositoryInterface + +import scala.concurrent.Future + +trait InfluencerRepositoryInterface extends CRUDRepositoryInterface[CertifiedInfluencer] { + def get(name: String, socialNetwork: SocialNetworkSlug): Future[Seq[CertifiedInfluencer]] +} diff --git a/app/repositories/influencer/InfluencerTable.scala b/app/repositories/influencer/InfluencerTable.scala new file mode 100644 index 000000000..04741655b --- /dev/null +++ b/app/repositories/influencer/InfluencerTable.scala @@ -0,0 +1,20 @@ +package repositories.influencer + +import models.report.SocialNetworkSlug +import models.report.socialnetwork.CertifiedInfluencer +import repositories.DatabaseTable +import repositories.PostgresProfile.api._ +import slick.ast.TypedType + +class InfluencerTable(tag: Tag)(implicit tt: TypedType[SocialNetworkSlug]) + extends DatabaseTable[CertifiedInfluencer](tag, "influencers") { + def socialNetwork = column[SocialNetworkSlug]("social_network") + + def name = column[String]("name") + + override def * = (id, socialNetwork, name) <> ((CertifiedInfluencer.apply _).tupled, CertifiedInfluencer.unapply) +} + +object InfluencerTable { + val table = TableQuery[InfluencerTable] +} diff --git a/build.sbt b/build.sbt index 3fdda2fc9..fb665bccc 100644 --- a/build.sbt +++ b/build.sbt @@ -31,6 +31,7 @@ routesImport ++= Seq( "models.UserRole", "models.website.IdentificationStatus", "models.report.ReportFileOrigin", + "models.report.SocialNetworkSlug", "java.time.OffsetDateTime", "models.investigation.InvestigationStatus", "models.website.WebsiteId", @@ -40,6 +41,7 @@ routesImport ++= Seq( "models.report.delete.ReportAdminActionType", "models.PublicStat", "controllers.IdentificationStatusQueryStringBindable", + "controllers.SocialNetworkSlugQueryStringBindable", "controllers.WebsiteIdPathBindable", "controllers.UUIDPathBindable", "controllers.OffsetDateTimeQueryStringBindable", diff --git a/conf/db/migration/default/V19__influencer_table.sql b/conf/db/migration/default/V19__influencer_table.sql new file mode 100644 index 000000000..ecab6ccc1 --- /dev/null +++ b/conf/db/migration/default/V19__influencer_table.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS influencers ( + id UUID NOT NULL PRIMARY KEY, + name VARCHAR NOT NULL, + social_network VARCHAR NOT NULL +); diff --git a/conf/routes b/conf/routes index 895b6fc92..23b1813c1 100644 --- a/conf/routes +++ b/conf/routes @@ -85,6 +85,8 @@ GET /api/async-files controll # Constants API GET /api/constants/countries controllers.ConstantController.getCountries() GET /api/constants/categories controllers.ConstantController.getCategories() +#GET /api/social-blade controllers.ConstantController.getSocialBlade() +GET /api/certified-influencer controllers.SocialNetworkController.get(name:String,socialNetwork: SocialNetworkSlug) GET /api/constants/categoriesByStatus controllers.ConstantController.getCategoriesByStatus() # Mobile app specific API From 2ce51b300f6adef1de1390a8b9e0193fc58c387b Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Sun, 4 Feb 2024 13:22:37 +0100 Subject: [PATCH 03/10] TRELLO-2174 : CertifiedInfluencer api --- app/loader/SignalConsoApplicationLoader.scala | 118 ++++++------------ 1 file changed, 35 insertions(+), 83 deletions(-) diff --git a/app/loader/SignalConsoApplicationLoader.scala b/app/loader/SignalConsoApplicationLoader.scala index 2339b5a7f..e6aa2ed1a 100644 --- a/app/loader/SignalConsoApplicationLoader.scala +++ b/app/loader/SignalConsoApplicationLoader.scala @@ -6,109 +6,61 @@ import actors._ import akka.actor.typed import akka.actor.typed.scaladsl.adapter.ClassicActorSystemOps import akka.util.Timeout -import authentication.APIKeyAuthenticator -import authentication.BCryptPasswordHasher -import authentication.CookieAuthenticator -import authentication.CredentialsProvider -import authentication.FingerprintGenerator -import authentication.JcaCrypter -import authentication.JcaSigner -import authentication.PasswordHasherRegistry -import com.typesafe.config.Config -import com.typesafe.config.ConfigFactory +import authentication._ +import com.typesafe.config.{Config, ConfigFactory} import config._ import models.report.ArborescenceNode import orchestrators._ import orchestrators.socialmedia.InfluencerOrchestrator -import orchestrators.socialmedia.SocialBladeClient import org.flywaydb.core.Flyway import play.api._ -import play.api.db.slick.DbName -import play.api.db.slick.SlickComponents -import play.api.libs.json.JsArray -import play.api.libs.json.Json +import play.api.db.slick.{DbName, SlickComponents} +import play.api.libs.json.{JsArray, Json} import play.api.libs.mailer.MailerComponents import play.api.libs.ws.ahc.AhcWSComponents -import play.api.mvc.Cookie -import play.api.mvc.EssentialFilter +import play.api.mvc.{Cookie, EssentialFilter} import play.api.routing.Router import play.filters.HttpFiltersComponents -import pureconfig.ConfigConvert -import pureconfig.ConfigReader -import pureconfig.ConfigSource +import pureconfig.{ConfigConvert, ConfigReader, ConfigSource} import pureconfig.configurable.localTimeConfigConvert import pureconfig.generic.auto._ import pureconfig.generic.semiauto.deriveReader -import repositories.accesstoken.AccessTokenRepository -import repositories.accesstoken.AccessTokenRepositoryInterface -import repositories.asyncfiles.AsyncFileRepository -import repositories.asyncfiles.AsyncFileRepositoryInterface -import repositories.authattempt.AuthAttemptRepository -import repositories.authattempt.AuthAttemptRepositoryInterface -import repositories.authtoken.AuthTokenRepository -import repositories.authtoken.AuthTokenRepositoryInterface -import repositories.blacklistedemails.BlacklistedEmailsRepository -import repositories.blacklistedemails.BlacklistedEmailsRepositoryInterface -import repositories.company.CompanyRepository -import repositories.company.CompanyRepositoryInterface -import repositories.company.CompanySyncRepository -import repositories.company.CompanySyncRepositoryInterface -import repositories.companyaccess.CompanyAccessRepository -import repositories.companyaccess.CompanyAccessRepositoryInterface -import repositories.companyactivationattempt.CompanyActivationAttemptRepository -import repositories.companyactivationattempt.CompanyActivationAttemptRepositoryInterface -import repositories.consumer.ConsumerRepository -import repositories.consumer.ConsumerRepositoryInterface -import repositories.dataeconomie.DataEconomieRepository -import repositories.dataeconomie.DataEconomieRepositoryInterface -import repositories.emailvalidation.EmailValidationRepository -import repositories.emailvalidation.EmailValidationRepositoryInterface -import repositories.event.EventRepository -import repositories.event.EventRepositoryInterface +import repositories.accesstoken.{AccessTokenRepository, AccessTokenRepositoryInterface} +import repositories.asyncfiles.{AsyncFileRepository, AsyncFileRepositoryInterface} +import repositories.authattempt.{AuthAttemptRepository, AuthAttemptRepositoryInterface} +import repositories.authtoken.{AuthTokenRepository, AuthTokenRepositoryInterface} import repositories.barcode.BarcodeProductRepository -import repositories.influencer.InfluencerRepository -import repositories.influencer.InfluencerRepositoryInterface +import repositories.blacklistedemails.{BlacklistedEmailsRepository, BlacklistedEmailsRepositoryInterface} +import repositories.company.{CompanyRepository, CompanyRepositoryInterface, CompanySyncRepository, CompanySyncRepositoryInterface} +import repositories.companyaccess.{CompanyAccessRepository, CompanyAccessRepositoryInterface} +import repositories.companyactivationattempt.{CompanyActivationAttemptRepository, CompanyActivationAttemptRepositoryInterface} +import repositories.consumer.{ConsumerRepository, ConsumerRepositoryInterface} +import repositories.dataeconomie.{DataEconomieRepository, DataEconomieRepositoryInterface} +import repositories.emailvalidation.{EmailValidationRepository, EmailValidationRepositoryInterface} +import repositories.event.{EventRepository, EventRepositoryInterface} +import repositories.influencer.{InfluencerRepository, InfluencerRepositoryInterface} import repositories.probe.ProbeRepository -import repositories.rating.RatingRepository -import repositories.rating.RatingRepositoryInterface -import repositories.report.ReportRepository -import repositories.report.ReportRepositoryInterface -import repositories.reportblockednotification.ReportNotificationBlockedRepository -import repositories.reportblockednotification.ReportNotificationBlockedRepositoryInterface -import repositories.reportconsumerreview.ResponseConsumerReviewRepository -import repositories.reportconsumerreview.ResponseConsumerReviewRepositoryInterface -import repositories.reportfile.ReportFileRepository -import repositories.reportfile.ReportFileRepositoryInterface -import repositories.reportmetadata.ReportMetadataRepository -import repositories.reportmetadata.ReportMetadataRepositoryInterface -import repositories.signalconsoreview.SignalConsoReviewRepository -import repositories.signalconsoreview.SignalConsoReviewRepositoryInterface -import repositories.socialnetwork.SocialNetworkRepository -import repositories.socialnetwork.SocialNetworkRepositoryInterface -import repositories.subscription.SubscriptionRepository -import repositories.subscription.SubscriptionRepositoryInterface +import repositories.rating.{RatingRepository, RatingRepositoryInterface} +import repositories.report.{ReportRepository, ReportRepositoryInterface} +import repositories.reportblockednotification.{ReportNotificationBlockedRepository, ReportNotificationBlockedRepositoryInterface} +import repositories.reportconsumerreview.{ResponseConsumerReviewRepository, ResponseConsumerReviewRepositoryInterface} +import repositories.reportfile.{ReportFileRepository, ReportFileRepositoryInterface} +import repositories.reportmetadata.{ReportMetadataRepository, ReportMetadataRepositoryInterface} +import repositories.signalconsoreview.{SignalConsoReviewRepository, SignalConsoReviewRepositoryInterface} +import repositories.socialnetwork.{SocialNetworkRepository, SocialNetworkRepositoryInterface} +import repositories.subscription.{SubscriptionRepository, SubscriptionRepositoryInterface} import repositories.tasklock.TaskRepository -import repositories.user.UserRepository -import repositories.user.UserRepositoryInterface -import repositories.usersettings.UserReportsFiltersRepository -import repositories.usersettings.UserReportsFiltersRepositoryInterface -import repositories.website.WebsiteRepository -import repositories.website.WebsiteRepositoryInterface +import repositories.user.{UserRepository, UserRepositoryInterface} +import repositories.usersettings.{UserReportsFiltersRepository, UserReportsFiltersRepositoryInterface} +import repositories.website.{WebsiteRepository, WebsiteRepositoryInterface} import services._ import slick.basic.DatabaseConfig import slick.jdbc.JdbcProfile -import tasks.account.InactiveAccountTask -import tasks.account.InactiveDgccrfAccountReminderTask -import tasks.account.InactiveDgccrfAccountRemoveTask +import tasks.account.{InactiveAccountTask, InactiveDgccrfAccountReminderTask, InactiveDgccrfAccountRemoveTask} import tasks.company._ -import tasks.probe.LowRateLanceurDAlerteTask -import tasks.probe.LowRateReponseConsoTask -import tasks.report.ReportClosureTask -import tasks.report.ReportNotificationTask -import tasks.report.ReportRemindersTask -import utils.EmailAddress -import utils.FrontRoute -import utils.LoggingFilter +import tasks.probe.{LowRateLanceurDAlerteTask, LowRateReponseConsoTask} +import tasks.report.{ReportClosureTask, ReportNotificationTask, ReportRemindersTask} +import utils.{EmailAddress, FrontRoute, LoggingFilter} import java.time.LocalTime import java.time.format.DateTimeFormatter From f2b8beb16d6c24cf9d24a3716c96183a4d9989ff Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Sat, 10 Feb 2024 10:52:53 +0100 Subject: [PATCH 04/10] TRELLO-2174 : CertifiedInfluencer api --- app/controllers/SocialNetworkController.scala | 15 ++- app/controllers/package.scala | 17 +-- app/loader/SignalConsoApplicationLoader.scala | 108 ++++++++++++------ .../influencer/InfluencerRepository.scala | 2 +- build.sbt | 2 - conf/routes | 3 +- 6 files changed, 92 insertions(+), 55 deletions(-) diff --git a/app/controllers/SocialNetworkController.scala b/app/controllers/SocialNetworkController.scala index 2b7343ab4..612212456 100644 --- a/app/controllers/SocialNetworkController.scala +++ b/app/controllers/SocialNetworkController.scala @@ -1,6 +1,7 @@ package controllers import authentication.Authenticator +import cats.implicits.toTraverseOps import models.User import models.report.SocialNetworkSlug import orchestrators.socialmedia.InfluencerOrchestrator @@ -19,10 +20,16 @@ class SocialNetworkController( ) extends BaseController(authenticator, controllerComponents) { val logger: Logger = Logger(this.getClass) - def get(name: String, socialNetwork: SocialNetworkSlug) = Action.async { _ => - influencerOrchestrator - .get(name, socialNetwork) - .map(product => Ok(Json.toJson(product))) + def get(name: String, socialNetwork: String) = Action.async { _ => + SocialNetworkSlug + .withNameInsensitiveOption(socialNetwork) + .traverse(socialNetworkSlug => + influencerOrchestrator + .get(name, socialNetworkSlug) + ) + .map(_.getOrElse(Seq.empty)) + .map(result => Ok(Json.toJson(result))) + } } diff --git a/app/controllers/package.scala b/app/controllers/package.scala index e62517b11..471ac6c3a 100644 --- a/app/controllers/package.scala +++ b/app/controllers/package.scala @@ -32,17 +32,10 @@ package object controllers { implicit val IdentificationStatusQueryStringBindable: QueryStringBindable[IdentificationStatus] = QueryStringBindable.bindableString .transform[IdentificationStatus]( - identificationStatus => IdentificationStatus.withName(identificationStatus), + identificationStatus => IdentificationStatus.withNameInsensitive(identificationStatus), identificationStatus => identificationStatus.entryName ) - implicit val SocialNetworkSlugQueryStringBindable: QueryStringBindable[SocialNetworkSlug] = - QueryStringBindable.bindableString - .transform[SocialNetworkSlug]( - socialNetwork => SocialNetworkSlug.withName(socialNetwork), - socialNetwork => socialNetwork.entryName - ) - implicit val UUIDPathBindable: PathBindable[UUID] = PathBindable.bindableString .transform[UUID]( @@ -81,28 +74,28 @@ package object controllers { implicit val ReportResponseTypeQueryStringBindable: QueryStringBindable[ReportResponseType] = QueryStringBindable.bindableString .transform[ReportResponseType]( - reportResponseType => ReportResponseType.withName(reportResponseType), + reportResponseType => ReportResponseType.withNameInsensitive(reportResponseType), reportResponseType => reportResponseType.entryName ) implicit val ReportFileOriginQueryStringBindable: QueryStringBindable[ReportFileOrigin] = QueryStringBindable.bindableString .transform[ReportFileOrigin]( - reportFileOrigin => ReportFileOrigin.withName(reportFileOrigin), + reportFileOrigin => ReportFileOrigin.withNameInsensitive(reportFileOrigin), reportFileOrigin => reportFileOrigin.entryName ) implicit val ReportAdminActionTypeQueryStringBindable: QueryStringBindable[ReportAdminActionType] = QueryStringBindable.bindableString .transform[ReportAdminActionType]( - reportAdminActionType => ReportAdminActionType.withName(reportAdminActionType), + reportAdminActionType => ReportAdminActionType.withNameInsensitive(reportAdminActionType), reportAdminActionType => reportAdminActionType.entryName ) implicit val PublicStatQueryStringBindable: QueryStringBindable[PublicStat] = QueryStringBindable.bindableString .transform[PublicStat]( - publicStat => PublicStat.withName(publicStat), + publicStat => PublicStat.withNameInsensitive(publicStat), publicStat => publicStat.entryName ) diff --git a/app/loader/SignalConsoApplicationLoader.scala b/app/loader/SignalConsoApplicationLoader.scala index e6aa2ed1a..132250d69 100644 --- a/app/loader/SignalConsoApplicationLoader.scala +++ b/app/loader/SignalConsoApplicationLoader.scala @@ -7,60 +7,100 @@ import akka.actor.typed import akka.actor.typed.scaladsl.adapter.ClassicActorSystemOps import akka.util.Timeout import authentication._ -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.Config +import com.typesafe.config.ConfigFactory import config._ import models.report.ArborescenceNode import orchestrators._ import orchestrators.socialmedia.InfluencerOrchestrator import org.flywaydb.core.Flyway import play.api._ -import play.api.db.slick.{DbName, SlickComponents} -import play.api.libs.json.{JsArray, Json} +import play.api.db.slick.DbName +import play.api.db.slick.SlickComponents +import play.api.libs.json.JsArray +import play.api.libs.json.Json import play.api.libs.mailer.MailerComponents import play.api.libs.ws.ahc.AhcWSComponents -import play.api.mvc.{Cookie, EssentialFilter} +import play.api.mvc.Cookie +import play.api.mvc.EssentialFilter import play.api.routing.Router import play.filters.HttpFiltersComponents -import pureconfig.{ConfigConvert, ConfigReader, ConfigSource} +import pureconfig.ConfigConvert +import pureconfig.ConfigReader +import pureconfig.ConfigSource import pureconfig.configurable.localTimeConfigConvert import pureconfig.generic.auto._ import pureconfig.generic.semiauto.deriveReader -import repositories.accesstoken.{AccessTokenRepository, AccessTokenRepositoryInterface} -import repositories.asyncfiles.{AsyncFileRepository, AsyncFileRepositoryInterface} -import repositories.authattempt.{AuthAttemptRepository, AuthAttemptRepositoryInterface} -import repositories.authtoken.{AuthTokenRepository, AuthTokenRepositoryInterface} +import repositories.accesstoken.AccessTokenRepository +import repositories.accesstoken.AccessTokenRepositoryInterface +import repositories.asyncfiles.AsyncFileRepository +import repositories.asyncfiles.AsyncFileRepositoryInterface +import repositories.authattempt.AuthAttemptRepository +import repositories.authattempt.AuthAttemptRepositoryInterface +import repositories.authtoken.AuthTokenRepository +import repositories.authtoken.AuthTokenRepositoryInterface import repositories.barcode.BarcodeProductRepository -import repositories.blacklistedemails.{BlacklistedEmailsRepository, BlacklistedEmailsRepositoryInterface} -import repositories.company.{CompanyRepository, CompanyRepositoryInterface, CompanySyncRepository, CompanySyncRepositoryInterface} -import repositories.companyaccess.{CompanyAccessRepository, CompanyAccessRepositoryInterface} -import repositories.companyactivationattempt.{CompanyActivationAttemptRepository, CompanyActivationAttemptRepositoryInterface} -import repositories.consumer.{ConsumerRepository, ConsumerRepositoryInterface} -import repositories.dataeconomie.{DataEconomieRepository, DataEconomieRepositoryInterface} -import repositories.emailvalidation.{EmailValidationRepository, EmailValidationRepositoryInterface} -import repositories.event.{EventRepository, EventRepositoryInterface} -import repositories.influencer.{InfluencerRepository, InfluencerRepositoryInterface} +import repositories.blacklistedemails.BlacklistedEmailsRepository +import repositories.blacklistedemails.BlacklistedEmailsRepositoryInterface +import repositories.company.CompanyRepository +import repositories.company.CompanyRepositoryInterface +import repositories.company.CompanySyncRepository +import repositories.company.CompanySyncRepositoryInterface +import repositories.companyaccess.CompanyAccessRepository +import repositories.companyaccess.CompanyAccessRepositoryInterface +import repositories.companyactivationattempt.CompanyActivationAttemptRepository +import repositories.companyactivationattempt.CompanyActivationAttemptRepositoryInterface +import repositories.consumer.ConsumerRepository +import repositories.consumer.ConsumerRepositoryInterface +import repositories.dataeconomie.DataEconomieRepository +import repositories.dataeconomie.DataEconomieRepositoryInterface +import repositories.emailvalidation.EmailValidationRepository +import repositories.emailvalidation.EmailValidationRepositoryInterface +import repositories.event.EventRepository +import repositories.event.EventRepositoryInterface +import repositories.influencer.InfluencerRepository +import repositories.influencer.InfluencerRepositoryInterface import repositories.probe.ProbeRepository -import repositories.rating.{RatingRepository, RatingRepositoryInterface} -import repositories.report.{ReportRepository, ReportRepositoryInterface} -import repositories.reportblockednotification.{ReportNotificationBlockedRepository, ReportNotificationBlockedRepositoryInterface} -import repositories.reportconsumerreview.{ResponseConsumerReviewRepository, ResponseConsumerReviewRepositoryInterface} -import repositories.reportfile.{ReportFileRepository, ReportFileRepositoryInterface} -import repositories.reportmetadata.{ReportMetadataRepository, ReportMetadataRepositoryInterface} -import repositories.signalconsoreview.{SignalConsoReviewRepository, SignalConsoReviewRepositoryInterface} -import repositories.socialnetwork.{SocialNetworkRepository, SocialNetworkRepositoryInterface} -import repositories.subscription.{SubscriptionRepository, SubscriptionRepositoryInterface} +import repositories.rating.RatingRepository +import repositories.rating.RatingRepositoryInterface +import repositories.report.ReportRepository +import repositories.report.ReportRepositoryInterface +import repositories.reportblockednotification.ReportNotificationBlockedRepository +import repositories.reportblockednotification.ReportNotificationBlockedRepositoryInterface +import repositories.reportconsumerreview.ResponseConsumerReviewRepository +import repositories.reportconsumerreview.ResponseConsumerReviewRepositoryInterface +import repositories.reportfile.ReportFileRepository +import repositories.reportfile.ReportFileRepositoryInterface +import repositories.reportmetadata.ReportMetadataRepository +import repositories.reportmetadata.ReportMetadataRepositoryInterface +import repositories.signalconsoreview.SignalConsoReviewRepository +import repositories.signalconsoreview.SignalConsoReviewRepositoryInterface +import repositories.socialnetwork.SocialNetworkRepository +import repositories.socialnetwork.SocialNetworkRepositoryInterface +import repositories.subscription.SubscriptionRepository +import repositories.subscription.SubscriptionRepositoryInterface import repositories.tasklock.TaskRepository -import repositories.user.{UserRepository, UserRepositoryInterface} -import repositories.usersettings.{UserReportsFiltersRepository, UserReportsFiltersRepositoryInterface} -import repositories.website.{WebsiteRepository, WebsiteRepositoryInterface} +import repositories.user.UserRepository +import repositories.user.UserRepositoryInterface +import repositories.usersettings.UserReportsFiltersRepository +import repositories.usersettings.UserReportsFiltersRepositoryInterface +import repositories.website.WebsiteRepository +import repositories.website.WebsiteRepositoryInterface import services._ import slick.basic.DatabaseConfig import slick.jdbc.JdbcProfile -import tasks.account.{InactiveAccountTask, InactiveDgccrfAccountReminderTask, InactiveDgccrfAccountRemoveTask} +import tasks.account.InactiveAccountTask +import tasks.account.InactiveDgccrfAccountReminderTask +import tasks.account.InactiveDgccrfAccountRemoveTask import tasks.company._ -import tasks.probe.{LowRateLanceurDAlerteTask, LowRateReponseConsoTask} -import tasks.report.{ReportClosureTask, ReportNotificationTask, ReportRemindersTask} -import utils.{EmailAddress, FrontRoute, LoggingFilter} +import tasks.probe.LowRateLanceurDAlerteTask +import tasks.probe.LowRateReponseConsoTask +import tasks.report.ReportClosureTask +import tasks.report.ReportNotificationTask +import tasks.report.ReportRemindersTask +import utils.EmailAddress +import utils.FrontRoute +import utils.LoggingFilter import java.time.LocalTime import java.time.format.DateTimeFormatter diff --git a/app/repositories/influencer/InfluencerRepository.scala b/app/repositories/influencer/InfluencerRepository.scala index 67967bbf8..f9f41a943 100644 --- a/app/repositories/influencer/InfluencerRepository.scala +++ b/app/repositories/influencer/InfluencerRepository.scala @@ -24,7 +24,7 @@ class InfluencerRepository(override val dbConfig: DatabaseConfig[JdbcProfile])(i .filter { result => (result.name <-> (name: String)).<(0.55) } - .filter(_.socialNetwork === socialNetwork) + .filter(_.socialNetwork === (socialNetwork: SocialNetworkSlug)) .result ) diff --git a/build.sbt b/build.sbt index fb665bccc..3fdda2fc9 100644 --- a/build.sbt +++ b/build.sbt @@ -31,7 +31,6 @@ routesImport ++= Seq( "models.UserRole", "models.website.IdentificationStatus", "models.report.ReportFileOrigin", - "models.report.SocialNetworkSlug", "java.time.OffsetDateTime", "models.investigation.InvestigationStatus", "models.website.WebsiteId", @@ -41,7 +40,6 @@ routesImport ++= Seq( "models.report.delete.ReportAdminActionType", "models.PublicStat", "controllers.IdentificationStatusQueryStringBindable", - "controllers.SocialNetworkSlugQueryStringBindable", "controllers.WebsiteIdPathBindable", "controllers.UUIDPathBindable", "controllers.OffsetDateTimeQueryStringBindable", diff --git a/conf/routes b/conf/routes index 23b1813c1..05c5ca412 100644 --- a/conf/routes +++ b/conf/routes @@ -85,8 +85,7 @@ GET /api/async-files controll # Constants API GET /api/constants/countries controllers.ConstantController.getCountries() GET /api/constants/categories controllers.ConstantController.getCategories() -#GET /api/social-blade controllers.ConstantController.getSocialBlade() -GET /api/certified-influencer controllers.SocialNetworkController.get(name:String,socialNetwork: SocialNetworkSlug) +GET /api/certified-influencer controllers.SocialNetworkController.get(name:String, socialNetwork: String) GET /api/constants/categoriesByStatus controllers.ConstantController.getCategoriesByStatus() # Mobile app specific API From 3c2f66bc08e845cdd30fe9c95c64506a86452e13 Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Sat, 10 Feb 2024 11:41:19 +0100 Subject: [PATCH 05/10] TRELLO-2174 : CertifiedInfluencer api --- app/controllers/package.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/package.scala b/app/controllers/package.scala index 471ac6c3a..544ab1c69 100644 --- a/app/controllers/package.scala +++ b/app/controllers/package.scala @@ -14,7 +14,6 @@ import models.PublicStat import models.extractUUID import models.report.ReportFileOrigin import models.report.ReportResponseType -import models.report.SocialNetworkSlug import models.report.delete.ReportAdminActionType import models.report.reportfile.ReportFileId import utils.DateUtils From 6d5f313d07c36b8f9c588b6e6ad421f116bcacd5 Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Sat, 10 Feb 2024 11:47:51 +0100 Subject: [PATCH 06/10] TRELLO-2174 : fix script --- .../{V19__influencer_table.sql => V20__influencer_table.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename conf/db/migration/default/{V19__influencer_table.sql => V20__influencer_table.sql} (100%) diff --git a/conf/db/migration/default/V19__influencer_table.sql b/conf/db/migration/default/V20__influencer_table.sql similarity index 100% rename from conf/db/migration/default/V19__influencer_table.sql rename to conf/db/migration/default/V20__influencer_table.sql From f03199fab56266fba1acc273873d9bf89f880e49 Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Mon, 12 Feb 2024 09:08:21 +0100 Subject: [PATCH 07/10] TRELLO-2174 : fix endpoint --- app/controllers/SocialNetworkController.scala | 2 +- .../socialmedia/InfluencerOrchestrator.scala | 5 +- .../socialmedia/SocialBladeClient.scala | 70 +++++++++++++++++++ .../influencer/InfluencerRepository.scala | 2 +- generatesql.sh | 22 ++++++ getRequest.js | 27 +++++++ project/Dependencies.scala | 42 +++++------ test.http | 7 ++ 8 files changed, 152 insertions(+), 25 deletions(-) create mode 100644 app/orchestrators/socialmedia/SocialBladeClient.scala create mode 100644 generatesql.sh create mode 100644 getRequest.js create mode 100644 test.http diff --git a/app/controllers/SocialNetworkController.scala b/app/controllers/SocialNetworkController.scala index 612212456..356455a1c 100644 --- a/app/controllers/SocialNetworkController.scala +++ b/app/controllers/SocialNetworkController.scala @@ -27,7 +27,7 @@ class SocialNetworkController( influencerOrchestrator .get(name, socialNetworkSlug) ) - .map(_.getOrElse(Seq.empty)) + .map(_.getOrElse(false)) .map(result => Ok(Json.toJson(result))) } diff --git a/app/orchestrators/socialmedia/InfluencerOrchestrator.scala b/app/orchestrators/socialmedia/InfluencerOrchestrator.scala index 56f16d127..35074bd30 100644 --- a/app/orchestrators/socialmedia/InfluencerOrchestrator.scala +++ b/app/orchestrators/socialmedia/InfluencerOrchestrator.scala @@ -1,7 +1,6 @@ package orchestrators.socialmedia import models.report.SocialNetworkSlug -import models.report.socialnetwork.CertifiedInfluencer import repositories.influencer.InfluencerRepositoryInterface import scala.concurrent.ExecutionContext @@ -13,6 +12,6 @@ class InfluencerOrchestrator( val executionContext: ExecutionContext ) { - def get(name: String, socialNetwork: SocialNetworkSlug): Future[Seq[CertifiedInfluencer]] = - influencerRepository.get(name, socialNetwork) + def get(name: String, socialNetwork: SocialNetworkSlug): Future[Boolean] = + influencerRepository.get(name, socialNetwork).map(_.nonEmpty) } diff --git a/app/orchestrators/socialmedia/SocialBladeClient.scala b/app/orchestrators/socialmedia/SocialBladeClient.scala new file mode 100644 index 000000000..05a858945 --- /dev/null +++ b/app/orchestrators/socialmedia/SocialBladeClient.scala @@ -0,0 +1,70 @@ +package orchestrators.socialmedia + +import cats.implicits.catsSyntaxApplicativeId +import models.report.SocialNetworkSlug +import play.api.Logger +import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend +import sttp.client3.HttpURLConnectionBackend +import sttp.client3.UriContext +import sttp.client3.basicRequest + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future + +class SocialBladeClient()(implicit ec: ExecutionContext) { + + val logger = Logger(this.getClass) + AsyncHttpClientFutureBackend() + + def checkSocialNetworkUsername( + platform: SocialNetworkSlug, + username: String + ): Future[Option[String]] = { + + val request = basicRequest + .get(uri"https://socialblade.com/youtube/user/carolinereceveur") + .header( + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" + ) + + val backend = HttpURLConnectionBackend() + val response = request.send(backend) + +// val lowercaseUsername = username.toLowerCase() +// val url = uri"https://socialblade.com/${platform.entryName.toLowerCase}/user/$lowercaseUsername" +// val request = emptyRequest +// .headers( +// Header.userAgent( +// "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" +// ) +// ) +// .get(url) +// .response(asString) + + println(s"------------------ request.toCurl = ${request.toCurl} ------------------") +// +// request +// .send(backend) +// .flatMap { response => +// if (response.code.isSuccess) { +// response.body match { +// case Right(product) => +// logger.debug(s"Call success") +// Future.successful(Option.when(findUserName(product, lowercaseUsername))(lowercaseUsername)) +// case Left(error) => +// logger.warnWithTitle("social_blade_error", s"Error while calling Socialblade: ${error}") +// Future.successful(None) +// } +// } else { +// logger.warnWithTitle("social_blade_error", s"Error while calling Socialblade: ${response.code}") +// Future.successful(None) +// } +// } + + println(response) + + response.body.toOption.pure[Future] + } + +} diff --git a/app/repositories/influencer/InfluencerRepository.scala b/app/repositories/influencer/InfluencerRepository.scala index f9f41a943..c78c201f3 100644 --- a/app/repositories/influencer/InfluencerRepository.scala +++ b/app/repositories/influencer/InfluencerRepository.scala @@ -22,7 +22,7 @@ class InfluencerRepository(override val dbConfig: DatabaseConfig[JdbcProfile])(i override def get(name: String, socialNetwork: SocialNetworkSlug) = db.run( table .filter { result => - (result.name <-> (name: String)).<(0.55) + result.name.toLowerCase === (name.toLowerCase: String) } .filter(_.socialNetwork === (socialNetwork: SocialNetworkSlug)) .result diff --git a/generatesql.sh b/generatesql.sh new file mode 100644 index 000000000..e33e368b2 --- /dev/null +++ b/generatesql.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Path to your CSV file +CSV_FILE="20240203_155429_checked.csv" + +# Path to the output file +outputFile="$(date +"%Y%m%d_%H%M%S")_output.sql" + +# Ensure the output file is empty before starting to append data +> "$outputFile" + +# Skip the header row and process each line +tail -n +2 "$CSV_FILE" | while IFS=, read -r social_network name +do + # Generate a UUID for each row. Adjust based on your system's uuid generation command + ID=$(uuidgen) + + # Capitalize the first letter of social_network using awk + social_network=$(echo "$social_network" | awk '{print toupper(substr($0, 1, 1)) tolower(substr($0, 2))}') + + # Print the INSERT statement and append to the output file + echo "INSERT INTO influencers (id, social_network, name) VALUES ('$ID', '$social_network', '$name');" >> "$outputFile" +done diff --git a/getRequest.js b/getRequest.js new file mode 100644 index 000000000..0a6c46824 --- /dev/null +++ b/getRequest.js @@ -0,0 +1,27 @@ +const https = require('https'); + +const options = { + hostname: 'socialblade.com', + path: '/youtube/user/carolinereceveur', + method: 'GET', + headers: { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0', + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3', + } +}; + +const req = https.request(options, res => { + console.log(`statusCode: ${res.statusCode}`); + + res.on('data', d => { + process.stdout.write(d); + }); +}); + +req.on('error', error => { + console.error(error); +}); + +req.end(); + diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1e1cbd68a..6a3ff64f7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -56,26 +56,27 @@ object Dependencies { val janino = "org.codehaus.janino" % "janino" % Versions.janino // Needed for the in logback conf val commonsCompiler = "org.codehaus.janino" % "commons-compiler" % Versions.janino // Needed for janino val logstashLogBackEncoder = "net.logstash.logback" % "logstash-logback-encoder" % Versions.logstashLogbackEncoder - val sttpPlayJson = "com.softwaremill.sttp.client3" %% "play-json" % Versions.sttpPlayJson - val sttp = "com.softwaremill.sttp.client3" %% "core" % Versions.sttp - val sentry = "io.sentry" % "sentry-logback" % Versions.sentryVersion - val catsCore = "org.typelevel" %% "cats-core" % Versions.catsCoreVersion - val pureConfig = "com.github.pureconfig" %% "pureconfig" % Versions.pureConfigVersion - val playJsonExtensions = "ai.x" %% "play-json-extensions" % Versions.playJsonExtensionsVersion - val playSlick = "com.typesafe.play" %% "play-slick" % Versions.playSlickVersion - val slickPg = "com.github.tminglei" %% "slick-pg" % Versions.slickPgVersion - val slickPgPlayJson = "com.github.tminglei" %% "slick-pg_play-json" % Versions.slickPgVersion - val alpakkaSlick = "com.lightbend.akka" %% "akka-stream-alpakka-slick" % Versions.alpakkaVersion - val playMailer = "org.playframework" %% "play-mailer" % Versions.playMailerVersion - val alpakkaS3 = "com.lightbend.akka" %% "akka-stream-alpakka-s3" % Versions.alpakkaVersion - val alpakkaCSV = "com.lightbend.akka" %% "akka-stream-alpakka-csv" % Versions.alpakkaVersion - val alpakkaFile = "com.lightbend.akka" %% "akka-stream-alpakka-file" % Versions.alpakkaVersion - val akkaHttp = "com.typesafe.akka" %% "akka-http" % Versions.akkaHttpVersion - val akkaHttpXml = "com.typesafe.akka" %% "akka-http-xml" % Versions.akkaHttpVersion - val jbcrypt = "org.mindrot" % "jbcrypt" % "0.4" - val enumeratum = "com.beachape" %% "enumeratum" % Versions.enumeratumVersion - val enumeratumPlay = "com.beachape" %% "enumeratum-play" % Versions.enumeratumVersion - val awsJavaSdkS3 = "com.amazonaws" % "aws-java-sdk-s3" % Versions.awsJavaSdkS3Version + val sttpPlayJson = "com.softwaremill.sttp.client3" %% "play-json" % Versions.sttpPlayJson + val sttp = "com.softwaremill.sttp.client3" %% "core" % Versions.sttp + val asyncSttp = "com.softwaremill.sttp.client3" %% "async-http-client-backend-future" % Versions.sttp + val sentry = "io.sentry" % "sentry-logback" % Versions.sentryVersion + val catsCore = "org.typelevel" %% "cats-core" % Versions.catsCoreVersion + val pureConfig = "com.github.pureconfig" %% "pureconfig" % Versions.pureConfigVersion + val playJsonExtensions = "ai.x" %% "play-json-extensions" % Versions.playJsonExtensionsVersion + val playSlick = "com.typesafe.play" %% "play-slick" % Versions.playSlickVersion + val slickPg = "com.github.tminglei" %% "slick-pg" % Versions.slickPgVersion + val slickPgPlayJson = "com.github.tminglei" %% "slick-pg_play-json" % Versions.slickPgVersion + val alpakkaSlick = "com.lightbend.akka" %% "akka-stream-alpakka-slick" % Versions.alpakkaVersion + val playMailer = "org.playframework" %% "play-mailer" % Versions.playMailerVersion + val alpakkaS3 = "com.lightbend.akka" %% "akka-stream-alpakka-s3" % Versions.alpakkaVersion + val alpakkaCSV = "com.lightbend.akka" %% "akka-stream-alpakka-csv" % Versions.alpakkaVersion + val alpakkaFile = "com.lightbend.akka" %% "akka-stream-alpakka-file" % Versions.alpakkaVersion + val akkaHttp = "com.typesafe.akka" %% "akka-http" % Versions.akkaHttpVersion + val akkaHttpXml = "com.typesafe.akka" %% "akka-http-xml" % Versions.akkaHttpVersion + val jbcrypt = "org.mindrot" % "jbcrypt" % "0.4" + val enumeratum = "com.beachape" %% "enumeratum" % Versions.enumeratumVersion + val enumeratumPlay = "com.beachape" %% "enumeratum-play" % Versions.enumeratumVersion + val awsJavaSdkS3 = "com.amazonaws" % "aws-java-sdk-s3" % Versions.awsJavaSdkS3Version val jacksonModuleScala = "com.fasterxml.jackson.module" %% "jackson-module-scala" % Versions.jacksonModuleScalaVersion val postgresql = "org.postgresql" % "postgresql" % Versions.postgresqlVersion @@ -92,6 +93,7 @@ object Dependencies { Compile.commonsCompiler, Compile.logstashLogBackEncoder, Compile.sttp, + Compile.asyncSttp, Compile.sttpPlayJson, Compile.sentry, Compile.catsCore, diff --git a/test.http b/test.http new file mode 100644 index 000000000..ca868cbf6 --- /dev/null +++ b/test.http @@ -0,0 +1,7 @@ + +GET https://socialblade.com/youtube/user/carolinereceveur +User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' +### + + +GET localhost:9000/api/certified-influencer?name=orang&socialNetwork=Facebook From 10116db0112fb41345dbbb3e52074c54848c3b5f Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Mon, 12 Feb 2024 09:14:14 +0100 Subject: [PATCH 08/10] TRELLO-2174 : remove unused --- .gitignore | 3 +++ getRequest.js | 27 --------------------------- 2 files changed, 3 insertions(+), 27 deletions(-) delete mode 100644 getRequest.js diff --git a/.gitignore b/.gitignore index ad098fbdb..41427f09c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ conf/local.application.conf /.bsp javaOptions.sbt *.sc +!/test.http +!/zip.http +!/generatesql.sh diff --git a/getRequest.js b/getRequest.js deleted file mode 100644 index 0a6c46824..000000000 --- a/getRequest.js +++ /dev/null @@ -1,27 +0,0 @@ -const https = require('https'); - -const options = { - hostname: 'socialblade.com', - path: '/youtube/user/carolinereceveur', - method: 'GET', - headers: { - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0', - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', - 'Accept-Language': 'fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3', - } -}; - -const req = https.request(options, res => { - console.log(`statusCode: ${res.statusCode}`); - - res.on('data', d => { - process.stdout.write(d); - }); -}); - -req.on('error', error => { - console.error(error); -}); - -req.end(); - From 1612428d6f72dfa3d8a98a6de1a359bd19980893 Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Mon, 12 Feb 2024 09:19:09 +0100 Subject: [PATCH 09/10] TRELLO-2174 : remove unused --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 41427f09c..666ce0af6 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ javaOptions.sbt !/test.http !/zip.http !/generatesql.sh +!/app/orchestrators/socialmedia/SocialBladeClient.scala From f651fd8e2189a8c38e090d5f8679e0d93eb9c15c Mon Sep 17 00:00:00 2001 From: ssedoudbgouv Date: Mon, 12 Feb 2024 09:25:05 +0100 Subject: [PATCH 10/10] TRELLO-2174 : remove unused --- .../socialmedia/SocialBladeClient.scala | 70 ------------------- generatesql.sh | 22 ------ 2 files changed, 92 deletions(-) delete mode 100644 app/orchestrators/socialmedia/SocialBladeClient.scala delete mode 100644 generatesql.sh diff --git a/app/orchestrators/socialmedia/SocialBladeClient.scala b/app/orchestrators/socialmedia/SocialBladeClient.scala deleted file mode 100644 index 05a858945..000000000 --- a/app/orchestrators/socialmedia/SocialBladeClient.scala +++ /dev/null @@ -1,70 +0,0 @@ -package orchestrators.socialmedia - -import cats.implicits.catsSyntaxApplicativeId -import models.report.SocialNetworkSlug -import play.api.Logger -import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend -import sttp.client3.HttpURLConnectionBackend -import sttp.client3.UriContext -import sttp.client3.basicRequest - -import scala.concurrent.ExecutionContext -import scala.concurrent.Future - -class SocialBladeClient()(implicit ec: ExecutionContext) { - - val logger = Logger(this.getClass) - AsyncHttpClientFutureBackend() - - def checkSocialNetworkUsername( - platform: SocialNetworkSlug, - username: String - ): Future[Option[String]] = { - - val request = basicRequest - .get(uri"https://socialblade.com/youtube/user/carolinereceveur") - .header( - "User-Agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" - ) - - val backend = HttpURLConnectionBackend() - val response = request.send(backend) - -// val lowercaseUsername = username.toLowerCase() -// val url = uri"https://socialblade.com/${platform.entryName.toLowerCase}/user/$lowercaseUsername" -// val request = emptyRequest -// .headers( -// Header.userAgent( -// "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" -// ) -// ) -// .get(url) -// .response(asString) - - println(s"------------------ request.toCurl = ${request.toCurl} ------------------") -// -// request -// .send(backend) -// .flatMap { response => -// if (response.code.isSuccess) { -// response.body match { -// case Right(product) => -// logger.debug(s"Call success") -// Future.successful(Option.when(findUserName(product, lowercaseUsername))(lowercaseUsername)) -// case Left(error) => -// logger.warnWithTitle("social_blade_error", s"Error while calling Socialblade: ${error}") -// Future.successful(None) -// } -// } else { -// logger.warnWithTitle("social_blade_error", s"Error while calling Socialblade: ${response.code}") -// Future.successful(None) -// } -// } - - println(response) - - response.body.toOption.pure[Future] - } - -} diff --git a/generatesql.sh b/generatesql.sh deleted file mode 100644 index e33e368b2..000000000 --- a/generatesql.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Path to your CSV file -CSV_FILE="20240203_155429_checked.csv" - -# Path to the output file -outputFile="$(date +"%Y%m%d_%H%M%S")_output.sql" - -# Ensure the output file is empty before starting to append data -> "$outputFile" - -# Skip the header row and process each line -tail -n +2 "$CSV_FILE" | while IFS=, read -r social_network name -do - # Generate a UUID for each row. Adjust based on your system's uuid generation command - ID=$(uuidgen) - - # Capitalize the first letter of social_network using awk - social_network=$(echo "$social_network" | awk '{print toupper(substr($0, 1, 1)) tolower(substr($0, 2))}') - - # Print the INSERT statement and append to the output file - echo "INSERT INTO influencers (id, social_network, name) VALUES ('$ID', '$social_network', '$name');" >> "$outputFile" -done