(feat): add politician table to scheduler

src/api/repository.py

@@ -8,6 +8,7 @@
 
 # local
 import src.db.models as models
+from src.domain.entities import Entity
 
 
 class AbstractRepository(abc.ABC):
@@ -151,48 +152,84 @@ def __init__(self, session):
         super().__init__(session, models.Vote)
 
 
+class SqlAlchemyPoliticianRepository(SqlAlchemyBaseRepository):
+    def __init__(self, session):
+        super().__init__(session, models.Politician)
+
+
 class SqlAlchemyFactory:
     """Factory class to create repositories"""
 
     def __init__(self, session):
         self.session = session
+        """Must add new repositories to the map to be able to create them"""
+        self.entity_factory_map = {
+            Entity.PARTY.value: self.create_party_repository,
+            Entity.PARTY_STYLE.value: self.create_party_style_repository,
+            Entity.POLITICIAN.value: self.create_politician_repository,
+            Entity.PARLIAMENT.value: self.create_parliament_repository,
+            Entity.PARLIAMENT_PERIOD.value: self.create_parliament_period_repository,
+            Entity.TOPIC.value: self.create_topic_repository,
+            Entity.COMMITTEE.value: self.create_committee_repository,
+            Entity.COMMITTEE_HAS_TOPIC.value: self.create_committee_has_topic_repository,
+            Entity.POLL.value: self.create_poll_repository,
+        }
+
+    def get_repository(
+        self, entity: Entity | str
+    ) -> Optional[SqlAlchemyBaseRepository]:
+        if entity in self.entity_factory_map:
+            return self.entity_factory_map[entity]()
+        else:
+            return None
 
     def create_party_donation_repository(self) -> SqlAlchemyPartyDonationRepository:
         return SqlAlchemyPartyDonationRepository(self.session)
 
+    # in the map
     def create_party_repository(self) -> SqlAlchemyPartyRepository:
         return SqlAlchemyPartyRepository(self.session)
 
+    # in the map
     def create_party_style_repository(self) -> SqlAlchemyPartyStyleRepository:
         return SqlAlchemyPartyStyleRepository(self.session)
 
+    # in the map
     def create_parliament_repository(self) -> SqlAlchemyParliamentRepository:
         return SqlAlchemyParliamentRepository(self.session)
 
+    # in the map
     def create_parliament_period_repository(
         self,
     ) -> SqlAlchemyParliamentPeriodRepository:
         return SqlAlchemyParliamentPeriodRepository(self.session)
 
+    # in the map
     def create_topic_repository(
         self,
     ) -> SqlAlchemyTopicRepository:
         return SqlAlchemyTopicRepository(self.session)
 
+    # in the map
     def create_committee_repository(
         self,
     ) -> SqlAlchemyCommitteeRepository:
         return SqlAlchemyCommitteeRepository(self.session)
 
+    # in the map
     def create_committee_has_topic_repository(
         self,
     ) -> SqlAlchemyCommitteeHasTopicRepository:
         return SqlAlchemyCommitteeHasTopicRepository(self.session)
 
+    # in the map
     def create_poll_repository(
         self,
     ) -> SqlAlchemyPollRepository:
         return SqlAlchemyPollRepository(self.session)
 
     def create_vote_repository(self) -> SqlAlchemyVoteRepository:
         return SqlAlchemyVoteRepository(self.session)
+
+    def create_politician_repository(self) -> SqlAlchemyPoliticianRepository:
+        return SqlAlchemyPoliticianRepository(self.session)

src/domain/entities.py

@@ -15,3 +15,4 @@ class Entity(Enum):
     COMMITTEE = "committees"
     COMMITTEE_HAS_TOPIC = "committee-has-topics"
     POLL = "polls"
+    POLITICIAN = "politicians"

src/entrypoints/redis_scheduled_tasks.py

@@ -33,7 +33,7 @@ def subscribe_missing_entity_fetched():
     pubsub.subscribe("missing_entity_fetched")
 
     start_time = time.time()
-    duration = 120
+    duration = 360
 
     while time.time() - start_time <= duration:
         message = pubsub.get_message(timeout=1)
@@ -77,6 +77,7 @@ def publish_entity():
     entity_list = [
         Entity.PARTY,
         Entity.PARLIAMENT,
+        Entity.POLITICIAN,
         Entity.PARLIAMENT_PERIOD,
         Entity.TOPIC,
         Entity.COMMITTEE,
@@ -111,7 +112,7 @@ def run_tasks():
 
 
 if __name__ == "__main__":
-    schedule.every().wednesday.at("19:00").do(run_tasks)
+    schedule.every().friday.at("19:00").do(run_tasks)
     while True:
         schedule.run_pending()
         time.sleep(1)

src/service_layer/handlers.py

@@ -17,6 +17,7 @@
 
 ENTITY_HANDLERS = {
     Entity.PARTY.value: [utils.prepare_party_style_data, utils.prepare_party_data],
+    Entity.POLITICIAN.value: [utils.prepare_politician_data],
     Entity.PARLIAMENT.value: [utils.prepare_parliament_data],
     Entity.PARLIAMENT_PERIOD.value: [utils.prepare_parliament_period_data],
     Entity.TOPIC.value: [utils.prepare_topic_data],
@@ -29,6 +30,7 @@
 
 DATA_HANDLERS = {
     Entity.PARTY.value: [Entity.PARTY_STYLE.value, Entity.PARTY.value],
+    Entity.POLITICIAN.value: [Entity.POLITICIAN.value],
     Entity.PARLIAMENT.value: [Entity.PARLIAMENT.value],
     Entity.PARLIAMENT_PERIOD.value: [Entity.PARLIAMENT_PERIOD.value],
     Entity.TOPIC.value: [Entity.TOPIC.value],
@@ -39,10 +41,8 @@
 
 # Step 1
 def fetch_missing_entity(command: commands.FetchMissingEntity) -> List[Any]:
-    repo = utils.get_repository(
-        repository.SqlAlchemyFactory(command.session), command.entity
-    )
-
+    factory = repository.SqlAlchemyFactory(command.session)
+    repo = factory.get_repository(command.entity)
     if repo is not None:
         missing_entity_data = utils.FetchMissingEntity(
             command.entity, repo
@@ -78,7 +78,7 @@ def update_table(command: commands.UpdateTable):
         )
         return
     for entity, data in zip(command.entities, command.data):
-        repo = utils.get_repository(factory, entity)
+        repo = factory.get_repository(entity)
 
         if repo is not None and len(data) != 0:
             repo.add_or_update_list(data)

src/service_layer/utils.py

@@ -98,25 +98,6 @@ def fetch_missing_entities(self) -> List[models.Party]:
         return data_list
 
 
-def get_repository(
-    factory: SqlAlchemyFactory, entity: str
-) -> SqlAlchemyBaseRepository | None:
-    entity_factory_map = {
-        Entity.PARTY.value: factory.create_party_repository,
-        Entity.PARTY_STYLE.value: factory.create_party_style_repository,
-        Entity.PARLIAMENT.value: factory.create_parliament_repository,
-        Entity.PARLIAMENT_PERIOD.value: factory.create_parliament_period_repository,
-        Entity.TOPIC.value: factory.create_topic_repository,
-        Entity.COMMITTEE.value: factory.create_committee_repository,
-        Entity.COMMITTEE_HAS_TOPIC.value: factory.create_committee_has_topic_repository,
-        Entity.POLL.value: factory.create_poll_repository,
-    }
-    if entity in entity_factory_map:
-        return entity_factory_map[entity]()
-    else:
-        return None
-
-
 def prepare_party_style_data(api_parties):
     party_styles = [
         {
@@ -255,3 +236,37 @@ def prepare_poll_data(api_polls):
         for api_poll in api_polls
     ]
     return polls
+
+
+def prepare_politician_data(api_politicians):
+    politicians = [
+        {
+            "id": api_politician["id"],
+            "entity_type": api_politician["entity_type"],
+            "label": api_politician["label"],
+            "api_url": api_politician["api_url"],
+            "abgeordnetenwatch_url": api_politician["abgeordnetenwatch_url"],
+            "first_name": api_politician["first_name"],
+            "last_name": api_politician["last_name"],
+            "birth_name": api_politician["birth_name"],
+            "sex": api_politician["sex"],
+            "year_of_birth": api_politician["year_of_birth"],
+            "party_id": (
+                api_politician["party"]["id"] if api_politician["party"] else None
+            ),
+            "party_past": api_politician["party_past"],
+            "deceased": None,
+            "deceased_date": None,
+            "education": api_politician["education"],
+            "residence": api_politician["residence"],
+            "occupation": api_politician["occupation"],
+            "statistic_questions": api_politician["statistic_questions"],
+            "statistic_questions_answered": api_politician[
+                "statistic_questions_answered"
+            ],
+            "qid_wikidata": api_politician["qid_wikidata"],
+            "field_title": api_politician["field_title"],
+        }
+        for api_politician in api_politicians
+    ]
+    return politicians