diff --git a/src/domain/entities.py b/src/domain/entities.py new file mode 100644 index 0000000..561e819 --- /dev/null +++ b/src/domain/entities.py @@ -0,0 +1,17 @@ +from enum import Enum + + +class Entity(Enum): + """ + value is the name of Abgeordnetenwatch API endpoint + kebab-case and pluralized + """ + + PARTY = "parties" + PARTY_STYLE = "party-styles" + PARLIAMENT = "parliaments" + PARLIAMENT_PERIOD = "parliament-periods" + TOPIC = "topics" + COMMITTEE = "committees" + COMMITTEE_HAS_TOPIC = "committee-has-topics" + POLL = "polls" diff --git a/src/entrypoints/redis_scheduled_tasks.py b/src/entrypoints/redis_scheduled_tasks.py index aa7aa78..086fc9f 100644 --- a/src/entrypoints/redis_scheduled_tasks.py +++ b/src/entrypoints/redis_scheduled_tasks.py @@ -1,6 +1,5 @@ # std import logging -import os import time import threading @@ -16,6 +15,7 @@ ) from src.db.connection import Session from src.logging_config import configure_logging +from src.domain.entities import Entity # Caution: This task needs multiple threads to run. @@ -75,17 +75,17 @@ def publish_entity(): """Run the Redis event publisher.""" logger.info("Running the Redis event publisher") entity_list = [ - "party", - "parliament", - "parliament-period", - "topic", - "committee", - "polls", + Entity.PARTY, + Entity.PARLIAMENT, + Entity.PARLIAMENT_PERIOD, + Entity.TOPIC, + Entity.COMMITTEE, + Entity.POLL, ] for entity in entity_list: redis_eventpublisher.initiate_fetch_missing_data( - entity=entity, session=session, redis_client=redis_client + entity=entity.value, session=session, redis_client=redis_client ) diff --git a/src/service_layer/handlers.py b/src/service_layer/handlers.py index e26f887..ef77535 100644 --- a/src/service_layer/handlers.py +++ b/src/service_layer/handlers.py @@ -6,6 +6,7 @@ # local from src.domain import events, commands from src.api import repository +from src.domain.entities import Entity from src.logging_config import configure_logging from src.service_layer import utils @@ -14,109 +15,62 @@ logger = logging.getLogger(__name__) +ENTITY_HANDLERS = { + Entity.PARTY.value: utils.prepare_party_data, + Entity.PARLIAMENT.value: utils.prepare_parliament_data, + Entity.PARLIAMENT_PERIOD.value: utils.prepare_parliament_period_data, + Entity.TOPIC.value: utils.prepare_topic_data, + Entity.COMMITTEE.value: utils.prepare_committee_data, + Entity.POLL.value: utils.prepare_poll_data, +} + +DATA_HANDLERS = { + Entity.PARTY.value: [Entity.PARTY_STYLE.value, Entity.PARTY.value], + Entity.PARLIAMENT.value: [Entity.PARLIAMENT.value], + Entity.PARLIAMENT_PERIOD.value: [Entity.PARLIAMENT_PERIOD.value], + Entity.TOPIC.value: [Entity.TOPIC.value], + Entity.COMMITTEE.value: [Entity.COMMITTEE.value, Entity.COMMITTEE_HAS_TOPIC.value], + Entity.POLL.value: [Entity.POLL.value], +} + + # Step 1 def fetch_missing_entity(command: commands.FetchMissingEntity) -> List[Any]: - if command.entity == "party": - repo = repository.SqlAlchemyFactory(command.session).create_party_repository() - missing_party_data = utils.FetchMissingEntity( - "parties", repo - ).fetch_missing_entities() - return missing_party_data - - if command.entity == "parliament": - repo = repository.SqlAlchemyFactory( - command.session - ).create_parliament_repository() - missing_parliament_data = utils.FetchMissingEntity( - "parliaments", repo - ).fetch_missing_entities() - return missing_parliament_data - - if command.entity == "parliament-period": - repo = repository.SqlAlchemyFactory( - command.session - ).create_parliament_period_repository() - missing_parliament_period_data = utils.FetchMissingEntity( - "parliament-periods", repo - ).fetch_missing_entities() - return missing_parliament_period_data - if command.entity == "topic": - repo = repository.SqlAlchemyFactory(command.session).create_topic_repository() - missing_topic_data = utils.FetchMissingEntity( - "topics", repo - ).fetch_missing_entities() - return missing_topic_data - if command.entity == "committee": - repo = repository.SqlAlchemyFactory( - command.session - ).create_committee_repository() - missing_committee_data = utils.FetchMissingEntity( - "committees", repo - ).fetch_missing_entities() - return missing_committee_data - if command.entity == "poll": - repo = repository.SqlAlchemyFactory(command.session).create_poll_repository() - missing_poll_data = utils.FetchMissingEntity( - "polls", repo + repo = utils.get_repository( + repository.SqlAlchemyFactory(command.session), command.entity + ) + + if repo is not None: + missing_entity_data = utils.FetchMissingEntity( + command.entity, repo ).fetch_missing_entities() - return missing_poll_data + return missing_entity_data + return [] # Step 2 def prepare_update_data(command: commands.PrepareUpdateData): - if command.entity == "party": - party_styles, parties = utils.prepare_party_data(command.data) - return {"entities": ["party_style", "party"], "data": [party_styles, parties]} - - if command.entity == "parliament": - parliaments = utils.prepare_parliament_data(command.data) - return {"entities": ["parliament"], "data": [parliaments]} - - if command.entity == "parliament-period": - parliament_periods = utils.prepare_parliament_period_data(command.data) - return {"entities": ["parliament_period"], "data": [parliament_periods]} - if command.entity == "topic": - topics = utils.prepare_topic_data(command.data) - return {"entities": ["topic"], "data": [topics]} - if command.entity == "committee": - committees = utils.prepare_committee_data(command.data) - committee_has_topics = utils.prepare_committee_has_topic_data(command.data) - return {"entities": ["committee"], "data": [committees, committee_has_topics]} - if command.entity == "poll": - polls = utils.prepare_poll_data(command.data) - return {"entities": ["poll"], "data": [polls]} + entity = command.entity + data = command.data + + if entity in ENTITY_HANDLERS: + entity_data = ENTITY_HANDLERS[entity](data) + return {"entities": DATA_HANDLERS[entity], "data": [entity_data]} + else: + return {"entities": [], "data": []} # Step 3 def update_table(command: commands.UpdateTable): factory = repository.SqlAlchemyFactory(command.session) - if command.entities == ["party_style", "party"]: - party_style_repo = factory.create_party_style_repository() - party_repo = factory.create_party_repository() - party_style_repo.add_or_update_list(command.data[0]) # type: ignore - party_repo.add_or_update_list(command.data[1]) # type: ignore - - if command.entities == ["parliament"]: - parliament_repo = factory.create_parliament_repository() - parliament_repo.add_or_update_list(command.data[0]) - - if command.entities == ["parliament_period"]: - parliament_period_repo = factory.create_parliament_period_repository() - parliament_period_repo.add_or_update_list(command.data[0]) - if command.entities == ["topic"]: - topic_repo = factory.create_topic_repository() - topic_repo.add_or_update_list(command.data[0]) - if command.entities == ["committee"]: - committee_repo = factory.create_committee_repository() - committee_repo.add_or_update_list(command.data[0]) - committee_has_topic_repo = factory.create_committee_has_topic_repository() - committee_has_topic_repo.add_or_update_list(command.data[1]) - if command.entities == ["poll"]: - poll_repo = factory.create_poll_repository() - poll_repo.add_or_update_list(command.data[0]) - # add poll has topic - # add poll has field_link + + for entity, data in zip(command.entities, command.data): + repo = utils.get_repository(factory, entity) + if repo is not None: + repo.add_or_update_list(data) + else: + logger.error(f"Repository for entity {entity} not found.") def publish_missing_entity_fetched_event( diff --git a/src/service_layer/utils.py b/src/service_layer/utils.py index aa1628d..fe9f137 100644 --- a/src/service_layer/utils.py +++ b/src/service_layer/utils.py @@ -5,8 +5,9 @@ from typing import List # local +from src.domain.entities import Entity from src.logging_config import configure_logging -from src.api.repository import SqlAlchemyBaseRepository +from src.api.repository import SqlAlchemyBaseRepository, SqlAlchemyFactory import src.db.models as models @@ -97,6 +98,21 @@ def fetch_missing_entities(self) -> List[models.Party]: return data_list +def get_repository(factory: SqlAlchemyFactory, entity: str): + entity_factory_map = { + Entity.PARTY.value: factory.create_party_repository, + Entity.PARLIAMENT.value: factory.create_parliament_repository, + Entity.PARLIAMENT_PERIOD.value: factory.create_parliament_period_repository, + Entity.TOPIC.value: factory.create_topic_repository, + Entity.COMMITTEE.value: factory.create_committee_repository, + Entity.POLL.value: factory.create_poll_repository, + } + if entity in entity_factory_map: + return entity_factory_map[entity]() + else: + return None + + def prepare_party_data(api_parties): party_styles = [ {