From e232be02488bc539bad7608fb5a7f9d571f3d277 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Fri, 19 Apr 2024 12:55:33 +0200 Subject: [PATCH 01/13] feat: create vote repo --- src/api/repository.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/api/repository.py b/src/api/repository.py index 10f56ea..e702bb7 100644 --- a/src/api/repository.py +++ b/src/api/repository.py @@ -138,6 +138,11 @@ def __init__(self, session): super().__init__(session, models.Poll) +class SqlAlchemyVoteRepository(SqlAlchemyBaseRepository): + def __init__(self, session): + super().__init__(session, models.Vote) + + class SqlAlchemyFactory: """Factory class to create repositories""" @@ -180,3 +185,6 @@ def create_poll_repository( self, ) -> SqlAlchemyPollRepository: return SqlAlchemyPollRepository(self.session) + + def create_vote_repository(self) -> SqlAlchemyVoteRepository: + return SqlAlchemyVoteRepository(self.session) From 650986822a8973db32a4abd8a1df1aa0a01a83e3 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Fri, 19 Apr 2024 13:59:18 +0200 Subject: [PATCH 02/13] fix: fix data list --- src/service_layer/handlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/service_layer/handlers.py b/src/service_layer/handlers.py index ef77535..d9d9676 100644 --- a/src/service_layer/handlers.py +++ b/src/service_layer/handlers.py @@ -22,6 +22,7 @@ Entity.TOPIC.value: utils.prepare_topic_data, Entity.COMMITTEE.value: utils.prepare_committee_data, Entity.POLL.value: utils.prepare_poll_data, + Entity.VOTE.value: utils.prepare_vote_data, } DATA_HANDLERS = { @@ -31,6 +32,7 @@ Entity.TOPIC.value: [Entity.TOPIC.value], Entity.COMMITTEE.value: [Entity.COMMITTEE.value, Entity.COMMITTEE_HAS_TOPIC.value], Entity.POLL.value: [Entity.POLL.value], + Entity.VOTE.value: [Entity.VOTE.value], } @@ -56,7 +58,7 @@ def prepare_update_data(command: commands.PrepareUpdateData): if entity in ENTITY_HANDLERS: entity_data = ENTITY_HANDLERS[entity](data) - return {"entities": DATA_HANDLERS[entity], "data": [entity_data]} + return {"entities": DATA_HANDLERS[entity], "data": entity_data} else: return {"entities": [], "data": []} From 1a6225beaa77bfec63a6dc5471e3936a03061c03 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Mon, 22 Apr 2024 13:59:18 +0200 Subject: [PATCH 03/13] feat: add vote to entities --- src/domain/entities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/domain/entities.py b/src/domain/entities.py index 561e819..aa1abc4 100644 --- a/src/domain/entities.py +++ b/src/domain/entities.py @@ -15,3 +15,4 @@ class Entity(Enum): COMMITTEE = "committees" COMMITTEE_HAS_TOPIC = "committee-has-topics" POLL = "polls" + VOTE = "votes" From ddc59abf066b2b8981103ff59523bfe251e1ef4a Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Mon, 22 Apr 2024 15:17:10 +0200 Subject: [PATCH 04/13] fix: fix entity handlers --- src/service_layer/handlers.py | 42 ++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/service_layer/handlers.py b/src/service_layer/handlers.py index d9d9676..55738b5 100644 --- a/src/service_layer/handlers.py +++ b/src/service_layer/handlers.py @@ -16,13 +16,16 @@ ENTITY_HANDLERS = { - Entity.PARTY.value: utils.prepare_party_data, - Entity.PARLIAMENT.value: utils.prepare_parliament_data, - Entity.PARLIAMENT_PERIOD.value: utils.prepare_parliament_period_data, - Entity.TOPIC.value: utils.prepare_topic_data, - Entity.COMMITTEE.value: utils.prepare_committee_data, - Entity.POLL.value: utils.prepare_poll_data, - Entity.VOTE.value: utils.prepare_vote_data, + Entity.PARTY.value: [utils.prepare_party_data], + Entity.PARLIAMENT.value: [utils.prepare_parliament_data], + Entity.PARLIAMENT_PERIOD.value: [utils.prepare_parliament_period_data], + Entity.TOPIC.value: [utils.prepare_topic_data], + Entity.COMMITTEE.value: [ + utils.prepare_committee_data, + utils.prepare_committee_has_topic_data, + ], + Entity.POLL.value: [utils.prepare_poll_data], + Entity.VOTE.value: [utils.prepare_vote_data], } DATA_HANDLERS = { @@ -57,7 +60,10 @@ def prepare_update_data(command: commands.PrepareUpdateData): data = command.data if entity in ENTITY_HANDLERS: - entity_data = ENTITY_HANDLERS[entity](data) + entity_data = [] + for prepare_data_func in ENTITY_HANDLERS[entity]: + if prepare_data_func and len(data) != 0: + entity_data.append(prepare_data_func(data)) return {"entities": DATA_HANDLERS[entity], "data": entity_data} else: return {"entities": [], "data": []} @@ -66,13 +72,19 @@ def prepare_update_data(command: commands.PrepareUpdateData): # Step 3 def update_table(command: commands.UpdateTable): factory = repository.SqlAlchemyFactory(command.session) - - for entity, data in zip(command.entities, command.data): - repo = utils.get_repository(factory, entity) - if repo is not None: - repo.add_or_update_list(data) - else: - logger.error(f"Repository for entity {entity} not found.") + logger.info(len(command.data)) + logger.info(command.data[1]) + logger.info(command.entities[0]) + logger.info(command.entities[1]) + # logger.info("command.data", factory) + # logger.info("command.data", command.data) + # for entity, data in zip(command.entities, command.data): + # repo = utils.get_repository(factory, entity) + # if repo is not None: + # logger.info("data", data) + # repo.add_or_update_list(data) + # else: + # logger.error(f"Repository for entity {entity} not found.") def publish_missing_entity_fetched_event( From 3362663f552df8edde2521e8df7c464a539d083a Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 10:15:55 +0200 Subject: [PATCH 05/13] fix: fix committee_has_topic repo --- src/api/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/repository.py b/src/api/repository.py index e702bb7..69335e9 100644 --- a/src/api/repository.py +++ b/src/api/repository.py @@ -130,7 +130,7 @@ def __init__(self, session): class SqlAlchemyCommitteeHasTopicRepository(SqlAlchemyBaseRepository): def __init__(self, session): - super().__init__(session, models.Committee) + super().__init__(session, models.CommitteeHasTopic) class SqlAlchemyPollRepository(SqlAlchemyBaseRepository): From 0a0a8c6e83436261ebfb8fdcb4000018cd9ba3f4 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 11:31:45 +0200 Subject: [PATCH 06/13] feat: update list --- src/entrypoints/redis_scheduled_tasks.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/entrypoints/redis_scheduled_tasks.py b/src/entrypoints/redis_scheduled_tasks.py index 086fc9f..0051244 100644 --- a/src/entrypoints/redis_scheduled_tasks.py +++ b/src/entrypoints/redis_scheduled_tasks.py @@ -75,12 +75,13 @@ def publish_entity(): """Run the Redis event publisher.""" logger.info("Running the Redis event publisher") entity_list = [ - Entity.PARTY, - Entity.PARLIAMENT, - Entity.PARLIAMENT_PERIOD, - Entity.TOPIC, - Entity.COMMITTEE, - Entity.POLL, + # Entity.PARTY, + Entity.PARLIAMENT, # works + Entity.PARLIAMENT_PERIOD, # works + Entity.TOPIC, # works + # Entity.COMMITTEE, + # Entity.POLL, + Entity.VOTE, ] for entity in entity_list: @@ -111,7 +112,7 @@ def run_tasks(): if __name__ == "__main__": - schedule.every().wednesday.at("19:00").do(run_tasks) + schedule.every().tuesday.at("11:28").do(run_tasks) while True: schedule.run_pending() time.sleep(1) From f91d5489a816759ae706bbc4b51a570fb40ecc20 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 11:43:09 +0200 Subject: [PATCH 07/13] feat: split into two functions --- src/service_layer/handlers.py | 33 +++++++++++++++++---------------- src/service_layer/utils.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/service_layer/handlers.py b/src/service_layer/handlers.py index 55738b5..40fcdf9 100644 --- a/src/service_layer/handlers.py +++ b/src/service_layer/handlers.py @@ -16,7 +16,7 @@ ENTITY_HANDLERS = { - Entity.PARTY.value: [utils.prepare_party_data], + Entity.PARTY.value: [utils.prepare_party_style_data, utils.prepare_party_data], Entity.PARLIAMENT.value: [utils.prepare_parliament_data], Entity.PARLIAMENT_PERIOD.value: [utils.prepare_parliament_period_data], Entity.TOPIC.value: [utils.prepare_topic_data], @@ -25,7 +25,7 @@ utils.prepare_committee_has_topic_data, ], Entity.POLL.value: [utils.prepare_poll_data], - Entity.VOTE.value: [utils.prepare_vote_data], + # Entity.VOTE.value: [utils.prepare_vote_data], } DATA_HANDLERS = { @@ -35,7 +35,7 @@ Entity.TOPIC.value: [Entity.TOPIC.value], Entity.COMMITTEE.value: [Entity.COMMITTEE.value, Entity.COMMITTEE_HAS_TOPIC.value], Entity.POLL.value: [Entity.POLL.value], - Entity.VOTE.value: [Entity.VOTE.value], + # Entity.VOTE.value: [Entity.VOTE.value], } @@ -72,19 +72,20 @@ def prepare_update_data(command: commands.PrepareUpdateData): # Step 3 def update_table(command: commands.UpdateTable): factory = repository.SqlAlchemyFactory(command.session) - logger.info(len(command.data)) - logger.info(command.data[1]) - logger.info(command.entities[0]) - logger.info(command.entities[1]) - # logger.info("command.data", factory) - # logger.info("command.data", command.data) - # for entity, data in zip(command.entities, command.data): - # repo = utils.get_repository(factory, entity) - # if repo is not None: - # logger.info("data", data) - # repo.add_or_update_list(data) - # else: - # logger.error(f"Repository for entity {entity} not found.") + if len(command.entities) != len(command.data): + logger.error( + "Number of entities and data does not match. Entities: %s, Data: %s", + len(command.entities), + len(command.data), + ) + return + for entity, data in zip(command.entities, command.data): + repo = utils.get_repository(factory, entity) + + if repo is not None and len(data) != 0: + repo.add_or_update_list(data) + else: + logger.error(f"Repository for {entity} not found.") def publish_missing_entity_fetched_event( diff --git a/src/service_layer/utils.py b/src/service_layer/utils.py index fe9f137..08a42d7 100644 --- a/src/service_layer/utils.py +++ b/src/service_layer/utils.py @@ -98,14 +98,19 @@ def fetch_missing_entities(self) -> List[models.Party]: return data_list -def get_repository(factory: SqlAlchemyFactory, entity: str): +def get_repository( + factory: SqlAlchemyFactory, entity: str +) -> SqlAlchemyBaseRepository | None: entity_factory_map = { Entity.PARTY.value: factory.create_party_repository, + Entity.PARTY_STYLE.value: factory.create_party_style_repository, Entity.PARLIAMENT.value: factory.create_parliament_repository, Entity.PARLIAMENT_PERIOD.value: factory.create_parliament_period_repository, Entity.TOPIC.value: factory.create_topic_repository, Entity.COMMITTEE.value: factory.create_committee_repository, + Entity.COMMITTEE_HAS_TOPIC.value: factory.create_committee_has_topic_repository, Entity.POLL.value: factory.create_poll_repository, + # Entity.VOTE.value: factory.create_vote_repository, } if entity in entity_factory_map: return entity_factory_map[entity]() @@ -113,7 +118,7 @@ def get_repository(factory: SqlAlchemyFactory, entity: str): return None -def prepare_party_data(api_parties): +def prepare_party_style_data(api_parties): party_styles = [ { "id": ap.get("id"), @@ -124,6 +129,11 @@ def prepare_party_data(api_parties): } for ap in api_parties ] + + return party_styles + + +def prepare_party_data(api_parties): parties = [ { "id": ap["id"], @@ -136,7 +146,7 @@ def prepare_party_data(api_parties): } for ap in api_parties ] - return party_styles, parties + return parties def prepare_parliament_data(api_parliaments): @@ -244,3 +254,21 @@ def prepare_poll_data(api_polls): for api_poll in api_polls ] return polls + + +# def prepare_vote_data(api_votes): +# return [ +# { +# "id": ap.get("id"), +# "entity_type": ap.get("entity_type"), +# "label": ap.get("label"), +# "api_url": ap.get("api_url"), +# "mandate_id": ap["mandate"]["id"] if ap.get("mandate") else None, +# "fraction_id": ap["fraction"]["id"] if ap.get("fraction") else None, +# "poll_id": ap["poll"]["id"] if ap.get("poll") else None, +# "vote": ap.get("vote"), +# "reason_no_show": ap.get("reason_no_show"), +# "reason_no_show_other": ap.get("reason_no_show_other"), +# } +# for ap in api_votes +# ] From d402a554202594963ef72e9a3e6fa910dbe704a5 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 11:57:35 +0200 Subject: [PATCH 08/13] feat: remove vote --- src/domain/entities.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/domain/entities.py b/src/domain/entities.py index aa1abc4..561e819 100644 --- a/src/domain/entities.py +++ b/src/domain/entities.py @@ -15,4 +15,3 @@ class Entity(Enum): COMMITTEE = "committees" COMMITTEE_HAS_TOPIC = "committee-has-topics" POLL = "polls" - VOTE = "votes" From 17466d87545a382a841a9dbdeb1e6181bafa03b5 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 12:37:16 +0200 Subject: [PATCH 09/13] fix: fix repo to update bridge tables --- src/api/repository.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/api/repository.py b/src/api/repository.py index 69335e9..b3f2e09 100644 --- a/src/api/repository.py +++ b/src/api/repository.py @@ -4,6 +4,7 @@ # third-party from sqlalchemy.dialects.postgresql import insert +from sqlalchemy import inspect # local import src.db.models as models @@ -27,6 +28,7 @@ class SqlAlchemyBaseRepository(AbstractRepository): def __init__(self, session, model_class): self.session = session self.model_class = model_class + self.primary_keys = [key.name for key in inspect(self.model_class).primary_key] def add(self, entity): self.session.add(entity) @@ -46,10 +48,16 @@ def add_or_update(self, entity): def add_or_update_list(self, entities: List[Dict[str, Any]]): stmt = insert(self.model_class).values(entities) - stmt = stmt.on_conflict_do_update( - index_elements=["id"], - set_={col.name: col for col in stmt.excluded if not col.primary_key}, - ) + update_dict = {col.name: col for col in stmt.excluded if not col.primary_key} + + if update_dict: + stmt = stmt.on_conflict_do_update( + index_elements=self.primary_keys, + set_=update_dict, + ) + else: + stmt = stmt.on_conflict_do_nothing(index_elements=self.primary_keys) + self.session.execute(stmt) self.session.commit() From 5a9c29ce5f5c5a446971314b95fc8030cfc6831b Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 12:58:33 +0200 Subject: [PATCH 10/13] feat: add polls --- src/entrypoints/redis_scheduled_tasks.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/entrypoints/redis_scheduled_tasks.py b/src/entrypoints/redis_scheduled_tasks.py index 0051244..325964b 100644 --- a/src/entrypoints/redis_scheduled_tasks.py +++ b/src/entrypoints/redis_scheduled_tasks.py @@ -75,13 +75,12 @@ def publish_entity(): """Run the Redis event publisher.""" logger.info("Running the Redis event publisher") entity_list = [ - # Entity.PARTY, - Entity.PARLIAMENT, # works - Entity.PARLIAMENT_PERIOD, # works - Entity.TOPIC, # works - # Entity.COMMITTEE, - # Entity.POLL, - Entity.VOTE, + Entity.PARTY, + Entity.PARLIAMENT, + Entity.PARLIAMENT_PERIOD, + Entity.TOPIC, + Entity.COMMITTEE, + Entity.POLL, ] for entity in entity_list: @@ -112,7 +111,7 @@ def run_tasks(): if __name__ == "__main__": - schedule.every().tuesday.at("11:28").do(run_tasks) + schedule.every().tuesday.at("12:57").do(run_tasks) while True: schedule.run_pending() time.sleep(1) From c9dd9774e31941ede9117386301d39cc8156a4de Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 12:59:33 +0200 Subject: [PATCH 11/13] fix: fix prepare_committee_has_topic_data --- src/service_layer/utils.py | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/src/service_layer/utils.py b/src/service_layer/utils.py index 08a42d7..d39d515 100644 --- a/src/service_layer/utils.py +++ b/src/service_layer/utils.py @@ -110,7 +110,6 @@ def get_repository( Entity.COMMITTEE.value: factory.create_committee_repository, Entity.COMMITTEE_HAS_TOPIC.value: factory.create_committee_has_topic_repository, Entity.POLL.value: factory.create_poll_repository, - # Entity.VOTE.value: factory.create_vote_repository, } if entity in entity_factory_map: return entity_factory_map[entity]() @@ -215,19 +214,21 @@ def prepare_committee_data(api_committees): } for api_committee in api_committees ] + return committees def prepare_committee_has_topic_data(api_committees): committee_topics = [] for api_committee in api_committees: - for topic in api_committee["field_topics"]: - committee_topics.append( - { - "committee_id": api_committee["id"], - "topic_id": topic["id"], - } - ) + if api_committee["field_topics"] is not None: + for topic in api_committee["field_topics"]: + committee_topics.append( + { + "committee_id": api_committee["id"], + "topic_id": topic["id"], + } + ) return committee_topics @@ -254,21 +255,3 @@ def prepare_poll_data(api_polls): for api_poll in api_polls ] return polls - - -# def prepare_vote_data(api_votes): -# return [ -# { -# "id": ap.get("id"), -# "entity_type": ap.get("entity_type"), -# "label": ap.get("label"), -# "api_url": ap.get("api_url"), -# "mandate_id": ap["mandate"]["id"] if ap.get("mandate") else None, -# "fraction_id": ap["fraction"]["id"] if ap.get("fraction") else None, -# "poll_id": ap["poll"]["id"] if ap.get("poll") else None, -# "vote": ap.get("vote"), -# "reason_no_show": ap.get("reason_no_show"), -# "reason_no_show_other": ap.get("reason_no_show_other"), -# } -# for ap in api_votes -# ] From d2788c8f05ed3ef287a4e0275dcb706318ed0158 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 13:01:03 +0200 Subject: [PATCH 12/13] feat: remove votes --- src/service_layer/handlers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/service_layer/handlers.py b/src/service_layer/handlers.py index 40fcdf9..09b1738 100644 --- a/src/service_layer/handlers.py +++ b/src/service_layer/handlers.py @@ -25,7 +25,6 @@ utils.prepare_committee_has_topic_data, ], Entity.POLL.value: [utils.prepare_poll_data], - # Entity.VOTE.value: [utils.prepare_vote_data], } DATA_HANDLERS = { @@ -35,7 +34,6 @@ Entity.TOPIC.value: [Entity.TOPIC.value], Entity.COMMITTEE.value: [Entity.COMMITTEE.value, Entity.COMMITTEE_HAS_TOPIC.value], Entity.POLL.value: [Entity.POLL.value], - # Entity.VOTE.value: [Entity.VOTE.value], } From 1690f809a4970d4f15b04dadca9eb19bc6da1000 Mon Sep 17 00:00:00 2001 From: Takahiro Mitsui Date: Tue, 23 Apr 2024 13:02:18 +0200 Subject: [PATCH 13/13] feat: update schedule --- src/entrypoints/redis_scheduled_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/entrypoints/redis_scheduled_tasks.py b/src/entrypoints/redis_scheduled_tasks.py index 325964b..086fc9f 100644 --- a/src/entrypoints/redis_scheduled_tasks.py +++ b/src/entrypoints/redis_scheduled_tasks.py @@ -111,7 +111,7 @@ def run_tasks(): if __name__ == "__main__": - schedule.every().tuesday.at("12:57").do(run_tasks) + schedule.every().wednesday.at("19:00").do(run_tasks) while True: schedule.run_pending() time.sleep(1)