diff --git a/fec.py b/fec.py index beb4a6b..589ae4d 100755 --- a/fec.py +++ b/fec.py @@ -71,14 +71,12 @@ def init(data: str | None = None): for committee in Committee.from_data_manager(data_manager): session.add(committee) print("Adding individual contributions...") + batch_size = 1_000 with session_for_data_manager(data_manager) as session: - for contributions in batched( - tqdm( - Contribution.from_data_manager(data_manager), - unit="contribution", - total=70_659_611, - ), - 5_000, + for contributions in tqdm( + batched(Contribution.from_data_manager(data_manager), batch_size), + unit="batch", + total=70_659_611 // batch_size, ): with session.begin(): session.add_all(contributions) diff --git a/server/data/models.py b/server/data/models.py index 98d77c5..00ff7a0 100644 --- a/server/data/models.py +++ b/server/data/models.py @@ -360,7 +360,11 @@ def from_data_manager( def engine_for_data_manager(data_manager: DataManager) -> sa.Engine: """Return an engine for the given data manager.""" - return sa.create_engine(f"sqlite:///{data_manager.path / 'fec.db'}") + return sa.create_engine( + f"sqlite:///{data_manager.path / 'fec.db'}", + # connect_args={"check_same_thread": False}, + # execution_options={"sqlite_synchronous": False, "sqlite_journal_mode": "WAL"}, + ) def session_for_data_manager(data_manager: DataManager) -> sao.Session: