Skip to content

Commit

Permalink
This is slow and I am sad.
Browse files Browse the repository at this point in the history
  • Loading branch information
davepeck committed Nov 29, 2023
1 parent 838de52 commit 63a5c09
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 8 deletions.
12 changes: 5 additions & 7 deletions fec.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,12 @@ def init(data: str | None = None):
for committee in Committee.from_data_manager(data_manager):
session.add(committee)
print("Adding individual contributions...")
batch_size = 1_000
with session_for_data_manager(data_manager) as session:
for contributions in batched(
tqdm(
Contribution.from_data_manager(data_manager),
unit="contribution",
total=70_659_611,
),
5_000,
for contributions in tqdm(
batched(Contribution.from_data_manager(data_manager), batch_size),
unit="batch",
total=70_659_611 // batch_size,
):
with session.begin():
session.add_all(contributions)
Expand Down
6 changes: 5 additions & 1 deletion server/data/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,11 @@ def from_data_manager(

def engine_for_data_manager(data_manager: DataManager) -> sa.Engine:
"""Return an engine for the given data manager."""
return sa.create_engine(f"sqlite:///{data_manager.path / 'fec.db'}")
return sa.create_engine(
f"sqlite:///{data_manager.path / 'fec.db'}",
# connect_args={"check_same_thread": False},
# execution_options={"sqlite_synchronous": False, "sqlite_journal_mode": "WAL"},
)


def session_for_data_manager(data_manager: DataManager) -> sao.Session:
Expand Down

0 comments on commit 63a5c09

Please sign in to comment.