perf: optimize db access (#282)

Signed-off-by: Eiko Wagenknecht <[email protected]>
eikowagenknecht · Oct 26, 2023 · f1f1bcf · f1f1bcf
1 parent d6b16fb
commit f1f1bcf
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 49 deletions.
diff --git a/src/lootscraper/database.py b/src/lootscraper/database.py
@@ -292,48 +292,70 @@ def initialize_or_update(self) -> None:
         )
         command.upgrade(alembic_cfg, "head")
 
-    def read_active_offers(self, time: datetime) -> Sequence[Offer]:
+    def read_active_offers(
+        self,
+        time: datetime,
+        *,
+        type_: str | None = None,
+        source: str | None = None,
+        duration: str | None = None,
+        last_offer_id: str | None = None,
+    ) -> Sequence[Offer]:
         session: Session = self.Session()
+
+        # Build prefiltered query to reduce database load.
+        # The detailled filtering is then handled after execution.
+        query = sa.select(Offer).where(
+            sa.and_(
+                sa.or_(
+                    # Offers that are definitely still active
+                    Offer.valid_from <= time,
+                    # For some offers we don't really know,
+                    # we will filter this later.
+                    Offer.valid_from.is_(None),
+                ),
+                sa.or_(
+                    # Offers that are definitely still active
+                    Offer.valid_to >= time,
+                    # For some offers we don't really know, but...
+                    Offer.valid_to.is_(None),
+                    # ... when they have been seen in the last 24
+                    # hours, we consider them active.
+                    Offer.seen_last >= time - timedelta(days=1),
+                ),
+            ),
+        )
+
+        filter_conditions = []
+        if type_ is not None:
+            filter_conditions.append(Offer.type == type_)
+        if source is not None:
+            filter_conditions.append(Offer.source == source)
+        if duration is not None:
+            filter_conditions.append(Offer.duration == duration)
+        if last_offer_id is not None:
+            filter_conditions.append(Offer.id > last_offer_id)
+
+        if filter_conditions:
+            query = query.where(sa.and_(*filter_conditions))
+
+        filtered_offers = []
+
         try:
-            # TODO: Add option for custom prefiltering because this is a lot!
-            # Prefilter to reduce database load. The details are handled below.
-            offers = (
-                session.execute(
-                    sa.select(Offer).where(
-                        sa.and_(
-                            sa.or_(
-                                # Offers that are definitely still active
-                                Offer.valid_from <= time,
-                                # For some offers we don't really know,
-                                # we will filter this later.
-                                Offer.valid_from.is_(None),
-                            ),
-                            sa.or_(
-                                # Offers that are definitely still active
-                                Offer.valid_to >= time,
-                                # For some offers we don't really know, but...
-                                Offer.valid_to.is_(None),
-                                # ... when they have been seen in the last 24
-                                # hours, we consider them active.
-                                Offer.seen_last >= time - timedelta(days=1),
-                            ),
-                        ),
-                    ),
-                )
-                .scalars()
-                .all()
-            )
+            offers = session.execute(query).scalars().all()
 
             # Filter out offers that have a real end date that is in the future
-            filtered_offers = []
-            for offer in offers:
-                real_valid_to = offer.real_valid_to()
-                if real_valid_to is None or real_valid_to > time:
-                    filtered_offers.append(offer)
+            filtered_offers = [
+                offer
+                for offer in offers
+                if (real_valid_to := offer.real_valid_to()) is None
+                or real_valid_to > time
+            ]
 
         except Exception:
             session.rollback()
             raise
+
         return filtered_offers
 
     def read_all(self) -> Sequence[Offer]:

diff --git a/src/lootscraper/telegrambot.py b/src/lootscraper/telegrambot.py
@@ -1096,31 +1096,25 @@ async def send_new_offers(self, user: User) -> bool:
             for subscription in subscriptions:
                 offers: Sequence[Offer] = self.database.read_active_offers(
                     datetime.now(tz=timezone.utc),
+                    type_=subscription.type,
+                    source=subscription.source,
+                    duration=subscription.duration,
+                    last_offer_id=subscription.last_offer_id,
                 )
-                filtered_offers = []
 
-                for offer in offers:
-                    if (
-                        offer.type == subscription.type
-                        and offer.source == subscription.source
-                        and offer.duration == subscription.duration
-                        and offer.id > subscription.last_offer_id
-                    ):
-                        filtered_offers.append(offer)
-
-                if len(filtered_offers) == 0:
+                if len(offers) == 0:
                     continue
 
-                offers_sent += len(filtered_offers)
+                offers_sent += len(offers)
 
                 # Send the offers
-                for offer in filtered_offers:
+                for offer in offers:
                     await self.send_offer(offer, user)
 
                 # Update the last offer id
-                subscription.last_offer_id = filtered_offers[-1].id
+                subscription.last_offer_id = offers[-1].id
                 user.offers_received_count = user.offers_received_count + len(
-                    filtered_offers,
+                    offers,
                 )
                 session.commit()
         except Exception: