diff --git a/src/lootscraper/database.py b/src/lootscraper/database.py index 77339789..3a3fef64 100644 --- a/src/lootscraper/database.py +++ b/src/lootscraper/database.py @@ -292,48 +292,70 @@ def initialize_or_update(self) -> None: ) command.upgrade(alembic_cfg, "head") - def read_active_offers(self, time: datetime) -> Sequence[Offer]: + def read_active_offers( + self, + time: datetime, + *, + type_: str | None = None, + source: str | None = None, + duration: str | None = None, + last_offer_id: str | None = None, + ) -> Sequence[Offer]: session: Session = self.Session() + + # Build prefiltered query to reduce database load. + # The detailled filtering is then handled after execution. + query = sa.select(Offer).where( + sa.and_( + sa.or_( + # Offers that are definitely still active + Offer.valid_from <= time, + # For some offers we don't really know, + # we will filter this later. + Offer.valid_from.is_(None), + ), + sa.or_( + # Offers that are definitely still active + Offer.valid_to >= time, + # For some offers we don't really know, but... + Offer.valid_to.is_(None), + # ... when they have been seen in the last 24 + # hours, we consider them active. + Offer.seen_last >= time - timedelta(days=1), + ), + ), + ) + + filter_conditions = [] + if type_ is not None: + filter_conditions.append(Offer.type == type_) + if source is not None: + filter_conditions.append(Offer.source == source) + if duration is not None: + filter_conditions.append(Offer.duration == duration) + if last_offer_id is not None: + filter_conditions.append(Offer.id > last_offer_id) + + if filter_conditions: + query = query.where(sa.and_(*filter_conditions)) + + filtered_offers = [] + try: - # TODO: Add option for custom prefiltering because this is a lot! - # Prefilter to reduce database load. The details are handled below. - offers = ( - session.execute( - sa.select(Offer).where( - sa.and_( - sa.or_( - # Offers that are definitely still active - Offer.valid_from <= time, - # For some offers we don't really know, - # we will filter this later. - Offer.valid_from.is_(None), - ), - sa.or_( - # Offers that are definitely still active - Offer.valid_to >= time, - # For some offers we don't really know, but... - Offer.valid_to.is_(None), - # ... when they have been seen in the last 24 - # hours, we consider them active. - Offer.seen_last >= time - timedelta(days=1), - ), - ), - ), - ) - .scalars() - .all() - ) + offers = session.execute(query).scalars().all() # Filter out offers that have a real end date that is in the future - filtered_offers = [] - for offer in offers: - real_valid_to = offer.real_valid_to() - if real_valid_to is None or real_valid_to > time: - filtered_offers.append(offer) + filtered_offers = [ + offer + for offer in offers + if (real_valid_to := offer.real_valid_to()) is None + or real_valid_to > time + ] except Exception: session.rollback() raise + return filtered_offers def read_all(self) -> Sequence[Offer]: diff --git a/src/lootscraper/telegrambot.py b/src/lootscraper/telegrambot.py index 85afad19..65d07bbb 100644 --- a/src/lootscraper/telegrambot.py +++ b/src/lootscraper/telegrambot.py @@ -1096,31 +1096,25 @@ async def send_new_offers(self, user: User) -> bool: for subscription in subscriptions: offers: Sequence[Offer] = self.database.read_active_offers( datetime.now(tz=timezone.utc), + type_=subscription.type, + source=subscription.source, + duration=subscription.duration, + last_offer_id=subscription.last_offer_id, ) - filtered_offers = [] - for offer in offers: - if ( - offer.type == subscription.type - and offer.source == subscription.source - and offer.duration == subscription.duration - and offer.id > subscription.last_offer_id - ): - filtered_offers.append(offer) - - if len(filtered_offers) == 0: + if len(offers) == 0: continue - offers_sent += len(filtered_offers) + offers_sent += len(offers) # Send the offers - for offer in filtered_offers: + for offer in offers: await self.send_offer(offer, user) # Update the last offer id - subscription.last_offer_id = filtered_offers[-1].id + subscription.last_offer_id = offers[-1].id user.offers_received_count = user.offers_received_count + len( - filtered_offers, + offers, ) session.commit() except Exception: