From a5898f37272b6913643cf5158b6a39ae0a896243 Mon Sep 17 00:00:00 2001 From: Matthew Timms Date: Sun, 27 Feb 2022 11:18:15 +1100 Subject: [PATCH] fix: support changes to woolies & for 2Up --- .env.example | 1 - README.md | 9 ++++-- changelog.md | 19 +++++++++++ requirements.txt | 9 +++--- src/up_woolies/main.py | 20 ++++++++---- src/up_woolies/up.py | 29 +++++++++++++---- src/up_woolies/utils.py | 13 ++++++++ src/up_woolies/woolies.py | 67 +++++++++++++++++++++++++++++++++++---- 8 files changed, 140 insertions(+), 27 deletions(-) create mode 100644 changelog.md diff --git a/.env.example b/.env.example index c497344..19d4002 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,2 @@ -WOOLIES_CLIENT_ID= WOOLIES_TOKEN= UP_TOKEN= \ No newline at end of file diff --git a/README.md b/README.md index 90b0155..931622a 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ customers with ✨e-receipts✨, which is more than I can say for their competit 1. Head to [Up Banking's API](https://developer.up.com.au/#welcome) page & grab your personal API token 2. Login to [Woolworth's Everyday Rewards](https://www.woolworthsrewards.com.au/#login) site & navigate around with dev-tools monitoring network traffic. Filter network traffic with `api.woolworthsrewards.com.au` & find any request - that has `client_id` & `authorization` headers. + that has `authorization` header.

@@ -52,7 +52,6 @@ customers with ✨e-receipts✨, which is more than I can say for their competit 3. Copy `.env.example` to `.env` & place those three tokens inside: ``` -WOOLIES_CLIENT_ID=cXDN... WOOLIES_TOKEN=8h41... UP_TOKEN=up:yeah:1234abcd... ``` @@ -93,13 +92,19 @@ suggest the feature through support chat in-app 🙏 that many more banks had begun supporting Open Banking than when I last checked. * Unfortunately, despite the title _"Consumer Data Rights"_, the process of authenticating myself with these CDR data holders for my _own_ consumer data is a mystery to me. If you know, then please reach out to me. + * [Update] I've learnt that the support/access I'm looking for fell outside the scope of CDR, and banks have no + obligation to support it. I would have to hold out for Data Holders or Recipients to provide. * 👩‍💼 Talk to someone about Woolworths' API * I tried reaching out to Woolworths to talk about their API: EverdayRewards support, Quantium (the tech subsidiary managing the program), even cold-messaged people on LinkedIn associate with WooliesX. No luck. + * [Update] A login endpoint was shared to me via the repo's issues. It worked like a charm, allowing user/pass + flows. However, it suddenly started returning 403s & I have yet to find a solution. * ⚡ Talk to someone about Up Bank's smart receipts * A friend pointed out on [The Tree of Up](https://up.com.au/tree/) a leaf call _smart receipts_ & the existing integration with AfterPay. It would be interesting to hear how it was implemented, & if this proof-of-concept shares any similarities. + * [Update] Dom, Co-founder of Up, gave some insight about Up + Bank's [smart receipts](https://twitter.com/dompym/status/1418792235559235589) integration with AfterPay * 👫 Support 2Up * Please read [help wanted](#help-wanted) on how you can help push for API support of 2Up. * ⚖ Interpret item weights diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..1bc8e6a --- /dev/null +++ b/changelog.md @@ -0,0 +1,19 @@ +## v1.1.0 - 20/02/2022 + +- Added changelog 📑 +- ~~Added Woolworth's email:password login via env. vars. or CLI~~ + - Nevermind, the woolies' login endpoint shared via [#1](https://github.com/MattTimms/up_woolies/issues/1) has begun + rejecting requests & instead returns 403 forbbiden +- Added use of Up API's category-filter for off-loading some filtering compute to them +- Added python rich library for prettier print-outs +- Added scaffolding for accessing 2Up data +- Updated README +- Fixed missing/new requirement for `User-Agent` header for Woolworth's API +- Fixed default Up spending account name from `Up Account` to `Spending` as per + 💕 [2Up Support](https://github.com/up-banking/api/issues/31#issuecomment-1008441619) update +- Fixed indefinite requests with default timeout adapter on request sessions +- Fixed missing dependency versions + +## v1.0.0 - 24/07/2021 + +- ⚡ initial release \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9930b99..b392068 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -python-dotenv -requests -prance[osv] -pydantic \ No newline at end of file +python-dotenv==0.17.1 +requests==2.25.1 +rich==11.2.0 +prance[osv]==0.21.2 +pydantic==1.8.2 \ No newline at end of file diff --git a/src/up_woolies/main.py b/src/up_woolies/main.py index 07311d0..2a6789f 100644 --- a/src/up_woolies/main.py +++ b/src/up_woolies/main.py @@ -1,19 +1,25 @@ import datetime import json -from pprint import pprint +from rich import print import up import woolies +up_account = up.SpendingAccount() +# two_up_account = up.TwoUpAccount() + + def find_corresponding_up_transaction(woolies_transaction: woolies.Transaction) -> up.Transaction: """ Returns corresponding Up Bank transaction from a Woolies transaction""" + _up_category = 'groceries' _up_description = 'Woolworths' # How Up manages human-readable merchant Id for Woolworths # Request transactions within window of the Woolies transaction transaction_datetime = woolies_transaction.transaction_date for up_transactions in up_account.get_transactions(until=transaction_datetime + datetime.timedelta(seconds=10), - since=transaction_datetime - datetime.timedelta(seconds=10)): + since=transaction_datetime - datetime.timedelta(seconds=10), + category=_up_category): for up_transaction in up_transactions: # Validate transactions match is_merchant_woolies = up_transaction.description == _up_description @@ -26,9 +32,7 @@ def find_corresponding_up_transaction(woolies_transaction: woolies.Transaction) raise FileNotFoundError("could not find corresponding transaction with up bank") -if __name__ == '__main__': - - up_account = up.SpendingAccount() +def example(): for transactions in woolies.list_transactions(): for woolies_transaction in transactions: @@ -50,4 +54,8 @@ def find_corresponding_up_transaction(woolies_transaction: woolies.Transaction) woolies_receipt = woolies_transaction.get_receipt() # Print it but make it pretty - pprint({'date': up_transaction.createdAt.astimezone().isoformat('T'), **json.loads(woolies_receipt.json())}) + print({'date': up_transaction.createdAt.astimezone().isoformat('T'), **json.loads(woolies_receipt.json())}) + + +if __name__ == '__main__': + example() diff --git a/src/up_woolies/up.py b/src/up_woolies/up.py index 0a2412c..2525cc4 100644 --- a/src/up_woolies/up.py +++ b/src/up_woolies/up.py @@ -9,7 +9,7 @@ from prance import ResolvingParser from pydantic import BaseModel, Extra, UUID4 -from utils import parse_money +from utils import DefaultTimeoutAdapter, parse_money # Get token from environment variables for fp in ['../../.env', '.env']: @@ -18,6 +18,7 @@ # Define endpoint & headers endpoint = "https://api.up.com.au/api/v1/" session = requests.session() +session.mount('https://', DefaultTimeoutAdapter(timeout=5)) session.headers.update({ "Authorization": f"Bearer {os.environ['UP_TOKEN']}" }) @@ -64,13 +65,22 @@ def value(self) -> Decimal: class Account: """ Base account class """ - def __init__(self, name: str): + def __init__(self, *, + display_name: str, + account_type: Literal['TRANSACTIONAL', 'SAVER'] = None, + ownership_type: Literal['INDIVIDUAL', 'JOINT'] = None): + attributes = {k: v for k, v in { + 'displayName': display_name, + 'accountType': account_type, + 'ownershipType': ownership_type, + }.items() if v is not None} + # Find account details by name for account in list_accounts(): - if name in account['attributes']['displayName']: + if attributes.items() <= account['attributes'].items(): break else: - raise ValueError(f"could not find account {name=}") + raise ValueError(f"could not find account matching {attributes=}") self.account = account self.transaction_url = account['relationships']['transactions']['links']['related'] @@ -78,13 +88,15 @@ def __init__(self, name: str): def get_transactions(self, page_size: int = 10, since: datetime = None, - until: datetime = None) -> Generator[List[Transaction], None, None]: + until: datetime = None, + category: str = None) -> Generator[List[Transaction], None, None]: """ Yields list of transactions based off input filters """ response = session.get(url=self.transaction_url, params={ 'page[size]': page_size, 'filter[since]': since.astimezone().isoformat('T') if since is not None else since, 'filter[until]': until.astimezone().isoformat('T') if until is not None else until, + 'filter[category]': category }).json() yield [Transaction.from_response(transaction) for transaction in response['data']] @@ -95,10 +107,13 @@ def get_transactions(self, class SpendingAccount(Account): - name = "Up Account" + def __init__(self): + super().__init__(display_name='Spending', account_type='TRANSACTIONAL', ownership_type='INDIVIDUAL') + +class TwoUpAccount(Account): def __init__(self): - super().__init__(name=self.name) + super().__init__(display_name='2Up Spending', account_type='TRANSACTIONAL', ownership_type='JOINT') # diff --git a/src/up_woolies/utils.py b/src/up_woolies/utils.py index 2a1e637..ddee51a 100644 --- a/src/up_woolies/utils.py +++ b/src/up_woolies/utils.py @@ -1,6 +1,19 @@ from decimal import Decimal from re import sub +from requests.adapters import HTTPAdapter +from requests import PreparedRequest, Response + def parse_money(money_str: str) -> Decimal: return Decimal(sub(r'[^\d.]', '', money_str)) + + +class DefaultTimeoutAdapter(HTTPAdapter): + def __init__(self, *args, timeout: float, **kwargs): + self.timeout = timeout + super().__init__(*args, **kwargs) + + def send(self, request: PreparedRequest, **kwargs) -> Response: + kwargs['timeout'] = kwargs.get('timeout') or self.timeout + return super().send(request, **kwargs) diff --git a/src/up_woolies/woolies.py b/src/up_woolies/woolies.py index cad48ba..426fae5 100644 --- a/src/up_woolies/woolies.py +++ b/src/up_woolies/woolies.py @@ -1,5 +1,6 @@ import os import re +import warnings from datetime import datetime from decimal import Decimal from typing import Dict, List, Any, Generator, Optional @@ -8,25 +9,77 @@ import requests from dotenv import load_dotenv from pydantic import BaseModel, Extra, condecimal, PositiveInt +from rich.console import Console +from rich.prompt import Prompt -from utils import parse_money +from utils import DefaultTimeoutAdapter, parse_money # Get token from environment variables for fp in ['../../.env', '.env']: load_dotenv(dotenv_path=fp) # Define endpoint & headers -endpoint = "https://api.woolworthsrewards.com.au/wx/v1/" +endpoint = "https://api.woolworthsrewards.com.au/wx/" session = requests.session() +session.mount('https://', DefaultTimeoutAdapter(timeout=5)) session.headers.update({ - 'client_id': os.environ['WOOLIES_CLIENT_ID'], - 'Authorization': f"Bearer {os.environ['WOOLIES_TOKEN']}" + 'client_id': '8h41mMOiDULmlLT28xKSv5ITpp3XBRvH', # some universal client API ID key + 'User-Agent': 'up_woolies' # some User-Agent }) session.hooks = { 'response': lambda r, *args, **kwargs: r.raise_for_status() } +def __init(): + if (email := os.getenv('WOOLIES_EMAIL')) is not None and (password := os.getenv('WOOLIES_PASS')): + auth = Auth.login(email, password) # TODO implement token refresh + elif (token := os.getenv('WOOLIES_TOKEN')) is not None: + warnings.warn("WOOLIES_TOKEN is deprecated, use WOOLIES_[EMAIL|PASS] instead", DeprecationWarning) + session.headers.update({'Authorization': f"Bearer {token}"}) + return + else: + auth = Auth.login_cli() # TODO implement token refresh + session.headers.update({'Authorization': f"Bearer {auth.bearer}"}) + + +class Auth(BaseModel): + bearer: str + refresh: str + bearerExpiredInSeconds: int + refreshExpiredInSeconds: int + passwordResetRequired: bool + + @classmethod + def login(cls, email: str, password: str): + url = urljoin(endpoint, 'v2/security/login/rewards') + body = {'username': email, 'password': password} # email/pass + res = session.post(url=url, json=body) + return cls.parse_obj(res.json()['data']) + + @classmethod + def login_cli(cls): + Console().print('Woolworths Login') + email = Prompt.ask("Email") + password = Prompt.ask("Password", password=True) + return cls.login(email, password) # TODO retry bad pass + + def refresh_token(self): + url = urljoin(endpoint, 'v2/security/refreshLogin') + body = {'refresh_token': self.refresh} + res = session.post(url=url, json=body) + + _auth = self.parse_obj(res.json()['data']) + for attr in self.__annotations__.keys(): + setattr(self, attr, getattr(_auth, attr)) + return self + + +# N.B. new login options are disabled as explain in changelog +# __init() +session.headers.update({'Authorization': f"Bearer {os.environ['WOOLIES_TOKEN']}"}) + + class Purchase(BaseModel, extra=Extra.ignore): """ Dataclass of an unique purchased item """ description: str @@ -134,8 +187,8 @@ def value(self) -> Decimal: def list_transactions(page: int = 0) -> Generator[List[Transaction], None, None]: - """ Yields list of Transactions for global Woolies account """ - url = urljoin(endpoint, 'rewards/member/ereceipts/transactions/list') + """ Yields list ("page") of Transactions for global Woolies account """ + url = urljoin(endpoint, 'v1/rewards/member/ereceipts/transactions/list') while True: page += 1 # Endpoint indexes at 1 response = session.get(url=url, params={"page": page}) @@ -146,7 +199,7 @@ def list_transactions(page: int = 0) -> Generator[List[Transaction], None, None] def get_receipt(receipt_key: str) -> ReceiptDetails: - url = urljoin(endpoint, 'rewards/member/ereceipts/transactions/details') + url = urljoin(endpoint, 'v1/rewards/member/ereceipts/transactions/details') body = {"receiptKey": receipt_key} response = session.post(url=url, json=body) return ReceiptDetails.from_raw(response.json()['data'])