Skip to content

Commit

Permalink
Notebook - Add in STRR (#1857)
Browse files Browse the repository at this point in the history
  • Loading branch information
@seeker25
seeker25 authored Dec 16, 2024
1 parent d7a2794 commit fba9ff4
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 39 deletions.
5 changes: 3 additions & 2 deletions jobs/notebook-report/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,14 @@ class Config(object):
CSO_MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv("CSO_MONTHLY_RECONCILIATION_RECIPIENTS", "")
RPT_MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv("RPT_MONTHLY_RECONCILIATION_RECIPIENTS", "")
ESRA_MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv("ESRA_MONTHLY_RECONCILIATION_RECIPIENTS", "")
STRR_MONTHLY_RECONCILIATION_RECIPIENTS = os.getenv("STRR_MONTHLY_RECONCILIATION_RECIPIENTS", "")

EMAIL_SMTP = os.getenv("EMAIL_SMTP", "")
ENVIRONMENT = os.getenv("ENVIRONMENT", "")
WEEKLY_REPORT_DATES = os.getenv("WEEKLY_REPORT_DATES", "[1]")
MONTHLY_REPORT_DATES = os.getenv("MONTHLY_REPORT_DATES", "[1]")
PARTNER_CODES = os.getenv("PARTNER_CODES", "CSO,VS,RPT,ESRA")
PARTNER_CODES_DISBURSEMENT = os.getenv("PARTER_CODES_DISBURSEMENT", "CSO,VS")
PARTNER_CODES = os.getenv("PARTNER_CODES", "CSO,VS,RPT,ESRA,STRR")
PARTNER_CODES_DISBURSEMENT = os.getenv("PARTNER_CODES_DISBURSEMENT", "CSO,VS,STRR")
REPORT_API_URL = os.getenv("REPORT_API_URL", "") + os.getenv("REPORT_API_VERSION", "/api/v1")
NOTEBOOK_SERVICE_ACCOUNT_ID = os.getenv("NOTEBOOK_SERVICE_ACCOUNT_ID", "")
NOTEBOOK_SERVICE_ACCOUNT_SECRET = os.getenv("NOTEBOOK_SERVICE_ACCOUNT_SECRET", "")
Expand Down
92 changes: 57 additions & 35 deletions jobs/notebook-report/monthly/reconciliation_summary.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,23 +157,26 @@
"from dateutil.relativedelta import relativedelta\n",
"import pytz\n",
"\n",
"\n",
"def get_utc_timezone_adjusted_date(target_date) -> str:\n",
" target_datetime = datetime.combine(target_date, datetime.min.time())\n",
" hours = target_datetime.astimezone(pytz.timezone(\"America/Vancouver\")).utcoffset().total_seconds() / 60 / 60\n",
" target_date = target_datetime.replace(tzinfo=timezone.utc) + relativedelta(hours=-hours)\n",
" return target_date.replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S')\n",
" return target_date.replace(tzinfo=None).strftime(\"%Y-%m-%d %H:%M:%S\")\n",
"\n",
"\n",
"def get_first_last_month_dates_in_utc() -> tuple[str, str]:\n",
" current_time = datetime.now(pytz.timezone(\"America/Vancouver\"))\n",
" last_month = current_time - relativedelta(months=1)\n",
" from_date = last_month.replace(day=1)\n",
" from_date = get_utc_timezone_adjusted_date(from_date)\n",
" to_date = (last_month.replace(day=1) + relativedelta(months=1)) - timedelta(days=1)\n",
" to_date = get_utc_timezone_adjusted_date(to_date)\n",
" return from_date, to_date\n",
" current_time = datetime.now(pytz.timezone(\"America/Vancouver\"))\n",
" last_month = current_time - relativedelta(months=1)\n",
" from_date = last_month.replace(day=1)\n",
" from_date = get_utc_timezone_adjusted_date(from_date)\n",
" to_date = (last_month.replace(day=1) + relativedelta(months=1)) - timedelta(days=1)\n",
" to_date = get_utc_timezone_adjusted_date(to_date)\n",
" return from_date, to_date\n",
"\n",
"\n",
"from_date, to_date = get_first_last_month_dates_in_utc()\n",
"print(f'Using from_date: {from_date} and to_date: {to_date}')"
"print(f\"Using from_date: {from_date} and to_date: {to_date}\")"
]
},
{
Expand Down Expand Up @@ -207,14 +210,15 @@
" created_by,\n",
" payment_date,\n",
" refund_date,\n",
" invoice_status_code\n",
" invoice_status_code,\n",
" (select string_agg(quantity || 'x ' || description || ' $' || pli.total , ',') from payment_line_items pli where invoice_id = i.id) as payment_line_items\n",
"FROM\n",
" invoices\n",
" invoices i\n",
"WHERE\n",
" corp_type_code = :partner_code\n",
" AND total > 0\n",
" AND invoice_status_code in ('PAID', 'CREDITED', 'REFUNDED', 'CANCELLED', 'REFUND_REQUESTED')\n",
" AND payment_method_code in ('PAD','EJV') \n",
" AND payment_method_code in ('PAD','EJV', 'EFT', 'DIRECT_PAY') \n",
" AND ((created_on >= :from_date and created_on <= :to_date) \n",
" OR (refund_date >= :from_date and refund_date <= :to_date))\n",
"ORDER BY\n",
Expand All @@ -240,14 +244,20 @@
},
"outputs": [],
"source": [
"filename_summary = os.path.join(os.getcwd(), r'data/')+partner_code+'_monthly_reconciliation_summary_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m') +'.csv'\n",
"filename_summary = (\n",
" os.path.join(os.getcwd(), r\"data/\")\n",
" + partner_code\n",
" + \"_monthly_reconciliation_summary_\"\n",
" + datetime.strftime(datetime.now() - timedelta(1), \"%Y-%m\")\n",
" + \".csv\"\n",
")\n",
"df_summary = monthly_reconciliation_summary.DataFrame()\n",
"with open(filename_summary, 'w') as f:\n",
" f.write('Monthly Reconciliation Summary:\\n\\n')\n",
"with open(filename_summary, \"w\") as f:\n",
" f.write(\"Monthly Reconciliation Summary:\\n\\n\")\n",
" if df_summary.empty:\n",
" f.write('No Data Retrieved')\n",
" f.write(\"No Data Retrieved\")\n",
" else:\n",
" df_summary.to_csv(f, sep=',', encoding='utf-8', index=False)"
" df_summary.to_csv(f, sep=\",\", encoding=\"utf-8\", index=False)"
]
},
{
Expand Down Expand Up @@ -287,7 +297,7 @@
" FROM invoices\n",
" WHERE corp_type_code = :partner_code\n",
" AND invoice_status_code = 'PAID'\n",
" AND payment_method_code in ('PAD','EJV')\n",
" AND payment_method_code in ('PAD','EJV', 'EFT', 'DIRECT_PAY')\n",
" AND ((disbursement_date >= :from_date AND disbursement_date <= :to_date)\n",
" OR (disbursement_reversal_date >= :from_date AND disbursement_reversal_date <= :to_date))\n",
" order by 1;\n",
Expand Down Expand Up @@ -321,14 +331,20 @@
"outputs": [],
"source": [
"if partner_code in partners_for_disbursement_summary:\n",
" filename_disbursed = os.path.join(os.getcwd(), 'data', f'{partner_code}_monthly_reconciliation_disbursed_' + datetime.strftime(datetime.now() - timedelta(1), '%Y-%m') + '.csv')\n",
" filename_disbursed = os.path.join(\n",
" os.getcwd(),\n",
" \"data\",\n",
" f\"{partner_code}_monthly_reconciliation_disbursed_\"\n",
" + datetime.strftime(datetime.now() - timedelta(1), \"%Y-%m\")\n",
" + \".csv\",\n",
" )\n",
" print(f\"Saving CSV for partner_code: {partner_code} at {filename_disbursed}\")\n",
" with open(filename_disbursed, 'w') as f:\n",
" f.write('Monthly Reconciliation Disbursed:\\n\\n')\n",
" with open(filename_disbursed, \"w\") as f:\n",
" f.write(\"Monthly Reconciliation Disbursed:\\n\\n\")\n",
" if monthly_reconciliation_disbursed is None or monthly_reconciliation_disbursed.empty:\n",
" f.write('No Data Retrieved')\n",
" f.write(\"No Data Retrieved\")\n",
" else:\n",
" monthly_reconciliation_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n"
" monthly_reconciliation_disbursed.to_csv(f, sep=\",\", encoding=\"utf-8\", index=False)"
]
},
{
Expand All @@ -347,14 +363,13 @@
"payload = \"grant_type=client_credentials\"\n",
"basic_hash = base64.b64encode(f\"{Config.NOTEBOOK_SERVICE_ACCOUNT_ID}:{Config.NOTEBOOK_SERVICE_ACCOUNT_SECRET}\".encode())\n",
"\n",
"headers = {\n",
" 'Content-Type': 'application/x-www-form-urlencoded',\n",
" 'Authorization': f'Basic {basic_hash.decode()}'\n",
"}\n",
"response = requests.request(\"POST\", f\"{Config.JWT_OIDC_ISSUER}/protocol/openid-connect/token\", headers=headers, data=payload)\n",
"headers = {\"Content-Type\": \"application/x-www-form-urlencoded\", \"Authorization\": f\"Basic {basic_hash.decode()}\"}\n",
"response = requests.request(\n",
" \"POST\", f\"{Config.JWT_OIDC_ISSUER}/protocol/openid-connect/token\", headers=headers, data=payload\n",
")\n",
"\n",
"assert response.status_code == 200\n",
"notebook_service_account_token = response.json().get('access_token')"
"notebook_service_account_token = response.json().get(\"access_token\")"
]
},
{
Expand All @@ -365,39 +380,46 @@
"source": [
"\n",
"partner_details = {\n",
" 'CSO': {\n",
" \"CSO\": {\n",
" \"companyName\": \"Ministry of Justice\",\n",
" \"addressLine1\": \"PO Box 9249, Stn Prov Govt\",\n",
" \"addressLine2\": \"6th Floor, 850 Burdett Avenue\",\n",
" \"city\": \"VICTORIA\",\n",
" \"province\": \"BC\",\n",
" \"areaCode\": \"V8W 9J2\"\n",
" },\n",
" 'VS': {\n",
" \"VS\": {\n",
" \"companyName\": \"Vital Statistics Agency\",\n",
" \"addressLine1\": \"PO Box 9657, Stn Prov Govt\",\n",
" \"addressLine2\": \"\",\n",
" \"city\": \"VICTORIA\",\n",
" \"province\": \"BC\",\n",
" \"areaCode\": \"V8W 9P3\"\n",
" },\n",
" 'RPT': {\n",
" \"RPT\": {\n",
" \"companyName\": \"Property Taxation Branch\",\n",
" \"addressLine1\": \"Ministry of Provincial Revenue\",\n",
" \"addressLine2\": \"4th Floor, 1802 Douglas Street\",\n",
" \"city\": \"VICTORIA\",\n",
" \"province\": \"BC\",\n",
" \"areaCode\": \"V8T 4K6\"\n",
" },\n",
" 'ESRA': {\n",
" \"ESRA\": {\n",
" \"companyName\": \"Ministry of Environment\",\n",
" \"addressLine1\": \"Environmental Emergencies and Land Remediation\",\n",
" \"addressLine2\": \"2nd Floor 10470-152nd Street\",\n",
" \"city\": \"SURREY\",\n",
" \"province\": \"BC\",\n",
" \"areaCode\": \"V3R 0Y3\"\n",
" },\n",
" \"STRR\": {\n",
" \"companyName\": \"Ministry of Housing and Municipal Affairs\",\n",
" \"addressLine1\": \"PO BOX 9844, STN PROV GOVT\",\n",
" \"addressLine2\": \"4th Floor, 614 Humboldt Street\",\n",
" \"city\": \"VICTORIA\",\n",
" \"province\": \"BC\",\n",
" \"areaCode\": \"V8W 1A4\"\n",
" }\n",
"\n",
"}\n",
"\n",
"def generate_report(partner_code):\n",
Expand All @@ -423,7 +445,7 @@
" WHERE\n",
" corp_type_code = :partner_code\n",
" AND invoice_status_code = 'PAID'\n",
" AND payment_method_code IN ('PAD', 'EJV')\n",
" AND payment_method_code IN ('PAD', 'EJV', 'EFT', 'DIRECT_PAY')\n",
" AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n",
" GROUP BY\n",
" corp_type_code\n",
Expand Down
2 changes: 1 addition & 1 deletion jobs/notebook-report/notebookreport.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ def process_partner_notebooks(notebookdirectory: str, data_dir: str, partner_cod
logging.info("Processing daily notebooks for partner: %s", partner_code)
execute_notebook(notebookdirectory, data_dir, partner_code)

logging.info(f'Monthly report dates: {monthly_report_dates}')
logging.info(f"Monthly report dates: {monthly_report_dates}")
if notebookdirectory == "monthly" and today in monthly_report_dates:
logging.info("Processing monthly notebooks for partner: %s", partner_code)
execute_notebook(notebookdirectory, data_dir, partner_code, is_monthly=True)
Expand Down
3 changes: 2 additions & 1 deletion jobs/notebook-report/weekly/pay.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
"from config import Config\n",
"\n",
"%load_ext sql\n",
"%config SqlMagic.displaylimit = 5"
"%config SqlMagic.displaylimit = 5\n",
"%config SqlMagic.style = '_DEPRECATED_DEFAULT'"
]
},
{
Expand Down

0 comments on commit fba9ff4

Please sign in to comment.