From 6a8d4378d47a4d36fd7a597c93e39adbd95481c5 Mon Sep 17 00:00:00 2001 From: Avni Salhotra Date: Wed, 26 Jun 2024 15:16:29 -0700 Subject: [PATCH 1/4] adding changes to notebook report --- .../monthly/cso_reconciliation_summary.ipynb | 136 ++++++++++++++++- .../monthly/vs_reconciliation_summary.ipynb | 137 ++++++++++++++++-- jobs/notebook-report/notebookreport.py | 3 +- 3 files changed, 259 insertions(+), 17 deletions(-) diff --git a/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb b/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb index fe2a927aa..8ddd8a2da 100644 --- a/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb +++ b/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb @@ -49,9 +49,11 @@ }, "outputs": [], "source": [ - "import os\n", - "from datetime import datetime, timedelta\n", + "import base64\n", "from config import Config\n", + "from datetime import datetime, timedelta\n", + "import os\n", + "import requests\n", "\n", "%load_ext sql\n", "%config SqlMagic.displaylimit = 5" @@ -173,7 +175,7 @@ "FROM \n", " invoices\n", "WHERE \n", - " corp_type_code = 'CSO'\n", + " corp_type_code = :partner_code\n", " AND total > 0\n", " AND invoice_status_code = 'PAID'\n", " AND payment_method_code in ('PAD','EJV')\n", @@ -264,6 +266,134 @@ " else:\n", " df_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Authenticate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "payload = \"grant_type=client_credentials\"\n", + "basic_hash = base64.b64encode(f\"{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_ID')}:{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_SECRET')}\".encode())\n", + " \n", + "headers = {\n", + " 'Content-Type': 'application/x-www-form-urlencoded',\n", + " 'Authorization': f'Basic {basic_hash.decode()}'\n", + "}\n", + "response = requests.request(\"POST\", f\"{os.getenv('JWT_OIDC_ISSUER')}/protocol/openid-connect/token\", headers=headers, data=payload)\n", + "\n", + "assert response.status_code == 200\n", + "notebook_service_account_token = response.json().get('access_token')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get the API base URL from the environment variable\n", + "API_BASE_URL = os.getenv('REPORT_API_URL', '')\n", + "if not API_BASE_URL:\n", + " raise ValueError(\"The REPORT_API_URL environment variable is not set or is empty\")\n", + "\n", + "url = API_BASE_URL\n", + "headers = {\n", + " 'Authorization': f'Bearer {notebook_service_account_token}',\n", + " 'Content-Type': 'application/json',\n", + " 'Accept': 'application/pdf'\n", + "}\n", + "\n", + "# SQL query\n", + "query = \"\"\"\n", + "SELECT\n", + " COUNT(*) AS transaction_count,\n", + " SUM(total) AS total,\n", + " TO_CHAR(DATE_TRUNC('month',current_date) - INTERVAL '1 month','Month') as month,\n", + " corp_type_code\n", + "FROM \n", + " invoices\n", + "WHERE \n", + " corp_type_code = :partner_code\n", + " AND invoice_status_code = 'PAID'\n", + " AND payment_method_code IN ('PAD', 'EJV')\n", + " AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n", + "GROUP BY \n", + " corp_type_code\n", + "ORDER BY \n", + " month;\n", + "\"\"\"\n", + "\n", + "# Execute the SQL query and fetch results\n", + "result = %sql $query\n", + "\n", + "# Print the result to verify\n", + "print(result)\n", + "\n", + "# Check if the result is not None\n", + "if result:\n", + " # Convert the result to a DataFrame\n", + " df = result.DataFrame()\n", + "\n", + " # Rename columns to match the expected names in the template\n", + " df.rename(columns={\n", + " 'corp_type_code': 'Registry',\n", + " 'transaction_count': 'Trans_Counts',\n", + " 'total': 'Amount'\n", + " }, inplace=True)\n", + "\n", + " # Add a Symbol column with a dollar sign\n", + " df['Symbol'] = '$'\n", + "\n", + " # Convert DataFrame to JSON-compatible format\n", + " table_rows = df.to_dict(orient='records')\n", + "\n", + " # Get the current date\n", + " current_date = datetime.now().strftime(\"%B %d, %Y\")\n", + "\n", + " # Define the request body\n", + " data = {\n", + " \"templateVars\": {\n", + " \"Date\": current_date,\n", + " \"Company_Name\": \"Ministry of Justice\",\n", + " \"Address_Line_1\": \"PO Box 9249, Stn Prov Govt\",\n", + " \"Address_Line_2\": \"6th Floor, 850 Burdett Avenue\",\n", + " \"City\": \"VICTORIA\",\n", + " \"Province\": \"BC\",\n", + " \"Area_Code\": \"V8W 9J2\",\n", + " \"First_Name\": partner_code,\n", + " \"enter_month\": df['month'][0] if not df.empty else \"N/A\",\n", + " \"table_rows\": table_rows\n", + " },\n", + " \"templateName\": \"revenue_letter\",\n", + " \"reportName\": \"revenue_letter\"\n", + " }\n", + "\n", + " # Send the POST request\n", + " response = requests.post(url, headers=headers, json=data)\n", + "\n", + " # Check if the response is successful\n", + " if response.status_code == 200:\n", + " # Get the PDF content from the response\n", + " pdf_content = response.content\n", + " \n", + " # Save the PDF content to a file\n", + " with open(partner_code+'_revenue_letter.pdf', 'wb') as pdf_file:\n", + " pdf_file.write(pdf_content)\n", + " \n", + " print(\"PDF report saved successfully as 'payment_receipt.pdf'\")\n", + " else:\n", + " print('Failed to get the report:', response.text)\n", + "else:\n", + " print('No results returned from the SQL query')" + ] } ], "metadata": { diff --git a/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb b/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb index 5ec9c112d..a929df88f 100644 --- a/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb +++ b/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb @@ -49,9 +49,11 @@ }, "outputs": [], "source": [ - "import os\n", - "from datetime import datetime, timedelta\n", + "import base64\n", "from config import Config\n", + "from datetime import datetime, timedelta\n", + "import os\n", + "import requests\n", "\n", "%load_ext sql\n", "%config SqlMagic.displaylimit = 5" @@ -138,7 +140,7 @@ "outputs": [], "source": [ "%%sql\n", - "select now() AT TIME ZONE 'america/vancouver' as current_date" + "set time zone 'UTC';" ] }, { @@ -165,7 +167,7 @@ "sum(pli.total) AS subtotal,\n", "sum(pli.service_fees) AS service_fees,\n", "sum(pli.total + pli.service_fees) AS total,\n", - "(i.payment_date at time zone 'utc' at time zone 'america/vancouver')::date,\n", + "(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date,\n", "pli.description,\n", "i.payment_method_code,\n", "i.corp_type_code\n", @@ -174,10 +176,10 @@ "WHERE i.corp_type_code = :partner_code\n", "AND i.invoice_status_code = 'PAID'\n", "AND i.payment_method_code IN ('PAD', 'EJV', 'DIRECT_PAY', 'DRAWDOWN')\n", - "AND date(i.payment_date at time zone 'utc' at time zone 'america/vancouver') > date(current_date - 1 - interval '1 months')\n", - "AND date(i.payment_date at time zone 'utc' at time zone 'america/vancouver') <= date(current_date - 1)\n", - "GROUP BY (i.payment_date at time zone 'utc' at time zone 'america/vancouver')::date, i.payment_method_code, i.corp_type_code, pli.description\n", - "ORDER BY (i.payment_date at time zone 'utc' at time zone 'america/vancouver')::date, pli.description, i.payment_method_code" + "AND date(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') > date(current_date - 1 - interval '1 months')\n", + "AND date(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') <= date(current_date - 1)\n", + "GROUP BY (i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, i.payment_method_code, i.corp_type_code, pli.description\n", + "ORDER BY (i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, pli.description, i.payment_method_code" ] }, { @@ -226,7 +228,7 @@ "sum(pli.total) AS sub_total,\n", "sum(pli.service_fees) AS service_fees,\n", "sum(pli.total + pli.service_fees) AS total,\n", - "(i.disbursement_date at time zone 'utc' at time zone 'america/vancouver')::date,\n", + "(i.disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date,\n", "pli.description,\n", "i.payment_method_code,\n", "i.corp_type_code\n", @@ -236,10 +238,10 @@ "AND i.invoice_status_code = 'PAID'\n", "AND i.payment_method_code IN ('PAD', 'EJV', 'DIRECT_PAY')\n", "AND i.disbursement_status_code = 'COMPLETED'\n", - "AND date(disbursement_date at time zone 'utc' at time zone 'america/vancouver') > date(current_date - 1 - interval '1 months')\n", - "AND date(disbursement_date at time zone 'utc' at time zone 'america/vancouver') <= date(current_date - 1)\n", - "GROUP BY (disbursement_date at time zone 'utc' at time zone 'america/vancouver')::date, payment_method_code, corp_type_code, pli.description\n", - "ORDER BY (disbursement_date at time zone 'utc' at time zone 'america/vancouver')::date, pli.description, i.payment_method_code;" + "AND date(disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') > date(current_date - 1 - interval '1 months')\n", + "AND date(disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') <= date(current_date - 1)\n", + "GROUP BY (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, payment_method_code, corp_type_code, pli.description\n", + "ORDER BY (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, pli.description, i.payment_method_code;" ] }, { @@ -270,6 +272,115 @@ " else:\n", " df_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Authenticate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get the API base URL from the environment variable\n", + "API_BASE_URL = os.getenv('REPORT_API_URL', '')\n", + "if not API_BASE_URL:\n", + " raise ValueError(\"The RREPORT_API_URL environment variable is not set or is empty\")\n", + "\n", + "url = API_BASE_URL\n", + "headers = {\n", + " 'Authorization': 'Bearer your_jwt_token',\n", + " 'Content-Type': 'application/json',\n", + " 'Accept': 'application/pdf'\n", + "}\n", + "\n", + "# SQL query\n", + "query = \"\"\"\n", + "SELECT\n", + " COUNT(*) AS transaction_count,\n", + " SUM(total) AS total,\n", + " TO_CHAR(DATE_TRUNC('month',current_date) - INTERVAL '1 month','Month') as month,\n", + " corp_type_code\n", + "FROM \n", + " invoices\n", + "WHERE \n", + " corp_type_code = :partner_code\n", + " AND invoice_status_code = 'PAID'\n", + " AND payment_method_code IN ('PAD', 'EJV')\n", + " AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n", + "GROUP BY \n", + " corp_type_code\n", + "ORDER BY \n", + " month;\n", + "\"\"\"\n", + "\n", + "# Execute the SQL query and fetch results\n", + "result = %sql $query\n", + "\n", + "# Print the result to verify\n", + "print(result)\n", + "\n", + "# Check if the result is not None\n", + "if result:\n", + " # Convert the result to a DataFrame\n", + " df = result.DataFrame()\n", + "\n", + " # Rename columns to match the expected names in the template\n", + " df.rename(columns={\n", + " 'corp_type_code': 'Registry',\n", + " 'transaction_count': 'Trans_Counts',\n", + " 'total': 'Amount'\n", + " }, inplace=True)\n", + "\n", + " # Add a Symbol column with a dollar sign\n", + " df['Symbol'] = '$'\n", + "\n", + " # Convert DataFrame to JSON-compatible format\n", + " table_rows = df.to_dict(orient='records')\n", + "\n", + " # Get the current date\n", + " current_date = datetime.now().strftime(\"%B %d, %Y\")\n", + "\n", + " # Define the request body\n", + " data = {\n", + " \"templateVars\": {\n", + " \"Date\": current_date,\n", + " \"Company_Name\": \"Vital Statistics Agency\",\n", + " \"Address_Line_1\": \"PO Box 9657, Stn Prov Govt\",\n", + " \"Address_Line_2\": \"\",\n", + " \"City\": \"VICTORIA\",\n", + " \"Province\": \"BC\",\n", + " \"Area_Code\": \"V8W 9P3\",\n", + " \"First_Name\": partner_code,\n", + " \"enter_month\": df['month'][0] if not df.empty else \"N/A\",\n", + " \"table_rows\": table_rows\n", + " },\n", + " \"templateName\": \"revenue_letter\",\n", + " \"reportName\": \"revenue_letter\"\n", + " }\n", + "\n", + " # Send the POST request\n", + " response = requests.post(url, headers=headers, json=data)\n", + "\n", + " # Check if the response is successful\n", + " if response.status_code == 200:\n", + " # Get the PDF content from the response\n", + " pdf_content = response.content\n", + " \n", + " # Save the PDF content to a file\n", + " with open(partner_code+'_revenue_letter.pdf', 'wb') as pdf_file:\n", + " pdf_file.write(pdf_content)\n", + " \n", + " print(\"PDF report saved successfully as 'payment_receipt.pdf'\")\n", + " else:\n", + " print('Failed to get the report:', response.text)\n", + "else:\n", + " print('No results returned from the SQL query')" + ] } ], "metadata": { diff --git a/jobs/notebook-report/notebookreport.py b/jobs/notebook-report/notebookreport.py index c8fff20e4..f9b2f3a73 100644 --- a/jobs/notebook-report/notebookreport.py +++ b/jobs/notebook-report/notebookreport.py @@ -73,7 +73,8 @@ def send_email(file_processing, emailtype, errormessage, partner_code=None): year_month = datetime.strftime(datetime.now() - timedelta(1), '%Y-%m') subject = 'Monthly Reconciliation Stats ' + year_month + ext filenames = [f'{partner_code}_monthly_reconciliation_summary_' + year_month + '.csv', - f'{partner_code}_monthly_reconciliation_disbursed_' + year_month + '.csv'] + f'{partner_code}_monthly_reconciliation_disbursed_' + year_month + '.csv', + f'{partner_code}_revenue_letter.pdf'] recipients = get_partner_recipients(file_processing, partner_code) # Add body to email From 3fceea203c21c4c29a991992acec1fbd37ae9cf7 Mon Sep 17 00:00:00 2001 From: Avni Salhotra Date: Wed, 26 Jun 2024 15:19:53 -0700 Subject: [PATCH 2/4] adding partner_code to query --- jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb | 2 +- jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb b/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb index 8ddd8a2da..f81b6deec 100644 --- a/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb +++ b/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb @@ -229,7 +229,7 @@ "%%sql monthly_reconciliation_disbursed <<\n", "SELECT id, (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, total, service_fees, payment_method_code, corp_type_code,created_by\n", "FROM invoices\n", - "WHERE corp_type_code = 'CSO'\n", + "WHERE corp_type_code = :partner_code\n", "AND invoice_status_code = 'PAID'\n", "AND payment_method_code in ('PAD','EJV')\n", "AND disbursement_status_code = 'COMPLETED'\n", diff --git a/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb b/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb index a929df88f..82655a8f0 100644 --- a/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb +++ b/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb @@ -234,7 +234,7 @@ "i.corp_type_code\n", "FROM invoices i\n", "JOIN payment_line_items pli ON i.id = pli.invoice_id\n", - "WHERE i.corp_type_code = 'VS'\n", + "WHERE i.corp_type_code = :partner_code\n", "AND i.invoice_status_code = 'PAID'\n", "AND i.payment_method_code IN ('PAD', 'EJV', 'DIRECT_PAY')\n", "AND i.disbursement_status_code = 'COMPLETED'\n", From d212f9a0ce53001c7b92d296c073f518219bdce4 Mon Sep 17 00:00:00 2001 From: Avni Salhotra Date: Fri, 5 Jul 2024 12:58:57 -0700 Subject: [PATCH 3/4] revenue letter and single notebook now processing all partners --- .../monthly/cso_reconciliation_summary.ipynb | 435 --------------- .../monthly/reconciliation_summary.ipynb | 509 ++++++++++++++++++ .../monthly/vs_reconciliation_summary.ipynb | 422 --------------- jobs/notebook-report/notebookreport.py | 30 +- 4 files changed, 534 insertions(+), 862 deletions(-) delete mode 100644 jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb create mode 100644 jobs/notebook-report/monthly/reconciliation_summary.ipynb delete mode 100644 jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb diff --git a/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb b/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb deleted file mode 100644 index f81b6deec..000000000 --- a/jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb +++ /dev/null @@ -1,435 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "editable": true, - "jupyter": { - "outputs_hidden": false - }, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "# Reconciliation Monthly Stats" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "We need to load in these libraries into our notebook in order to query, load, manipulate and view the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "editable": true, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "is_executing": false, - "name": "#%%\n" - }, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import base64\n", - "from config import Config\n", - "from datetime import datetime, timedelta\n", - "import os\n", - "import requests\n", - "\n", - "%load_ext sql\n", - "%config SqlMagic.displaylimit = 5" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [ - "parameters" - ] - }, - "source": [ - "# Parameters cell for external parameters via papermill (job running this notebook will insert a parameter cell below this). This cell has a tag of with the name \"parameters\" that is used by papermill\n", - "\n", - "e.g.\n", - "param1 = \"some_value\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "This will create the connection to the database and prep the jupyter magic for SQL" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "editable": true, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "is_executing": false, - "name": "#%%\n" - }, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%sql $Config.SQLALCHEMY_DATABASE_URI" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "Simplest query to run to ensure our libraries are loaded and our DB connection is working" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "is_executing": false, - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "%%sql\n", - "set time zone 'UTC';" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Query ..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%%sql monthly_reconciliation_summary <<\n", - "SELECT \n", - " id, \n", - " (created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date AS created_date, \n", - " total, \n", - " service_fees, \n", - " payment_method_code, \n", - " corp_type_code, \n", - " created_by, \n", - " payment_date\n", - "FROM \n", - " invoices\n", - "WHERE \n", - " corp_type_code = :partner_code\n", - " AND total > 0\n", - " AND invoice_status_code = 'PAID'\n", - " AND payment_method_code in ('PAD','EJV')\n", - " AND created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' > (current_date - 1 - interval '1 months' - interval '5 days')::date\n", - " AND created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' <= (current_date - 1)::date\n", - "ORDER BY \n", - " 1;" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Save to CSV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "filename_summary = os.path.join(os.getcwd(), r'data/')+partner_code+'_monthly_reconciliation_summary_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m') +'.csv'\n", - "df_summary = monthly_reconciliation_summary.DataFrame()\n", - "with open(filename_summary, 'w') as f:\n", - " f.write('Monthly Reconciliation Summary:\\n\\n')\n", - " if df_summary.empty:\n", - " f.write('No Data Retrieved')\n", - " else:\n", - " df_summary.to_csv(f, sep=',', encoding='utf-8', index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%%sql monthly_reconciliation_disbursed <<\n", - "SELECT id, (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, total, service_fees, payment_method_code, corp_type_code,created_by\n", - "FROM invoices\n", - "WHERE corp_type_code = :partner_code\n", - "AND invoice_status_code = 'PAID'\n", - "AND payment_method_code in ('PAD','EJV')\n", - "AND disbursement_status_code = 'COMPLETED'\n", - "AND disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' > (current_date - 1 - interval '1 months')::date\n", - "AND disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' <= (current_date - 1)::date\n", - "order by 1;" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Save to another CSV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "filename_disbursed = os.path.join(os.getcwd(), r'data/')+partner_code+'_monthly_reconciliation_disbursed_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m') +'.csv'\n", - "df_disbursed = monthly_reconciliation_disbursed.DataFrame()\n", - "with open(filename_disbursed, 'a') as f:\n", - " f.write('Monthly Reconciliation Disbursed:\\n\\n')\n", - " if df_disbursed.empty:\n", - " f.write('No Data Retrieved')\n", - " else:\n", - " df_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Authenticate" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "payload = \"grant_type=client_credentials\"\n", - "basic_hash = base64.b64encode(f\"{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_ID')}:{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_SECRET')}\".encode())\n", - " \n", - "headers = {\n", - " 'Content-Type': 'application/x-www-form-urlencoded',\n", - " 'Authorization': f'Basic {basic_hash.decode()}'\n", - "}\n", - "response = requests.request(\"POST\", f\"{os.getenv('JWT_OIDC_ISSUER')}/protocol/openid-connect/token\", headers=headers, data=payload)\n", - "\n", - "assert response.status_code == 200\n", - "notebook_service_account_token = response.json().get('access_token')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get the API base URL from the environment variable\n", - "API_BASE_URL = os.getenv('REPORT_API_URL', '')\n", - "if not API_BASE_URL:\n", - " raise ValueError(\"The REPORT_API_URL environment variable is not set or is empty\")\n", - "\n", - "url = API_BASE_URL\n", - "headers = {\n", - " 'Authorization': f'Bearer {notebook_service_account_token}',\n", - " 'Content-Type': 'application/json',\n", - " 'Accept': 'application/pdf'\n", - "}\n", - "\n", - "# SQL query\n", - "query = \"\"\"\n", - "SELECT\n", - " COUNT(*) AS transaction_count,\n", - " SUM(total) AS total,\n", - " TO_CHAR(DATE_TRUNC('month',current_date) - INTERVAL '1 month','Month') as month,\n", - " corp_type_code\n", - "FROM \n", - " invoices\n", - "WHERE \n", - " corp_type_code = :partner_code\n", - " AND invoice_status_code = 'PAID'\n", - " AND payment_method_code IN ('PAD', 'EJV')\n", - " AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n", - "GROUP BY \n", - " corp_type_code\n", - "ORDER BY \n", - " month;\n", - "\"\"\"\n", - "\n", - "# Execute the SQL query and fetch results\n", - "result = %sql $query\n", - "\n", - "# Print the result to verify\n", - "print(result)\n", - "\n", - "# Check if the result is not None\n", - "if result:\n", - " # Convert the result to a DataFrame\n", - " df = result.DataFrame()\n", - "\n", - " # Rename columns to match the expected names in the template\n", - " df.rename(columns={\n", - " 'corp_type_code': 'Registry',\n", - " 'transaction_count': 'Trans_Counts',\n", - " 'total': 'Amount'\n", - " }, inplace=True)\n", - "\n", - " # Add a Symbol column with a dollar sign\n", - " df['Symbol'] = '$'\n", - "\n", - " # Convert DataFrame to JSON-compatible format\n", - " table_rows = df.to_dict(orient='records')\n", - "\n", - " # Get the current date\n", - " current_date = datetime.now().strftime(\"%B %d, %Y\")\n", - "\n", - " # Define the request body\n", - " data = {\n", - " \"templateVars\": {\n", - " \"Date\": current_date,\n", - " \"Company_Name\": \"Ministry of Justice\",\n", - " \"Address_Line_1\": \"PO Box 9249, Stn Prov Govt\",\n", - " \"Address_Line_2\": \"6th Floor, 850 Burdett Avenue\",\n", - " \"City\": \"VICTORIA\",\n", - " \"Province\": \"BC\",\n", - " \"Area_Code\": \"V8W 9J2\",\n", - " \"First_Name\": partner_code,\n", - " \"enter_month\": df['month'][0] if not df.empty else \"N/A\",\n", - " \"table_rows\": table_rows\n", - " },\n", - " \"templateName\": \"revenue_letter\",\n", - " \"reportName\": \"revenue_letter\"\n", - " }\n", - "\n", - " # Send the POST request\n", - " response = requests.post(url, headers=headers, json=data)\n", - "\n", - " # Check if the response is successful\n", - " if response.status_code == 200:\n", - " # Get the PDF content from the response\n", - " pdf_content = response.content\n", - " \n", - " # Save the PDF content to a file\n", - " with open(partner_code+'_revenue_letter.pdf', 'wb') as pdf_file:\n", - " pdf_file.write(pdf_content)\n", - " \n", - " print(\"PDF report saved successfully as 'payment_receipt.pdf'\")\n", - " else:\n", - " print('Failed to get the report:', response.text)\n", - "else:\n", - " print('No results returned from the SQL query')" - ] - } - ], - "metadata": { - "celltoolbar": "Tags", - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - }, - "vscode": { - "interpreter": { - "hash": "fcb35bce15c55b4cacb5112e543368f86c7f98ed17acd45e6841ee83ed1df6e3" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/jobs/notebook-report/monthly/reconciliation_summary.ipynb b/jobs/notebook-report/monthly/reconciliation_summary.ipynb new file mode 100644 index 000000000..b671ac63a --- /dev/null +++ b/jobs/notebook-report/monthly/reconciliation_summary.ipynb @@ -0,0 +1,509 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "collapsed": false, + "editable": true, + "jupyter": { + "outputs_hidden": false + }, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "# Reconciliation Monthly Stats" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "We need to load in these libraries into our notebook in order to query, load, manipulate and view the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true, + "jupyter": { + "outputs_hidden": false + }, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + }, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import base64\n", + "from config import Config\n", + "from datetime import datetime, timedelta\n", + "import os\n", + "import requests\n", + "from IPython import get_ipython\n", + "from IPython.display import display, Markdown\n", + "\n", + "%load_ext sql\n", + "%config SqlMagic.displaylimit = 5" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "parameters" + ] + }, + "source": [ + "# Parameters cell for external parameters via papermill (job running this notebook will insert a parameter cell below this). This cell has a tag of with the name \"parameters\" that is used by papermill\n", + "\n", + "e.g.\n", + "param1 = \"some_value\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "This will create the connection to the database and prep the jupyter magic for SQL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true, + "jupyter": { + "outputs_hidden": false + }, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + }, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%sql $Config.SQLALCHEMY_DATABASE_URI" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "source": [ + "Simplest query to run to ensure our libraries are loaded and our DB connection is working" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "%%sql\n", + "set time zone 'UTC';" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Query ..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "%%sql monthly_reconciliation_summary <<\n", + "SELECT \n", + " id, \n", + " (created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date AS created_date, \n", + " total, \n", + " service_fees, \n", + " payment_method_code, \n", + " corp_type_code, \n", + " created_by, \n", + " payment_date\n", + "FROM \n", + " invoices\n", + "WHERE \n", + " corp_type_code = :partner_code\n", + " AND total > 0\n", + " AND invoice_status_code = 'PAID'\n", + " AND payment_method_code in ('PAD','EJV')\n", + " AND created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' > (current_date - 1 - interval '1 months')::date\n", + " AND created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' <= (current_date - 1)::date\n", + "ORDER BY \n", + " 1;" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Save to CSV" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "filename_summary = os.path.join(os.getcwd(), r'data/')+partner_code+'_monthly_reconciliation_summary_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m') +'.csv'\n", + "df_summary = monthly_reconciliation_summary.DataFrame()\n", + "with open(filename_summary, 'w') as f:\n", + " f.write('Monthly Reconciliation Summary:\\n\\n')\n", + " if df_summary.empty:\n", + " f.write('No Data Retrieved')\n", + " else:\n", + " df_summary.to_csv(f, sep=',', encoding='utf-8', index=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "disbursement summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# Define the partner codes for which the disbursement summary should be executed\n", + "partners_for_disbursement_summary = ['CSO', 'VS']\n", + "\n", + "# Print the partner_code and the list to confirm their values\n", + "print(f\"Processing for partner_code: {partner_code}\")\n", + "print(f\"Partners for disbursement summary: {partners_for_disbursement_summary}\")\n", + "\n", + "if partner_code in partners_for_disbursement_summary:\n", + " print(f\"Partner code {partner_code} found in the list, executing SQL query.\")\n", + " query = f\"\"\"\n", + " SELECT id, (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, total, service_fees, payment_method_code, corp_type_code,created_by\n", + " FROM invoices\n", + " WHERE corp_type_code = :partner_code\n", + " AND invoice_status_code = 'PAID'\n", + " AND payment_method_code in ('PAD','EJV')\n", + " AND disbursement_status_code = 'COMPLETED'\n", + " AND disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' > (current_date - 1 - interval '1 months'- interval '5 days')::date\n", + " AND disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver' <= (current_date - 1)::date\n", + " order by 1;\n", + " \"\"\"\n", + " \n", + " display(Markdown(f\"## Running query for partner: {partner_code}\"))\n", + " results = get_ipython().run_cell_magic('sql', '', query)\n", + " monthly_reconciliation_disbursed = results.DataFrame() # Convert the results to a DataFrame for later use\n", + "else:\n", + " print(f\"Partner code {partner_code} not in the list, skipping SQL query.\")\n", + " monthly_reconciliation_disbursed = None # Assign None if not executed" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Save to another CSV" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "if monthly_reconciliation_disbursed is not None and not monthly_reconciliation_disbursed.empty:\n", + " filename_disbursed = os.path.join(os.getcwd(), 'data', f'{partner_code}_monthly_reconciliation_disbursed_' + datetime.strftime(datetime.now() - timedelta(1), '%Y-%m-%d') + '.csv')\n", + " print(f\"Saving CSV for partner_code: {partner_code} at {filename_disbursed}\")\n", + " with open(filename_disbursed, 'w') as f:\n", + " f.write('Monthly Reconciliation Disbursed:\\n\\n')\n", + " if monthly_reconciliation_disbursed.empty:\n", + " f.write('No Data Retrieved')\n", + " else:\n", + " monthly_reconciliation_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n", + "else:\n", + " print(f\"Partner code {partner_code} not in the list or no data to save, skipping CSV save.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Authenticate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "payload = \"grant_type=client_credentials\"\n", + "basic_hash = base64.b64encode(f\"{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_ID')}:{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_SECRET')}\".encode())\n", + " \n", + "headers = {\n", + " 'Content-Type': 'application/x-www-form-urlencoded',\n", + " 'Authorization': f'Basic {basic_hash.decode()}'\n", + "}\n", + "response = requests.request(\"POST\", f\"{os.getenv('JWT_OIDC_ISSUER')}/protocol/openid-connect/token\", headers=headers, data=payload)\n", + "\n", + "assert response.status_code == 200\n", + "notebook_service_account_token = response.json().get('access_token')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "partner_details = {\n", + " 'CSO': {\n", + " \"companyName\": \"Ministry of Justice\",\n", + " \"addressLine1\": \"PO Box 9249, Stn Prov Govt\",\n", + " \"addressLine2\": \"6th Floor, 850 Burdett Avenue\",\n", + " \"city\": \"VICTORIA\",\n", + " \"province\": \"BC\",\n", + " \"areaCode\": \"V8W 9J2\"\n", + " },\n", + " 'VS': {\n", + " \"companyName\": \"Vital Statistics Agency\",\n", + " \"addressLine1\": \"PO Box 9657, Stn Prov Govt\",\n", + " \"addressLine2\": \"\",\n", + " \"city\": \"VICTORIA\",\n", + " \"province\": \"BC\",\n", + " \"areaCode\": \"V8W 9P3\"\n", + " },\n", + " 'RPT': {\n", + " \"companyName\": \"Property Taxation Branch\",\n", + " \"addressLine1\": \"Ministry of Provincial Revenue\",\n", + " \"addressLine2\": \"4th Floor, 1802 Douglas Street\",\n", + " \"city\": \"VICTORIA\",\n", + " \"province\": \"BC\",\n", + " \"areaCode\": \"V8T 4K6\"\n", + " },\n", + " 'ESRA': {\n", + " \"companyName\": \"Ministry of Enviornment\",\n", + " \"addressLine1\": \"Environmental Emergencies and Land Remediation\",\n", + " \"addressLine2\": \"2nd Floorm 10470-152nd Street\",\n", + " \"city\": \"SURREY\",\n", + " \"province\": \"BC\",\n", + " \"areaCode\": \"V3R 0Y3\"\n", + " }\n", + " \n", + "}\n", + "\n", + "def generate_report(partner_code):\n", + " # Get the API base URL from the environment variable\n", + " API_BASE_URL = os.getenv('REPORT_API_URL', '')\n", + " if not API_BASE_URL:\n", + " raise ValueError(\"The REPORT_API_URL environment variable is not set or is empty\")\n", + "\n", + " url = API_BASE_URL\n", + " headers = {\n", + " 'Authorization': f'Bearer {notebook_service_account_token}',\n", + " 'Content-Type': 'application/json',\n", + " 'Accept': 'application/pdf'\n", + " }\n", + "\n", + " # SQL query\n", + " query = \"\"\"\n", + " SELECT\n", + " COUNT(*) AS transaction_count,\n", + " SUM(total) AS total,\n", + " TO_CHAR(DATE_TRUNC('month',current_date) - INTERVAL '1 month','Month') as month,\n", + " corp_type_code\n", + " FROM \n", + " invoices\n", + " WHERE \n", + " corp_type_code = :partner_code\n", + " AND invoice_status_code = 'PAID'\n", + " AND payment_method_code IN ('PAD', 'EJV')\n", + " AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n", + " GROUP BY \n", + " corp_type_code\n", + " ORDER BY \n", + " month;\n", + " \"\"\"\n", + "\n", + " # Execute the SQL query and fetch results\n", + " result = %sql $query\n", + "\n", + " # Print the result to verify\n", + " print(result)\n", + "\n", + " # Check if the result is not None\n", + " if result:\n", + " # Convert the result to a DataFrame\n", + " df = result.DataFrame()\n", + "\n", + " # Rename columns to match the expected names in the template\n", + " df.rename(columns={\n", + " 'corp_type_code': 'registry',\n", + " 'transaction_count': 'transCounts',\n", + " 'total': 'amount'\n", + " }, inplace=True)\n", + "\n", + " # Add a Symbol column with a dollar sign\n", + " df['symbol'] = '$'\n", + "\n", + " # Convert DataFrame to JSON-compatible format\n", + " tableRows = df.to_dict(orient='records')\n", + "\n", + " # Get the current date\n", + " current_date = datetime.now().strftime(\"%B %d, %Y\")\n", + " \n", + " # Get the details for the current partner\n", + " details = partner_details.get(partner_code, {})\n", + " if not details:\n", + " raise ValueError(f\"No details found for partner code: {partner_code}\")\n", + "\n", + " # Define the request body\n", + " data = {\n", + " \"templateVars\": {\n", + " \"date\": current_date,\n", + " \"companyName\": details[\"companyName\"],\n", + " \"addressLine1\": details[\"addressLine1\"],\n", + " \"addressLine2\": details[\"addressLine2\"],\n", + " \"city\": details[\"city\"],\n", + " \"province\": details[\"province\"],\n", + " \"areaCode\": details[\"areaCode\"],\n", + " \"firstName\": partner_code,\n", + " \"enterMonth\": df['month'][0] if not df.empty else \"N/A\",\n", + " \"tableRows\": tableRows\n", + " },\n", + " \"templateName\": \"revenue_letter\",\n", + " \"reportName\": \"revenue_letter\"\n", + " }\n", + "\n", + " # Send the POST request\n", + " response = requests.post(url, headers=headers, json=data)\n", + "\n", + " # Check if the response is successful\n", + " if response.status_code == 200:\n", + " # Get the PDF content from the response\n", + " pdf_content = response.content\n", + " \n", + " # Save the PDF content to a file\n", + " pdf_filename = os.path.join(os.getcwd(), 'data', f'{partner_code}_revenue_letter.pdf')\n", + " with open(pdf_filename, 'wb') as pdf_file:\n", + " pdf_file.write(pdf_content)\n", + " \n", + " print(\"PDF report saved successfully as 'payment_receipt.pdf'\")\n", + " else:\n", + " print('Failed to get the report:', response.text)\n", + " else:\n", + " print('No results returned from the SQL query')\n", + "generate_report(partner_code)" + ] + } + ], + "metadata": { + "celltoolbar": "Tags", + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "metadata": { + "collapsed": false + }, + "source": [] + } + }, + "vscode": { + "interpreter": { + "hash": "fcb35bce15c55b4cacb5112e543368f86c7f98ed17acd45e6841ee83ed1df6e3" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb b/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb deleted file mode 100644 index 82655a8f0..000000000 --- a/jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb +++ /dev/null @@ -1,422 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "editable": true, - "jupyter": { - "outputs_hidden": false - }, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "source": [ - "# Reconciliation Monthly Stats" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "We need to load in these libraries into our notebook in order to query, load, manipulate and view the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "editable": true, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "is_executing": false, - "name": "#%%\n" - }, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "import base64\n", - "from config import Config\n", - "from datetime import datetime, timedelta\n", - "import os\n", - "import requests\n", - "\n", - "%load_ext sql\n", - "%config SqlMagic.displaylimit = 5" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [ - "parameters" - ] - }, - "source": [ - "# Parameters cell for external parameters via papermill (job running this notebook will insert a parameter cell below this). This cell has a tag of with the name \"parameters\" that is used by papermill\n", - "\n", - "e.g.\n", - "param1 = \"some_value\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "This will create the connection to the database and prep the jupyter magic for SQL" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "editable": true, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "is_executing": false, - "name": "#%%\n" - }, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%sql $Config.SQLALCHEMY_DATABASE_URI" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "Simplest query to run to ensure our libraries are loaded and our DB connection is working" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - }, - "pycharm": { - "is_executing": false, - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "%%sql\n", - "set time zone 'UTC';" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Query ..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%%sql monthly_reconciliation_summary <<\n", - "SELECT\n", - "sum(pli.total) AS subtotal,\n", - "sum(pli.service_fees) AS service_fees,\n", - "sum(pli.total + pli.service_fees) AS total,\n", - "(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date,\n", - "pli.description,\n", - "i.payment_method_code,\n", - "i.corp_type_code\n", - "FROM invoices i\n", - "JOIN payment_line_items pli ON i.id = pli.invoice_id\n", - "WHERE i.corp_type_code = :partner_code\n", - "AND i.invoice_status_code = 'PAID'\n", - "AND i.payment_method_code IN ('PAD', 'EJV', 'DIRECT_PAY', 'DRAWDOWN')\n", - "AND date(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') > date(current_date - 1 - interval '1 months')\n", - "AND date(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') <= date(current_date - 1)\n", - "GROUP BY (i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, i.payment_method_code, i.corp_type_code, pli.description\n", - "ORDER BY (i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, pli.description, i.payment_method_code" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Save to CSV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "filename_summary = os.path.join(os.getcwd(), r'data/')+partner_code+'_monthly_reconciliation_summary_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m') +'.csv'\n", - "df_summary = monthly_reconciliation_summary.DataFrame()\n", - "with open(filename_summary, 'w') as f:\n", - " f.write('Monthly Reconciliation Summary:\\n\\n')\n", - " if df_summary.empty:\n", - " f.write('No Data Retrieved')\n", - " else:\n", - " df_summary.to_csv(f, sep=',', encoding='utf-8', index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "%%sql monthly_reconciliation_disbursed <<\n", - "SELECT count(*) AS transaction_count,\n", - "sum(pli.total) AS sub_total,\n", - "sum(pli.service_fees) AS service_fees,\n", - "sum(pli.total + pli.service_fees) AS total,\n", - "(i.disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date,\n", - "pli.description,\n", - "i.payment_method_code,\n", - "i.corp_type_code\n", - "FROM invoices i\n", - "JOIN payment_line_items pli ON i.id = pli.invoice_id\n", - "WHERE i.corp_type_code = :partner_code\n", - "AND i.invoice_status_code = 'PAID'\n", - "AND i.payment_method_code IN ('PAD', 'EJV', 'DIRECT_PAY')\n", - "AND i.disbursement_status_code = 'COMPLETED'\n", - "AND date(disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') > date(current_date - 1 - interval '1 months')\n", - "AND date(disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') <= date(current_date - 1)\n", - "GROUP BY (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, payment_method_code, corp_type_code, pli.description\n", - "ORDER BY (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, pli.description, i.payment_method_code;" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Save to another CSV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "editable": true, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [], - "source": [ - "filename_disbursed = os.path.join(os.getcwd(), r'data/')+partner_code+'_monthly_reconciliation_disbursed_' + datetime.strftime(datetime.now()-timedelta(1), '%Y-%m') +'.csv'\n", - "df_disbursed = monthly_reconciliation_disbursed.DataFrame()\n", - "with open(filename_disbursed, 'a') as f:\n", - " f.write('Monthly Reconciliation Disbursed:\\n\\n')\n", - " if df_disbursed.empty:\n", - " f.write('No Data Retrieved')\n", - " else:\n", - " df_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Authenticate" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get the API base URL from the environment variable\n", - "API_BASE_URL = os.getenv('REPORT_API_URL', '')\n", - "if not API_BASE_URL:\n", - " raise ValueError(\"The RREPORT_API_URL environment variable is not set or is empty\")\n", - "\n", - "url = API_BASE_URL\n", - "headers = {\n", - " 'Authorization': 'Bearer your_jwt_token',\n", - " 'Content-Type': 'application/json',\n", - " 'Accept': 'application/pdf'\n", - "}\n", - "\n", - "# SQL query\n", - "query = \"\"\"\n", - "SELECT\n", - " COUNT(*) AS transaction_count,\n", - " SUM(total) AS total,\n", - " TO_CHAR(DATE_TRUNC('month',current_date) - INTERVAL '1 month','Month') as month,\n", - " corp_type_code\n", - "FROM \n", - " invoices\n", - "WHERE \n", - " corp_type_code = :partner_code\n", - " AND invoice_status_code = 'PAID'\n", - " AND payment_method_code IN ('PAD', 'EJV')\n", - " AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n", - "GROUP BY \n", - " corp_type_code\n", - "ORDER BY \n", - " month;\n", - "\"\"\"\n", - "\n", - "# Execute the SQL query and fetch results\n", - "result = %sql $query\n", - "\n", - "# Print the result to verify\n", - "print(result)\n", - "\n", - "# Check if the result is not None\n", - "if result:\n", - " # Convert the result to a DataFrame\n", - " df = result.DataFrame()\n", - "\n", - " # Rename columns to match the expected names in the template\n", - " df.rename(columns={\n", - " 'corp_type_code': 'Registry',\n", - " 'transaction_count': 'Trans_Counts',\n", - " 'total': 'Amount'\n", - " }, inplace=True)\n", - "\n", - " # Add a Symbol column with a dollar sign\n", - " df['Symbol'] = '$'\n", - "\n", - " # Convert DataFrame to JSON-compatible format\n", - " table_rows = df.to_dict(orient='records')\n", - "\n", - " # Get the current date\n", - " current_date = datetime.now().strftime(\"%B %d, %Y\")\n", - "\n", - " # Define the request body\n", - " data = {\n", - " \"templateVars\": {\n", - " \"Date\": current_date,\n", - " \"Company_Name\": \"Vital Statistics Agency\",\n", - " \"Address_Line_1\": \"PO Box 9657, Stn Prov Govt\",\n", - " \"Address_Line_2\": \"\",\n", - " \"City\": \"VICTORIA\",\n", - " \"Province\": \"BC\",\n", - " \"Area_Code\": \"V8W 9P3\",\n", - " \"First_Name\": partner_code,\n", - " \"enter_month\": df['month'][0] if not df.empty else \"N/A\",\n", - " \"table_rows\": table_rows\n", - " },\n", - " \"templateName\": \"revenue_letter\",\n", - " \"reportName\": \"revenue_letter\"\n", - " }\n", - "\n", - " # Send the POST request\n", - " response = requests.post(url, headers=headers, json=data)\n", - "\n", - " # Check if the response is successful\n", - " if response.status_code == 200:\n", - " # Get the PDF content from the response\n", - " pdf_content = response.content\n", - " \n", - " # Save the PDF content to a file\n", - " with open(partner_code+'_revenue_letter.pdf', 'wb') as pdf_file:\n", - " pdf_file.write(pdf_content)\n", - " \n", - " print(\"PDF report saved successfully as 'payment_receipt.pdf'\")\n", - " else:\n", - " print('Failed to get the report:', response.text)\n", - "else:\n", - " print('No results returned from the SQL query')" - ] - } - ], - "metadata": { - "celltoolbar": "Tags", - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - }, - "vscode": { - "interpreter": { - "hash": "fcb35bce15c55b4cacb5112e543368f86c7f98ed17acd45e6841ee83ed1df6e3" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/jobs/notebook-report/notebookreport.py b/jobs/notebook-report/notebookreport.py index f9b2f3a73..c864fb345 100644 --- a/jobs/notebook-report/notebookreport.py +++ b/jobs/notebook-report/notebookreport.py @@ -112,14 +112,31 @@ def process_partner_notebooks(notebookdirectory: str, data_dir: str, partner_cod try: monthly_report_dates = ast.literal_eval(Config.MONTHLY_REPORT_DATES) except Exception: # noqa: B902 - logging.exception('Error: %s.', notebookdirectory) + logging.exception('Error parsing monthly report dates for: %s', notebookdirectory) send_email(notebookdirectory, 'ERROR', traceback.format_exc()) + return # Early exit if there is an error in configuration # First day of the month is 1 - if notebookdirectory == 'daily' \ - or (notebookdirectory == 'monthly' and date.today().day in monthly_report_dates): + today = date.today().day + logging.info('Today\'s date: %s', today) + + if notebookdirectory == 'daily': + logging.info('Processing daily notebooks for partner: %s', partner_code) execute_notebook(notebookdirectory, data_dir, partner_code) + if notebookdirectory == 'monthly' and today in monthly_report_dates: + logging.info('Processing monthly notebooks for partner: %s', partner_code) + execute_notebook(notebookdirectory, data_dir, partner_code, is_monthly=True) + + # Ensure both daily and monthly run on the 1st of the month + if today == 1: + if notebookdirectory == 'daily': + logging.info('Also processing daily notebooks on the 1st of the month for partner: %s', partner_code) + execute_notebook(notebookdirectory, data_dir, partner_code) + elif notebookdirectory == 'monthly': + logging.info('Also processing monthly notebooks on the 1st of the month for partner: %s', partner_code) + execute_notebook(notebookdirectory, data_dir, partner_code, is_monthly=True) + def process_notebooks(notebookdirectory: str, data_dir: str): """Process Notebook.""" @@ -136,10 +153,13 @@ def process_notebooks(notebookdirectory: str, data_dir: str): execute_notebook(notebookdirectory, data_dir) -def execute_notebook(notebookdirectory: str, data_dir: str, partner_code: str = None): +def execute_notebook(notebookdirectory: str, data_dir: str, partner_code: str = None, is_monthly:bool = False): """Execute notebook and send emails.""" parameters = {'partner_code': partner_code} if partner_code else None - pattern = f'{partner_code.lower()}_*.ipynb' if partner_code else '*.ipynb' + if is_monthly: + pattern = 'reconciliation_summary.ipynb' + else: + pattern = f'{partner_code.lower()}_*.ipynb' if partner_code else '*.ipynb' for file in findfiles(notebookdirectory, pattern): try: From 18d503fe3b807856517ce6c0b6fa07a70f8516f7 Mon Sep 17 00:00:00 2001 From: Avni Salhotra Date: Mon, 8 Jul 2024 09:08:02 -0700 Subject: [PATCH 4/4] format decimal digits displayed --- .../monthly/reconciliation_summary.ipynb | 22 +++++-------------- jobs/notebook-report/notebookreport.py | 3 +-- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/jobs/notebook-report/monthly/reconciliation_summary.ipynb b/jobs/notebook-report/monthly/reconciliation_summary.ipynb index b671ac63a..e9b310187 100644 --- a/jobs/notebook-report/monthly/reconciliation_summary.ipynb +++ b/jobs/notebook-report/monthly/reconciliation_summary.ipynb @@ -238,7 +238,6 @@ "# Define the partner codes for which the disbursement summary should be executed\n", "partners_for_disbursement_summary = ['CSO', 'VS']\n", "\n", - "# Print the partner_code and the list to confirm their values\n", "print(f\"Processing for partner_code: {partner_code}\")\n", "print(f\"Partners for disbursement summary: {partners_for_disbursement_summary}\")\n", "\n", @@ -261,7 +260,7 @@ " monthly_reconciliation_disbursed = results.DataFrame() # Convert the results to a DataFrame for later use\n", "else:\n", " print(f\"Partner code {partner_code} not in the list, skipping SQL query.\")\n", - " monthly_reconciliation_disbursed = None # Assign None if not executed" + " monthly_reconciliation_disbursed = None " ] }, { @@ -357,7 +356,7 @@ " 'ESRA': {\n", " \"companyName\": \"Ministry of Enviornment\",\n", " \"addressLine1\": \"Environmental Emergencies and Land Remediation\",\n", - " \"addressLine2\": \"2nd Floorm 10470-152nd Street\",\n", + " \"addressLine2\": \"2nd Floor 10470-152nd Street\",\n", " \"city\": \"SURREY\",\n", " \"province\": \"BC\",\n", " \"areaCode\": \"V3R 0Y3\"\n", @@ -366,7 +365,6 @@ "}\n", "\n", "def generate_report(partner_code):\n", - " # Get the API base URL from the environment variable\n", " API_BASE_URL = os.getenv('REPORT_API_URL', '')\n", " if not API_BASE_URL:\n", " raise ValueError(\"The REPORT_API_URL environment variable is not set or is empty\")\n", @@ -378,7 +376,6 @@ " 'Accept': 'application/pdf'\n", " }\n", "\n", - " # SQL query\n", " query = \"\"\"\n", " SELECT\n", " COUNT(*) AS transaction_count,\n", @@ -398,15 +395,11 @@ " month;\n", " \"\"\"\n", "\n", - " # Execute the SQL query and fetch results\n", " result = %sql $query\n", "\n", - " # Print the result to verify\n", " print(result)\n", "\n", - " # Check if the result is not None\n", " if result:\n", - " # Convert the result to a DataFrame\n", " df = result.DataFrame()\n", "\n", " # Rename columns to match the expected names in the template\n", @@ -416,16 +409,15 @@ " 'total': 'amount'\n", " }, inplace=True)\n", "\n", - " # Add a Symbol column with a dollar sign\n", " df['symbol'] = '$'\n", + " \n", + " df['amount'] = df['amount'].apply(lambda x: f\"{x:.2f}\")\n", "\n", " # Convert DataFrame to JSON-compatible format\n", " tableRows = df.to_dict(orient='records')\n", "\n", - " # Get the current date\n", " current_date = datetime.now().strftime(\"%B %d, %Y\")\n", " \n", - " # Get the details for the current partner\n", " details = partner_details.get(partner_code, {})\n", " if not details:\n", " raise ValueError(f\"No details found for partner code: {partner_code}\")\n", @@ -448,15 +440,11 @@ " \"reportName\": \"revenue_letter\"\n", " }\n", "\n", - " # Send the POST request\n", " response = requests.post(url, headers=headers, json=data)\n", - "\n", - " # Check if the response is successful\n", + " \n", " if response.status_code == 200:\n", - " # Get the PDF content from the response\n", " pdf_content = response.content\n", " \n", - " # Save the PDF content to a file\n", " pdf_filename = os.path.join(os.getcwd(), 'data', f'{partner_code}_revenue_letter.pdf')\n", " with open(pdf_filename, 'wb') as pdf_file:\n", " pdf_file.write(pdf_content)\n", diff --git a/jobs/notebook-report/notebookreport.py b/jobs/notebook-report/notebookreport.py index c864fb345..491e0d8b5 100644 --- a/jobs/notebook-report/notebookreport.py +++ b/jobs/notebook-report/notebookreport.py @@ -114,9 +114,8 @@ def process_partner_notebooks(notebookdirectory: str, data_dir: str, partner_cod except Exception: # noqa: B902 logging.exception('Error parsing monthly report dates for: %s', notebookdirectory) send_email(notebookdirectory, 'ERROR', traceback.format_exc()) - return # Early exit if there is an error in configuration + return - # First day of the month is 1 today = date.today().day logging.info('Today\'s date: %s', today)