Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

21622 changes to notebook report for generating revenue letter #1590

Closed
wants to merge 4 commits into from
Closed

21622 changes to notebook report for generating revenue letter #1590

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6a8d437
adding changes to notebook report
avni-work Jun 26, 2024
3fceea2
adding partner_code to query
avni-work Jun 26, 2024
d212f9a
revenue letter and single notebook now processing all partners
avni-work Jul 5, 2024
18d503f
format decimal digits displayed
avni-work Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 134 additions & 4 deletions jobs/notebook-report/monthly/cso_reconciliation_summary.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,11 @@
},
"outputs": [],
"source": [
"import os\n",
"from datetime import datetime, timedelta\n",
"import base64\n",
"from config import Config\n",
"from datetime import datetime, timedelta\n",
"import os\n",
"import requests\n",
"\n",
"%load_ext sql\n",
"%config SqlMagic.displaylimit = 5"
Expand Down Expand Up @@ -173,7 +175,7 @@
"FROM \n",
" invoices\n",
"WHERE \n",
" corp_type_code = 'CSO'\n",
" corp_type_code = :partner_code\n",
" AND total > 0\n",
" AND invoice_status_code = 'PAID'\n",
" AND payment_method_code in ('PAD','EJV')\n",
Expand Down Expand Up @@ -227,7 +229,7 @@
"%%sql monthly_reconciliation_disbursed <<\n",
"SELECT id, (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, total, service_fees, payment_method_code, corp_type_code,created_by\n",
"FROM invoices\n",
"WHERE corp_type_code = 'CSO'\n",
"WHERE corp_type_code = :partner_code\n",
"AND invoice_status_code = 'PAID'\n",
"AND payment_method_code in ('PAD','EJV')\n",
"AND disbursement_status_code = 'COMPLETED'\n",
Expand Down Expand Up @@ -264,6 +266,134 @@
" else:\n",
" df_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Authenticate"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"payload = \"grant_type=client_credentials\"\n",
"basic_hash = base64.b64encode(f\"{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_ID')}:{os.getenv('NOTEBOOK_SERVICE_ACCOUNT_SECRET')}\".encode())\n",
" \n",
"headers = {\n",
" 'Content-Type': 'application/x-www-form-urlencoded',\n",
" 'Authorization': f'Basic {basic_hash.decode()}'\n",
"}\n",
"response = requests.request(\"POST\", f\"{os.getenv('JWT_OIDC_ISSUER')}/protocol/openid-connect/token\", headers=headers, data=payload)\n",
"\n",
"assert response.status_code == 200\n",
"notebook_service_account_token = response.json().get('access_token')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Get the API base URL from the environment variable\n",
"API_BASE_URL = os.getenv('REPORT_API_URL', '')\n",
"if not API_BASE_URL:\n",
" raise ValueError(\"The REPORT_API_URL environment variable is not set or is empty\")\n",
"\n",
"url = API_BASE_URL\n",
"headers = {\n",
" 'Authorization': f'Bearer {notebook_service_account_token}',\n",
" 'Content-Type': 'application/json',\n",
" 'Accept': 'application/pdf'\n",
"}\n",
"\n",
"# SQL query\n",
"query = \"\"\"\n",
"SELECT\n",
" COUNT(*) AS transaction_count,\n",
" SUM(total) AS total,\n",
" TO_CHAR(DATE_TRUNC('month',current_date) - INTERVAL '1 month','Month') as month,\n",
" corp_type_code\n",
"FROM \n",
" invoices\n",
"WHERE \n",
" corp_type_code = :partner_code\n",
" AND invoice_status_code = 'PAID'\n",
" AND payment_method_code IN ('PAD', 'EJV')\n",
" AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n",
"GROUP BY \n",
" corp_type_code\n",
"ORDER BY \n",
" month;\n",
"\"\"\"\n",
"\n",
"# Execute the SQL query and fetch results\n",
"result = %sql $query\n",
"\n",
"# Print the result to verify\n",
"print(result)\n",
"\n",
"# Check if the result is not None\n",
"if result:\n",
" # Convert the result to a DataFrame\n",
" df = result.DataFrame()\n",
"\n",
" # Rename columns to match the expected names in the template\n",
" df.rename(columns={\n",
" 'corp_type_code': 'Registry',\n",
" 'transaction_count': 'Trans_Counts',\n",
" 'total': 'Amount'\n",
" }, inplace=True)\n",
"\n",
" # Add a Symbol column with a dollar sign\n",
" df['Symbol'] = '$'\n",
"\n",
" # Convert DataFrame to JSON-compatible format\n",
" table_rows = df.to_dict(orient='records')\n",
"\n",
" # Get the current date\n",
" current_date = datetime.now().strftime(\"%B %d, %Y\")\n",
"\n",
" # Define the request body\n",
" data = {\n",
" \"templateVars\": {\n",
" \"Date\": current_date,\n",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these should be camelCase

" \"Company_Name\": \"Ministry of Justice\",\n",
" \"Address_Line_1\": \"PO Box 9249, Stn Prov Govt\",\n",
" \"Address_Line_2\": \"6th Floor, 850 Burdett Avenue\",\n",
" \"City\": \"VICTORIA\",\n",
" \"Province\": \"BC\",\n",
" \"Area_Code\": \"V8W 9J2\",\n",
" \"First_Name\": partner_code,\n",
" \"enter_month\": df['month'][0] if not df.empty else \"N/A\",\n",
" \"table_rows\": table_rows\n",
" },\n",
" \"templateName\": \"revenue_letter\",\n",
" \"reportName\": \"revenue_letter\"\n",
" }\n",
"\n",
" # Send the POST request\n",
" response = requests.post(url, headers=headers, json=data)\n",
"\n",
" # Check if the response is successful\n",
" if response.status_code == 200:\n",
" # Get the PDF content from the response\n",
" pdf_content = response.content\n",
" \n",
" # Save the PDF content to a file\n",
" with open(partner_code+'_revenue_letter.pdf', 'wb') as pdf_file:\n",
" pdf_file.write(pdf_content)\n",
" \n",
" print(\"PDF report saved successfully as 'payment_receipt.pdf'\")\n",
" else:\n",
" print('Failed to get the report:', response.text)\n",
"else:\n",
" print('No results returned from the SQL query')"
]
}
],
"metadata": {
Expand Down
139 changes: 125 additions & 14 deletions jobs/notebook-report/monthly/vs_reconciliation_summary.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,11 @@
},
"outputs": [],
"source": [
"import os\n",
"from datetime import datetime, timedelta\n",
"import base64\n",
"from config import Config\n",
"from datetime import datetime, timedelta\n",
"import os\n",
"import requests\n",
"\n",
"%load_ext sql\n",
"%config SqlMagic.displaylimit = 5"
Expand Down Expand Up @@ -138,7 +140,7 @@
"outputs": [],
"source": [
"%%sql\n",
"select now() AT TIME ZONE 'america/vancouver' as current_date"
"set time zone 'UTC';"
]
},
{
Expand All @@ -165,7 +167,7 @@
"sum(pli.total) AS subtotal,\n",
"sum(pli.service_fees) AS service_fees,\n",
"sum(pli.total + pli.service_fees) AS total,\n",
"(i.payment_date at time zone 'utc' at time zone 'america/vancouver')::date,\n",
"(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date,\n",
"pli.description,\n",
"i.payment_method_code,\n",
"i.corp_type_code\n",
Expand All @@ -174,10 +176,10 @@
"WHERE i.corp_type_code = :partner_code\n",
"AND i.invoice_status_code = 'PAID'\n",
"AND i.payment_method_code IN ('PAD', 'EJV', 'DIRECT_PAY', 'DRAWDOWN')\n",
"AND date(i.payment_date at time zone 'utc' at time zone 'america/vancouver') > date(current_date - 1 - interval '1 months')\n",
"AND date(i.payment_date at time zone 'utc' at time zone 'america/vancouver') <= date(current_date - 1)\n",
"GROUP BY (i.payment_date at time zone 'utc' at time zone 'america/vancouver')::date, i.payment_method_code, i.corp_type_code, pli.description\n",
"ORDER BY (i.payment_date at time zone 'utc' at time zone 'america/vancouver')::date, pli.description, i.payment_method_code"
"AND date(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') > date(current_date - 1 - interval '1 months')\n",
"AND date(i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') <= date(current_date - 1)\n",
"GROUP BY (i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, i.payment_method_code, i.corp_type_code, pli.description\n",
"ORDER BY (i.payment_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, pli.description, i.payment_method_code"
]
},
{
Expand Down Expand Up @@ -226,20 +228,20 @@
"sum(pli.total) AS sub_total,\n",
"sum(pli.service_fees) AS service_fees,\n",
"sum(pli.total + pli.service_fees) AS total,\n",
"(i.disbursement_date at time zone 'utc' at time zone 'america/vancouver')::date,\n",
"(i.disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date,\n",
"pli.description,\n",
"i.payment_method_code,\n",
"i.corp_type_code\n",
"FROM invoices i\n",
"JOIN payment_line_items pli ON i.id = pli.invoice_id\n",
"WHERE i.corp_type_code = 'VS'\n",
"WHERE i.corp_type_code = :partner_code\n",
"AND i.invoice_status_code = 'PAID'\n",
"AND i.payment_method_code IN ('PAD', 'EJV', 'DIRECT_PAY')\n",
"AND i.disbursement_status_code = 'COMPLETED'\n",
"AND date(disbursement_date at time zone 'utc' at time zone 'america/vancouver') > date(current_date - 1 - interval '1 months')\n",
"AND date(disbursement_date at time zone 'utc' at time zone 'america/vancouver') <= date(current_date - 1)\n",
"GROUP BY (disbursement_date at time zone 'utc' at time zone 'america/vancouver')::date, payment_method_code, corp_type_code, pli.description\n",
"ORDER BY (disbursement_date at time zone 'utc' at time zone 'america/vancouver')::date, pli.description, i.payment_method_code;"
"AND date(disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') > date(current_date - 1 - interval '1 months')\n",
"AND date(disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') <= date(current_date - 1)\n",
"GROUP BY (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, payment_method_code, corp_type_code, pli.description\n",
"ORDER BY (disbursement_date AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver')::date, pli.description, i.payment_method_code;"
]
},
{
Expand Down Expand Up @@ -270,6 +272,115 @@
" else:\n",
" df_disbursed.to_csv(f, sep=',', encoding='utf-8', index=False)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Authenticate"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Get the API base URL from the environment variable\n",
"API_BASE_URL = os.getenv('REPORT_API_URL', '')\n",
"if not API_BASE_URL:\n",
" raise ValueError(\"The RREPORT_API_URL environment variable is not set or is empty\")\n",
"\n",
"url = API_BASE_URL\n",
"headers = {\n",
" 'Authorization': 'Bearer your_jwt_token',\n",
" 'Content-Type': 'application/json',\n",
" 'Accept': 'application/pdf'\n",
"}\n",
"\n",
"# SQL query\n",
"query = \"\"\"\n",
"SELECT\n",
" COUNT(*) AS transaction_count,\n",
" SUM(total) AS total,\n",
" TO_CHAR(DATE_TRUNC('month',current_date) - INTERVAL '1 month','Month') as month,\n",
" corp_type_code\n",
"FROM \n",
" invoices\n",
"WHERE \n",
" corp_type_code = :partner_code\n",
" AND invoice_status_code = 'PAID'\n",
" AND payment_method_code IN ('PAD', 'EJV')\n",
" AND DATE_TRUNC('month', created_on AT TIME ZONE 'UTC' AT TIME ZONE 'America/Vancouver') = DATE_TRUNC('month', current_date - INTERVAL '1 month')\n",
"GROUP BY \n",
" corp_type_code\n",
"ORDER BY \n",
" month;\n",
"\"\"\"\n",
"\n",
"# Execute the SQL query and fetch results\n",
"result = %sql $query\n",
"\n",
"# Print the result to verify\n",
"print(result)\n",
"\n",
"# Check if the result is not None\n",
"if result:\n",
" # Convert the result to a DataFrame\n",
" df = result.DataFrame()\n",
"\n",
" # Rename columns to match the expected names in the template\n",
" df.rename(columns={\n",
" 'corp_type_code': 'Registry',\n",
" 'transaction_count': 'Trans_Counts',\n",
" 'total': 'Amount'\n",
" }, inplace=True)\n",
"\n",
" # Add a Symbol column with a dollar sign\n",
" df['Symbol'] = '$'\n",
"\n",
" # Convert DataFrame to JSON-compatible format\n",
" table_rows = df.to_dict(orient='records')\n",
"\n",
" # Get the current date\n",
" current_date = datetime.now().strftime(\"%B %d, %Y\")\n",
"\n",
" # Define the request body\n",
" data = {\n",
" \"templateVars\": {\n",
" \"Date\": current_date,\n",
" \"Company_Name\": \"Vital Statistics Agency\",\n",
" \"Address_Line_1\": \"PO Box 9657, Stn Prov Govt\",\n",
" \"Address_Line_2\": \"\",\n",
" \"City\": \"VICTORIA\",\n",
" \"Province\": \"BC\",\n",
" \"Area_Code\": \"V8W 9P3\",\n",
" \"First_Name\": partner_code,\n",
" \"enter_month\": df['month'][0] if not df.empty else \"N/A\",\n",
" \"table_rows\": table_rows\n",
" },\n",
" \"templateName\": \"revenue_letter\",\n",
" \"reportName\": \"revenue_letter\"\n",
" }\n",
"\n",
" # Send the POST request\n",
" response = requests.post(url, headers=headers, json=data)\n",
"\n",
" # Check if the response is successful\n",
" if response.status_code == 200:\n",
" # Get the PDF content from the response\n",
" pdf_content = response.content\n",
" \n",
" # Save the PDF content to a file\n",
" with open(partner_code+'_revenue_letter.pdf', 'wb') as pdf_file:\n",
" pdf_file.write(pdf_content)\n",
" \n",
" print(\"PDF report saved successfully as 'payment_receipt.pdf'\")\n",
" else:\n",
" print('Failed to get the report:', response.text)\n",
"else:\n",
" print('No results returned from the SQL query')"
]
}
],
"metadata": {
Expand Down
3 changes: 2 additions & 1 deletion jobs/notebook-report/notebookreport.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ def send_email(file_processing, emailtype, errormessage, partner_code=None):
year_month = datetime.strftime(datetime.now() - timedelta(1), '%Y-%m')
subject = 'Monthly Reconciliation Stats ' + year_month + ext
filenames = [f'{partner_code}_monthly_reconciliation_summary_' + year_month + '.csv',
f'{partner_code}_monthly_reconciliation_disbursed_' + year_month + '.csv']
f'{partner_code}_monthly_reconciliation_disbursed_' + year_month + '.csv',
f'{partner_code}_revenue_letter.pdf']
recipients = get_partner_recipients(file_processing, partner_code)

# Add body to email
Expand Down
Loading