diff --git a/.ebextensions/01_procfile.config b/.ebextensions/01_churroflowapi.config similarity index 68% rename from .ebextensions/01_procfile.config rename to .ebextensions/01_churroflowapi.config index 2a5ad09..4dcda7f 100644 --- a/.ebextensions/01_procfile.config +++ b/.ebextensions/01_churroflowapi.config @@ -1,14 +1,14 @@ -# .ebextensions/01_fastapi.config +# .ebextensions/01_churroflowapi.config option_settings: aws:elasticbeanstalk:application:environment: PYTHONPATH: "/var/app/current:$PYTHONPATH" aws:elasticbeanstalk:container:python: - WSGIPath: "src.main:app" - + WSGIPath: "main:app" + aws:elasticbeanstalk:environment:proxy: + ProxyServer: apache container_commands: 01_initdb: command: "source /var/app/venv/*/bin/activate && python3 src/database.py" leader_only: true - diff --git a/.ebextensions/01_packages.config b/.ebextensions/01_packages.config deleted file mode 100644 index 9c4cf0e..0000000 --- a/.ebextensions/01_packages.config +++ /dev/null @@ -1,16 +0,0 @@ -packages: - yum: - cairo-devel: [] - pango: [] - pango-devel : [] - libicu-devel: [] - -commands: - 01_postgres_libs: - command: rpm -ivh --force https://yum.postgresql.org/10/redhat/rhel-6.9-x86_64/postgresql10-libs-10.7-1PGDG.rhel6.x86_64.rpm - 02_postgres_install: - command: rpm -ivh --force https://yum.postgresql.org/10/redhat/rhel-6.9-x86_64/postgresql10-10.7-1PGDG.rhel6.x86_64.rpm - 03_symink_pg_config: - command: sudo ln -sf /usr/pgsql-10/bin/pg_config /usr/bin/pg_config - 04_postgres_devel: - command: sudo rpm -ivh --force https://yum.postgresql.org/10/redhat/rhel-6.9-x86_64/postgresql10-devel-10.7-1PGDG.rhel6.x86_64.rpm diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml index 2ca3353..ae0941c 100644 --- a/.elasticbeanstalk/config.yml +++ b/.elasticbeanstalk/config.yml @@ -1,15 +1,10 @@ branch-defaults: - main: - environment: churros - group_suffix: null -environment-defaults: - churros: - branch: null - repository: null + jeremy-data_generation: + environment: ChurroFlowAPI-dev global: - application_name: CHURROS_2021 + application_name: ChurroFlowAPI branch: null - default_ec2_keyname: ricardo + default_ec2_keyname: aws-eb default_platform: Python 3.8 running on 64bit Amazon Linux 2 default_region: ap-southeast-2 include_git_submodules: true diff --git a/.gitignore b/.gitignore index f25182b..ba162de 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,8 @@ bin/ pyvenv.cfg lib64 .vscode + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml diff --git a/.platform/httpd/conf.d/ssl_rewrite.conf b/.platform/httpd/conf.d/ssl_rewrite.conf new file mode 100644 index 0000000..10ec086 --- /dev/null +++ b/.platform/httpd/conf.d/ssl_rewrite.conf @@ -0,0 +1,6 @@ +# .platform/httpd/conf.d/ssl_rewrite.conf + +RewriteEngine On + +RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [R,L] + \ No newline at end of file diff --git a/dummy_data/AUInvoice_template.xml b/dummy_data/AUInvoice_template.xml new file mode 100644 index 0000000..99f1892 --- /dev/null +++ b/dummy_data/AUInvoice_template.xml @@ -0,0 +1,272 @@ + + + urn:cen.eu:en16931:2017#conformant#urn:fdc:peppol.eu:2017:poacc:billing:international:aunz:3.0 + urn:fdc:peppol.eu:2017:poacc:billing:01:1.0 + {{name}} + {{issue_date}} + {{due_date}} + 380 + Tax invoice + AUD + 4025:123:4343 + 0150abc + + {{invoice_start_date}} + {{invoice_end_date}} + + + {{order_ref}} + 12345678 + + + + {{order_ref}} + {{issue_date}} + + + + + {{supplier_abn}} + + {{supplier_abn}} + + + {{supplier_name}} + + + {{supplier_road}} + {{supplier_suburb}} + {{supplier_city}} {{supplier_state}} + {{supplier_postcode}} + + AU + + + + {{supplier_abn}} + + GST + + + + {{supplier_name}} + {{supplier_abn}} + Partnership + + + + Ronald MacDonald + Mobile 0430123456 + ronald.macdonald@qualitygoods.com.au + + + + + + + {{customer_abn}} + + {{customer_abn}} + + + {{customer_name}} + + + 100 Queen Street + Po box 878 + Sydney + 2000 + + AU + + + + {{customer_abn}} + + GST + + + + {{customer_name}} + {{customer_abn}} + + + {{customer_contact_name}} + {{customer_contact_phone}} + {{customer_contact_email}} + + + + + + + {{customer_abn}} + + + Mr Anderson + + + + {{customer_abn}} + + + + + + Mr Wilson + + + 16 Stout Street + Po box 878 + Sydney + 2000 + NSW + + Unit 1 + + + AU + + + + {{customer_abn}} + + GST + + + + + + + {{delivery_date}} + + {{customer_abn}} + + {{delivery_road}} + {{delivery_suburb}} + {{delivery_city}} + {{delivery_postcode}} + {{delivery_state}} + + AU + + + + + + Delivery party Name + + + + + 30 + PaymentReferenceText + + AccountNumber + AccountName + + BSB Number + + + + + Payment within 30 days + + + true + SAA + Shipping and Handling + 0 + 0 + 0 + + S + 10 + + GST + + + + + + + {{tax_amount}} + + {{pre_tax_total}} + {{tax_amount}} + + S + 10 + + GST + + + + + + + + + {{pre_tax_total}} + {{pre_tax_total}} + {{total_amount}} + 0.00 + 0.00 + {{total_amount}} + + + + + 1 + 1 + {{pre_tax_total}} + Consulting Fees + + {{invoice_start_date}} + {{invoice_end_date}} + + + 123 + + + 9000074677 + 130 + + + + {{description}} + {{description}} + + W659590 + + + WG546767 + + + WG546767 + + + AU + + + 09348023 + + + S + 10 + + GST + + + + + + {{pre_tax_total}} + + + + + + \ No newline at end of file diff --git a/generate_data.py b/dummy_data/generate_data.py similarity index 100% rename from generate_data.py rename to dummy_data/generate_data.py diff --git a/dummy_data/generate_invoices.py b/dummy_data/generate_invoices.py new file mode 100644 index 0000000..9601941 --- /dev/null +++ b/dummy_data/generate_invoices.py @@ -0,0 +1,306 @@ +from time import sleep +from faker import Faker +import random +import datetime +import binascii +import requests +import sys + +# Check for file input and folder input + +if len(sys.argv) < 3: + print("Usage: python3 generate_invoices.py ") + sys.exit(1) + +TEMPLATE_FILE = sys.argv[1] +OUTPUT_FOLDER = sys.argv[2] +START_AT = 1 + +# Instantiate a Faker object +fake = Faker() + +NUM_INVOICES = 500 +NUM_LINE_ITEMS = 300 + +weighting = [10, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19] + +def validate(abn): + """ + Validate that the provided number is indeed an ABN. + """ + values = list(map(int, list(abn))) + values[0] -= 1 + total = sum([x * w for (x, w) in zip(values, weighting)]) + return total % 89 == 0 + +def abn(): + """ + Generate a random ABN + """ + value = ''.join([str(int(random.random() * 10)) for i in range(9)]) + temp = list('00%s' % value) + total = sum([w * x for (w,x) in zip(weighting, map(int, temp))]) + remainder = total % 89 + prefix = 10 + (89 - remainder) + abn = '%s%s' % (prefix, value) + if not validate(abn): + return abn() + return abn + +def random_lat(): + min_lat, max_lat = -34.05, -33.568 + return round(random.uniform(min_lat, max_lat), 6) + +def random_lon(): + min_lon, max_lon = 150.52, 151.34 + return round(random.uniform(min_lon, max_lon), 6) + +supplier = ("Churros Pty Ltd", abn()) +supplier_warehouses = [ + (-33.913944, 151.022874), + (-33.848536, 150.901258), + (-33.791443, 151.072199), +] + +# -33.913944, 151.022874 +# -33.848536, 150.901258 +# -33.791443, 151.072199 + +fake_customers = [(fake.company(), + abn(), + random_lat(), + random_lon(), + fake.name(), + fake.email(), + fake.phone_number(), + ) for _ in range(20)] + +# Generate fake data for the invoices table +invoices = [] +for i in range(START_AT, NUM_INVOICES + 1): + customer = random.choice(fake_customers) + + issue_date = fake.date_time_between(start_date="-2y", end_date="now") + due_date = fake.date_time_between(start_date=issue_date, end_date="now") + + start_date = fake.date_time_between(start_date=issue_date - datetime.timedelta(days=30), end_date=issue_date) + end_date = fake.date_time_between(start_date=start_date, end_date=start_date + datetime.timedelta(days=30)) + + delivery_date = fake.date_time_between(start_date=start_date, end_date=start_date + datetime.timedelta(days=10)) + + supplier_warehouse = random.choice(supplier_warehouses) + + invoices.append(( + "invoice{}.xml".format(i), + 1, + fake.date_time_between(start_date="-1d", end_date="now").strftime('%Y-%m-%d'), + fake.date_time_between(start_date="-1d", end_date="now").strftime('%Y-%m-%d'), + 0, + 0, + True, + "", + "Invoice #{}".format(i), + issue_date.strftime('%Y-%m-%d'), + due_date.strftime('%Y-%m-%d'), + random.randint(1, 100), + start_date.strftime('%Y-%m-%d'), + end_date.strftime('%Y-%m-%d'), + supplier[0], + supplier[1], + supplier_warehouse[0], + supplier_warehouse[1], + customer[0], + customer[1], + delivery_date.strftime('%Y-%m-%d'), + # random_lat(), + # random_lon(), + round(random.uniform(supplier_warehouse[0], customer[2]), 6), + round(random.uniform(supplier_warehouse[1], customer[3]), 6), + # customer[2], + # customer[3], + customer[4], + customer[5], + customer[6], + round(random.uniform(100, 10000), 2) + )) + +# List of stationary supply items and their price +possible_items = [ + ('Pens', 1000.00), + ('Pencils', 1000.00), + ('Paper', 0.10), + ('Stapler', 5.00), + ('Staples', 1.00), + ('Paper Clips', 1.00), + ('Ruler', 0.50), + ('Eraser', 0.10), + ('Glue', 3.00), + ('Scissors', 6.00), + ('Tape', 5.00), + ('Sticky Notes', 4.00), +] + +# Generate fake data for the lineitems table +lineitems = [] +for i in range(1, NUM_LINE_ITEMS + 1): + item = random.choice(possible_items) + quantity = random.randint(1, 10) + + lineitems.append(( + random.randint(1, NUM_INVOICES), + item[0], + quantity, + item[1], + # round(quantity * item[1], 2) + random.randint(50, 500) + )) + + + +TEMPLATE_INVOICE = None +with open(TEMPLATE_FILE, 'r') as f: + TEMPLATE_INVOICE = f.read() + +def get_address_data(lat, lon): + sleep(0.5) + + addy = requests.get(f"https://geocode.maps.co/reverse", params={ + "lat": str(lat), + "lon": str(lon), + }).json()['address'] + + for key in ['road', 'suburb', 'postcode', 'country']: + if key not in addy: + return False + + return addy + +addresses = [] +for i in range(3): + addy = get_address_data(supplier_warehouses[i][0], supplier_warehouses[i][1]) + while not addy: + print('retrying') + addy = get_address_data(supplier_warehouses[i][0], supplier_warehouses[i][1]) + addresses.append(addy) + +# supplier_warehouses +warehouse_addresses = { + supplier_warehouses[0][0]: addresses[0], + supplier_warehouses[1][0]: addresses[1], + supplier_warehouses[2][0]: addresses[2], +} + + +for invoice in invoices: + print('supplier coords', invoice[16], invoice[17]) + print('delivery coords', invoice[21], invoice[22]) + + # supplier_address_data = random.choice(warehouse_addresses) + supplier_address_data = warehouse_addresses[invoice[16]] + + # flag = False + # for key in ['road', 'postcode', 'country']: + # if key not in supplier_address_data: + # flag = True + # if flag: + # continue + + delivery_address_data = get_address_data(invoice[21], invoice[22]) + + if not delivery_address_data: + continue + # flag = False + # for key in ['road', 'postcode', 'country']: + # if key not in delivery_address_data: + # flag = True + # if flag: + # continue + + # if supplier_address_data['road'] == delivery_address_data['road']: + # print('Same road' + supplier_address_data['road']) + # continue + + with open(f'{OUTPUT_FOLDER}/{invoice[0]}', 'w') as f: + invoice_text = TEMPLATE_INVOICE + supplier_road = supplier_address_data["road"] + supplier_suburb = supplier_address_data["suburb"] + supplier_city = supplier_address_data["city"] + supplier_state = supplier_address_data["state"] + supplier_postcode = supplier_address_data["postcode"] + supplier_country = supplier_address_data["country"] + + delivery_road = delivery_address_data["road"] + delivery_suburb = delivery_address_data["suburb"] if "suburb" in delivery_address_data else "" + delivery_city = delivery_address_data["city"] if "city" in delivery_address_data else "" + delivery_state = delivery_address_data["state"] if "state" in delivery_address_data else "" + delivery_postcode = delivery_address_data["postcode"] + + invoice_text = invoice_text.replace("{{name}}", invoice[8]) + invoice_text = invoice_text.replace("{{issue_date}}", invoice[9]) + invoice_text = invoice_text.replace("{{due_date}}", invoice[10]) + invoice_text = invoice_text.replace("{{order_id}}", str(invoice[11])) + invoice_text = invoice_text.replace("{{invoice_start_date}}", invoice[12]) + invoice_text = invoice_text.replace("{{invoice_end_date}}", invoice[13]) + invoice_text = invoice_text.replace("{{supplier_name}}", invoice[14]) + invoice_text = invoice_text.replace("{{supplier_abn}}", invoice[15]) + invoice_text = invoice_text.replace("{{order_ref}}", 'CF%06X' % random.randint(0, 256**3-1)) + + invoice_text = invoice_text.replace("{{supplier_road}}", supplier_road) + invoice_text = invoice_text.replace("{{supplier_suburb}}", supplier_suburb if supplier_suburb else "") + invoice_text = invoice_text.replace("{{supplier_city}}", supplier_city if supplier_city else "") + invoice_text = invoice_text.replace("{{supplier_state}}", supplier_state if supplier_state else "") + invoice_text = invoice_text.replace("{{supplier_postcode}}", supplier_postcode) + invoice_text = invoice_text.replace("{{supplier_country}}", "Australia") + + invoice_text = invoice_text.replace("{{customer_name}}", invoice[18]) + invoice_text = invoice_text.replace("{{customer_abn}}", str(invoice[19])) + invoice_text = invoice_text.replace("{{delivery_date}}", invoice[20]) + + invoice_text = invoice_text.replace("{{delivery_road}}", delivery_road) + invoice_text = invoice_text.replace("{{delivery_suburb}}", delivery_suburb) + invoice_text = invoice_text.replace("{{delivery_city}}", delivery_city) + invoice_text = invoice_text.replace("{{delivery_state}}", delivery_state) + invoice_text = invoice_text.replace("{{delivery_postcode}}", delivery_postcode) + invoice_text = invoice_text.replace("{{delivery_country}}", "Australia") + + invoice_text = invoice_text.replace("{{customer_contact_name}}", invoice[23]) + invoice_text = invoice_text.replace("{{customer_contact_email}}", invoice[24]) + invoice_text = invoice_text.replace("{{customer_contact_phone}}", invoice[25]) + + + line_item = random.choice(lineitems) + + # description,quantity,unit_price,total_price + invoice_text = invoice_text.replace("{{description}}", line_item[1]) + + + pre_tax_total = line_item[4] + tax_amount = round(pre_tax_total * 0.1, 2) + total_amount = round(pre_tax_total + tax_amount, 2) + + invoice_text = invoice_text.replace("{{pre_tax_total}}", str(pre_tax_total)) + invoice_text = invoice_text.replace("{{tax_amount}}", str(tax_amount)) + invoice_text = invoice_text.replace("{{total_amount}}", str(total_amount)) + + f.write(invoice_text) + # break + +# save the fake data to a csv +# with open('invoices.csv', 'w') as f: +# f.write("id,name,owner_id,date_last_modified,date_added,num_warnings,num_errors,is_valid,text_content,invoice_title,issue_date,due_date,order_id,invoice_start_date,invoice_end_date,supplier_name,supplier_abn,supplier_latitude,supplier_longitude,customer_name,customer_abn,delivery_date,delivery_latitude,delivery_longitude,customer_contact_name,customer_contact_email,customer_contact_phone,total_amount") +# for invoice in invoices: +# f.write(f"\n{','.join(str(x) for x in invoice)}") + +# with open('lineitems.csv', 'w') as f: +# f.write("id,invoice_id,description,quantity,unit_price,total_price") +# for lineitem in lineitems: +# f.write(f"\n{','.join(str(x) for x in lineitem)}") + +# Clear Invoices and LineItems tables +# db.drop_tables([Invoices, LineItems]) +# db.create_tables([Invoices, LineItems]) + +# # Add the fake data to the database +# Invoices.insert_many(invoices).execute() +# LineItems.insert_many(lineitems).execute() + diff --git a/requirements.txt b/requirements.txt index 8d4f7ba..aef49b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ lxml==4.9.2 saxonche==12.0.0 fastapi==0.92.0 uvicorn==0.20.0 -weasyprint==58.1 +weasyprint==52.5 beautifulsoup4==4.11.2 peewee==3.16.0 python-multipart==0.0.6 diff --git a/src/authentication.py b/src/authentication.py index 64930fb..ec9d8e6 100644 --- a/src/authentication.py +++ b/src/authentication.py @@ -86,7 +86,8 @@ def auth_login_v2(email, password) -> AuthReturnV2: id = auth_login_v1(email, password).auth_user_id now = datetime.now() token = hashlib.sha256(id.to_bytes(8, 'big') + now.strftime("%s").encode("utf-8")).hexdigest() - Sessions.create(user=id, token=token, date_created=now, date_expires=now + timedelta(days=1)) + + Sessions.create(user=id, token=token, date_created=now, date_expires=now + timedelta(days=10)) return AuthReturnV2(token=token, id=id) @@ -94,5 +95,5 @@ def auth_register_v2(name, email, password) -> AuthReturnV2: id = auth_register_v1(name, email, password).auth_user_id now = datetime.now() token = hashlib.sha256(id.to_bytes(8, 'big') + now.strftime("%s").encode("utf-8")).hexdigest() - Sessions.create(user=id, token=token, date_created=now, date_expires=now + timedelta(days=1)) + Sessions.create(user=id, token=token, date_created=now, date_expires=now + timedelta(days=10)) return AuthReturnV2(token=token, id=id) \ No newline at end of file diff --git a/src/database.py b/src/database.py index 0a7e7e7..93e6bc9 100644 --- a/src/database.py +++ b/src/database.py @@ -130,6 +130,7 @@ class Invoices(BaseModel): delivery_date = DateField(null=True,default=None) delivery_latitude = FloatField(null=True,default=None) delivery_longitude = FloatField(null=True,default=None) + delivery_suburb = TextField(null=True,default=None) customer_contact_name = TextField(null=True,default=None) customer_contact_email = TextField(null=True,default=None) @@ -203,13 +204,9 @@ def to_json(self): # Create the tables in the database def create_tables(): with db: - if db.table_exists('users'): - # add name column to users table if it doesn't exist - db.execute_sql('ALTER TABLE users ADD COLUMN IF NOT EXISTS name TEXT;') - - if db.table_exists('reports'): - # add owner column to reports table if it doesn't exist - db.execute_sql('ALTER TABLE reports ADD COLUMN IF NOT EXISTS owner_id INTEGER REFERENCES users(id);') + if db.table_exists('invoices'): + # add delivery_suburb column to invoices table if it doesn't exist + db.execute_sql('ALTER TABLE invoices ADD COLUMN IF NOT EXISTS delivery_suburb TEXT NULL DEFAULT NULL;') db.create_tables(tables) diff --git a/src/invoice_processing.py b/src/invoice_processing.py index 7777649..4af5d2d 100644 --- a/src/invoice_processing.py +++ b/src/invoice_processing.py @@ -5,6 +5,11 @@ import requests from src.generation import generate_diagnostic_list from peewee import DoesNotExist, fn +from collections import defaultdict +import calendar + + +TOLERANCE = 0.001 def get_invoice_field(invoice_data: dict, field: str) -> str: @@ -42,8 +47,8 @@ def process_and_update_invoice(invoice_text: str, invoice: Invoices): raise InputError(detail="Could not parse invoice, please check your invoice") else: invoice_data = response.json() - supplier_latitude, supplier_longitude = get_lat_long_from_address(invoice_data["AccountingSupplierParty"]["Party"]["PostalAddress"]) - delivery_latitude, delivery_longitude = get_lat_long_from_address(invoice_data["Delivery"]["DeliveryLocation"]["Address"]) + supplier_latitude, supplier_longitude, _ = get_lat_long_from_address(invoice_data["AccountingSupplierParty"]["Party"]["PostalAddress"]) + delivery_latitude, delivery_longitude, delivery_suburb = get_lat_long_from_address(invoice_data["Delivery"]["DeliveryLocation"]["Address"]) invoice.date_last_modified = datetime.now() invoice.num_warnings = num_warnings @@ -69,6 +74,7 @@ def process_and_update_invoice(invoice_text: str, invoice: Invoices): invoice.delivery_date = invoice_data["Delivery"]["ActualDeliveryDate"] invoice.delivery_latitude = delivery_latitude invoice.delivery_longitude = delivery_longitude + invoice.delivery_suburb = delivery_suburb invoice.customer_contact_name = invoice_data["AccountingCustomerParty"]["Party"]["Contact"]["Name"] invoice.customer_contact_email = invoice_data["AccountingCustomerParty"]["Party"]["Contact"]["ElectronicMail"] @@ -132,8 +138,13 @@ def store_and_process_invoice(invoice_name: str, invoice_text: str, owner: int) raise InputError(detail="Could not parse invoice, please check your invoice") else: invoice_data = response.json() - supplier_latitude, supplier_longitude = get_lat_long_from_address(invoice_data["AccountingSupplierParty"]["Party"]["PostalAddress"]) - delivery_latitude, delivery_longitude = get_lat_long_from_address(invoice_data["Delivery"]["DeliveryLocation"]["Address"]) + supplier_latitude, supplier_longitude, _ = get_lat_long_from_address(invoice_data["AccountingSupplierParty"]["Party"]["PostalAddress"]) + delivery_latitude, delivery_longitude, delivery_suburb = get_lat_long_from_address(invoice_data["Delivery"]["DeliveryLocation"]["Address"]) + + print(invoice_data["AccountingSupplierParty"]["Party"]["PostalAddress"]) + print("supplier location", supplier_latitude, supplier_longitude) + print(invoice_data["Delivery"]["DeliveryLocation"]["Address"]) + print("delivery location", delivery_latitude, delivery_longitude) invoice = Invoices.create( name=invoice_name, @@ -165,6 +176,7 @@ def store_and_process_invoice(invoice_name: str, invoice_text: str, owner: int) delivery_date=invoice_data["Delivery"]["ActualDeliveryDate"], delivery_latitude=delivery_latitude, delivery_longitude=delivery_longitude, + delivery_suburb=delivery_suburb, customer_contact_name=invoice_data["AccountingCustomerParty"]["Party"]["Contact"]["Name"], customer_contact_email=invoice_data["AccountingCustomerParty"]["Party"]["Contact"]["ElectronicMail"], @@ -187,6 +199,7 @@ def store_and_process_invoice(invoice_name: str, invoice_text: str, owner: int) def get_lat_long_from_address(data: str) -> tuple: query = [] + suburb = None for field in ["StreetName", "AdditionalStreetName", "CityName", "PostalZone", "CountrySubentity", "AddressLine", "Country"]: if field in data: @@ -194,6 +207,12 @@ def get_lat_long_from_address(data: str) -> tuple: query.append(str(list(data[field].values())[0])) else: query.append(str(data[field])) + + if field == "AdditionalStreetName": + suburb = str(data[field]) + + if not suburb: + raise InputError(detail="No suburb found") response = requests.get(f"https://geocode.maps.co/search", params={ "q": " ".join(query), @@ -202,7 +221,7 @@ def get_lat_long_from_address(data: str) -> tuple: if response.status_code == 200: data = response.json() if data: - return data[0]["lat"], data[0]["lon"] + return data[0]["lat"], data[0]["lon"], suburb raise InputError(detail="Could not find location of party. Address: " + " ".join(query)) def invoice_processing_upload_text_v2(invoice_name: str, invoice_text: str, owner: int) -> InvoiceID: @@ -288,7 +307,7 @@ def coord_distance(lat1, lon1, lat2, lon2): return R * c -def invoice_processing_query_v2(query: str, from_date: str, to_date: str, owner: int): +def invoice_processing_query_v2(query: str, from_date: str, to_date: str, owner: int, warehouse_lat: str = None, warehouse_long = None): if query == "numActiveCustomers": from_date = datetime.strptime(from_date, "%Y-%m-%d") to_date = datetime.strptime(to_date, "%Y-%m-%d") @@ -488,6 +507,123 @@ def invoice_processing_query_v2(query: str, from_date: str, to_date: str, owner: "data": client_data } + elif query == "suburbDataTable": + from_date = datetime.strptime(from_date, "%Y-%m-%d") + to_date = datetime.strptime(to_date, "%Y-%m-%d") + + # Get suburb name, total deliveries, total revenue and average delivery time for each suburb + + if warehouse_lat and warehouse_long: + warehouse_lat = float(warehouse_lat) + warehouse_long = float(warehouse_long) + suburb_query = (Invoices + .select(Invoices.delivery_suburb, + fn.COUNT('*').alias('total_deliveries'), + fn.SUM(Invoices.total_amount).alias('total_revenue'), + fn.AVG(Invoices.delivery_date - Invoices.invoice_start_date).alias('avg_delivery_time')) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) & + (Invoices.invoice_start_date >= from_date) & + (Invoices.invoice_start_date <= to_date)) + .group_by(Invoices.delivery_suburb)) + + result = {"data": []} + + for i, suburb in enumerate(suburb_query): + suburb_data = { + "id": i, + "name": suburb.delivery_suburb if suburb.delivery_suburb else "Not Specified", + "total-deliveries": suburb.total_deliveries, + "total-revenue": suburb.total_revenue, + "avg-delivery-time": suburb.avg_delivery_time + } + result["data"].append(suburb_data) + else: + suburb_query = (Invoices + .select(Invoices.delivery_suburb, + fn.COUNT('*').alias('total_deliveries'), + fn.SUM(Invoices.total_amount).alias('total_revenue'), + fn.AVG(Invoices.delivery_date - Invoices.invoice_start_date).alias('avg_delivery_time')) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + (Invoices.invoice_start_date >= from_date) & + (Invoices.invoice_start_date <= to_date)) + .group_by(Invoices.delivery_suburb)) + + result = {"data": []} + + for i, suburb in enumerate(suburb_query): + suburb_data = { + "id": i, + "name": suburb.delivery_suburb if suburb.delivery_suburb else "Not Specified", + "total-deliveries": suburb.total_deliveries, + "total-revenue": suburb.total_revenue, + "avg-delivery-time": suburb.avg_delivery_time + } + result["data"].append(suburb_data) + + return result + + elif query == "warehouseProductDataTable": + # Convert from_date and to_date to datetime objects + from_date = datetime.strptime(from_date, "%Y-%m-%d") + to_date = datetime.strptime(to_date, "%Y-%m-%d") + + if warehouse_lat and warehouse_long: + warehouse_lat = float(warehouse_lat) + warehouse_long = float(warehouse_long) + + # Query the database and calculate the desired values + results = [] + line_items = (LineItems.select(LineItems.description, + fn.SUM(LineItems.quantity).alias('total_units'), + fn.SUM(LineItems.total_price).alias('total_value')) + .join(Invoices) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) & + (Invoices.invoice_start_date >= from_date) & + (Invoices.invoice_start_date <= to_date)) + .group_by(LineItems.description)) + + for i, item in enumerate(line_items): + # Append the result to the list + result = { + "id": i, + "name": item.description, + "total-units": item.total_units, + "total-value": item.total_value, + } + results.append(result) + else: + # Query the database and calculate the desired values + results = [] + line_items = (LineItems.select(LineItems.description, + fn.SUM(LineItems.quantity).alias('total_units'), + fn.SUM(LineItems.total_price).alias('total_value')) + .join(Invoices) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + (Invoices.invoice_start_date >= from_date) & + (Invoices.invoice_start_date <= to_date)) + .group_by(LineItems.description)) + + for i, item in enumerate(line_items): + # Append the result to the list + result = { + "id": i, + "name": item.description, + "total-units": item.total_units, + "total-value": item.total_value, + } + results.append(result) + + # Create the final output dictionary + return {"data": results} + elif query == "heatmapCoords": from_date = datetime.strptime(from_date, "%Y-%m-%d") to_date = datetime.strptime(to_date, "%Y-%m-%d") @@ -521,11 +657,341 @@ def invoice_processing_query_v2(query: str, from_date: str, to_date: str, owner: warehouse_coords.append({ "lat": invoice.supplier_latitude, "lon": invoice.supplier_longitude, - "value": invoice.total_amount + "value": invoice.total_amount, + "name": invoice.supplier_name, }) return { "data": warehouse_coords } + elif query == "deliveriesMadeMonthly": + from_date = datetime.strptime(from_date, "%Y-%m-%d") + to_date = datetime.strptime(to_date, "%Y-%m-%d") + + if warehouse_lat and warehouse_long: + warehouse_lat = float(warehouse_lat) + warehouse_long = float(warehouse_long) + query = (LineItems + .select(fn.TO_CHAR(Invoices.delivery_date, 'Mon').alias('month'), + fn.COUNT('*').alias('count')) + .join(Invoices) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) & + Invoices.delivery_date.between(from_date, to_date) + ) + .group_by(fn.TO_CHAR(Invoices.delivery_date, 'Mon')) + .order_by(fn.MIN(Invoices.delivery_date))) + else: + query = (LineItems + .select(fn.TO_CHAR(Invoices.delivery_date, 'Mon').alias('month'), + fn.COUNT('*').alias('count')) + .join(Invoices) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + Invoices.delivery_date.between(from_date, to_date) + ) + .group_by(fn.TO_CHAR(Invoices.delivery_date, 'Mon')) + .order_by(fn.MIN(Invoices.delivery_date))) + + result = query.dicts() + labels = [item['month'] for item in result] + data = [item['count'] for item in result] + + return { + "labels": labels, + "data": data + } + + elif query == "warehouseMonthlyAvgDeliveryTime": + from_date = datetime.strptime(from_date, "%Y-%m-%d") + to_date = datetime.strptime(to_date, "%Y-%m-%d") + + if warehouse_lat and warehouse_long: + warehouse_lat = float(warehouse_lat) + warehouse_long = float(warehouse_long) + query = (Invoices + .select(fn.TO_CHAR(Invoices.delivery_date, 'Mon').alias('month'), + fn.AVG(Invoices.delivery_date - Invoices.invoice_start_date).alias('average_delivery_time')) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) & + Invoices.delivery_date.between(from_date, to_date)) + .group_by(fn.TO_CHAR(Invoices.delivery_date, 'Mon')) + .order_by(fn.MIN(Invoices.delivery_date))) + else: + query = (Invoices + .select(fn.TO_CHAR(Invoices.delivery_date, 'Mon').alias('month'), + fn.AVG(Invoices.delivery_date - Invoices.invoice_start_date).alias('average_delivery_time')) + .where((Invoices.is_valid == True) & + (Invoices.owner == owner) & + Invoices.delivery_date.between(from_date, to_date)) + .group_by(fn.TO_CHAR(Invoices.delivery_date, 'Mon')) + .order_by(fn.MIN(Invoices.delivery_date))) + + result = query.dicts() + labels = [item['month'] for item in result] + data = [item['average_delivery_time'] for item in result] + + return { + "labels": labels, + "data": data + } + + elif query == "warehouseMonthlyAvgDeliveryDistance": + from_date = datetime.strptime(from_date, "%Y-%m-%d") + to_date = datetime.strptime(to_date, "%Y-%m-%d") + + if warehouse_lat and warehouse_long: + warehouse_lat = float(warehouse_lat) + warehouse_long = float(warehouse_long) + invoices = ( + Invoices.select( + Invoices.delivery_date, + Invoices.supplier_latitude, + Invoices.supplier_longitude, + Invoices.delivery_latitude, + Invoices.delivery_longitude, + ) + .where( + (Invoices.is_valid == True) & + (Invoices.owner == owner) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) & + (Invoices.delivery_date >= from_date) & + (Invoices.delivery_date <= to_date) + ) + .order_by(Invoices.delivery_date) + ) + else: + invoices = ( + Invoices.select( + Invoices.delivery_date, + Invoices.supplier_latitude, + Invoices.supplier_longitude, + Invoices.delivery_latitude, + Invoices.delivery_longitude, + ) + .where( + (Invoices.is_valid == True) & + (Invoices.owner == owner) & + (Invoices.delivery_date >= from_date) + & (Invoices.delivery_date <= to_date) + ) + .order_by(Invoices.delivery_date) + ) + + monthly_distances = defaultdict(list) + for invoice in invoices: + month = invoice.delivery_date.strftime("%b") + distance = coord_distance( + invoice.supplier_latitude, + invoice.supplier_longitude, + invoice.delivery_latitude, + invoice.delivery_longitude, + ) + monthly_distances[month].append(distance) + + avg_monthly_distances = { + month: sum(distances) / len(distances) + for month, distances in monthly_distances.items() + } + + result = { + "labels": list(avg_monthly_distances.keys()), + "data": list(avg_monthly_distances.values()), + } + return result + + elif query == "numUniqueCustomers": + from_date = datetime.strptime(from_date, "%Y-%m-%d") + to_date = datetime.strptime(to_date, "%Y-%m-%d") + + if warehouse_lat and warehouse_long: + warehouse_lat = float(warehouse_lat) + warehouse_long = float(warehouse_long) + + # Query the database for active customers within the date range + active_customers = Invoices.select().where( + (Invoices.is_valid == True) & + (Invoices.invoice_end_date >= from_date) & + (Invoices.invoice_end_date <= to_date) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) & + (Invoices.owner == owner) + ).distinct(Invoices.customer_name) + + # Count the number of active customers + num_active_customers = active_customers.count() + + # Define the date range to query for the previous 12 months + prev_year_to_date = to_date - timedelta(days=365) + prev_year_from_date = prev_year_to_date - timedelta(days=90) + + # Query the database for active customers within the previous 12 months + prev_year_active_customers = Invoices.select().where( + (Invoices.is_valid == True) & + (Invoices.invoice_end_date >= prev_year_from_date) & + (Invoices.invoice_end_date <= prev_year_to_date) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) & + (Invoices.owner == owner) + ).distinct(Invoices.customer_name) + + # Count the number of active customers in the previous 12 months + num_prev_year_active_customers = prev_year_active_customers.count() + + # Calculate the percentage change in active customers from the previous 12 months + if num_prev_year_active_customers == 0: + percentage_change = 0 + else: + percentage_change = ((num_active_customers - num_prev_year_active_customers) / num_prev_year_active_customers) * 100 + + return { + "value": num_active_customers, + "change": percentage_change, + } + else: + # Query the database for active customers within the date range + active_customers = Invoices.select().where( + (Invoices.is_valid == True) & + (Invoices.invoice_end_date >= from_date) & + (Invoices.invoice_end_date <= to_date) & + (Invoices.owner == owner) + ).distinct(Invoices.customer_name) + + # Count the number of active customers + num_active_customers = active_customers.count() + + # Define the date range to query for the previous 12 months + prev_year_to_date = to_date - timedelta(days=365) + prev_year_from_date = prev_year_to_date - timedelta(days=90) + + # Query the database for active customers within the previous 12 months + prev_year_active_customers = Invoices.select().where( + (Invoices.is_valid == True) & + (Invoices.invoice_end_date >= prev_year_from_date) & + (Invoices.invoice_end_date <= prev_year_to_date) & + (Invoices.owner == owner) + ).distinct(Invoices.customer_name) + + # Count the number of active customers in the previous 12 months + num_prev_year_active_customers = prev_year_active_customers.count() + + # Calculate the percentage change in active customers from the previous 12 months + if num_prev_year_active_customers == 0: + percentage_change = 0 + else: + percentage_change = ((num_active_customers - num_prev_year_active_customers) / num_prev_year_active_customers) * 100 + + return { + "value": num_active_customers, + "change": percentage_change, + } + elif query == "totalRevenue": + from_date = datetime.strptime(from_date, "%Y-%m-%d") + to_date = datetime.strptime(to_date, "%Y-%m-%d") + + if warehouse_lat and warehouse_long: + warehouse_lat = float(warehouse_lat) + warehouse_long = float(warehouse_long) + + invoices = ( + Invoices.select( + fn.SUM(LineItems.total_price).alias('total_revenue') + ) + .join(LineItems, on=(Invoices.id == LineItems.invoice)) + .where( + (Invoices.delivery_date >= from_date) & + (Invoices.delivery_date <= to_date) & + (Invoices.is_valid == True) & + (Invoices.owner == owner) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) + ) + ) + + total_revenue = invoices[0].total_revenue if invoices else 0 + + # Define the date range to query for the previous 12 months + prev_year_to_date = to_date - timedelta(days=365) + prev_year_from_date = prev_year_to_date - timedelta(days=90) + + prev_year_invoices = ( + Invoices.select( + fn.SUM(LineItems.total_price).alias('total_revenue') + ) + .join(LineItems, on=(Invoices.id == LineItems.invoice)) + .where( + (Invoices.delivery_date >= prev_year_from_date) & + (Invoices.delivery_date <= prev_year_to_date) & + (Invoices.is_valid == True) & + (Invoices.owner == owner) & + (fn.ABS(Invoices.supplier_latitude - warehouse_lat) <= TOLERANCE) & + (fn.ABS(Invoices.supplier_longitude - warehouse_long) <= TOLERANCE) + ) + ) + + prev_year_total_revenue = prev_year_invoices[0].total_revenue if prev_year_invoices else 0 + + # Calculate the percentage change in total revenue from the previous 12 months + if prev_year_total_revenue == 0: + percentage_change = 0 + else: + percentage_change = ((total_revenue - prev_year_total_revenue) / prev_year_total_revenue) * 100 + + return { + "value": total_revenue, + "change": percentage_change, + } + + else: + invoices = ( + Invoices.select( + fn.SUM(LineItems.total_price).alias('total_revenue') + ) + .join(LineItems, on=(Invoices.id == LineItems.invoice)) + .where( + (Invoices.is_valid == True) & + (Invoices.owner == owner) & + (Invoices.delivery_date >= from_date) & + (Invoices.delivery_date <= to_date) + ) + ) + + total_revenue = invoices[0].total_revenue if invoices else 0 + + # Define the date range to query for the previous 12 months + prev_year_to_date = to_date - timedelta(days=365) + prev_year_from_date = prev_year_to_date - timedelta(days=90) + + prev_year_invoices = ( + Invoices.select( + fn.SUM(LineItems.total_price).alias('total_revenue') + ) + .join(LineItems, on=(Invoices.id == LineItems.invoice)) + .where( + (Invoices.is_valid == True) & + (Invoices.owner == owner) & + (Invoices.delivery_date >= prev_year_from_date) & + (Invoices.delivery_date <= prev_year_to_date) + ) + ) + + prev_year_total_revenue = prev_year_invoices[0].total_revenue if prev_year_invoices else 0 + + # Calculate the percentage change in total revenue from the previous 12 months + if prev_year_total_revenue == 0: + percentage_change = 0 + else: + percentage_change = ((total_revenue - prev_year_total_revenue) / prev_year_total_revenue) * 100 + + return { + "value": total_revenue, + "change": percentage_change, + } + return {} diff --git a/src/main.py b/src/main.py index c34ca1b..64a074a 100644 --- a/src/main.py +++ b/src/main.py @@ -392,8 +392,8 @@ async def api_invoice_processing_delete_v2(invoice_id: int, token = Depends(get_ return invoice_processing_delete_v2(invoice_id=invoice_id, owner=Sessions.get(token=token).user) @app.get("/invoice_processing/query/v2", tags=["v2 invoice_processing"]) -async def api_invoice_processing_query_v2(query: str, from_date: str, to_date: str, token = Depends(get_token)): - return invoice_processing_query_v2(query=query, from_date=from_date, to_date=to_date, owner=Sessions.get(token=token).user) +async def api_invoice_processing_query_v2(query: str, from_date: str, to_date: str, warehouse_lat: str = None, warehouse_long = None, token = Depends(get_token)): + return invoice_processing_query_v2(query=query, from_date=from_date, to_date=to_date, warehouse_lat=warehouse_lat, warehouse_long=warehouse_long, owner=Sessions.get(token=token).user) @app.get("/virtual_warehouse_coords") async def get_virtual_warehouse_data(n_clusters: int, from_date: str, to_date: str, token = Depends(get_token)): diff --git a/src/validation.py b/src/validation.py index 89273c2..81aefbc 100644 --- a/src/validation.py +++ b/src/validation.py @@ -4,6 +4,7 @@ from src.helpers import create_temp_file from os import unlink from src.database import Violations +from src.error import InputError def get_wellformedness_violations(invoice_text: str) -> List[Violation]: @@ -51,7 +52,7 @@ def get_xslt_violations(executable, invoice_text: str): unlink(tmp_filename) if not schematron_output: - raise Exception("Could not generate evaluation due to invalid XML!") + raise InputError("Could not generate evaluation due to invalid XML!") violations = [] diff --git a/tests/report/peppol_test.py b/tests/report/peppol_test.py index 37a9c36..543ba96 100644 --- a/tests/report/peppol_test.py +++ b/tests/report/peppol_test.py @@ -52,9 +52,6 @@ def test_peppol_single_violation(): abn_violation = peppol_evaluation.violations[0] - # From 'A-NZ_Invoice_Extension_v1.0.8.docx' file: - # PEPPOL-COMMON-R050 | Australian Business Number (ABN) MUST be stated in the correct format. | Same | warning - # Check that the violation is for the correct rule and is flagged as fatal assert abn_violation.rule_id == "PEPPOL-COMMON-R050" assert abn_violation.is_fatal == False