Skip to content

Commit

Permalink
Testing and implementation of report database query endpoints (#17)
Browse files Browse the repository at this point in the history
* Moved validation docs outside functions to be parsed only once

* Adding tests for change name, delete, list all and list by

* Adding server calls for new report endpoints

* Fixing report tests + implementation of list_all endpoint

* Fixing list by test to better test functionality

* Implementation of list_by endpoint

* Fixes to change_name tests

* Implementation of change name endpoint

* Implementation of report/delete endpoint

* Adding tests for check validity endpoint

* Implementation of check validity endpoint

* Small changes to bulk upload tests

* Implementation of bulk upload passing tests

* Updating tests to work with server

* Initial implementation of bulk_export endpoint

* Fixing file upload endpoint

* Fixing bulk export implementation

---------

Co-authored-by: Mohamad Mohamad <[email protected]>
  • Loading branch information
jeremytraini and Mr-Squared authored Mar 12, 2023
1 parent 785f6ba commit 1a04d61
Show file tree
Hide file tree
Showing 22 changed files with 401 additions and 203 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# se2021-23t1-einvoicing-api-template
# SENG2021 23T1 CHURROS E-Invoicing Validation API


In order to run the server, we must
Expand Down
4 changes: 0 additions & 4 deletions src/classes/Sample.py

This file was deleted.

9 changes: 9 additions & 0 deletions src/constants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from lxml import etree
from saxonche import PySaxonProcessor

# Parse the XSD file
XSD_SCHEMA = etree.XMLSchema(etree.parse("src/xsd/maindoc/UBL-Invoice-2.1.xsd", parser=None))
proc = PySaxonProcessor(license=False)
xsltproc = proc.new_xslt30_processor()
SYNTAX_EXECUTABLE = xsltproc.compile_stylesheet(stylesheet_file="src/validation_artefacts/AUNZ-UBL-validation.xslt")
PEPPOL_EXECUTABLE = xsltproc.compile_stylesheet(stylesheet_file="src/validation_artefacts/AUNZ-PEPPOL-validation.xslt")
4 changes: 2 additions & 2 deletions src/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class Evaluations(BaseModel):
num_rules_failed = IntegerField()

def to_json(self):
violations = Violations.select().where(Violations.evaluation == self.id)
violations = Violations.select().where(Violations.evaluation == self.id) # type: ignore

return {
"is_valid": self.is_valid,
Expand All @@ -49,7 +49,7 @@ class Reports(BaseModel):

def to_json(self):
return {
"report_id": self.id,
"report_id": self.id, # type: ignore
"date_generated": self.date_generated,
"invoice_name": self.invoice_name,
"invoice_text": self.invoice_text,
Expand Down
9 changes: 9 additions & 0 deletions src/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,3 +162,12 @@ def export_csv_report_v1(report_id: int):

return csv_contents

def report_bulk_export_v1(report_ids, report_format) -> List:
report_format = report_format.lower()
print("Exporting reports")
if report_format == "json":
return [export_json_report_v1(report_id) for report_id in report_ids]
elif report_format == "html":
return [export_html_report_v1(report_id) for report_id in report_ids]
else:
raise Exception("Unknown report format")
148 changes: 67 additions & 81 deletions src/generation.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from src.type_structure import *
from src.constants import XSD_SCHEMA, SYNTAX_EXECUTABLE, PEPPOL_EXECUTABLE
from lxml import etree
from typing import Dict
from saxonche import PySaxonProcessor
Expand All @@ -21,7 +22,7 @@ def generate_wellformedness_evaluation(invoice_text: str) -> Evaluations:
violations = []

try:
etree.fromstring(invoice_text.encode("utf-8"))
etree.fromstring(invoice_text.encode("utf-8"), parser=None)
except etree.XMLSyntaxError as error:
evaluation.is_valid = False
evaluation.num_errors = 1
Expand Down Expand Up @@ -51,21 +52,17 @@ def generate_schema_evaluation(invoice_text: str) -> Evaluations:
num_errors=0,
num_rules_failed=0
)

# Parse the XSD file
xsd_doc = etree.parse("src/xsd/maindoc/UBL-Invoice-2.1.xsd")
xsd = etree.XMLSchema(xsd_doc)

# Parse the XML data
xml_doc = etree.fromstring(invoice_text.encode("utf-8"))
xml_doc = etree.fromstring(invoice_text.encode("utf-8"), parser=None)

violations = []

# Validate the XML against the XSD schema
if not xsd.validate(xml_doc):
if not XSD_SCHEMA.validate(xml_doc):
evaluation.is_valid = False

for error in xsd.error_log:
for error in XSD_SCHEMA.error_log:
evaluation.num_errors += 1
evaluation.num_rules_failed += 1

Expand Down Expand Up @@ -93,77 +90,69 @@ def generate_peppol_evaluation(invoice_text: str) -> Evaluations:
return generate_xslt_evaluation("peppol", invoice_text)

def generate_xslt_evaluation(aspect, invoice_text) -> Evaluations:
with PySaxonProcessor(license=False) as proc:
xsltproc = proc.new_xslt30_processor()

if aspect == "syntax":
xslt_path = "src/validation_artefacts/AUNZ-UBL-validation.xslt"
else:
xslt_path = "src/validation_artefacts/AUNZ-PEPPOL-validation.xslt"

executable = xsltproc.compile_stylesheet(stylesheet_file=xslt_path)

if xsltproc.exception_occurred:
raise Exception("XSLT failed to load! " + xsltproc.error_message)

tmp_filename = create_temp_file(invoice_text)
schematron_output = executable.transform_to_value(source_file=tmp_filename)
unlink(tmp_filename)

if not schematron_output:
raise Exception("Could not generate evaluation due to bad XML!")

violations = []

num_warnings = 0
num_errors = 0
rules_failed = set()

output = schematron_output.item_at(0).get_node_value().children[0].children

for item in output:
if item.name and item.name.endswith("failed-assert"):
id_name = item.get_attribute_value("id")
rules_failed.add(id_name)
is_fatal = item.get_attribute_value("flag") == "fatal"

if is_fatal:
num_errors += 1
else:
num_warnings += 1

xpath = item.get_attribute_value("location")
test = item.get_attribute_value("test")

message = ""
suggestion = ""
if item.children:
message = item.children[0].string_value

if len(item.children) > 1:
suggestion = item.children[1].string_value
if (aspect == "syntax"):
executable = SYNTAX_EXECUTABLE
else:
executable = PEPPOL_EXECUTABLE

tmp_filename = create_temp_file(invoice_text)
schematron_output = executable.transform_to_value(source_file=tmp_filename)
unlink(tmp_filename)

if not schematron_output:
raise Exception("Could not generate evaluation due to bad XML!")

violations = []

num_warnings = 0
num_errors = 0
rules_failed = set()

output = schematron_output.item_at(0).get_node_value().children[0].children

for item in output:
if item.name and item.name.endswith("failed-assert"):
id_name = item.get_attribute_value("id")
rules_failed.add(id_name)
is_fatal = item.get_attribute_value("flag") == "fatal"

if is_fatal:
num_errors += 1
else:
num_warnings += 1

xpath = item.get_attribute_value("location")
test = item.get_attribute_value("test")

message = ""
suggestion = ""
if item.children:
message = item.children[0].string_value

violations.append(Violations(
rule_id=id_name,
is_fatal=is_fatal,
xpath=xpath,
test=test,
message=message,
suggestion=suggestion
))

evaluation = Evaluations.create(
is_valid=num_errors == 0,
num_warnings=num_warnings,
num_errors=num_errors,
num_rules_failed=len(rules_failed)
)

for violation in violations:
violation.evaluation = evaluation.id
violation.save()

return evaluation
if len(item.children) > 1:
suggestion = item.children[1].string_value

violations.append(Violations(
rule_id=id_name,
is_fatal=is_fatal,
xpath=xpath,
test=test,
message=message,
suggestion=suggestion
))

evaluation = Evaluations.create(
is_valid=num_errors == 0,
num_warnings=num_warnings,
num_errors=num_errors,
num_rules_failed=len(rules_failed)
)

for violation in violations:
violation.evaluation = evaluation.id
violation.save()

return evaluation

def generate_report(invoice_name: str, invoice_text: str) -> int:
wellformedness_evaluation = None
Expand Down Expand Up @@ -206,7 +195,4 @@ def generate_report(invoice_name: str, invoice_text: str) -> int:
peppol=peppol_evaluation.id if peppol_evaluation else None
)

print(report)
print(report.id)

return report.id
1 change: 1 addition & 0 deletions src/helpers.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from tempfile import NamedTemporaryFile
from src.type_structure import *
import requests

def create_temp_file(invoice_text: str) -> str:
tmp = NamedTemporaryFile(mode='w', delete=False)
Expand Down
19 changes: 8 additions & 11 deletions src/invoice.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from typing import Dict
from src.type_structure import *
from src.report import report_get_v1
from src.database import Users, Reports, Violations, Evaluations, db
import requests
from src.generation import generate_report
Expand Down Expand Up @@ -29,24 +28,22 @@ def invoice_upload_url_v1(invoice_name: str, invoice_url: str):


def invoice_upload_file_v1(invoice_name: str, invoice_file):
with open(invoice_file, 'rb') as f:
invoice_text = f.read()

report_id = generate_report(invoice_name, invoice_text)
report_id = generate_report(invoice_name, invoice_file.decode("utf-8"))

return {
"report_id": report_id
}

def invoice_check_validity_v1(report_id: int) -> CheckValidReturn:
report = Reports.query.filter_by(id=report_id).one()
try:
report = Reports.get_by_id(report_id)
except DoesNotExist:
raise Exception(f"Report with id {report_id} not found")

return CheckValidReturn(is_valid=report.is_valid, invoice_hash=report.invoice_hash)
return CheckValidReturn(is_valid=report.is_valid)

def invoice_generate_hash_v1(invoice: Invoice) -> str:
return "hash"

def invoice_bulk_quick_fix_v1(invoices: List[Invoice]) -> List[Invoice]:
invoice = Invoice(name="invoice", source="text", data="")
invoice_list = [invoice]
return invoice_list
def invoice_file_upload_bulk_v1(invoices: List[Invoice]) -> List[int]:
return [generate_report(invoice.name, invoice.data) for invoice in invoices]
Loading

0 comments on commit 1a04d61

Please sign in to comment.