Skip to content

Commit

Permalink
add validate invoice 1 and 3
Browse files Browse the repository at this point in the history
  • Loading branch information
@mrzaizai2k
mrzaizai2k committed Sep 14, 2024
1 parent 69f6200 commit a42565e
Show file tree
Hide file tree
Showing 9 changed files with 601 additions and 171 deletions.
1 change: 0 additions & 1 deletion config/invoice_template.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
"document_subject": "",
"package_number": "",
"invoice_number": "",
"invoice_type": "",
"receipt_number": "",
"shop_number": "",
"transaction_number": "",
Expand Down
133 changes: 30 additions & 103 deletions config/page_1_template.txt
View file
Original file line number Diff line number Diff line change
@@ -1,107 +1,34 @@
```json
{
"invoice_info": {
"name": "Tümmler, Dirk",
"project_number": "240045",
"customer": "Magua",
"city": "Salzgitter",
"kw": "",
"land": "DE"
"lines": [
{
"date": "07.08.",
"start_time": "6.45",
"end_time": "7.30",
"break_time": "",
"description": "BS-SZ-Support",
"has_customer_signature": true
},
{
"date": "07.08.",
"start_time": "7.30",
"end_time": "16.00",
"break_time": "0.5",
"description": "",
"has_customer_signature": true
},
{
"date": "08.08.",
"start_time": "7.30",
"end_time": "7.45",
"break_time": "",
"description": "SZ-BS",
"has_customer_signature": true
},
{
"date": "08.08.",
"start_time": "7.45",
"end_time": "16.30",
"break_time": "0.5",
"description": "",
"has_customer_signature": true
},
{
"date": "09.08.",
"start_time": "7.00",
"end_time": "7.45",
"break_time": "",
"description": "Support",
"has_customer_signature": true
},
{
"date": "09.08.",
"start_time": "7.45",
"end_time": "16.30",
"break_time": "0.5",
"description": "Support",
"has_customer_signature": true
},
{
"date": "10.08.",
"start_time": "7.15",
"end_time": "8.00",
"break_time": "",
"description": "SZPBS",
"has_customer_signature": true
},
{
"date": "10.08.",
"start_time": "8.00",
"end_time": "17.00",
"break_time": "0.5",
"description": "",
"has_customer_signature": true
},
{
"date": "11.08.",
"start_time": "6.30",
"end_time": "7.15",
"break_time": "",
"description": "support",
"has_customer_signature": true
},
{
"date": "11.08.",
"start_time": "7.15",
"end_time": "16.00",
"break_time": "0.5",
"description": "support",
"has_customer_signature": true
},
{
"date": "11.08.",
"start_time": "16.00",
"end_time": "16.45",
"break_time": "",
"description": "Supports",
"has_customer_signature": true
}
],
"is_process_done": true,
"is_commissioned_work": true,
"is_without_measuring_technology": false,
"sign_date": "13.08.2024",
"has_employee_signature": true
}
"invoice_info": {
"name": "",
"project_number": "",
"customer": "",
"city": "",
"land": "",
"lines": [
{
"date": "",
"start_time": "",
"end_time": "",
"break_time": "",
"description": "",
"has_customer_signature": True
},
{
"date": "",
"start_time": "",
"end_time": "",
"break_time": "",
"description": "",
"has_customer_signature": False
}
],
"is_process_done": True,
"is_commissioned_work": True,
"is_without_measuring_technology": False,
"sign_date": "",
"has_employee_signature": True
}
}
```
49 changes: 27 additions & 22 deletions config/page_2_template.txt
View file
Original file line number Diff line number Diff line change
@@ -1,44 +1,49 @@
```json
payment_method has 3 values self paid, visa, invoice
``json
{
"invoice_info": {
"name": "Tümmler, Dirk",
"project_number": "240045",
"is_in_egw": true,
"name": "",
"project_number": "",
"is_in_egw": True,
"currency": "",
"lines": [
"has_employee_signature": True,
"sign_date": "",
"fixed_lines": [
{
"title": "Hotel",
"with_breakfast": true,
"amount": "412,50",
"with_breakfast": True,
"can_book_again": True,
"amount": ,
"payment_method": ""
},
{
"title": "Fuel",
"amount": "86,24",
"amount": ,
"payment_method": ""
},
{
"title": "Parking fees",
"amount": "4,",
"amount": ,
"payment_method": ""
},
{
"title": "Rental car",
"amount": "156,20",
"amount": ,
"payment_method": ""
},
{
"title": "Toll",
"amount": "20",
},
{
"title": "Train ticket",
"amount": "24,50",
"payment_method": "paid for yourself"
},
{
"title": "Bicycle ticket",
"amount": "9,80",
"payment_method": "paid for yourself"
"amount": ,
"payment_method": ""
}
],
"has_employee_signature": true
"lines": [
{
"title": "",
"amount": ,
"payment_method": ""
}
]
}
}
```
17 changes: 1 addition & 16 deletions config/sample_mongo_doc.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@
"document_subject": "Invoice Subject",
"package_number": "PKG123",
"invoice_number": "INV123",
"invoice_type": "Regular",
"receipt_number": "REC123",
"shop_number": "SHOP123",
"transaction_number": "TRANS123",
Expand All @@ -92,26 +91,12 @@
"merchant_website": "https://merchant.com",
"merchant_email": "[email protected]",
"merchant_address": "123 Merchant Street",
"merchant_street_name": "Merchant Street",
"merchant_house_number": "123",
"merchant_city": "Merchant City",
"merchant_municipality": "Merchant Municipality",
"merchant_province": "Merchant Province",
"merchant_country": "Merchant Country",
"merchant_country_code": "MC",
"merchant_phone": "+1234567890",
"merchant_main_activity_code": "ACT123",
"customer_name": "Customer Name",
"customer_number": "CUST123",
"customer_reference": "CUSTREF123",
"customer_address": "456 Customer Avenue",
"customer_street_name": "Customer Avenue",
"customer_house_number": "456",
"customer_city": "Customer City",
"customer_municipality": "Customer Municipality",
"customer_province": "Customer Province",
"customer_country": "Customer Country",
"customer_phone": "+0987654321",
"customer_phone": "0987654321",
"customer_vat_number": "CUSTVAT123",
"customer_coc_number": "COC456",
"customer_bank_account_number": "BANK987654",
Expand Down
16 changes: 0 additions & 16 deletions docs/invoice_info.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,15 +81,13 @@ This is the sample document saved in MongoDB for `invoice_type` == "invoice 3".
"document_subject": "string",
"package_number": "string",
"invoice_number": "string",
"invoice_type": "string",
"receipt_number": "string",
"shop_number": "string",
"transaction_number": "string",
"transaction_reference": "string",
"order_number": "string",
"table_number": "string",
"table_group": "string",
"server": "string",
"merchant_name": "string",
"merchant_id": "string",
"merchant_coc_number": "string",
Expand All @@ -103,25 +101,11 @@ This is the sample document saved in MongoDB for `invoice_type` == "invoice 3".
"merchant_website": "string",
"merchant_email": "string",
"merchant_address": "string",
"merchant_street_name": "string",
"merchant_house_number": "string",
"merchant_city": "string",
"merchant_municipality": "string",
"merchant_province": "string",
"merchant_country": "string",
"merchant_country_code": "string",
"merchant_phone": "string",
"merchant_main_activity_code": "string",
"customer_name": "string",
"customer_number": "string",
"customer_reference": "string",
"customer_address": "string",
"customer_street_name": "string",
"customer_house_number": "string",
"customer_city": "string",
"customer_municipality": "string",
"customer_province": "string",
"customer_country": "string",
"customer_phone": "string",
"customer_vat_number": "string",
"customer_coc_number": "string",
Expand Down
13 changes: 9 additions & 4 deletions src/base_extractors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,11 @@ def postprocess(self, invoice_template:str,

# Prepare the prompt for the LLM
prompt = f"""
You are a helpful assistant that responds in JSON format with the invoice information in English. Don't add any annotations there.
Remember to close any bracket. And just output the field that has value, don't return field that are empty. return the key names as in the template is a MUST.
Use the text from the model response and the text from OCR. Describe what's in the image as the template here:
You are a helpful assistant that responds in JSON format with the invoice information in English.
Don't add any annotations there. Remember to close any bracket. And just output the field that has value,
don't return field that are empty. number, price and amount should be number, date should be convert to dd/mm/yyyy,
time should be convert to HH:mm:ss, currency should be 3 chracters like VND, USD, EUR.
Use the text from the model response and the text from OCR. Return the key names as in the template is a MUST. Describe what's in the image as the template here:
{invoice_template}.
The OCR text is: {ocr_text}
The model response text is: {model_text}
Expand Down Expand Up @@ -107,7 +109,10 @@ def _extract_invoice_llm(self, ocr_text, base64_image:str, invoice_template:str)
response = self.client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful assistant that responds in JSON format with the invoice information in English. Don't add any annotations there. Remember to close any bracket. And just output the field that has value, don't return field that are empty. "},
{"role": "system", "content": """You are a helpful assistant that responds in JSON format with the invoice information in English.
Don't add any annotations there. Remember to close any bracket. And just output the field that has value,
don't return field that are empty. number, price and amount should be number, date should be convert to dd/mm/yyyy,
time should be convert to HH:mm:ss, currency should be 3 chracters like VND, USD, EUR"""},
{"role": "user", "content": [
{"type": "text", "text": f"From the image of the bill and the text from OCR, extract the information. The ocr text is: {ocr_text} \n. Return the key names as in the template is a MUST. The invoice template: \n {invoice_template}"},
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{base64_image}"}}
Expand Down
23 changes: 18 additions & 5 deletions src/invoice_extraction.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from src.qwen2_extract import Qwen2Extractor
from src.Utils.utils import (timeit, read_config, convert_img_path_to_base64,
get_current_time, convert_base64_to_pil_image, read_txt_file)
from src.validate_invoice import (validate_invoice_3, validate_invoice_1,validate_invoice_2,
Invoice3, Invoice2, Invoice1)


def preprocess_text(text: str) -> str:
Expand Down Expand Up @@ -109,7 +111,7 @@ def extract_invoice_info(base64_img:str, ocr_reader:OcrReader, invoice_extractor
rotate_image = ocr_reader.get_rotated_image(pil_img)
invoice_info = invoice_extractor.extract_invoice(ocr_text=ocr_result['text'], image=rotate_image,
invoice_template=invoice_template)

print(invoice_info)
invoice_info = validate_invoice(invoice_info, invoice_type)
result['translator'] = ocr_reader['translator']
result['ocr_detector'] = ocr_reader['ocr_detector']
Expand All @@ -127,11 +129,22 @@ def extract_invoice_info(base64_img:str, ocr_reader:OcrReader, invoice_extractor


def validate_invoice(invoice_info:dict, invoice_type:str) ->dict:
if invoice_type == "invoice 3":
return invoice_info

return invoice_info
if invoice_type == "invoice 1":
valid_invoice = validate_invoice_1(invoice_info)
full_invoice = Invoice1(invoice_info=valid_invoice['invoice_info'])


elif invoice_type == "invoice 2":
valid_invoice = validate_invoice_2(invoice_info)
full_invoice = Invoice2(invoice_info=valid_invoice['invoice_info'])

elif invoice_type == "invoice 3":
valid_invoice = validate_invoice_3(invoice_info)
full_invoice = Invoice3(invoice_info=valid_invoice['invoice_info'])

full_invoice_dict = full_invoice.model_dump(exclude_unset=False)
return full_invoice_dict


if __name__ == "__main__":
Expand All @@ -140,7 +153,7 @@ def validate_invoice(invoice_info:dict, invoice_type:str) ->dict:

ocr_reader = OcrReader(config_path=config_path, translator=GoogleTranslator())
invoice_extractor = OpenAIExtractor(config_path=config_path)
img_path = "test/images/page_6.png"
img_path = "test/images/009_2.png"
base64_img = convert_img_path_to_base64(img_path)
result = extract_invoice_info(base64_img=base64_img, ocr_reader=ocr_reader,
invoice_extractor=invoice_extractor, config=config)
Expand Down
Loading

0 comments on commit a42565e

Please sign in to comment.