Leveraging The Power Of LLMs For Receipt Extraction

python
from paddleocr import PaddleOCR
 
file_path = '/root/documents/receipts/text-receipt.pdf'
 
_ocr = PaddleOCR(
        use_angle_cls=True, 
        lang='fr', 
        show_log=True,
        )
 
try:
    formated_text = extract_receipts(_ocr, file_path)
except:
    # This is where the error would be handled once the added to the Flask API.
    return jsonify({"error": "Something went wrong when passing the file through OCR."}), 500
 
# Helper function to extract text from receipts, then format it to a giant string where chunks are separated by a newline.
def extract_receipts(ocr, file_path):
        results = ocr.ocr(file_path, cls=True)
        txts = []
 
        # PaddleORC returns a list of lists, where each sublist contains the text and the confidence score.
        for result in results:
            for line in result:
                txts.append(line[1][0])
        
        return "\n".join(txts)

python
from flask import Flask, request, abort, jsonify
from werkzeug.utils import secure_filename
from paddleocr import PaddleOCR
import os
 
app = Flask(__name__)
 
@app.route("/api/process-receipts", methods=["POST"])
def process_receipts():
    # Initializing an OCR object on every request that way the API can deal with multiple requests at once.
    _ocr = PaddleOCR(
        use_angle_cls=True, 
        lang='fr', 
        show_log=True,
        )
    
    # Get supplier name from header
    supplier = request.headers.get('X-Supplier')
 
    if 'file' not in request.files:
        return jsonify({"error": "No file part"}), 400
    
    file = request.files['file']
 
    if file.filename == '':
        return jsonify({"error": "No selected file"}), 400
 
    filename = secure_filename(file.filename)   
    file_path = os.path.join("/tmp", filename)
    file.save(file_path)
 
    try:
        formated_text = extract_receipts(_ocr, file_path)
    except:
        return jsonify({"error": "Something went wrong when passing the file through OCR."}), 500
 
    os.remove(file_path)
 
    # Once the text has been extracted, we can pass it to GPT-4o to format the data.
 
if __name__ == "__main__":
    app.run()

python
from flask import Flask, request, abort, jsonify
from flask_cors import CORS
from werkzeug.utils import secure_filename
from data.dao import ApiDataManager
from paddleocr import PaddleOCR
import os
 
app = Flask(__name__)
 
# Added a database to store the API keys.
db = ApiDataManager()
# This will only allow requests coming from the same domain.
CORS(app)
 
@app.route("/api/process-receipts", methods=["POST"])
def process_receipts():
    _ocr = PaddleOCR(
        use_angle_cls=True, 
        lang='fr', 
        show_log=True,
        )
 
    # Added two new headers to the request
    app_name = request.headers.get('X-App-Name')
    app_api_key = request.headers.get('X-Api-Key')
 
    supplier_notes = request.headers.get('X-Supplier-Notes')
 
    if app_api_key != db.get_application_api_key(app_name):
        abort(401)

python
import os
import openai
import json
import re
 
class Inferencer:
    def __init__(self):
        self.__client = openai.Client()
        self.__client.api_key = os.getenv("OPENAI_API_KEY")
 
    def inference(self, additional_notes, receipts):
        # This is the base prompt that will be used to make sense of the text data sent to GPT-4o.
        # The additional_notes variable is a string that contains additional notes that will be added to the prompt.
        prompt = f"The following text is from a receipt, convert it to a JSON object.There might be some errors in the \
            item description: For example, '5LB' might have been extracted as 'SLB' since the 5 closely resembles an S. \
            Fix those errors. The JSON object should only contain the items and their price per unit (if there are \
            two prices for the same item use the smallest of the 2). Units that are labeled GR should be changed to G. LT to L \
            CL is not an origin tag it should be ignored. It is possible for items to not have an origin. The QUANTITY-UNIT \
            should be in 2 unique fields also add a category field. The category should be logical with the item name, \
            and should be in the language of the receipt. Use the following categories: 'Fruit & Légume', \
            'Viande', 'Poisson', 'Produit Laitier', 'Pâtisserie', 'Cannes', 'Congeler', 'Sec', 'Fines Herbes'.\ 
            Make sure to filter duplicate items.If the item is not in the list use 'Autre'. A mushroom should be classified \ 
            as 'Fruit & Légume' The JSON structure for an item should be: 'name': 'NAME OF PRODUCT', 'quantity': number, \
            'unit': 'UNIT OF PRODUCT', 'origin': 'ORIGIN TAG OF PRODUCT', 'category': 'ONE OF THE CATEGORY', 'price': number \
            . Make sure to correct common french mistakes (e.g lls should be île) {additional_notes}"
    
        response = openai.chat.completions.create(
            model="gpt-4-0125-preview",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "text",
                            "text": prompt
                        },
                        {
                            "type": "text",
                            "text": receipts
                        }
                    ],
                }
            ],
            max_tokens=4096,
        )
        
        # In order to receive a JSON object we need to remove the markdown.
        json_string = response.choices[0].message.content.replace("```json\n", "").replace("\n```", "");
 
        # In some cases the JSON object was inside of an array, so I also need to remove that array.
        array =  re.search(r'\[(.*?)\]', json_string, re.DOTALL).group(0)
        json_data = json.loads(array)
 
        return json_data

python
from gpt4 import Inferencer
 
_gpt = Inferencer()
 
try:
    print("sending to GPT-4...")
    interpreted_receipt_data = _gpt.inference(supplier_notes, formated_text)
    return jsonify(interpreted_receipt_data), 200
except:
    return jsonify({"error": "Something went wrong when converting the response from GPT-4."}), 500
 
if __name__ == "__main__":
    app.run()

dockerfile
FROM python:3.10-slim
WORKDIR /app
 
RUN apt-get update && apt-get install -y \
    libgomp1 \
    && rm -rf /var/lib/apt/lists/*
 
RUN apt-get update && apt-get install -y \
    libgomp1 \
    libgl1-mesa-glx \
    libglib2.0-0 \
    libsm6 \
    libxext6 \
    libxrender-dev \
    && rm -rf /var/lib/apt/lists/*
 
COPY app/ .
RUN pip install -r requirements.txt
CMD gunicorn --bind 0.0.0.0:5000 app:app

dockerfile
FROM python:3.10-slim
WORKDIR /app
 
RUN apt-get update && apt-get install -y \
    libgomp1 \
    && rm -rf /var/lib/apt/lists/*
 
RUN apt-get update && apt-get install -y \
    libgomp1 \
    libgl1-mesa-glx \
    libglib2.0-0 \
    libsm6 \
    libxext6 \
    libxrender-dev \
    && rm -rf /var/lib/apt/lists/*
 
COPY app/ .
RUN pip install -r requirements.txt
COPY models/ /root/.paddleocr/whl/
CMD gunicorn --bind 0.0.0.0:5000 app:app

python
@app.route("/api/process-receipts", methods=["POST"])
def process_receipts():
    _ocr = PaddleOCR(
        use_angle_cls=True, 
        lang='fr', 
        show_log=True,
        det_model_dir='/root/.paddleocr/whl/det/en_PP-OCRv3_det_infer/',
        rec_model_dir='/root/.paddleocr/whl/rec/latin_PP-OCRv3_rec_infer/',
        cls_model_dir='/root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer/'
        )