Spaces:
Running
Running
Backend
Browse files- backend.py +75 -0
backend.py
ADDED
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import re
|
3 |
+
import pandas as pd
|
4 |
+
from pypdf import PdfReader
|
5 |
+
from typing import List, Dict
|
6 |
+
from langchain.prompts import PromptTemplate
|
7 |
+
# from langchain_google_genai import GoogleGenerativeAI
|
8 |
+
from langchain_openai import OpenAI
|
9 |
+
|
10 |
+
|
11 |
+
api_key = "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"
|
12 |
+
|
13 |
+
os.environ["OPENAI_API_KEY"] = api_key
|
14 |
+
class InvoicePipeline:
|
15 |
+
|
16 |
+
def __init__(self, paths):
|
17 |
+
# This is your file path
|
18 |
+
self._paths = paths
|
19 |
+
# This is your LLM (GPT)
|
20 |
+
self._llm = OpenAI(model = "gpt-4o-mini")
|
21 |
+
# This is prompt
|
22 |
+
self._prompt_template = self._get_default_prompt_template()
|
23 |
+
# This function will help in extracting and run the code, and will produce a dataframe for us
|
24 |
+
def run(self) -> pd.DataFrame:
|
25 |
+
# We have defined the way the data has to be returned
|
26 |
+
df = pd.DataFrame({
|
27 |
+
"Invoice ID": pd.Series(dtype = "int"),
|
28 |
+
"DESCRIPTION": pd.Series(dtype = "str"),
|
29 |
+
"Issue Data": pd.Series(dtype = "str"),
|
30 |
+
"UNIT PRICE": pd.Series(dtype = "str"),
|
31 |
+
"AMOUNT": pd.Series(dtype = "int"),
|
32 |
+
"Bill For": pd.Series(dtype = "str"),
|
33 |
+
"From": pd.Series(dtype ="str"),
|
34 |
+
"Terms": pd.Series(dtype = "str")}
|
35 |
+
)
|
36 |
+
|
37 |
+
for path in self._paths:
|
38 |
+
raw_text = self._get_raw_text_from_pdf(path) # This function needs to be created
|
39 |
+
llm_resp = self._extract_data_from_llm(raw_text) #
|
40 |
+
data = self._parse_response(llm_resp)
|
41 |
+
df = pd.concat([df, pd.DataFrame([data])], ignore_index = True)
|
42 |
+
|
43 |
+
return df
|
44 |
+
|
45 |
+
# The default template that the machine will take
|
46 |
+
def _get_default_prompt_template(self) -> PromptTemplate:
|
47 |
+
template = """Extract all the following values: Invoice ID, DESCRIPTION, Issue Data,UNIT PRICE, AMOUNT, Bill for, From and Terms for: {pages}
|
48 |
+
Expected Outcome: remove any dollar symbols {{"Invoice ID":"12341234", "DESCRIPTION": "UNIT PRICE", "AMOUNT": "3", "Date": "2/1/2021", "AMOUNT": "100", "Bill For": "Dev", "From": "Coca Cola", "Terms" : "Net for 30 days"}}
|
49 |
+
"""
|
50 |
+
|
51 |
+
prompt_template = PromptTemplate(input_variables = ["pages"], template = template)
|
52 |
+
return prompt_template
|
53 |
+
|
54 |
+
|
55 |
+
# We will try to extract the text from the PDF to a normal variable.
|
56 |
+
def _get_raw_text_from_pdf(self, path:str) -> str:
|
57 |
+
text = ""
|
58 |
+
pdf_reader = PdfReader(path)
|
59 |
+
for page in pdf_reader.pages:
|
60 |
+
text += page.extract_text()
|
61 |
+
return text
|
62 |
+
|
63 |
+
def _extract_data_from_llm(self, raw_data:str) -> str:
|
64 |
+
resp = self._llm(self._prompt_template.format(pages = raw_data))
|
65 |
+
return resp
|
66 |
+
|
67 |
+
def _parse_response(self, response: str) -> Dict[str, str]:
|
68 |
+
pattern = r'{(.+)}'
|
69 |
+
re_match = re.search(pattern, response, re.DOTALL)
|
70 |
+
if re_match:
|
71 |
+
extracted_text = re_match.group(1)
|
72 |
+
data = eval('{' + extracted_text + '}')
|
73 |
+
return data
|
74 |
+
else:
|
75 |
+
raise Exception("No match found.")
|