thisisdev commited on
Commit
6f79d72
·
verified ·
1 Parent(s): 2edbbcd
Files changed (1) hide show
  1. backend.py +75 -0
backend.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ import pandas as pd
4
+ from pypdf import PdfReader
5
+ from typing import List, Dict
6
+ from langchain.prompts import PromptTemplate
7
+ # from langchain_google_genai import GoogleGenerativeAI
8
+ from langchain_openai import OpenAI
9
+
10
+
11
+ api_key = "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"
12
+
13
+ os.environ["OPENAI_API_KEY"] = api_key
14
+ class InvoicePipeline:
15
+
16
+ def __init__(self, paths):
17
+ # This is your file path
18
+ self._paths = paths
19
+ # This is your LLM (GPT)
20
+ self._llm = OpenAI(model = "gpt-4o-mini")
21
+ # This is prompt
22
+ self._prompt_template = self._get_default_prompt_template()
23
+ # This function will help in extracting and run the code, and will produce a dataframe for us
24
+ def run(self) -> pd.DataFrame:
25
+ # We have defined the way the data has to be returned
26
+ df = pd.DataFrame({
27
+ "Invoice ID": pd.Series(dtype = "int"),
28
+ "DESCRIPTION": pd.Series(dtype = "str"),
29
+ "Issue Data": pd.Series(dtype = "str"),
30
+ "UNIT PRICE": pd.Series(dtype = "str"),
31
+ "AMOUNT": pd.Series(dtype = "int"),
32
+ "Bill For": pd.Series(dtype = "str"),
33
+ "From": pd.Series(dtype ="str"),
34
+ "Terms": pd.Series(dtype = "str")}
35
+ )
36
+
37
+ for path in self._paths:
38
+ raw_text = self._get_raw_text_from_pdf(path) # This function needs to be created
39
+ llm_resp = self._extract_data_from_llm(raw_text) #
40
+ data = self._parse_response(llm_resp)
41
+ df = pd.concat([df, pd.DataFrame([data])], ignore_index = True)
42
+
43
+ return df
44
+
45
+ # The default template that the machine will take
46
+ def _get_default_prompt_template(self) -> PromptTemplate:
47
+ template = """Extract all the following values: Invoice ID, DESCRIPTION, Issue Data,UNIT PRICE, AMOUNT, Bill for, From and Terms for: {pages}
48
+ Expected Outcome: remove any dollar symbols {{"Invoice ID":"12341234", "DESCRIPTION": "UNIT PRICE", "AMOUNT": "3", "Date": "2/1/2021", "AMOUNT": "100", "Bill For": "Dev", "From": "Coca Cola", "Terms" : "Net for 30 days"}}
49
+ """
50
+
51
+ prompt_template = PromptTemplate(input_variables = ["pages"], template = template)
52
+ return prompt_template
53
+
54
+
55
+ # We will try to extract the text from the PDF to a normal variable.
56
+ def _get_raw_text_from_pdf(self, path:str) -> str:
57
+ text = ""
58
+ pdf_reader = PdfReader(path)
59
+ for page in pdf_reader.pages:
60
+ text += page.extract_text()
61
+ return text
62
+
63
+ def _extract_data_from_llm(self, raw_data:str) -> str:
64
+ resp = self._llm(self._prompt_template.format(pages = raw_data))
65
+ return resp
66
+
67
+ def _parse_response(self, response: str) -> Dict[str, str]:
68
+ pattern = r'{(.+)}'
69
+ re_match = re.search(pattern, response, re.DOTALL)
70
+ if re_match:
71
+ extracted_text = re_match.group(1)
72
+ data = eval('{' + extracted_text + '}')
73
+ return data
74
+ else:
75
+ raise Exception("No match found.")