|
from langchain_openai import OpenAI |
|
from pypdf import PdfReader |
|
import pandas as pd |
|
import re |
|
from langchain.prompts import PromptTemplate |
|
from langchain_community.llms import CTransformers |
|
from ctransformers import AutoModelForCausalLM |
|
|
|
|
|
def get_pdf_text(pdf_doc): |
|
text = "" |
|
pdf_reader = PdfReader(pdf_doc) |
|
for page in pdf_reader.pages: |
|
text += page.extract_text() |
|
return text |
|
|
|
def extracted_data(pages_data): |
|
template = """Please Extract all the following values : invoice no., Description, Quantity, date, |
|
Unit price , Amount, Total, email, phone number and address from this data: {pages} |
|
|
|
Expected output: remove any dollar symbols {{'Invoice no.': '1001329','Description': 'Office Chair','Quantity': '2','Date': '5/4/2023','Unit price': '1100.00$','Amount': '2200.00$','Total': '2200.00$','Email': '[email protected]','Phone number': '9999999999','Address': 'Mumbai, India'}} |
|
""" |
|
|
|
|
|
|
|
|
|
|
|
|
|
llm = AutoModelForCausalLM.from_pretrained("TheBloke/Llama-2-7B-Chat-GGML", model_file="llama-2-7b-chat.ggmlv3.q8_0.bin") |
|
|
|
|
|
prompt = PromptTemplate( |
|
input_variables=["pages"], |
|
template=template,) |
|
|
|
|
|
response=llm(prompt.format(email_topic=form_input,sender=email_sender,recipient=email_recipient,style=email_style)) |
|
output_text=llm(prompt_template.format(pages=pages_data)) |
|
|
|
full_response = '' |
|
for item in output_text: |
|
full_response += item |
|
return full_response |
|
|
|
|
|
|
|
def create_docs(user_pdf_list): |
|
|
|
df = pd.DataFrame({'Invoice no.': pd.Series(dtype='str'), |
|
'Description': pd.Series(dtype='str'), |
|
'Quantity': pd.Series(dtype='str'), |
|
'Date': pd.Series(dtype='str'), |
|
'Unit price': pd.Series(dtype='str'), |
|
'Amount': pd.Series(dtype='int'), |
|
'Total': pd.Series(dtype='str'), |
|
'Email': pd.Series(dtype='str'), |
|
'Phone number': pd.Series(dtype='str'), |
|
'Address': pd.Series(dtype='str') |
|
}) |
|
|
|
|
|
|
|
|
|
for filename in user_pdf_list: |
|
|
|
|
|
raw_data=get_pdf_text(filename) |
|
print("pdf_Data",raw_data) |
|
|
|
|
|
llm_extracted_data=extracted_data(raw_data) |
|
print("llm_extracted_data",llm_extracted_data) |
|
|
|
|
|
|
|
|
|
pattern = r'{(.+)}' |
|
match = re.search(pattern, llm_extracted_data, re.DOTALL) |
|
|
|
|
|
|
|
if match: |
|
extracted_text = match.group(1) |
|
|
|
data_dict = eval('{' + extracted_text + '}') |
|
print(data_dict) |
|
else: |
|
print("No match found.") |
|
|
|
data_dict = {} |
|
|
|
|
|
df=df._append([data_dict], ignore_index=True) |
|
print("********************DONE***************") |
|
|
|
|
|
llm_extracted_data |
|
return llm_extracted_data |