Upload folder using huggingface_hub

app.py

@@ -0,0 +1,50 @@
+import streamlit as st
+import pandas as pd
+from PIL import Image
+import io
+from pdf_processing import process_comparison_data, extract_text_with_pypdf
+
+def simulate_processing(pdf1, pdf2, tags):
+    # This is a placeholder function. Replace with actual processing logic
+    return [
+        ('key', 'Sample Data 1', 'Sample Data 2'),
+        ('index', 'More Sample Data 1', 'More Sample Data 2')
+    ]
+
+# App title
+st.title("PDF Tag Processing")
+
+# Sidebar configuration
+st.sidebar.header("Input Configuration")
+uploaded_file1 = st.sidebar.file_uploader("Upload First PDF", type="pdf")
+uploaded_file2 = st.sidebar.file_uploader("Upload Second PDF", type="pdf")
+tags_input = st.sidebar.text_area("Enter Tags (comma-separated)")
+
+# Process button
+if st.button("Process"):
+    # pdf1_text = extract_text_with_pypdf(uploaded_file1)
+    if not uploaded_file1:
+        st.error("Please upload a PDF file in the first pdf space")
+    elif not uploaded_file2:
+        st.error("Please upload a PDF file in the second pdf space")
+    elif not tags_input:
+        st.error("Please add some tags in the text area")
+    else:
+        df = process_comparison_data(uploaded_file1, uploaded_file2, [t.strip() for t in tags_input.split(',') if t.strip()])
+    # Display results in a table
+    st.subheader("Results")
+    st.dataframe(df)
+
+
+
+# Display instructions
+st.write("""
+This app allows you to upload two PDF files and enter tags. When you click "Process",
+it extracts information related to the tags from both the pdfs and compares the information
+in each pdf for each tag and displays the results in a table.
+""")
+
+# Add an image to illustrate the concept
+image = Image.open('doaz_image.png')  # Make sure to replace with your own image
+st.image(image, caption='Doaz')
+

pdf_processing.py

@@ -0,0 +1,144 @@
+import fitz
+from PyPDF2 import PdfReader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from anthropic import Anthropic
+from prompts import INFORMATION_EXTRACTION_PROMPT, INFORMATION_EXTRACTION_TAG_FORMAT, verify_INFORMATION_EXTRACTION_PROMPT, extract_INFORMATION_EXTRACTION_PROMPT
+from prompts import verify_all_tags_present
+from prompts import COMPARISON_INPUT_FORMAT, COMPARISON_PROMPT, COMPARISON_TAG_FORMAT, verify_COMPARISON_PROMPT, extract_COMPARISON_PROMPT
+import pandas as pd
+from concurrent.futures import ThreadPoolExecutor
+import streamlit as st
+from dotenv import load_dotenv
+load_dotenv()
+
+def make_llm_api_call(messages):
+    print("Making LLM api call")
+    client = Anthropic()
+    message = client.messages.create(
+        model="claude-3-haiku-20240307",
+        max_tokens=4096,
+        temperature=0,
+        messages=messages,
+    )
+    print("LLM response received")
+    return message
+
+def loop_verify_format(answer_text, tag_format, messages, verify_func,root_tag):
+    i = 0
+    while not verify_func(answer_text):
+        print("Wrong format")
+        assistant_message = {"role": "assistant", "content": [{"type":"text", "text":answer_text}]}
+        corrective_message = {"role":"user", "content":[{"type": "text", "text": f"You did not provide your answer in the correct format. Please provide your answer in the following format:\n{tag_format}"}]}
+        messages.append(assistant_message)
+        messages.append(corrective_message)
+        message = make_llm_api_call(messages)
+        message_text = message.content[0].text
+        answer_text = f"<{root_tag}>\n{message_text.split(f'<{root_tag}>')[1].split(f'</{root_tag}>')[0].strip()}\n</{root_tag}>"
+        if i > 3:
+            raise Exception(f"LLM failed to provide a valid answer in {i-1} attempts")
+    return answer_text
+
+def loop_verify_all_tags_present(answer_text, tags, user_message, tag_format, verify_func, root_tag):
+    missing_tags, _ = verify_all_tags_present(answer_text, tags)
+    if missing_tags:
+        print("There are missing tags", missing_tags)
+        assistant_message = {"role":"assistant", "content":[{"type":"text", "text":answer_text}]}
+        corrective_message = [{"role":"user", "content":[{"type":"text", "text":("In your response, the following tags are missing:\n" + "\n".join([f"<tag>{tag}</tag>" for tag in missing_tags]) + "\n\nPlease add information about the above missing tags and give a complete correct response.")}]}]
+        messages = [user_message, assistant_message, corrective_message]
+        message = make_llm_api_call(messages)
+        message_text = message.content[0].text
+        answer_text = f"<{root_tag}>\n{message_text.split(f'<{root_tag}>')[1].split(f'</{root_tag}>')[0].strip()}\n</{root_tag}>"
+        answer_text = loop_verify_format(answer_text, tag_format, [user_message], verify_func, root_tag)
+        missing_tags, _ = verify_all_tags_present(answer_text, tags)
+    return answer_text
+
+def extract_information_from_pdf(pdf_text, tags):
+    tag_text = "\n".join([f"<tag>{tag}</tag>" for tag in tags])
+    prompt = INFORMATION_EXTRACTION_PROMPT.format(TEXT=pdf_text, TAGS=tag_text)
+    user_message = {"role": "user", "content": [{"type": "text", "text": prompt}]}
+    answer_text = ""
+    messages = [user_message]
+    message = make_llm_api_call(messages)
+    message_text = message.content[0].text
+    answer_text = f"<answer>\n{message_text.split('<answer>')[1].split('</answer>')[0].strip()}\n</answer>"
+    answer_text = loop_verify_format(answer_text, INFORMATION_EXTRACTION_TAG_FORMAT, messages, verify_INFORMATION_EXTRACTION_PROMPT, 'answer')
+    answer_text = loop_verify_all_tags_present(answer_text, tags, user_message, INFORMATION_EXTRACTION_PROMPT, verify_INFORMATION_EXTRACTION_PROMPT, 'answer')
+
+    return extract_INFORMATION_EXTRACTION_PROMPT(answer_text)
+
+
+def extract_text_with_pypdf(pdf_path):
+    reader = PdfReader(pdf_path)
+    text = ""
+    for page in reader.pages:
+        text += f"{page.extract_text()}\n"
+    return text.strip()
+
+def get_tag_info_for_pdf(pdf, tags):
+    text = extract_text_with_pypdf(pdf)
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=100000, chunk_overlap=0)
+    chunks = text_splitter.split_text(text)
+    tag_data = {tag:"" for tag in tags}
+    print("chunk length",len(chunks))
+    for chunk in chunks:
+        data = extract_information_from_pdf(chunk, tags)
+        for tag in tags:
+            tag_data.update({tag:f"{tag_data.get(tag)}\n{data.get(tag)}"})
+    return tag_data
+
+def do_comparison_process(pdf1_data, pdf2_data, tags):
+    tag_data_list = []
+    for tag in tags:
+        tag_info_text = COMPARISON_INPUT_FORMAT.format(tag=tag, pdf1_information=pdf1_data.get(tag), pdf2_information=pdf2_data.get(tag))
+        tag_data_list.append(tag_info_text)
+    tag_data_text = "\n".join(tag_data_list)
+    prompt = COMPARISON_PROMPT.format(TAG_INFO= tag_data_text)
+    user_message = {"role": "user", "content": [{"type": "text", "text": prompt}]}
+    message = make_llm_api_call([user_message])
+    message_text = message.content[0].text
+    comparison_text = f"<comparison>\n{message_text.split('<comparison>')[1].split('</comparison>')[0].strip()}\n</comparison>"
+    comparison_text = loop_verify_format(comparison_text, COMPARISON_TAG_FORMAT, [user_message], verify_COMPARISON_PROMPT, 'comparison')
+    comparison_text = loop_verify_all_tags_present(comparison_text, tags, user_message, COMPARISON_TAG_FORMAT, verify_COMPARISON_PROMPT, 'comparison')
+    
+    return extract_COMPARISON_PROMPT(comparison_text)
+
+# def get_pdf_data(pdf1, pdf2, tags):
+#     def get_tag_info_for_pdf(pdf, tags):
+#         text = extract_text_with_pypdf(pdf)
+#         text_splitter = RecursiveCharacterTextSplitter(chunk_size=100000, chunk_overlap=0)
+#         chunks = text_splitter.split_text(text)
+#         tag_data = {tag:"" for tag in tags}
+#         for chunk in chunks:
+#             data = extract_information_from_pdf(chunk, tags)
+#             for tag in tags:
+#                 tag_data.update({tag:f"{tag_data.get(tag)}\n{data.get(tag)}"})
+#         return tag_data
+
+#     # Create a ThreadPoolExecutor (or ProcessPoolExecutor for CPU-bound tasks)
+#     with ThreadPoolExecutor(max_workers=2) as executor:
+#         # Submit the functions to the executor
+#         pdf1_future = executor.submit(get_tag_info_for_pdf, pdf1, tags)
+#         pdf2_future = executor.submit(get_tag_info_for_pdf, pdf2, tags)
+
+#         # Collect the results
+#         pdf1_data = pdf1_future.result()
+#         pdf2_data = pdf2_future.result()
+
+#     return pdf1_data, pdf2_data
+
+
+def process_comparison_data(pdf1, pdf2, tags):
+    with st.spinner("Processing PDF 1"):
+        pdf1_data = get_tag_info_for_pdf(pdf1, tags)
+    with st.spinner("Processing PDF 2"):
+        pdf2_data = get_tag_info_for_pdf(pdf2, tags)
+    with st.spinner("Generating Comparison Data"):
+        comparison_data = do_comparison_process(pdf1_data, pdf2_data, tags)
+    # pdf1_data, pdf2_data = get_pdf_data(pdf1, pdf2, tags)
+    # comparison_data = do_comparison_process(pdf1_data, pdf2_data, tags)
+    table_data = []
+    for tag in tags:
+        table_data.append((tag, pdf1_data.get(tag), pdf2_data.get(tag), comparison_data.get(tag)))
+    df = pd.DataFrame(table_data, columns=['Tags', 'PDF 1', 'PDF 2', 'Difference'])
+    df.set_index('Tags', inplace=True)
+    return df

prompts.py

@@ -0,0 +1,342 @@
+import xml.etree.ElementTree as ET
+import re
+
+INFORMATION_EXTRACTION_PROMPT = """You will be given a text and a list of tags. Your task is to extract all important information from the text related to each tag separately and provide the information with each tag and their corresponding relevant information.
+
+Here is the input text:
+<text>
+{TEXT}
+</text>
+
+Here is the list of tags:
+<all_tags>
+{TAGS}
+</all_tags>
+
+Follow these steps to complete the task:
+
+1. Read through the entire text carefully.
+
+2. For each tag in the list:
+   a. Identify all relevant information in the text that relates to the tag.
+   b. Extract and summarize the important points related to that tag.
+   c. If no relevant information is found for a tag, note that as well.
+
+3. Format your output as follows:
+   - Use <tag_info> tags to enclose the information for each tag.
+   - Within each <tag_info> section:
+     - Start with the tag name in <tag> tags.
+     - Follow with the extracted information in <info> tags.
+   - If no relevant information is found for a tag, just leave section empty within the <info> tags but make sure the <info> tags are present.
+
+4. Ensure that the extracted information is concise yet comprehensive, capturing all important points related to each tag.
+
+Here's an example of how your output should be formatted:
+
+<answer>
+<tag_info>
+<tag>Tag1</tag>
+<info>Relevant information for Tag1 extracted from the text...</info>
+</tag_info>
+
+<tag_info>
+<tag>Tag2</tag>
+<info>Relevant information for Tag2 extracted from the text...</info>
+</tag_info>
+
+<tag_info>
+<tag>Tag3</tag>
+<info></info>
+</tag_info>
+</answer>
+
+Remember to include all tags from the provided list, even if no relevant information is found for some of them.
+
+Provide your complete answer within <answer> tags."""
+
+INFORMATION_EXTRACTION_TAG_FORMAT = """<answer>
+<tag_info>
+<tag>Tag1</tag>
+<info>Relevant information for Tag1 extracted from the text...</info>
+</tag_info>
+
+<tag_info>
+<tag>Tag2</tag>
+<info>Relevant information for Tag2 extracted from the text...</info>
+</tag_info>
+
+<tag_info>
+<tag>Tag3</tag>
+<info></info>
+</tag_info>
+</answer>"""
+
+COMPARISON_PROMPT = """You are tasked with comparing information from two PDFs for a given set of tags. The input will be provided in the following format:
+
+<tag_info>
+{TAG_INFO}
+</tag_info>
+
+Follow these steps to complete the task:
+
+1. Read through the entire input carefully.
+
+2. For each tag data, you will see the tag and the information related to the tag from pdf1 and pdf2. Your job is to compare this information and explain the differences.
+
+3. When comparing the information:
+   - Look for differences in content, wording, or details.
+   - Note if one PDF provides information that the other doesn't.
+   - Identify any contradictions between the two PDFs.
+
+4. Format your output as follows:
+   - Use <comparison> tags to enclose your entire response.
+   - For each tag, use <tag_difference> tags to enclose the explanation of differences.
+   - Within each <tag_difference> section, use <tag> to enclose the tag name, and <difference> to enclose the explanation of differences.
+   - Provide a clear, concise explanation of the differences between pdf1 and pdf2 for that tag.
+   - Your output should be in korean language
+
+5. Here's an example of how your output should look:
+
+<comparison>
+<tag_difference>
+<tag>Tag 1</tag>
+<difference>
+PDF1 states that the product is available in three colors (red, blue, green), while PDF2 mentions four colors (red, blue, green, yellow).
+</difference>
+</tag_difference>
+<tag_difference>
+<tag>Tag 2</tag>
+<difference>
+The information in PDF1 and PDF2 is identical for this tag, both describing the product's weight as 500g.
+</difference>
+</tag_difference>
+</comparison>
+
+6. Additional guidelines:
+   - Be thorough in your comparison, addressing all aspects of the information provided.
+   - If there are no differences for a particular tag, state this clearly.
+   - Use clear and concise language in your explanations.
+   - If one PDF lacks information that the other provides, explicitly mention this.
+
+Begin your analysis with the first tag in the input and proceed through all tags sequentially. Ensure that your output is comprehensive and accurately reflects the differences between the two PDFs for each tag and is in korean language."""
+
+COMPARISON_INPUT_FORMAT = """<tag_data>
+<tag>{tag}</tag>
+<pdf1>{pdf1_information}</pdf1>
+<pdf2>{pdf2_information}</pdf2>
+</tag_data>"""
+
+COMPARISON_TAG_FORMAT = """<comparison>
+<tag_difference>
+<tag>Tag 1</tag>
+<difference>
+PDF1 states that the product is available in three colors (red, blue, green), while PDF2 mentions four colors (red, blue, green, yellow).
+</difference>
+</tag_difference>
+<tag_difference>
+<tag>Tag 2</tag>
+<difference>
+The information in PDF1 and PDF2 is identical for this tag, both describing the product's weight as 500g.
+</difference>
+</tag_difference>
+</comparison>"""
+
+# def verify_INFORMATION_EXTRACTION_PROMPT(text):
+#     try:
+#         root = ET.fromstring(text)
+        
+#         # Check if the root element is 'answer'
+#         if root.tag != 'answer':
+#             print("root tag is wrong")
+#             return False
+        
+#         # Check if all child elements are 'tag_info'
+#         for child in root:
+#             if child.tag != 'tag_info':
+#                 print("tag_info tag is wrong")
+#                 return False
+        
+#         # Verify each tag_info element
+#         for tag_info in root.findall('.//tag_info'):
+#             tag = tag_info.find('tag')
+#             info = tag_info.find('info')
+            
+#             # Check if tag exists and is not empty
+#             if tag is None:
+#                 print("tag is missing")
+#                 return False
+            
+#             if not tag.text.strip():
+#                 print("tag is empty")
+#                 return False
+            
+#             # Check if info exists and is optional
+#             if info is None:
+#                 print("info is missing")
+#                 return False
+                    
+#         # If all checks pass, the format is valid
+#         return True
+    
+#     except ET.ParseError:
+#         # If parsing fails, the text doesn't follow the expected format
+#         print(text)
+#         return False
+
+
+def verify_INFORMATION_EXTRACTION_PROMPT(text):
+    # Regular expression pattern
+    pattern = r'<answer>(.*?)<\/answer>'
+    
+    # Try to match the overall structure
+    match = re.search(pattern, text, flags=re.DOTALL)
+    
+    # Extract the content between <answer> tags
+    content = match.group(1)
+    
+    # Pattern for tag_info blocks
+    tag_info_pattern = r'<tag_info>\s*<tag>(.*?)<\/tag>\s*<info>(.*?)<\/info>\s*<\/tag_info>'
+    
+    # Find all tag_info blocks
+    matches = re.findall(tag_info_pattern, content,flags=re.DOTALL)
+    # Check each match
+    for match in matches:
+        tag, info = match
+        
+        # Check if tag exists and is not empty
+        if not tag.strip():
+            print("Tag is empty")
+            return False
+        
+    
+    # If all checks pass, the format is valid
+    return True
+
+# def verify_COMPARISON_PROMPT(text):
+#     try:
+#         root = ET.fromstring(text)
+        
+#         # Check if the root element is 'answer'
+#         if root.tag != 'comparison':
+#             return False
+        
+#         # Check if all child elements are 'tag_difference'
+#         for child in root:
+#             if child.tag != 'tag_difference':
+#                 return False
+        
+#         # Verify each tag_difference element
+#         for tag_difference in root.findall('.//tag_difference'):
+#             tag = tag_difference.find('tag')
+#             difference = tag_difference.find('difference')
+            
+#             # Check if tag exists and is not empty
+#             if tag is None:
+#                 return False
+            
+#             if not tag.text.strip():
+#                 return False
+            
+#             # Check if info exists and is optional
+#             if difference is None:
+#                 return False
+                    
+#         # If all checks pass, the format is valid
+#         return True
+    
+#     except ET.ParseError:
+#         # If parsing fails, the text doesn't follow the expected format
+#         return False
+
+def verify_COMPARISON_PROMPT(text):
+    # Regular expression pattern
+    pattern = r'<comparison>(.*?)<\/comparison>'
+    
+    # Try to match the overall structure
+    match = re.search(pattern, text, flags=re.DOTALL)
+    if not match:
+        print("no comparison match")
+        return False
+    
+    # Extract the content between <comparison> tags
+    content = match.group(1)
+    
+    # Pattern for tag_difference blocks
+    tag_difference_pattern = r'<tag_difference>\s*<tag>(.*?)<\/tag>\s*<difference>(.*?)<\/difference>\s*<\/tag_difference>'
+    
+    # Find all tag_info blocks
+    matches = re.findall(tag_difference_pattern, content,flags=re.DOTALL)
+    # Check each match
+    for match in matches:
+        tag, difference = match
+        
+        # Check if tag exists and is not empty
+        if not tag.strip():
+            print("Tag is empty")
+            return False
+        
+    
+    # If all checks pass, the format is valid
+    return True
+
+def verify_all_tags_present(text, tags):
+    tag_pattern = r'<tag>(.*?)<\/tag>'
+    tags_llm = re.findall(tag_pattern, text,flags=re.DOTALL)
+    tags = [t.strip() for t in tags]
+    tags_llm = [t.strip() for t in tags_llm]
+    extra_tags = []
+    missing_tags = []
+    for tag in tags:
+        if tag not in tags_llm:
+            missing_tags.append(tag)
+    for tag in tags_llm:
+        if tag not in tags:
+            extra_tags.append(tag)
+    return missing_tags, extra_tags
+
+def extract_INFORMATION_EXTRACTION_PROMPT(text):
+    # Regular expression pattern
+    pattern = r'<answer>(.*?)<\/answer>'
+    
+    # Try to match the overall structure
+    match = re.search(pattern, text, flags=re.DOTALL)
+    
+    # Extract the content between <answer> tags
+    content = match.group(1)
+    
+    # Pattern for tag_info blocks
+    tag_info_pattern = r'<tag_info>\s*<tag>(.*?)<\/tag>\s*<info>(.*?)<\/info>\s*<\/tag_info>'
+    
+    # Find all tag_info blocks
+    matches = re.findall(tag_info_pattern, content,flags=re.DOTALL)
+    # Check each match
+    data = {}
+    for match in matches:
+        tag, info = match
+        
+        data.update({tag:info})
+
+    return data
+
+def extract_COMPARISON_PROMPT(text):
+    # Regular expression pattern
+    pattern = r'<comparison>(.*?)<\/comparison>'
+    
+    # Try to match the overall structure
+    match = re.search(pattern, text, flags=re.DOTALL)
+    
+    # Extract the content between <comparison> tags
+    content = match.group(1)
+    
+    # Pattern for tag_difference blocks
+    tag_difference_pattern = r'<tag_difference>\s*<tag>(.*?)<\/tag>\s*<difference>(.*?)<\/difference>\s*<\/tag_difference>'
+    
+    # Find all tag_info blocks
+    matches = re.findall(tag_difference_pattern, content,flags=re.DOTALL)
+    # Check each match
+    data = {}
+    for match in matches:
+        tag, difference = match
+        data.update({tag:difference})
+
+    return data

requirements.txt

@@ -0,0 +1,10 @@
+streamlit
+pandas
+pillow
+PyPDF2
+fitz
+PyMuPDF
+langchain-text-splitters
+anthropic
+python-dotenv
+tiktoken