Upload 18 files

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+chroma_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+documents/(최종본)[[:space:]]생성형[[:space:]]AI[[:space:]]저작권[[:space:]]안내서.pdf filter=lfs diff=lfs merge=lfs -text
+documents/1.[[:space:]](자료집)생성형[[:space:]]AI[[:space:]]교육자료-ChatGPT[[:space:]]사례[[:space:]]중심으로).pdf filter=lfs diff=lfs merge=lfs -text
+documents/챗GPT등[[:space:]]생성형[[:space:]]AI[[:space:]]활용[[:space:]]보안[[:space:]]가이드라인.pdf filter=lfs diff=lfs merge=lfs -text
+documents/docs1.pdf filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,175 @@
+import os
+import gradio as gr
+from dotenv import load_dotenv
+from rag_system import load_retrieval_qa_chain, get_answer, update_embeddings
+import json 
+import re
+from PyPDF2 import PdfReader
+from PIL import Image
+import io
+from pydantic_settings import BaseSettings
+
+# Load environment variables
+load_dotenv()
+
+# Set OpenAI API key
+openai_api_key = os.getenv("OPENAI_API_KEY")
+os.environ["OPENAI_API_KEY"] = openai_api_key
+
+# Ensure the static directory exists
+static_directory = "static"
+if not os.path.exists(static_directory):
+    os.makedirs(static_directory)
+
+# PDF utility functions
+def get_pdf_page_count(file_path):
+    with open(file_path, 'rb') as file:
+        pdf = PdfReader(file)
+        return len(pdf.pages)
+
+def render_pdf_page(file_path, page_num):
+    import fitz  # PyMuPDF
+    doc = fitz.open(file_path)
+    page = doc.load_page(page_num - 1)  # page numbers start from 0
+    pix = page.get_pixmap()
+    img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
+    return img
+
+# Load PDF data
+def load_pdf_data():
+    pdf_data = {}
+    pdf_files = [f for f in os.listdir("./documents") if f.endswith('.pdf')]
+    for pdf_file in pdf_files:
+        file_path = f"./documents/{pdf_file}"
+        pdf_data[pdf_file] = {
+            'path': file_path,
+            'num_pages': get_pdf_page_count(file_path)
+        }
+    return pdf_data
+
+# Update embeddings with new documents
+update_embeddings()
+
+# Load vector store and PDF data
+qa_chain = load_retrieval_qa_chain()
+pdf_data = load_pdf_data()
+
+def pdf_viewer_interface(pdf_state, page_number, action=None, page_input=None):
+    selected_pdf = pdf_state['selected_pdf']
+    current_page = page_number
+    max_pages = pdf_data[selected_pdf]['num_pages']
+
+    if action == "prev":
+        current_page = max(1, current_page - 1)
+    elif action == "next":
+        current_page = min(max_pages, current_page + 1)
+    elif page_input is not None:
+        try:
+            current_page = int(page_input)
+            current_page = max(1, min(current_page, max_pages))
+        except ValueError:
+            pass
+
+    pdf_state['page_number'] = current_page
+    pdf_path = pdf_data[selected_pdf]['path']
+    img = render_pdf_page(pdf_path, current_page)
+    return img, current_page, str(current_page)
+
+def chat_interface(user_input, chat_history, pdf_state):
+    chat_history_list = [item for sublist in chat_history for item in sublist]
+    
+    response = get_answer(qa_chain, user_input, chat_history_list)
+    full_response = response["answer"]
+    sources = response["sources"]
+    
+    chat_history.append((user_input, full_response))
+    return chat_history, sources
+
+def handle_source_click(evt: gr.SelectData, sources, pdf_state, page_number):
+    index = evt.index[0] if isinstance(evt.index, list) else evt.index
+    
+    if index >= len(sources):
+        return None, pdf_state, page_number, ""
+    
+    source = sources[index]
+    file_name, page_str = source.split(' (Page ')
+    page_str = page_str.rstrip(')')
+    page = int(page_str)
+    
+    if file_name not in pdf_data:
+        return None, pdf_state, page_number, ""
+    
+    pdf_state['selected_pdf'] = file_name
+    pdf_state['page_number'] = page
+    pdf_path = pdf_data[file_name]['path']
+    img = render_pdf_page(pdf_path, page)
+    return img, pdf_state, page, str(page)
+
+with gr.Blocks() as demo:
+    initial_pdf = list(pdf_data.keys())[0]
+    pdf_state = gr.State({'selected_pdf': initial_pdf, 'page_number': 1})
+    sources = gr.State([])
+    page_number = gr.State(1)
+    
+    with gr.Row():
+        with gr.Column(scale=3):
+            chat_history = gr.State([])
+            chatbot = gr.Chatbot()
+            user_input = gr.Textbox(show_label=False, placeholder="Enter your question...")
+            source_list = gr.Dataframe(
+                headers=["Source", "Page"],
+                datatype=["str", "number"],
+                row_count=4,
+                col_count=2,
+                interactive=False,
+                label="Sources"
+            )
+        
+        with gr.Column(scale=2):
+            pdf_dropdown = gr.Dropdown(choices=list(pdf_data.keys()), label="Select PDF", value=initial_pdf)
+            pdf_viewer = gr.Image(label="PDF Viewer", height=600)
+            pdf_page = gr.Number(label="Page Number", value=1)
+            with gr.Row():
+                prev_button = gr.Button("Previous Page")
+                next_button = gr.Button("Next Page")
+    
+    user_input.submit(chat_interface, [user_input, chat_history, pdf_state], [chatbot, sources]).then(
+        lambda s: [[src.split(' (Page ')[0], int(src.split(' (Page ')[1].rstrip(')'))] for src in s],
+        inputs=[sources],
+        outputs=[source_list]
+    )
+    
+    source_list.select(handle_source_click, [sources, pdf_state, page_number], [pdf_viewer, pdf_state, page_number, pdf_page])
+    
+    pdf_dropdown.change(
+        lambda x: {'selected_pdf': x, 'page_number': 1},
+        inputs=[pdf_dropdown],
+        outputs=[pdf_state]
+    ).then(
+        pdf_viewer_interface,
+        inputs=[pdf_state, gr.State(1)],
+        outputs=[pdf_viewer, page_number, pdf_page]
+    )
+    
+    prev_button.click(
+        pdf_viewer_interface,
+        inputs=[pdf_state, page_number, gr.State("prev")],
+        outputs=[pdf_viewer, page_number, pdf_page]
+    )
+    
+    next_button.click(
+        pdf_viewer_interface,
+        inputs=[pdf_state, page_number, gr.State("next")],
+        outputs=[pdf_viewer, page_number, pdf_page]
+    )
+    
+    pdf_page.submit(
+        pdf_viewer_interface,
+        inputs=[pdf_state, page_number, gr.State(None), pdf_page],
+        outputs=[pdf_viewer, page_number, pdf_page]
+    )
+    
+    chatbot.select(handle_source_click, [sources, pdf_state, page_number], [pdf_viewer, pdf_state, page_number, pdf_page])
+
+if __name__ == "__main__":
+    demo.launch()

chroma_db/92bcea26-ebd2-4410-bf8d-bc0f9f5546b7/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b012862b5134a19825524db76e88f9ed7753467e36912b781a4eedcc5b79e59d
+size 219940000

chroma_db/92bcea26-ebd2-4410-bf8d-bc0f9f5546b7/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aac1349cae78413d368be855bf1b1fe05c464a1466557a38049597c5ab1c2fd2
+size 100

chroma_db/92bcea26-ebd2-4410-bf8d-bc0f9f5546b7/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:558d9768335c4e2df650ce68b66561af9b993d605444d9979dd431ddec869835
+size 2028516

chroma_db/92bcea26-ebd2-4410-bf8d-bc0f9f5546b7/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:955616dc55327c057865719f90dae1fb9511e0350daef736668fe472a58313c9
+size 140000

chroma_db/92bcea26-ebd2-4410-bf8d-bc0f9f5546b7/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bec79adf817e9c72e98c967480193b6a182a6973ead638c1607b88972c4f661f
+size 303336

chroma_db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82c709d4b055db6c031121a4019fd81ecaac321cae4bf7fb3117e09baef6d03f
+size 518348800

documents/(최종본) 생성형 AI 저작권 안내서.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0c786f0992d3e227fef6471763d29fc14de622635568a5e7a46ea7bb02bc319
+size 8063005

documents/1. (자료집)생성형 AI 교육자료-ChatGPT 사례 중심으로).pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b99c1c27309f641c2926e8c49c8aba295017622df2c2de614f5c64d600abedee
+size 23096564

documents/docs1.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85bd8d387d03fda0e5d1fc079cabdea2f85ec4bc6d928ed71ac272fc0b8d6c37
+size 3476556

documents/챗GPT등 생성형 AI 활용 보안 가이드라인.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65d0c5200f2153b0a02f8ed366038b7ad7d1db9aad4386df2eea41fcf0cc2726
+size 2033387

pdf_viewer_component.py

@@ -0,0 +1,23 @@
+import streamlit.components.v1 as components
+import os
+import base64
+
+_RELEASE = False
+
+if not _RELEASE:
+    _component_func = components.declare_component(
+        "pdf_viewer",
+        url="http://localhost:3000",
+    )
+else:
+    parent_dir = os.path.dirname(os.path.abspath(__file__))
+    build_dir = os.path.join(parent_dir, "frontend/build")
+    _component_func = components.declare_component("pdf_viewer", path=build_dir)
+
+def pdf_viewer(pdf_base64, initial_page=1, key=None):
+    component_value = _component_func(pdfBase64=pdf_base64, initialPage=initial_page, key=key, default=initial_page)
+    return component_value
+
+def load_pdf_as_base64(file_path):
+    with open(file_path, "rb") as pdf_file:
+        return base64.b64encode(pdf_file.read()).decode("utf-8")

rag_system.py

@@ -0,0 +1,122 @@
+import os
+from dotenv import load_dotenv
+from langchain_openai import OpenAIEmbeddings
+from langchain_chroma import Chroma  # 이 줄을 수정
+from langchain.chains import ConversationalRetrievalChain
+from langchain_openai import ChatOpenAI
+from langchain.docstore.document import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import pdfplumber
+from concurrent.futures import ThreadPoolExecutor
+
+# Load environment variables
+load_dotenv()
+
+# Set OpenAI API key
+api_key = os.getenv("OPENAI_API_KEY")
+if not api_key:
+    raise ValueError("API key not found. Please set the OPENAI_API_KEY environment variable.")
+os.environ["OPENAI_API_KEY"] = api_key
+
+def load_retrieval_qa_chain():
+    # Load embeddings
+    embeddings = OpenAIEmbeddings()
+    
+    # Load vector store
+    vectorstore = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)
+
+    # Initialize ChatOpenAI model
+    llm = ChatOpenAI(model_name="gpt-4o-mini", temperature=0)  # "gpt-4o-mini
+
+    # Create ConversationalRetrievalChain
+    qa_chain = ConversationalRetrievalChain.from_llm(
+        llm,
+        vectorstore.as_retriever(),
+        return_source_documents=True
+    )
+
+    return qa_chain
+
+def extract_text_from_pdf(file_path):
+    documents = []
+    with pdfplumber.open(file_path) as pdf:
+        for page_num, page in enumerate(pdf.pages):
+            text = page.extract_text()
+            if text:
+                # Split text into chunks
+                text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+                chunks = text_splitter.split_text(text)
+                for chunk in chunks:
+                    doc = Document(page_content=chunk, metadata={"source": os.path.basename(file_path), "page": page_num + 1})
+                    documents.append(doc)
+    return documents
+
+def embed_documents():
+    embeddings = OpenAIEmbeddings(api_key=os.getenv("OPENAI_API_KEY"))
+    vectorstore = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)
+    
+    pdf_files = [f for f in os.listdir("./documents") if f.endswith('.pdf')]
+    documents = []
+    with ThreadPoolExecutor() as executor:
+        results = executor.map(extract_text_from_pdf, [f"./documents/{pdf_file}" for pdf_file in pdf_files])
+        for result in results:
+            documents.extend(result)
+    vectorstore.add_documents(documents)
+
+def update_embeddings():
+    embeddings = OpenAIEmbeddings(api_key=os.getenv("OPENAI_API_KEY"))
+    vectorstore = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)
+    
+    # Retrieve existing documents
+    existing_files = set()
+    for doc in vectorstore.similarity_search(""):
+        existing_files.add(doc.metadata["source"])
+    
+    pdf_files = [f for f in os.listdir("./documents") if f.endswith('.pdf')]
+    
+    new_files = [f for f in pdf_files if f not in existing_files]
+    documents = []
+    with ThreadPoolExecutor() as executor:
+        results = executor.map(extract_text_from_pdf, [f"./documents/{pdf_file}" for pdf_file in new_files])
+        for result in results:
+            documents.extend(result)
+    vectorstore.add_documents(documents)
+
+# Generate answer for a query
+def get_answer(qa_chain, query, chat_history):
+    formatted_history = [(q, a) for q, a in zip(chat_history[::2], chat_history[1::2])]
+    
+    response = qa_chain.invoke({"question": query, "chat_history": formatted_history})
+    
+    answer = response["answer"]
+    
+    source_docs = response.get("source_documents", [])
+    source_texts = [f"{os.path.basename(doc.metadata['source'])} (Page {doc.metadata['page']})" for doc in source_docs]
+    
+    return {"answer": answer, "sources": source_texts}
+
+# Example usage
+if __name__ == "__main__":
+    update_embeddings()  # Update embeddings with new documents
+    qa_chain = load_retrieval_qa_chain()
+    question = """당신은 RAG(Retrieval-Augmented Generation) 기반 AI 어시스턴트입니다. 다음 지침을 따라 사용자 질문에 답하세요:
+
+1. 검색 결과 활용: 제공된 검색 결과를 분석하고 관련 정보를 사용해 답변하세요.
+
+2. 정확성 유지: 정보의 정확성을 확인하고, 불확실한 경우 이를 명시하세요.
+
+3. 간결한 응답: 질문에 직접 답하고 핵심 내용에 집중하세요.
+
+4. 추가 정보 제안: 관련된 추가 정보가 있다면 언급하세요.
+
+5. 윤리성 고려: 객관적이고 중립적인 태도를 유지하세요.
+
+6. 한계 인정: 답변할 수 없는 경우 솔직히 인정하세요.
+
+7. 대화 유지: 자연스럽게 대화를 이어가고, 필요시 후속 질문을 제안하세요.
+항상 정확하고 유용한 정보를 제공하는 것을 목표로 하세요."""
+
+    response = get_answer(qa_chain, question, [])
+    print(f"Question: {question}")
+    print(f"Answer: {response['answer']}")
+    print(f"Sources: {response['sources']}")

requirements.txt

@@ -0,0 +1,35 @@
+cffi==1.17.0
+charset_normalizer==3.3.2
+constructor==0.1.0
+Cython==3.0.11
+cython==3.0.11
+fitz==0.0.1.dev2
+flint==0.0.1
+gmpy2==2.2.1
+gradio==4.42.0
+hypothesis==6.111.2
+ipython==8.12.3
+langchain==0.2.15
+langchain_chroma==0.1.3
+langchain_openai==0.1.23
+lll==0.0.1
+mpmath==1.3.0
+mtrand==0.1
+mypy==1.11.2
+numba==0.60.0
+numeric==24.2
+olefile==0.47
+pdfplumber==0.5.28
+pip==24.2
+PyInstaller==6.10.0
+PyJWT==2.9.0
+PyPDF2==3.0.1
+PyQt6==6.7.1
+PySide6==6.7.2
+pytest==8.3.2
+python-dotenv==1.0.1
+pytz==2024.1
+setuptools==74.0.0
+streamlit==1.38.0
+threadpoolctl==3.5.0
+typing_extensions==4.12.2