Upload 5 files

src/chunking_embedding.py

@@ -0,0 +1,30 @@
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter, TokenTextSplitter
+
+def setup_chunking_and_embedding(documents, chunking_strategy, chunk_size, chunk_overlap, embedding_model):
+    """Setup text chunking and embedding."""
+    embedding_models = {
+        'HuggingFace': HuggingFaceEmbeddings()
+    }
+    
+    embeddings = embedding_models.get(embedding_model)
+    if not embeddings:
+        raise ValueError("Unsupported embedding model.")
+
+    chunking_strategies = {
+        'Recursive': RecursiveCharacterTextSplitter,
+        'Character': CharacterTextSplitter,
+        'Token': TokenTextSplitter
+    }
+
+    text_splitter = chunking_strategies.get(chunking_strategy)
+    if not text_splitter:
+        raise ValueError("Unsupported chunking strategy.")
+    
+    splitter_instance = text_splitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    doc_chunks = splitter_instance.split_documents(documents)
+
+    if not doc_chunks:
+        raise ValueError("No document chunks created.")
+    
+    return doc_chunks, embeddings

src/document_loader.py

@@ -0,0 +1,26 @@
+import streamlit as st
+from langchain_community.document_loaders import (
+    PyPDFLoader,
+    PyMuPDFLoader,
+    PDFPlumberLoader,
+    PDFMinerLoader,
+    UnstructuredWordDocumentLoader,
+)
+
+def load_document(file_path, loader_type):
+    """Load document based on the selected loader type."""
+    loaders = {
+        'PyPDF': PyPDFLoader,
+        'PyMuPDF': PyMuPDFLoader,
+        'PDFPlumber': PDFPlumberLoader,
+        'PDFMiner': PDFMinerLoader,
+        'Unstructured Word': UnstructuredWordDocumentLoader
+    }
+    
+    loader = loaders.get(loader_type)
+    if not loader:
+        raise ValueError("Unsupported loader type.")
+    
+    documents = loader(file_path).load()
+    st.success(f"Number of documents loaded: {len(documents)}")
+    return documents

src/utils.py

@@ -0,0 +1,21 @@
+import json
+import os
+
+def log_interaction(question, response, params):
+    """Log interaction details to a JSON file."""
+    log_entry = {
+        "question": question,
+        "response": response,
+        "params": params
+    }
+    log_file_path = "interaction_log.json"
+    
+    logs = []
+    if os.path.exists(log_file_path):
+        with open(log_file_path, "r") as log_file:
+            logs = json.load(log_file)
+    
+    logs.append(log_entry)
+    
+    with open(log_file_path, "w") as log_file:
+        json.dump(logs, log_file, indent=4)

src/vector_store.py

@@ -0,0 +1,6 @@
+from langchain_community.vectorstores import FAISS
+
+def create_vectorstore(doc_chunks, embeddings):
+    """Create a FAISS vector store from document chunks."""
+    vectorstore = FAISS.from_documents(doc_chunks, embeddings)
+    return vectorstore