import tempfile import time import os from utils import compute_sha1_from_file from langchain.schema import Document import streamlit as st from langchain.text_splitter import RecursiveCharacterTextSplitter from stats import add_usage def process_file(vector_store, file, loader_class, file_suffix, stats_db=None): try: print("=== Starting file processing ===") print(f"Initial file details - Name: {file.name}, Size: {file.size}") documents = [] file_name = file.name file_size = file.size if st.secrets.self_hosted == "false": if file_size > 1000000: st.error("File size is too large. Please upload a file smaller than 1MB or self host.") return dateshort = time.strftime("%Y%m%d") # Debug loading print("=== Document Loading ===") with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file: tmp_file.write(file.getvalue()) tmp_file.flush() print(f"Temporary file created: {tmp_file.name}") loader = loader_class(tmp_file.name) documents = loader.load() print(f"Number of documents after loading: {len(documents)}") print("First document content preview:") if documents: print(documents[0].page_content[:200]) file_sha1 = compute_sha1_from_file(tmp_file.name) os.remove(tmp_file.name) # Debug splitting print("\n=== Document Splitting ===") chunk_size = st.session_state['chunk_size'] chunk_overlap = st.session_state['chunk_overlap'] print(f"Splitting with chunk_size: {chunk_size}, overlap: {chunk_overlap}") text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap) documents = text_splitter.split_documents(documents) print(f"Number of documents after splitting: {len(documents)}") # Debug metadata creation print("\n=== Creating Documents with Metadata ===") docs_with_metadata = [] for i, doc in enumerate(documents): if isinstance(doc.page_content, str): if "error" in doc.page_content.lower(): print(f"WARNING: Found potential error message in document {i}:") print(doc.page_content[:200]) continue # Skip this document new_doc = Document( page_content=doc.page_content, metadata={ "file_sha1": file_sha1, "file_size": file_size, "file_name": file_name, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort, "user": st.session_state["username"] } ) docs_with_metadata.append(new_doc) else: print(f"WARNING: Document {i} has non-string content type: {type(doc.page_content)}") print(f"Content: {str(doc.page_content)[:200]}") print(f"Final number of documents to be added: {len(docs_with_metadata)}") # Vector store addition try: vector_store.add_documents(docs_with_metadata) if stats_db: add_usage(stats_db, "embedding", "file", metadata={ "file_name": file_name, "file_type": file_suffix, "chunk_size": chunk_size, "chunk_overlap": chunk_overlap }) except Exception as e: print(f"\n=== Vector Store Addition Error ===") print(f"Exception: {str(e)}") print(f"Input details:") print(f"File name: {file_name}") print(f"File size: {file_size}") print(f"File SHA1: {file_sha1}") print(f"Number of documents: {len(docs_with_metadata)}") print(f"Vector store type: {type(vector_store).__name__}") raise except Exception as e: print(f"\n=== General Processing Error ===") print(f"Exception occurred during file processing: {str(e)}") raise