loaders/common.py

import tempfile
import time 
import os
from utils import compute_sha1_from_file
from langchain.schema import Document
import streamlit as st
from langchain.text_splitter import RecursiveCharacterTextSplitter
from stats import add_usage

def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
    try:
        print("=== Starting file processing ===")
        print(f"Initial file details - Name: {file.name}, Size: {file.size}")
        
        documents = []
        file_name = file.name
        file_size = file.size
        if st.secrets.self_hosted == "false":
            if file_size > 1000000:
                st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
                return
                
        dateshort = time.strftime("%Y%m%d")
        
        # Debug loading
        print("=== Document Loading ===")
        with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
            tmp_file.write(file.getvalue())
            tmp_file.flush()
            print(f"Temporary file created: {tmp_file.name}")
            
            loader = loader_class(tmp_file.name)
            documents = loader.load()
            print(f"Number of documents after loading: {len(documents)}")
            print("First document content preview:")
            if documents:
                print(documents[0].page_content[:200])
            
            file_sha1 = compute_sha1_from_file(tmp_file.name)
        os.remove(tmp_file.name)
        
        # Debug splitting
        print("\n=== Document Splitting ===")
        chunk_size = st.session_state['chunk_size']
        chunk_overlap = st.session_state['chunk_overlap']
        print(f"Splitting with chunk_size: {chunk_size}, overlap: {chunk_overlap}")
        
        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
        documents = text_splitter.split_documents(documents)
        print(f"Number of documents after splitting: {len(documents)}")
        
        # Debug metadata creation
        print("\n=== Creating Documents with Metadata ===")
        docs_with_metadata = []
        for i, doc in enumerate(documents):
            if isinstance(doc.page_content, str):
                if "error" in doc.page_content.lower():
                    print(f"WARNING: Found potential error message in document {i}:")
                    print(doc.page_content[:200])
                    continue  # Skip this document
                    
                new_doc = Document(
                    page_content=doc.page_content,
                    metadata={
                        "file_sha1": file_sha1,
                        "file_size": file_size,
                        "file_name": file_name, 
                        "chunk_size": chunk_size,
                        "chunk_overlap": chunk_overlap,
                        "date": dateshort,
                        "user": st.session_state["username"]
                    }
                )
                docs_with_metadata.append(new_doc)
            else:
                print(f"WARNING: Document {i} has non-string content type: {type(doc.page_content)}")
                print(f"Content: {str(doc.page_content)[:200]}")
        
        print(f"Final number of documents to be added: {len(docs_with_metadata)}")
        
        # Vector store addition
        try:
            vector_store.add_documents(docs_with_metadata)
            if stats_db:
                add_usage(stats_db, "embedding", "file", metadata={
                    "file_name": file_name,
                    "file_type": file_suffix, 
                    "chunk_size": chunk_size,
                    "chunk_overlap": chunk_overlap
                })
        except Exception as e:
            print(f"\n=== Vector Store Addition Error ===")
            print(f"Exception: {str(e)}")
            print(f"Input details:")
            print(f"File name: {file_name}")
            print(f"File size: {file_size}")
            print(f"File SHA1: {file_sha1}")
            print(f"Number of documents: {len(docs_with_metadata)}")
            print(f"Vector store type: {type(vector_store).__name__}")
            raise

    except Exception as e:
        print(f"\n=== General Processing Error ===")
        print(f"Exception occurred during file processing: {str(e)}")
        raise