File size: 4,450 Bytes
4e00df7 8056ec2 b84dd14 f65663c b84dd14 f65663c b84dd14 f65663c b84dd14 f65663c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
import tempfile
import time
import os
from utils import compute_sha1_from_file
from langchain.schema import Document
import streamlit as st
from langchain.text_splitter import RecursiveCharacterTextSplitter
from stats import add_usage
def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
try:
print("=== Starting file processing ===")
print(f"Initial file details - Name: {file.name}, Size: {file.size}")
documents = []
file_name = file.name
file_size = file.size
if st.secrets.self_hosted == "false":
if file_size > 1000000:
st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
return
dateshort = time.strftime("%Y%m%d")
# Debug loading
print("=== Document Loading ===")
with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
tmp_file.write(file.getvalue())
tmp_file.flush()
print(f"Temporary file created: {tmp_file.name}")
loader = loader_class(tmp_file.name)
documents = loader.load()
print(f"Number of documents after loading: {len(documents)}")
print("First document content preview:")
if documents:
print(documents[0].page_content[:200])
file_sha1 = compute_sha1_from_file(tmp_file.name)
os.remove(tmp_file.name)
# Debug splitting
print("\n=== Document Splitting ===")
chunk_size = st.session_state['chunk_size']
chunk_overlap = st.session_state['chunk_overlap']
print(f"Splitting with chunk_size: {chunk_size}, overlap: {chunk_overlap}")
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
documents = text_splitter.split_documents(documents)
print(f"Number of documents after splitting: {len(documents)}")
# Debug metadata creation
print("\n=== Creating Documents with Metadata ===")
docs_with_metadata = []
for i, doc in enumerate(documents):
if isinstance(doc.page_content, str):
if "error" in doc.page_content.lower():
print(f"WARNING: Found potential error message in document {i}:")
print(doc.page_content[:200])
continue # Skip this document
new_doc = Document(
page_content=doc.page_content,
metadata={
"file_sha1": file_sha1,
"file_size": file_size,
"file_name": file_name,
"chunk_size": chunk_size,
"chunk_overlap": chunk_overlap,
"date": dateshort,
"user": st.session_state["username"]
}
)
docs_with_metadata.append(new_doc)
else:
print(f"WARNING: Document {i} has non-string content type: {type(doc.page_content)}")
print(f"Content: {str(doc.page_content)[:200]}")
print(f"Final number of documents to be added: {len(docs_with_metadata)}")
# Vector store addition
try:
vector_store.add_documents(docs_with_metadata)
if stats_db:
add_usage(stats_db, "embedding", "file", metadata={
"file_name": file_name,
"file_type": file_suffix,
"chunk_size": chunk_size,
"chunk_overlap": chunk_overlap
})
except Exception as e:
print(f"\n=== Vector Store Addition Error ===")
print(f"Exception: {str(e)}")
print(f"Input details:")
print(f"File name: {file_name}")
print(f"File size: {file_size}")
print(f"File SHA1: {file_sha1}")
print(f"Number of documents: {len(docs_with_metadata)}")
print(f"Vector store type: {type(vector_store).__name__}")
raise
except Exception as e:
print(f"\n=== General Processing Error ===")
print(f"Exception occurred during file processing: {str(e)}")
raise |