Update loaders/common.py

loaders/common.py

@@ -15,32 +15,43 @@ def process_file(vector_store, file, loader_class, file_suffix, stats_db=None):
         if file_size > 1000000:
             st.error("File size is too large. Please upload a file smaller than 1MB or self host.")
             return
-
+            
     dateshort = time.strftime("%Y%m%d")
     with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
         tmp_file.write(file.getvalue())
         tmp_file.flush()
-
         loader = loader_class(tmp_file.name)
         documents = loader.load()
         file_sha1 = compute_sha1_from_file(tmp_file.name)
-
     os.remove(tmp_file.name)
     
     chunk_size = st.session_state['chunk_size']
     chunk_overlap = st.session_state['chunk_overlap']
-
     text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
     
     documents = text_splitter.split_documents(documents)
-
     # Add the document sha1 as metadata to each document
     docs_with_metadata = [Document(page_content=doc.page_content, metadata={"file_sha1": file_sha1,"file_size":file_size ,"file_name": file_name, 
                                                                             "chunk_size": chunk_size, "chunk_overlap": chunk_overlap, "date": dateshort,
                                                                             "user" : st.session_state["username"]}) 
                           for doc in documents]
     
-    vector_store.add_documents(docs_with_metadata)
-    if stats_db:
-        add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,"file_type": file_suffix, 
-                                                           "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
+    try:
+        vector_store.add_documents(docs_with_metadata)
+        if stats_db:
+            add_usage(stats_db, "embedding", "file", metadata={"file_name": file_name,"file_type": file_suffix, 
+                                                             "chunk_size": chunk_size, "chunk_overlap": chunk_overlap})
+    except Exception as e:
+        print(f"Error adding documents to vector store:")
+        print(f"Exception: {str(e)}")
+        print(f"Input details:")
+        print(f"File name: {file_name}")
+        print(f"File size: {file_size}")
+        print(f"File SHA1: {file_sha1}")
+        print(f"Number of documents: {len(docs_with_metadata)}")
+        print(f"Chunk size: {chunk_size}")
+        print(f"Chunk overlap: {chunk_overlap}")
+        print(f"First document preview (truncated):")
+        if docs_with_metadata:
+            print(docs_with_metadata[0].page_content[:500])
+        raise  # Re-raise the exception after logging