Update main.py

main.py

@@ -24,6 +24,12 @@ LLM_MODEL_NAME = "Qwen/Qwen2-7B-Instruct"
 
 # Load the dataset and convert to pandas
 data = pandas.read_parquet("hf://datasets/ccm/rag-idetc/data/train-00000-of-00001.parquet")
+# Filter out any publications without an abstract
+title_is_empty = [
+    title is "" for title in data["title"].values
+]
+data = data[~pandas.Series(title_is_empty)]
+data.reset_index(inplace=True)\
 
 # Load the model for later use in embeddings
 model = sentence_transformers.SentenceTransformer(EMBEDDING_MODEL_NAME)