Update main.py

main.py

@@ -25,13 +25,6 @@ LLM_MODEL_NAME = "Qwen/Qwen2-7B-Instruct"
 # Load the dataset and convert to pandas
 data = pandas.read_parquet("hf://datasets/ccm/rag-idetc/data/train-00000-of-00001.parquet")
 
-# Filter out any publications without an abstract
-abstract_is_null = [
-    '"abstract": null' in json.dumps(bibdict) for bibdict in data["bib_dict"].values
-]
-data = data[~pandas.Series(abstract_is_null)]
-data.reset_index(inplace=True)
-
 # Load the model for later use in embeddings
 model = sentence_transformers.SentenceTransformer(EMBEDDING_MODEL_NAME)