feat(summarization): add document summarization

.gitignore

@@ -1 +1,2 @@
-.env
+.env
+__pycache__/

data/processed/Fulgoroidea2008-Mascareignes-AttiéBourgoinVeslotSoulier-summary.txt

@@ -0,0 +1 @@
+ This article examines the trophic relationships between planthoppers (Hemiptera: Fulgoromorpha) and their host plants on the Mascarene Islands. It was found that the endemic fauna remains essentially on endemic plants and exotic planthoppers have not shifted to them, remaining on exotic plants. Additionally, a positive correlation between endemic planthopper diversity and endemic plant diversity was observed. The article also discusses the disparities between Cixiidae from La Re´union and Mauritius, the host plant data of planthoppers, the biological and ecological data available for the planthoppers, the percentage of records on dicots families for each family of Fulgoromorpha, the effects of alien plant invasions, the cost of polyphagy, and the conservation and restoration of the flora of Mauritius and Rodrigues.

load.py

@@ -18,18 +18,22 @@ def parse_documents(path):
     documents = []
 
     for file_path in pdf_files:
-        try:
-            loader = UnstructuredFileLoader(file_path)
-            document = loader.load()
-            documents.extend(document)
-            print(f"File added: {file_path}")
-
-        except Exception as e:
-            print(f"An error occurred while processing the file {file_path}: {str(e)}")
+        documents.extend(parse_document(file_path))
 
     return documents
 
 
+def parse_document(file_path):
+    try:
+        loader = UnstructuredFileLoader(file_path)
+        document = loader.load()
+        print(f"File parsed: {file_path}")
+        return document
+
+    except Exception as e:
+        print(f"An error occurred while processing the file {file_path}: {str(e)}")
+
+
 def split(documents):
     text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=20)
     return text_splitter.split_documents(documents)

summarize.py

@@ -0,0 +1,47 @@
+from dotenv import load_dotenv
+from langchain import OpenAI
+from langchain.chains.summarize import load_summarize_chain
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from load import parse_document
+
+load_dotenv()
+
+DOCUMENT_PATH = "data/raw/cixiidae"
+
+
+llm = OpenAI(temperature=0)
+
+
+def summarize(raw_documents):
+    text_splitter = RecursiveCharacterTextSplitter(
+        separators=["\n\n", "\n"], chunk_size=6000, chunk_overlap=300
+    )
+    docs = text_splitter.split_documents(raw_documents)
+
+    num_docs = len(docs)
+    num_tokens_first_doc = llm.get_num_tokens(docs[0].page_content)
+    print(
+        f"Now we have {num_docs} documents and the first one has {num_tokens_first_doc} tokens"
+    )
+
+    summary_chain = load_summarize_chain(llm=llm, chain_type="map_reduce")
+    output = summary_chain.run(docs)
+
+    return output
+
+
+def main():
+    raw_documents = parse_document(
+        "data/raw/cixiidae/Fulgoroidea2008-Mascareignes-AttiéBourgoinVeslotSoulier.pdf"
+    )
+    output = summarize(raw_documents)
+    print(output)
+    with open(
+        "data/processed/Fulgoroidea2008-Mascareignes-AttiéBourgoinVeslotSoulier-summary.txt",
+        "w",
+    ) as f:
+        f.write(output)
+
+
+if __name__ == "__main__":
+    main()