feat(*): make the whole project run locally

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 chroma filter=lfs diff=lfs merge=lfs -text
+data filter=lfs diff=lfs merge=lfs -text
+data/raw/cixiidae/*.pdf filter=lfs diff=lfs merge=lfs -text
+data/raw/cixiidae/Fulgoroidea1967-Galapagos-Fennah.PDF filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

README.md

@@ -10,3 +10,34 @@ pinned: false
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+### Desciption 
+
+This project use LLM to interact with documents describing fulgoroidea.
+
+### Quick start
+
+This project use poetry.
+
+To install packages
+
+```shell
+poetry install
+```
+
+To lauch costum env:
+```shell
+poetry shell
+```
+
+Then to lauch the app:
+
+```shell
+python app.py
+````
+
+App can be found at http://127.0.0.1:7860/
+
+
+

app.py

@@ -1,3 +1,8 @@
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
 from langchain.embeddings import OpenAIEmbeddings # for creating embeddings
 from langchain.vectorstores import Chroma # for the vectorization part
 from langchain.chains import ConversationalRetrievalChain

chroma/chroma-embeddings.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ccbf54215927a71ebb20809761fae745307f060abc93e49b48064750b5fcba1b
-size 45295396
+oid sha256:86ce62daaa8745aef49791108519965e063de1a1caa0611fefba9915330a8d38
+size 88840691

chroma/index/{id_to_uuid_63b0b7b3-7308-4629-ba5b-af235fc19082.pkl → id_to_uuid_62dbdb42-07cb-451b-94f8-85ab4e0a3aa9.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eba841025944f648bbeadabfcc703e1d3ba869e94bb188cf036f1476f41a3ff9
-size 115799
+oid sha256:38d1b4a85efa0adb626b6bef79faec65538f47da093d42fa4af15e6ad7bf8662
+size 116066

chroma/index/{index_63b0b7b3-7308-4629-ba5b-af235fc19082.bin → index_62dbdb42-07cb-451b-94f8-85ab4e0a3aa9.bin}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8aec36f0d959e3241c04693e550f7f26514c27894c58d9186e2b38d870f2079
-size 22465256
+oid sha256:9b312b85e2e2ba964451568d56a9235dd5f990451d8272ea5f2f72bda972c2d8
+size 22514608

chroma/index/{index_metadata_63b0b7b3-7308-4629-ba5b-af235fc19082.pkl → index_metadata_62dbdb42-07cb-451b-94f8-85ab4e0a3aa9.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:200b2c59698f3cd951e3fbb0c0be1adb214d5b471c5e9304e761f6ec8a4b546e
+oid sha256:08928359a065acd02acc0dd47b309a4a5cf7716dfad604edb06a3a52de955544
 size 74

chroma/index/{uuid_to_id_63b0b7b3-7308-4629-ba5b-af235fc19082.pkl → uuid_to_id_62dbdb42-07cb-451b-94f8-85ab4e0a3aa9.pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2261ce355c76b03fb84877e970454e3f80252fcb19f1491b9f3ddf1500e984eb
-size 135445
+oid sha256:92b9ed8b3ea549ce344b1171b5f50c3f636d33c7f58f6da5df0848c44e0f8cbb
+size 135749

data/raw/cixiidae/Fulgoroidea1906-FaunaBritishIndiaCelonBurma-Distant.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47374edb165ec953eceab55609918f6e4585c20960c42ad3f68374901c4930a3
+size 38324614

data/raw/cixiidae/Fulgoroidea1922-NewIndianHomoptera-Muir.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba42c7d09d4c5454270c50dc758edb295fccef72dbc8511670321256562c77a3
+size 3574936

data/raw/cixiidae/Fulgoroidea1934-NewLittleKnownFulgoroidea.Muir1934.pdf [missing#581, 587?].pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55dd9ceeefdcb184212b50a74760db0f067ebd24084f3221a846d24a9b67d232
+size 436216

data/raw/cixiidae/Fulgoroidea1942-HomoptèresChopardAfriqueOccidentale-Lallemand.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fdb1a26498d804939d71b7ab325b57be3f3b560d246dad3a8b2b816c4a9e71f
+size 188673

data/raw/cixiidae/Fulgoroidea1945-LanterfliesTrinidadSiuthAmerica-Fennah.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:000a43fb2e9d1560664f25b4a99038e2b0cb22b81e3be728198e981b13591a47
+size 62543484

data/raw/cixiidae/Fulgoroidea1945-PintaliaEquitosaAteson-Fennah.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50efdb4949fde4efeefc915512af757b2a59ba91b261c5fd6fbd105fc62edc00
+size 818526

data/raw/cixiidae/Fulgoroidea1952-FauneDeFrance-Ribaut .pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247c3656080830a0b64765b3bd793ca0d8bd7d892970dd36fa2a5ae8cdab0976
+size 43546668

data/raw/cixiidae/Fulgoroidea1954-CarolineIslands-Metcalf.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d59893291595ff0c2a3f33706e234d94c3fab35ddc0243a043e75a887ba0005a
+size 3327485

data/raw/cixiidae/Fulgoroidea1957-DieZikadenAfghanist-Dlabola.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94ad30ec6e163bf204c0ebe6093453509c6a9a552a12d9e3d7492f31af88607e
+size 11136356

data/raw/cixiidae/Fulgoroidea1958-MontNimba-Lallemand.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3b8395eb2aff419b9e660e1bc3026c0e9f48e7e3e9699d3d6f928834c242fd9
+size 621407

data/raw/cixiidae/Fulgoroidea1965-NewSpeciesWestIndies-Fennah.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:654a6efbbe8c6b743682a479cdf1b0a0d674d1b5a8e747d4a7e8eef192a912c4
+size 1557342

data/raw/cixiidae/Fulgoroidea1967-Galapagos-Fennah.PDF

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e6d6509361be6c9926f0f94d58e609fe882b8f290c8666df8e6466fcb50cac0
+size 1562170

data/raw/cixiidae/Fulgoroidea1967-New LittleKnownSouthAfrica-Fennah.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cb0394f0d7215d88fd2bc8eb4c52ec2a5d9cba3cb314a29d9bc8d68167c96c0
+size 2656903

data/raw/cixiidae/Fulgoroidea1969-NewCaledonia-Fennah.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dae6d0db9b5d3d1fa63a2d5d8b7e5c3e5a9723a6da8e6da479930d1b6a53c1d3
+size 12344035

data/raw/cixiidae/Fulgoroidea1982-ScientificResultsMountCameroonExpedition-VanStalle.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c09eecd6eeb8b6e746ce81d358faec03f6f2d7acd64d4d6b828834ec6932afb4
+size 5030611

data/raw/cixiidae/Fulgoroidea1985-EconomicInsectFaunaChina-ChouLuHuangWang.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:062434ec1f6eba2f1822a061aa25c731e25d80b66f8f88eb84d09159df926a35
+size 50565258

data/raw/cixiidae/Fulgoroidea1985-NewSynonymiesCombinationsNewWorldFulgoroidea-Obrien 1985 3491.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38fa7c85f06fe661e56979493b1df3dd0c142cbfcc4dc9a2f5d797bab1b24389
+size 763298

data/raw/cixiidae/Fulgoroidea2008-FulgoromorphaSeychellesPreliminaryChecklis-Holzinger-LöckerLöcker.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61dd802becbb724794a73e63b6fb38ba6bec931b199dfd5de67cc0a0fc6c43cc
+size 155971

data/raw/cixiidae/Fulgoroidea2008-Mascareignes-AttiéBourgoinVeslotSoulier.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b45f8118894317bfc092afdf4ecc91cbddcb6e00710725a875d5c293dd4957c5
+size 465630

example.env

@@ -0,0 +1 @@
+OPENAI_API_KEY=

load.py

@@ -0,0 +1,41 @@
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+from langchain.document_loaders import UnstructuredFileLoader # for loading the pdf
+from langchain.embeddings import OpenAIEmbeddings # for creating embeddings
+from langchain.vectorstores import Chroma # for the vectorization part
+from langchain.chains import ConversationalRetrievalChain
+from langchain.llms import OpenAI # the LLM model we'll use (CHatGPT)
+from langchain.text_splitter import CharacterTextSplitter
+from glob import glob
+import os
+
+DOCUMENT_PATH = "data/raw/cixiidae"
+DB_DIR = "chroma"
+
+pdf_files = glob(os.path.join(DOCUMENT_PATH, "*.pdf"))
+documents = []
+
+# Iterate through the list of PDF files
+for file_path in pdf_files:
+    try:
+        loader = UnstructuredFileLoader(file_path)
+        document = loader.load()
+        documents.extend(document)
+        print(f"File added: {file_path}")
+
+    except Exception as e:
+        print(f"An error occurred while processing the file {file_path}: {str(e)}")
+
+text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=20)
+documents = text_splitter.split_documents(documents)
+
+# Now, all_pages contains all the pages from every document
+print(f'Total pages: {len(documents)}')
+
+embeddings = OpenAIEmbeddings()
+vectordb = Chroma.from_documents(documents, embedding=embeddings,
+                                 persist_directory=DB_DIR)
+vectordb.persist()

pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "datak"
+version = "0.1.0"
+description = ""
+authors = ["LOUIS SANNA <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+openai = "^0.27.6"
+langchain = "^0.0.161"
+chromadb = "^0.3.21"
+gradio = "^3.28.3"
+python-dotenv = "^1.0.0"
+unstructured = "^0.6.3"
+tiktoken = "^0.4.0"
+pytesseract = "^0.3.10"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"