Spaces:
Sleeping
Sleeping
Delete build_retriever.py
Browse files- build_retriever.py +0 -21
build_retriever.py
DELETED
@@ -1,21 +0,0 @@
|
|
1 |
-
# build_retriever.py
|
2 |
-
|
3 |
-
import faiss
|
4 |
-
import json
|
5 |
-
from sentence_transformers import SentenceTransformer
|
6 |
-
|
7 |
-
# Load the extracted text
|
8 |
-
with open("apexcustoms.json", "r") as f:
|
9 |
-
documents = json.load(f)
|
10 |
-
|
11 |
-
# Load a pre-trained sentence transformer model
|
12 |
-
model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
|
13 |
-
document_embeddings = model.encode(documents)
|
14 |
-
|
15 |
-
# Create a FAISS index
|
16 |
-
index = faiss.IndexFlatL2(document_embeddings.shape[1])
|
17 |
-
index.add(document_embeddings)
|
18 |
-
|
19 |
-
# Save the FAISS index and the embeddings
|
20 |
-
faiss.write_index(index, "apexcustoms_index.faiss")
|
21 |
-
model.save("sentence_transformer_model")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|