Update pages/Подбор фильмов по описанию✏️🔍.py

pages/Подбор фильмов по описанию✏️🔍.py

@@ -5,6 +5,37 @@ from transformers import AutoTokenizer, AutoModel
 import faiss
 import numpy as np
 
+@st.cache_data
+def load_data(url):
+    df = pd.read_csv(url)  # 👈 Download the data
+    return df
+
+df = load_data('data/final_data.csv')
+
+@st.cache_data
+def embedding_and_index():
+    embeddings_array = np.load('data/embeddings_final.npy')
+    index = faiss.read_index('data/desc_faiss_index_final.index')
+    return(embeddings_array, index)
+
+embeddings_array, index = embedding_and_index()
+     
+@st.cache_resource
+def load_tokenizer_and_model():
+    tokenizer = AutoTokenizer.from_pretrained("DeepPavlov/rubert-base-cased-sentence")
+    model = AutoModel.from_pretrained("DeepPavlov/rubert-base-cased-sentence")
+    return tokenizer, model
+    
+tokenizer, model = load_tokenizer_and_model()
+
+@st.cache_resource
+def encode_description(description, tokenizer, model):
+    tokens = tokenizer(description, return_tensors="pt")
+    with torch.no_grad():
+        outputs = model(**tokens)
+    embeddings = outputs.last_hidden_state.mean(dim=1)
+    return embeddings.cpu().numpy().astype('float32')
+
 st.header("Подбор фильмов по описанию ✏️🔍")