nltk

find.py

@@ -3,7 +3,7 @@ import pandas as pd
 from transformers import AutoTokenizer, AutoModel,BertTokenizer,BertModel
 import numpy as np
 import pickle
-# import sklearn
+import nltk
 from nltk.stem import WordNetLemmatizer
 from nltk.tag import pos_tag
 from nltk.corpus import stopwords
@@ -14,9 +14,13 @@ import faiss
 from tqdm import tqdm
 tokenizer =  AutoTokenizer.from_pretrained("cointegrated/rubert-tiny2")
 model = AutoModel.from_pretrained("cointegrated/rubert-tiny2")
+
+nltk.download('stopwords')
+
 eng_stop_words = stopwords.words('english')
 with open('russian.txt', 'r') as f:
     ru_stop_words = f.read()
+
 ru_stop_words=ru_stop_words.split('\n')
 allow="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдеёжзийклмнопрстуфхцчшщъыьэюя0123456789-' \n\t"
 #Задаём стеммер