download models from hub

app.py

@@ -8,6 +8,11 @@ import time
 from transformers import AutoTokenizer, AutoModel
 import torch.nn.functional as F
 from openai import AzureOpenAI
+from huggingface_hub import hf_hub_download
+
+# Download the FastText model from Hugging Face
+model_path_fasttext_raw = hf_hub_download(repo_id="talhasarit41/fasttext", filename="fasttext_raw.bin")
+model_path_fasttext_preprocessed = hf_hub_download(repo_id="talhasarit41/fasttext", filename="fasttext_preprocessed.bin")
 
 # Azure OpenAI Configuration
 AZURE_API_VERSION = "2024-02-01"
@@ -17,9 +22,9 @@ MODEL_DIR = "models"
 
 # Initialize Azure OpenAI client
 azure_client = AzureOpenAI(
-    api_key=os.getenv(AZURE_OPENAI_API_KEY),
+    api_key=os.getenv("AZURE_OPENAI_API_KEY"),
     api_version=AZURE_API_VERSION,
-    azure_endpoint=os.getenv(AZURE_OPENAI_EMBEDDING_ENDPOINT)
+    azure_endpoint=os.getenv("AZURE_OPENAI_EMBEDDING_ENDPOINT")
 )
 
 def generate_e5_embedding(text, model_name='intfloat/multilingual-e5-large'):
@@ -123,8 +128,8 @@ def load_models():
         models['GTE Classifier'] = pickle.load(f)
     
     # Load FastText models
-    models['FastText Raw'] = fasttext.load_model(os.path.join(MODEL_DIR, 'fasttext_raw.bin'))
-    models['FastText Preprocessed'] = fasttext.load_model(os.path.join(MODEL_DIR, 'fasttext_preprocessed.bin'))
+    models['FastText Raw'] = fasttext.load_model(model_path_fasttext_raw)
+    models['FastText Preprocessed'] = fasttext.load_model(model_path_fasttext_preprocessed)
     
     return models
 
@@ -177,7 +182,7 @@ def get_confidence_color(confidence):
         return "#ff4444"  # Bright red for low confidence
 
 # [Add GTE embedding generation function]
-def generate_gte_embedding(text, model_name='Alibaba-NLP/gte-base'):
+def generate_gte_embedding(text, model_name='Alibaba-NLP/gte-multilingual-base'):
     """Generate GTE embeddings for a single text."""
     start_time = time.time()
     tokenizer = AutoTokenizer.from_pretrained(model_name)