added COOLDOWN_PERIOD

app.py

@@ -1,4 +1,5 @@
 import os
+import time
 from flask import Flask, jsonify, request
 from flask_cors import CORS 
 from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig  
@@ -9,11 +10,13 @@ os.environ["HF_HOME"] = "/workspace/huggingface_cache"  # Change this to a writa
 app = Flask(__name__)
 
 # Enable CORS for specific origins
-CORS(app, resources={r"api/predict/*": {"origins": ["http://localhost:3000", "https://main.dbn2ikif9ou3g.amplifyapp.com"]}})
+CORS(app, resources={r"/api/predict/*": {"origins": ["http://localhost:3000", "https://main.dbn2ikif9ou3g.amplifyapp.com"]}})
 
 # Global variables for model and tokenizer
 model = None
 tokenizer = None
+last_loaded_time = 0
+COOLDOWN_PERIOD = 300  # Set your cooldown period to 5 minutes (300 seconds)
 
 def get_model_and_tokenizer(model_id):
     global model, tokenizer
@@ -23,22 +26,29 @@ def get_model_and_tokenizer(model_id):
         tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False)
         tokenizer.pad_token = tokenizer.eos_token
         
-        print(f"Loading model and for model_id: {model_id}")
+        print(f"Loading model for model_id: {model_id}")
         # Load the model
-        model = AutoModelForCausalLM.from_pretrained(model_id) #, device_map="auto")
+        model = AutoModelForCausalLM.from_pretrained(model_id)  # , device_map="auto")
         model.config.use_cache = False
-   
+        print("Model loaded successfully!")
     except Exception as e:
         print(f"Error loading model: {e}")
 
+def is_model_loaded_and_fresh():
+    global last_loaded_time
+    current_time = time.time()
+    return model is not None and (current_time - last_loaded_time) < COOLDOWN_PERIOD
+
 def generate_response(user_input, model_id):
     prompt = formatted_prompt(user_input)
     
     global model, tokenizer
 
-    # Load the model and tokenizer if they are not already loaded or if the model_id has changed
-    if model is None or tokenizer is None or (model.config._name_or_path != model_id):
+    # Check if model is loaded and fresh
+    if not is_model_loaded_and_fresh():
         get_model_and_tokenizer(model_id)  # Load model and tokenizer
+        global last_loaded_time
+        last_loaded_time = time.time()  # Update the last load time
 
     # Prepare the input tensors
     inputs = tokenizer(prompt, return_tensors="pt")  # Move inputs to GPU if available