remove modernbert

app.py

@@ -73,22 +73,6 @@ def generate_e5_instruct_embedding(text, model_name='intfloat/multilingual-e5-la
     inference_time = time.time() - start_time
     return embeddings[0].numpy(), inference_time
 
-def generate_modernbert_embedding(text, model_name="answerdotai/ModernBERT-base"):
-    """Generate ModernBERT embeddings for a single text."""
-    start_time = time.time()
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModel.from_pretrained(model_name)
-    
-    # Tokenize and generate embedding
-    inputs = tokenizer(text, padding=True, truncation=True, return_tensors="pt")
-    with torch.no_grad():
-        outputs = model(**inputs)
-        # Take [CLS] token embedding
-        embeddings = outputs.last_hidden_state[:, 0, :]
-    
-    inference_time = time.time() - start_time
-    return embeddings[0].numpy(), inference_time
-
 def mean_pooling(token_embeddings, attention_mask):
     """Mean pooling function for E5 models."""
     input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
@@ -121,9 +105,6 @@ def load_models():
     with open(os.path.join(MODEL_DIR, 'azure_knn_classifier.pkl'), 'rb') as f:
         models['Azure KNN Classifier'] = pickle.load(f)
     
-    with open(os.path.join(MODEL_DIR, 'modernbert_rf_classifier.pkl'), 'rb') as f:
-        models['ModernBERT RF Classifier'] = pickle.load(f)
-
     with open(os.path.join(MODEL_DIR, 'gte_classifier.pkl'), 'rb') as f:
         models['GTE Classifier'] = pickle.load(f)
     
@@ -265,26 +246,8 @@ def predict_text_streaming(text):
             })
             yield format_progress(70, f"Completed {model_name}"), format_results(results)
         
-        # Process ModernBERT model
-        yield format_progress(80, "Processing ModernBERT RF Classifier..."), format_results(results)
-        modernbert_embedding, embed_time = generate_modernbert_embedding(text)
-        model = models['ModernBERT RF Classifier']
-        embedding_2d = modernbert_embedding.reshape(1, -1)
-        prediction = model.predict(embedding_2d)[0]
-        probabilities = model.predict_proba(embedding_2d)[0]
-        confidence = max(probabilities)
-        inference_time = time.time() - start_time
-        
-        results.append({
-            'model': 'ModernBERT RF Classifier',
-            'prediction': prediction,
-            'confidence': confidence,
-            'time': inference_time + embed_time
-        })
-        yield format_progress(90, "Completed ModernBERT RF Classifier"), format_results(results)
-
         # Process GTE model
-        yield format_progress(95, "Processing GTE Classifier..."), format_results(results)
+        yield format_progress(90, "Processing GTE Classifier..."), format_results(results)
         gte_embedding, embed_time = generate_gte_embedding(text)
         model = models['GTE Classifier']
         embedding_2d = gte_embedding.reshape(1, -1)