Muzammil6376 commited on
Commit
009e0ad
Β·
verified Β·
1 Parent(s): 31d7ac1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +31 -17
app.py CHANGED
@@ -44,23 +44,37 @@ vision_client = InferenceClient(model="Salesforce/blip2-opt-2.7b")
44
  # multimodal_client = InferenceClient(model="microsoft/DialoGPT-medium") # For conversational AI
45
  # multimodal_client = InferenceClient(model="facebook/opt-iml-max-30b") # For instruction following
46
 
47
- # ── Multimodal Embeddings ───────────────────────────────────────────────────
48
- # Primary: CLIP embeddings for excellent text-image alignment
49
- try:
50
- embeddings = HuggingFaceEmbeddings(
51
- model_name="sentence-transformers/clip-ViT-B-32",
52
- model_kwargs={'device': 'cpu'}, # Ensure CPU usage for HF Spaces
53
- encode_kwargs={'normalize_embeddings': True}
54
- )
55
- print("βœ… Using CLIP embeddings for multimodal support")
56
- except Exception as e:
57
- print(f"⚠️ CLIP failed, falling back to BGE: {e}")
58
- # Fallback to BGE embeddings
59
- embeddings = HuggingFaceEmbeddings(
60
- model_name="BAAI/bge-base-en-v1.5",
61
- model_kwargs={'device': 'cpu'},
62
- encode_kwargs={'normalize_embeddings': True}
63
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
 
65
  def create_multimodal_embeddings(text_chunks, image_descriptions):
66
  """
 
44
  # multimodal_client = InferenceClient(model="microsoft/DialoGPT-medium") # For conversational AI
45
  # multimodal_client = InferenceClient(model="facebook/opt-iml-max-30b") # For instruction following
46
 
47
+ # ── Open Source Multimodal Embeddings ──────────────────────────────────────
48
+ # Primary choices - all open source, no OpenAI dependency
49
+ embedding_models = [
50
+ "sentence-transformers/all-mpnet-base-v2", # Excellent general purpose
51
+ "BAAI/bge-large-en-v1.5", # Best Chinese model, great English
52
+ "intfloat/e5-large-v2", # Microsoft's open model
53
+ "sentence-transformers/all-MiniLM-L12-v2", # Good balance speed/quality
54
+ "BAAI/bge-base-en-v1.5" # Fallback option
55
+ ]
56
+
57
+ def initialize_embeddings():
58
+ """Initialize embeddings with fallback options"""
59
+ for model_name in embedding_models:
60
+ try:
61
+ embeddings = HuggingFaceEmbeddings(
62
+ model_name=model_name,
63
+ model_kwargs={'device': 'cpu', 'trust_remote_code': True},
64
+ encode_kwargs={'normalize_embeddings': True, 'batch_size': 16}
65
+ )
66
+ print(f"βœ… Successfully loaded: {model_name}")
67
+ return embeddings
68
+ except Exception as e:
69
+ print(f"⚠️ Failed to load {model_name}: {e}")
70
+ continue
71
+
72
+ # Ultimate fallback - should always work
73
+ print("πŸ”„ Using basic sentence-transformers model")
74
+ return HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
75
+
76
+ # Initialize embeddings
77
+ embeddings = initialize_embeddings()
78
 
79
  def create_multimodal_embeddings(text_chunks, image_descriptions):
80
  """