Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -44,23 +44,37 @@ vision_client = InferenceClient(model="Salesforce/blip2-opt-2.7b")
|
|
44 |
# multimodal_client = InferenceClient(model="microsoft/DialoGPT-medium") # For conversational AI
|
45 |
# multimodal_client = InferenceClient(model="facebook/opt-iml-max-30b") # For instruction following
|
46 |
|
47 |
-
# ββ Multimodal Embeddings
|
48 |
-
# Primary
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
64 |
|
65 |
def create_multimodal_embeddings(text_chunks, image_descriptions):
|
66 |
"""
|
|
|
44 |
# multimodal_client = InferenceClient(model="microsoft/DialoGPT-medium") # For conversational AI
|
45 |
# multimodal_client = InferenceClient(model="facebook/opt-iml-max-30b") # For instruction following
|
46 |
|
47 |
+
# ββ Open Source Multimodal Embeddings ββββββββββββββββββββββββββββββββββββββ
|
48 |
+
# Primary choices - all open source, no OpenAI dependency
|
49 |
+
embedding_models = [
|
50 |
+
"sentence-transformers/all-mpnet-base-v2", # Excellent general purpose
|
51 |
+
"BAAI/bge-large-en-v1.5", # Best Chinese model, great English
|
52 |
+
"intfloat/e5-large-v2", # Microsoft's open model
|
53 |
+
"sentence-transformers/all-MiniLM-L12-v2", # Good balance speed/quality
|
54 |
+
"BAAI/bge-base-en-v1.5" # Fallback option
|
55 |
+
]
|
56 |
+
|
57 |
+
def initialize_embeddings():
|
58 |
+
"""Initialize embeddings with fallback options"""
|
59 |
+
for model_name in embedding_models:
|
60 |
+
try:
|
61 |
+
embeddings = HuggingFaceEmbeddings(
|
62 |
+
model_name=model_name,
|
63 |
+
model_kwargs={'device': 'cpu', 'trust_remote_code': True},
|
64 |
+
encode_kwargs={'normalize_embeddings': True, 'batch_size': 16}
|
65 |
+
)
|
66 |
+
print(f"β
Successfully loaded: {model_name}")
|
67 |
+
return embeddings
|
68 |
+
except Exception as e:
|
69 |
+
print(f"β οΈ Failed to load {model_name}: {e}")
|
70 |
+
continue
|
71 |
+
|
72 |
+
# Ultimate fallback - should always work
|
73 |
+
print("π Using basic sentence-transformers model")
|
74 |
+
return HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
|
75 |
+
|
76 |
+
# Initialize embeddings
|
77 |
+
embeddings = initialize_embeddings()
|
78 |
|
79 |
def create_multimodal_embeddings(text_chunks, image_descriptions):
|
80 |
"""
|