Spaces:
Running
Running
mini mods after rebase
Browse files
examples/LynxScribe Image RAG
CHANGED
@@ -21,6 +21,13 @@
|
|
21 |
"target": "LynxScribe Image RAG Query 1",
|
22 |
"targetHandle": "text"
|
23 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
{
|
25 |
"id": "LynxScribe Image Describer 1 LynxScribe Image RAG Builder 1",
|
26 |
"source": "LynxScribe Image Describer 1",
|
@@ -34,13 +41,6 @@
|
|
34 |
"sourceHandle": "output",
|
35 |
"target": "LynxScribe Image RAG Builder 1",
|
36 |
"targetHandle": "rag_graph"
|
37 |
-
},
|
38 |
-
{
|
39 |
-
"id": "LynxScribe Image RAG Query 1 View image 1",
|
40 |
-
"source": "LynxScribe Image RAG Query 1",
|
41 |
-
"sourceHandle": "output",
|
42 |
-
"target": "View image 1",
|
43 |
-
"targetHandle": "embedding_similarities"
|
44 |
}
|
45 |
],
|
46 |
"env": "LynxScribe",
|
@@ -147,10 +147,6 @@
|
|
147 |
}
|
148 |
}
|
149 |
},
|
150 |
-
"position": {
|
151 |
-
"x": 979.0,
|
152 |
-
"y": 238.0
|
153 |
-
},
|
154 |
"type": "basic"
|
155 |
},
|
156 |
"params": {},
|
@@ -170,7 +166,7 @@
|
|
170 |
{
|
171 |
"data": {
|
172 |
"__execution_delay": 0.0,
|
173 |
-
"collapsed":
|
174 |
"display": null,
|
175 |
"error": null,
|
176 |
"meta": {
|
@@ -194,14 +190,10 @@
|
|
194 |
}
|
195 |
}
|
196 |
},
|
197 |
-
"position": {
|
198 |
-
"x": 1336.0,
|
199 |
-
"y": 378.0
|
200 |
-
},
|
201 |
"type": "basic"
|
202 |
},
|
203 |
"params": {
|
204 |
-
"chat": "Show me a picture about
|
205 |
},
|
206 |
"status": "done",
|
207 |
"title": "Input chat"
|
@@ -210,8 +202,8 @@
|
|
210 |
"height": 214.0,
|
211 |
"id": "Input chat 1",
|
212 |
"position": {
|
213 |
-
"x": -
|
214 |
-
"y": -
|
215 |
},
|
216 |
"type": "basic",
|
217 |
"width": 387.0
|
@@ -258,10 +250,6 @@
|
|
258 |
}
|
259 |
}
|
260 |
},
|
261 |
-
"position": {
|
262 |
-
"x": 1419.0,
|
263 |
-
"y": 371.0
|
264 |
-
},
|
265 |
"type": "basic"
|
266 |
},
|
267 |
"params": {
|
@@ -280,6 +268,41 @@
|
|
280 |
"type": "basic",
|
281 |
"width": 200.0
|
282 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
283 |
{
|
284 |
"data": {
|
285 |
"display": null,
|
@@ -297,13 +320,6 @@
|
|
297 |
}
|
298 |
},
|
299 |
"params": {
|
300 |
-
"api_key_name": {
|
301 |
-
"default": "OPENAI_API_KEY",
|
302 |
-
"name": "api_key_name",
|
303 |
-
"type": {
|
304 |
-
"type": "<class 'str'>"
|
305 |
-
}
|
306 |
-
},
|
307 |
"llm_interface": {
|
308 |
"default": "openai",
|
309 |
"name": "llm_interface",
|
@@ -319,7 +335,7 @@
|
|
319 |
}
|
320 |
},
|
321 |
"llm_prompt_path": {
|
322 |
-
"default": "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
|
323 |
"name": "llm_prompt_path",
|
324 |
"type": {
|
325 |
"type": "<class 'str'>"
|
@@ -334,30 +350,29 @@
|
|
334 |
}
|
335 |
},
|
336 |
"position": {
|
337 |
-
"x":
|
338 |
-
"y":
|
339 |
},
|
340 |
"type": "basic"
|
341 |
},
|
342 |
"params": {
|
343 |
-
"api_key_name": "OPENAI_API_KEY",
|
344 |
"llm_interface": "openai",
|
345 |
"llm_prompt_name": "cot_picture_descriptor",
|
346 |
-
"llm_prompt_path": "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
|
347 |
"llm_visual_model": "gpt-4o"
|
348 |
},
|
349 |
"status": "done",
|
350 |
"title": "LynxScribe Image Describer"
|
351 |
},
|
352 |
"dragHandle": ".bg-primary",
|
353 |
-
"height":
|
354 |
"id": "LynxScribe Image Describer 1",
|
355 |
"position": {
|
356 |
-
"x":
|
357 |
-
"y":
|
358 |
},
|
359 |
"type": "basic",
|
360 |
-
"width":
|
361 |
},
|
362 |
{
|
363 |
"data": {
|
@@ -376,13 +391,6 @@
|
|
376 |
}
|
377 |
},
|
378 |
"params": {
|
379 |
-
"api_key_name": {
|
380 |
-
"default": "OPENAI_API_KEY",
|
381 |
-
"name": "api_key_name",
|
382 |
-
"type": {
|
383 |
-
"type": "<class 'str'>"
|
384 |
-
}
|
385 |
-
},
|
386 |
"collection_name": {
|
387 |
"default": "lynx",
|
388 |
"name": "collection_name",
|
@@ -420,78 +428,30 @@
|
|
420 |
}
|
421 |
},
|
422 |
"position": {
|
423 |
-
"x":
|
424 |
-
"y":
|
425 |
},
|
426 |
"type": "basic"
|
427 |
},
|
428 |
"params": {
|
429 |
-
"api_key_name": "OPENAI_API_KEY",
|
430 |
"collection_name": "lynx",
|
431 |
"name": "faiss",
|
432 |
"num_dimensions": 3072.0,
|
433 |
"text_embedder_interface": "openai",
|
434 |
"text_embedder_model_name_or_path": "text-embedding-3-large"
|
435 |
},
|
436 |
-
"status": "
|
437 |
"title": "LynxScribe RAG Vector Store"
|
438 |
},
|
439 |
"dragHandle": ".bg-primary",
|
440 |
-
"height":
|
441 |
"id": "LynxScribe RAG Vector Store 1",
|
442 |
"position": {
|
443 |
-
"x":
|
444 |
-
"y":
|
445 |
},
|
446 |
"type": "basic",
|
447 |
-
"width":
|
448 |
-
},
|
449 |
-
{
|
450 |
-
"data": {
|
451 |
-
"__execution_delay": null,
|
452 |
-
"collapsed": false,
|
453 |
-
"display": null,
|
454 |
-
"error": null,
|
455 |
-
"meta": {
|
456 |
-
"inputs": {
|
457 |
-
"embedding_similarities": {
|
458 |
-
"name": "embedding_similarities",
|
459 |
-
"position": "left",
|
460 |
-
"type": {
|
461 |
-
"type": "<class 'inspect._empty'>"
|
462 |
-
}
|
463 |
-
}
|
464 |
-
},
|
465 |
-
"name": "View image",
|
466 |
-
"outputs": {},
|
467 |
-
"params": {
|
468 |
-
"idx": {
|
469 |
-
"default": 0.0,
|
470 |
-
"name": "idx",
|
471 |
-
"type": {
|
472 |
-
"type": "<class 'int'>"
|
473 |
-
}
|
474 |
-
}
|
475 |
-
},
|
476 |
-
"position": {
|
477 |
-
"x": 1418.0,
|
478 |
-
"y": 272.0
|
479 |
-
},
|
480 |
-
"type": "table_view"
|
481 |
-
},
|
482 |
-
"params": {},
|
483 |
-
"status": "done",
|
484 |
-
"title": "View image"
|
485 |
-
},
|
486 |
-
"dragHandle": ".bg-primary",
|
487 |
-
"height": 342.0,
|
488 |
-
"id": "View image 1",
|
489 |
-
"position": {
|
490 |
-
"x": 1424.7240705678887,
|
491 |
-
"y": -202.17696978464585
|
492 |
-
},
|
493 |
-
"type": "table_view",
|
494 |
-
"width": 652.0
|
495 |
}
|
496 |
]
|
497 |
}
|
|
|
21 |
"target": "LynxScribe Image RAG Query 1",
|
22 |
"targetHandle": "text"
|
23 |
},
|
24 |
+
{
|
25 |
+
"id": "LynxScribe Image RAG Query 1 View image 1",
|
26 |
+
"source": "LynxScribe Image RAG Query 1",
|
27 |
+
"sourceHandle": "output",
|
28 |
+
"target": "View image 1",
|
29 |
+
"targetHandle": "embedding_similarities"
|
30 |
+
},
|
31 |
{
|
32 |
"id": "LynxScribe Image Describer 1 LynxScribe Image RAG Builder 1",
|
33 |
"source": "LynxScribe Image Describer 1",
|
|
|
41 |
"sourceHandle": "output",
|
42 |
"target": "LynxScribe Image RAG Builder 1",
|
43 |
"targetHandle": "rag_graph"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
44 |
}
|
45 |
],
|
46 |
"env": "LynxScribe",
|
|
|
147 |
}
|
148 |
}
|
149 |
},
|
|
|
|
|
|
|
|
|
150 |
"type": "basic"
|
151 |
},
|
152 |
"params": {},
|
|
|
166 |
{
|
167 |
"data": {
|
168 |
"__execution_delay": 0.0,
|
169 |
+
"collapsed": false,
|
170 |
"display": null,
|
171 |
"error": null,
|
172 |
"meta": {
|
|
|
190 |
}
|
191 |
}
|
192 |
},
|
|
|
|
|
|
|
|
|
193 |
"type": "basic"
|
194 |
},
|
195 |
"params": {
|
196 |
+
"chat": "Show me a picture about healthy life"
|
197 |
},
|
198 |
"status": "done",
|
199 |
"title": "Input chat"
|
|
|
202 |
"height": 214.0,
|
203 |
"id": "Input chat 1",
|
204 |
"position": {
|
205 |
+
"x": -302.70349900314835,
|
206 |
+
"y": -140.67386384008685
|
207 |
},
|
208 |
"type": "basic",
|
209 |
"width": 387.0
|
|
|
250 |
}
|
251 |
}
|
252 |
},
|
|
|
|
|
|
|
|
|
253 |
"type": "basic"
|
254 |
},
|
255 |
"params": {
|
|
|
268 |
"type": "basic",
|
269 |
"width": 200.0
|
270 |
},
|
271 |
+
{
|
272 |
+
"data": {
|
273 |
+
"__execution_delay": null,
|
274 |
+
"collapsed": false,
|
275 |
+
"display": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test/food-405521_1280.jpg",
|
276 |
+
"error": null,
|
277 |
+
"meta": {
|
278 |
+
"inputs": {
|
279 |
+
"embedding_similarities": {
|
280 |
+
"name": "embedding_similarities",
|
281 |
+
"position": "left",
|
282 |
+
"type": {
|
283 |
+
"type": "<class 'inspect._empty'>"
|
284 |
+
}
|
285 |
+
}
|
286 |
+
},
|
287 |
+
"name": "View image",
|
288 |
+
"outputs": {},
|
289 |
+
"params": {},
|
290 |
+
"type": "image"
|
291 |
+
},
|
292 |
+
"params": {},
|
293 |
+
"status": "done",
|
294 |
+
"title": "View image"
|
295 |
+
},
|
296 |
+
"dragHandle": ".bg-primary",
|
297 |
+
"height": 1170.0,
|
298 |
+
"id": "View image 1",
|
299 |
+
"position": {
|
300 |
+
"x": 1426.7020124006506,
|
301 |
+
"y": -293.16229409169125
|
302 |
+
},
|
303 |
+
"type": "image",
|
304 |
+
"width": 750.0
|
305 |
+
},
|
306 |
{
|
307 |
"data": {
|
308 |
"display": null,
|
|
|
320 |
}
|
321 |
},
|
322 |
"params": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
323 |
"llm_interface": {
|
324 |
"default": "openai",
|
325 |
"name": "llm_interface",
|
|
|
335 |
}
|
336 |
},
|
337 |
"llm_prompt_path": {
|
338 |
+
"default": "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
|
339 |
"name": "llm_prompt_path",
|
340 |
"type": {
|
341 |
"type": "<class 'str'>"
|
|
|
350 |
}
|
351 |
},
|
352 |
"position": {
|
353 |
+
"x": 1066.0,
|
354 |
+
"y": 713.0
|
355 |
},
|
356 |
"type": "basic"
|
357 |
},
|
358 |
"params": {
|
|
|
359 |
"llm_interface": "openai",
|
360 |
"llm_prompt_name": "cot_picture_descriptor",
|
361 |
+
"llm_prompt_path": "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
|
362 |
"llm_visual_model": "gpt-4o"
|
363 |
},
|
364 |
"status": "done",
|
365 |
"title": "LynxScribe Image Describer"
|
366 |
},
|
367 |
"dragHandle": ".bg-primary",
|
368 |
+
"height": 363.0,
|
369 |
"id": "LynxScribe Image Describer 1",
|
370 |
"position": {
|
371 |
+
"x": 16.941021961247145,
|
372 |
+
"y": 664.8693601063997
|
373 |
},
|
374 |
"type": "basic",
|
375 |
+
"width": 401.0
|
376 |
},
|
377 |
{
|
378 |
"data": {
|
|
|
391 |
}
|
392 |
},
|
393 |
"params": {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
394 |
"collection_name": {
|
395 |
"default": "lynx",
|
396 |
"name": "collection_name",
|
|
|
428 |
}
|
429 |
},
|
430 |
"position": {
|
431 |
+
"x": 831.0,
|
432 |
+
"y": 734.0
|
433 |
},
|
434 |
"type": "basic"
|
435 |
},
|
436 |
"params": {
|
|
|
437 |
"collection_name": "lynx",
|
438 |
"name": "faiss",
|
439 |
"num_dimensions": 3072.0,
|
440 |
"text_embedder_interface": "openai",
|
441 |
"text_embedder_model_name_or_path": "text-embedding-3-large"
|
442 |
},
|
443 |
+
"status": "done",
|
444 |
"title": "LynxScribe RAG Vector Store"
|
445 |
},
|
446 |
"dragHandle": ".bg-primary",
|
447 |
+
"height": 445.0,
|
448 |
"id": "LynxScribe RAG Vector Store 1",
|
449 |
"position": {
|
450 |
+
"x": 485.48864677068343,
|
451 |
+
"y": 669.7387202127995
|
452 |
},
|
453 |
"type": "basic",
|
454 |
+
"width": 442.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
455 |
}
|
456 |
]
|
457 |
}
|
lynxkite-lynxscribe/src/lynxkite_lynxscribe/lynxscribe_ops.py
CHANGED
@@ -66,7 +66,7 @@ def gcp_image_loader(
|
|
66 |
|
67 |
@output_on_top
|
68 |
@op("LynxScribe RAG Vector Store")
|
69 |
-
|
70 |
def ls_rag_graph(
|
71 |
*,
|
72 |
name: str = "faiss",
|
@@ -74,7 +74,7 @@ def ls_rag_graph(
|
|
74 |
collection_name: str = "lynx",
|
75 |
text_embedder_interface: str = "openai",
|
76 |
text_embedder_model_name_or_path: str = "text-embedding-3-large",
|
77 |
-
api_key_name: str = "OPENAI_API_KEY",
|
78 |
):
|
79 |
"""
|
80 |
Returns with a vector store instance.
|
@@ -82,8 +82,8 @@ def ls_rag_graph(
|
|
82 |
|
83 |
# getting the text embedder instance
|
84 |
llm_params = {"name": text_embedder_interface}
|
85 |
-
if api_key_name:
|
86 |
-
|
87 |
llm = get_llm_engine(**llm_params)
|
88 |
text_embedder = TextEmbedder(llm=llm, model=text_embedder_model_name_or_path)
|
89 |
|
@@ -105,14 +105,14 @@ def ls_rag_graph(
|
|
105 |
|
106 |
@output_on_top
|
107 |
@op("LynxScribe Image Describer")
|
108 |
-
|
109 |
def ls_image_describer(
|
110 |
*,
|
111 |
llm_interface: str = "openai",
|
112 |
llm_visual_model: str = "gpt-4o",
|
113 |
-
llm_prompt_path: str = "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
|
114 |
llm_prompt_name: str = "cot_picture_descriptor",
|
115 |
-
api_key_name: str = "OPENAI_API_KEY",
|
116 |
):
|
117 |
"""
|
118 |
Returns with an image describer instance.
|
@@ -120,8 +120,8 @@ def ls_image_describer(
|
|
120 |
"""
|
121 |
|
122 |
llm_params = {"name": llm_interface}
|
123 |
-
if api_key_name:
|
124 |
-
|
125 |
llm = get_llm_engine(**llm_params)
|
126 |
|
127 |
prompt_base = load_config(llm_prompt_path)[llm_prompt_name]
|
@@ -137,7 +137,7 @@ def ls_image_describer(
|
|
137 |
|
138 |
@ops.input_position(image_describer="bottom", rag_graph="bottom")
|
139 |
@op("LynxScribe Image RAG Builder")
|
140 |
-
|
141 |
async def ls_image_rag_builder(
|
142 |
image_urls,
|
143 |
image_describer,
|
|
|
66 |
|
67 |
@output_on_top
|
68 |
@op("LynxScribe RAG Vector Store")
|
69 |
+
@mem.cache
|
70 |
def ls_rag_graph(
|
71 |
*,
|
72 |
name: str = "faiss",
|
|
|
74 |
collection_name: str = "lynx",
|
75 |
text_embedder_interface: str = "openai",
|
76 |
text_embedder_model_name_or_path: str = "text-embedding-3-large",
|
77 |
+
# api_key_name: str = "OPENAI_API_KEY",
|
78 |
):
|
79 |
"""
|
80 |
Returns with a vector store instance.
|
|
|
82 |
|
83 |
# getting the text embedder instance
|
84 |
llm_params = {"name": text_embedder_interface}
|
85 |
+
# if api_key_name:
|
86 |
+
# llm_params["api_key"] = os.getenv(api_key_name)
|
87 |
llm = get_llm_engine(**llm_params)
|
88 |
text_embedder = TextEmbedder(llm=llm, model=text_embedder_model_name_or_path)
|
89 |
|
|
|
105 |
|
106 |
@output_on_top
|
107 |
@op("LynxScribe Image Describer")
|
108 |
+
@mem.cache
|
109 |
def ls_image_describer(
|
110 |
*,
|
111 |
llm_interface: str = "openai",
|
112 |
llm_visual_model: str = "gpt-4o",
|
113 |
+
llm_prompt_path: str = "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
|
114 |
llm_prompt_name: str = "cot_picture_descriptor",
|
115 |
+
# api_key_name: str = "OPENAI_API_KEY",
|
116 |
):
|
117 |
"""
|
118 |
Returns with an image describer instance.
|
|
|
120 |
"""
|
121 |
|
122 |
llm_params = {"name": llm_interface}
|
123 |
+
# if api_key_name:
|
124 |
+
# llm_params["api_key"] = os.getenv(api_key_name)
|
125 |
llm = get_llm_engine(**llm_params)
|
126 |
|
127 |
prompt_base = load_config(llm_prompt_path)[llm_prompt_name]
|
|
|
137 |
|
138 |
@ops.input_position(image_describer="bottom", rag_graph="bottom")
|
139 |
@op("LynxScribe Image RAG Builder")
|
140 |
+
@mem.cache
|
141 |
async def ls_image_rag_builder(
|
142 |
image_urls,
|
143 |
image_describer,
|