mszel commited on
Commit
5b603ce
·
1 Parent(s): b3b67d2

mini mods after rebase

Browse files
examples/LynxScribe Image RAG CHANGED
@@ -21,6 +21,13 @@
21
  "target": "LynxScribe Image RAG Query 1",
22
  "targetHandle": "text"
23
  },
 
 
 
 
 
 
 
24
  {
25
  "id": "LynxScribe Image Describer 1 LynxScribe Image RAG Builder 1",
26
  "source": "LynxScribe Image Describer 1",
@@ -34,13 +41,6 @@
34
  "sourceHandle": "output",
35
  "target": "LynxScribe Image RAG Builder 1",
36
  "targetHandle": "rag_graph"
37
- },
38
- {
39
- "id": "LynxScribe Image RAG Query 1 View image 1",
40
- "source": "LynxScribe Image RAG Query 1",
41
- "sourceHandle": "output",
42
- "target": "View image 1",
43
- "targetHandle": "embedding_similarities"
44
  }
45
  ],
46
  "env": "LynxScribe",
@@ -147,10 +147,6 @@
147
  }
148
  }
149
  },
150
- "position": {
151
- "x": 979.0,
152
- "y": 238.0
153
- },
154
  "type": "basic"
155
  },
156
  "params": {},
@@ -170,7 +166,7 @@
170
  {
171
  "data": {
172
  "__execution_delay": 0.0,
173
- "collapsed": null,
174
  "display": null,
175
  "error": null,
176
  "meta": {
@@ -194,14 +190,10 @@
194
  }
195
  }
196
  },
197
- "position": {
198
- "x": 1336.0,
199
- "y": 378.0
200
- },
201
  "type": "basic"
202
  },
203
  "params": {
204
- "chat": "Show me a picture about meditation!"
205
  },
206
  "status": "done",
207
  "title": "Input chat"
@@ -210,8 +202,8 @@
210
  "height": 214.0,
211
  "id": "Input chat 1",
212
  "position": {
213
- "x": -310.1420152146455,
214
- "y": -139.39548490290966
215
  },
216
  "type": "basic",
217
  "width": 387.0
@@ -258,10 +250,6 @@
258
  }
259
  }
260
  },
261
- "position": {
262
- "x": 1419.0,
263
- "y": 371.0
264
- },
265
  "type": "basic"
266
  },
267
  "params": {
@@ -280,6 +268,41 @@
280
  "type": "basic",
281
  "width": 200.0
282
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
283
  {
284
  "data": {
285
  "display": null,
@@ -297,13 +320,6 @@
297
  }
298
  },
299
  "params": {
300
- "api_key_name": {
301
- "default": "OPENAI_API_KEY",
302
- "name": "api_key_name",
303
- "type": {
304
- "type": "<class 'str'>"
305
- }
306
- },
307
  "llm_interface": {
308
  "default": "openai",
309
  "name": "llm_interface",
@@ -319,7 +335,7 @@
319
  }
320
  },
321
  "llm_prompt_path": {
322
- "default": "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
323
  "name": "llm_prompt_path",
324
  "type": {
325
  "type": "<class 'str'>"
@@ -334,30 +350,29 @@
334
  }
335
  },
336
  "position": {
337
- "x": 879.0,
338
- "y": 686.0
339
  },
340
  "type": "basic"
341
  },
342
  "params": {
343
- "api_key_name": "OPENAI_API_KEY",
344
  "llm_interface": "openai",
345
  "llm_prompt_name": "cot_picture_descriptor",
346
- "llm_prompt_path": "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
347
  "llm_visual_model": "gpt-4o"
348
  },
349
  "status": "done",
350
  "title": "LynxScribe Image Describer"
351
  },
352
  "dragHandle": ".bg-primary",
353
- "height": 457.0,
354
  "id": "LynxScribe Image Describer 1",
355
  "position": {
356
- "x": 41.706162883849856,
357
- "y": 609.2665142696401
358
  },
359
  "type": "basic",
360
- "width": 357.0
361
  },
362
  {
363
  "data": {
@@ -376,13 +391,6 @@
376
  }
377
  },
378
  "params": {
379
- "api_key_name": {
380
- "default": "OPENAI_API_KEY",
381
- "name": "api_key_name",
382
- "type": {
383
- "type": "<class 'str'>"
384
- }
385
- },
386
  "collection_name": {
387
  "default": "lynx",
388
  "name": "collection_name",
@@ -420,78 +428,30 @@
420
  }
421
  },
422
  "position": {
423
- "x": 1010.0,
424
- "y": 713.0
425
  },
426
  "type": "basic"
427
  },
428
  "params": {
429
- "api_key_name": "OPENAI_API_KEY",
430
  "collection_name": "lynx",
431
  "name": "faiss",
432
  "num_dimensions": 3072.0,
433
  "text_embedder_interface": "openai",
434
  "text_embedder_model_name_or_path": "text-embedding-3-large"
435
  },
436
- "status": "active",
437
  "title": "LynxScribe RAG Vector Store"
438
  },
439
  "dragHandle": ".bg-primary",
440
- "height": 515.0,
441
  "id": "LynxScribe RAG Vector Store 1",
442
  "position": {
443
- "x": 470.84700620705013,
444
- "y": 609.2665142696401
445
  },
446
  "type": "basic",
447
- "width": 350.0
448
- },
449
- {
450
- "data": {
451
- "__execution_delay": null,
452
- "collapsed": false,
453
- "display": null,
454
- "error": null,
455
- "meta": {
456
- "inputs": {
457
- "embedding_similarities": {
458
- "name": "embedding_similarities",
459
- "position": "left",
460
- "type": {
461
- "type": "<class 'inspect._empty'>"
462
- }
463
- }
464
- },
465
- "name": "View image",
466
- "outputs": {},
467
- "params": {
468
- "idx": {
469
- "default": 0.0,
470
- "name": "idx",
471
- "type": {
472
- "type": "<class 'int'>"
473
- }
474
- }
475
- },
476
- "position": {
477
- "x": 1418.0,
478
- "y": 272.0
479
- },
480
- "type": "table_view"
481
- },
482
- "params": {},
483
- "status": "done",
484
- "title": "View image"
485
- },
486
- "dragHandle": ".bg-primary",
487
- "height": 342.0,
488
- "id": "View image 1",
489
- "position": {
490
- "x": 1424.7240705678887,
491
- "y": -202.17696978464585
492
- },
493
- "type": "table_view",
494
- "width": 652.0
495
  }
496
  ]
497
  }
 
21
  "target": "LynxScribe Image RAG Query 1",
22
  "targetHandle": "text"
23
  },
24
+ {
25
+ "id": "LynxScribe Image RAG Query 1 View image 1",
26
+ "source": "LynxScribe Image RAG Query 1",
27
+ "sourceHandle": "output",
28
+ "target": "View image 1",
29
+ "targetHandle": "embedding_similarities"
30
+ },
31
  {
32
  "id": "LynxScribe Image Describer 1 LynxScribe Image RAG Builder 1",
33
  "source": "LynxScribe Image Describer 1",
 
41
  "sourceHandle": "output",
42
  "target": "LynxScribe Image RAG Builder 1",
43
  "targetHandle": "rag_graph"
 
 
 
 
 
 
 
44
  }
45
  ],
46
  "env": "LynxScribe",
 
147
  }
148
  }
149
  },
 
 
 
 
150
  "type": "basic"
151
  },
152
  "params": {},
 
166
  {
167
  "data": {
168
  "__execution_delay": 0.0,
169
+ "collapsed": false,
170
  "display": null,
171
  "error": null,
172
  "meta": {
 
190
  }
191
  }
192
  },
 
 
 
 
193
  "type": "basic"
194
  },
195
  "params": {
196
+ "chat": "Show me a picture about healthy life"
197
  },
198
  "status": "done",
199
  "title": "Input chat"
 
202
  "height": 214.0,
203
  "id": "Input chat 1",
204
  "position": {
205
+ "x": -302.70349900314835,
206
+ "y": -140.67386384008685
207
  },
208
  "type": "basic",
209
  "width": 387.0
 
250
  }
251
  }
252
  },
 
 
 
 
253
  "type": "basic"
254
  },
255
  "params": {
 
268
  "type": "basic",
269
  "width": 200.0
270
  },
271
+ {
272
+ "data": {
273
+ "__execution_delay": null,
274
+ "collapsed": false,
275
+ "display": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test/food-405521_1280.jpg",
276
+ "error": null,
277
+ "meta": {
278
+ "inputs": {
279
+ "embedding_similarities": {
280
+ "name": "embedding_similarities",
281
+ "position": "left",
282
+ "type": {
283
+ "type": "<class 'inspect._empty'>"
284
+ }
285
+ }
286
+ },
287
+ "name": "View image",
288
+ "outputs": {},
289
+ "params": {},
290
+ "type": "image"
291
+ },
292
+ "params": {},
293
+ "status": "done",
294
+ "title": "View image"
295
+ },
296
+ "dragHandle": ".bg-primary",
297
+ "height": 1170.0,
298
+ "id": "View image 1",
299
+ "position": {
300
+ "x": 1426.7020124006506,
301
+ "y": -293.16229409169125
302
+ },
303
+ "type": "image",
304
+ "width": 750.0
305
+ },
306
  {
307
  "data": {
308
  "display": null,
 
320
  }
321
  },
322
  "params": {
 
 
 
 
 
 
 
323
  "llm_interface": {
324
  "default": "openai",
325
  "name": "llm_interface",
 
335
  }
336
  },
337
  "llm_prompt_path": {
338
+ "default": "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
339
  "name": "llm_prompt_path",
340
  "type": {
341
  "type": "<class 'str'>"
 
350
  }
351
  },
352
  "position": {
353
+ "x": 1066.0,
354
+ "y": 713.0
355
  },
356
  "type": "basic"
357
  },
358
  "params": {
 
359
  "llm_interface": "openai",
360
  "llm_prompt_name": "cot_picture_descriptor",
361
+ "llm_prompt_path": "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
362
  "llm_visual_model": "gpt-4o"
363
  },
364
  "status": "done",
365
  "title": "LynxScribe Image Describer"
366
  },
367
  "dragHandle": ".bg-primary",
368
+ "height": 363.0,
369
  "id": "LynxScribe Image Describer 1",
370
  "position": {
371
+ "x": 16.941021961247145,
372
+ "y": 664.8693601063997
373
  },
374
  "type": "basic",
375
+ "width": 401.0
376
  },
377
  {
378
  "data": {
 
391
  }
392
  },
393
  "params": {
 
 
 
 
 
 
 
394
  "collection_name": {
395
  "default": "lynx",
396
  "name": "collection_name",
 
428
  }
429
  },
430
  "position": {
431
+ "x": 831.0,
432
+ "y": 734.0
433
  },
434
  "type": "basic"
435
  },
436
  "params": {
 
437
  "collection_name": "lynx",
438
  "name": "faiss",
439
  "num_dimensions": 3072.0,
440
  "text_embedder_interface": "openai",
441
  "text_embedder_model_name_or_path": "text-embedding-3-large"
442
  },
443
+ "status": "done",
444
  "title": "LynxScribe RAG Vector Store"
445
  },
446
  "dragHandle": ".bg-primary",
447
+ "height": 445.0,
448
  "id": "LynxScribe RAG Vector Store 1",
449
  "position": {
450
+ "x": 485.48864677068343,
451
+ "y": 669.7387202127995
452
  },
453
  "type": "basic",
454
+ "width": 442.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
455
  }
456
  ]
457
  }
lynxkite-lynxscribe/src/lynxkite_lynxscribe/lynxscribe_ops.py CHANGED
@@ -66,7 +66,7 @@ def gcp_image_loader(
66
 
67
  @output_on_top
68
  @op("LynxScribe RAG Vector Store")
69
- # @mem.cache
70
  def ls_rag_graph(
71
  *,
72
  name: str = "faiss",
@@ -74,7 +74,7 @@ def ls_rag_graph(
74
  collection_name: str = "lynx",
75
  text_embedder_interface: str = "openai",
76
  text_embedder_model_name_or_path: str = "text-embedding-3-large",
77
- api_key_name: str = "OPENAI_API_KEY",
78
  ):
79
  """
80
  Returns with a vector store instance.
@@ -82,8 +82,8 @@ def ls_rag_graph(
82
 
83
  # getting the text embedder instance
84
  llm_params = {"name": text_embedder_interface}
85
- if api_key_name:
86
- llm_params["api_key"] = os.getenv(api_key_name)
87
  llm = get_llm_engine(**llm_params)
88
  text_embedder = TextEmbedder(llm=llm, model=text_embedder_model_name_or_path)
89
 
@@ -105,14 +105,14 @@ def ls_rag_graph(
105
 
106
  @output_on_top
107
  @op("LynxScribe Image Describer")
108
- # @mem.cache
109
  def ls_image_describer(
110
  *,
111
  llm_interface: str = "openai",
112
  llm_visual_model: str = "gpt-4o",
113
- llm_prompt_path: str = "lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
114
  llm_prompt_name: str = "cot_picture_descriptor",
115
- api_key_name: str = "OPENAI_API_KEY",
116
  ):
117
  """
118
  Returns with an image describer instance.
@@ -120,8 +120,8 @@ def ls_image_describer(
120
  """
121
 
122
  llm_params = {"name": llm_interface}
123
- if api_key_name:
124
- llm_params["api_key"] = os.getenv(api_key_name)
125
  llm = get_llm_engine(**llm_params)
126
 
127
  prompt_base = load_config(llm_prompt_path)[llm_prompt_name]
@@ -137,7 +137,7 @@ def ls_image_describer(
137
 
138
  @ops.input_position(image_describer="bottom", rag_graph="bottom")
139
  @op("LynxScribe Image RAG Builder")
140
- # @mem.cache
141
  async def ls_image_rag_builder(
142
  image_urls,
143
  image_describer,
 
66
 
67
  @output_on_top
68
  @op("LynxScribe RAG Vector Store")
69
+ @mem.cache
70
  def ls_rag_graph(
71
  *,
72
  name: str = "faiss",
 
74
  collection_name: str = "lynx",
75
  text_embedder_interface: str = "openai",
76
  text_embedder_model_name_or_path: str = "text-embedding-3-large",
77
+ # api_key_name: str = "OPENAI_API_KEY",
78
  ):
79
  """
80
  Returns with a vector store instance.
 
82
 
83
  # getting the text embedder instance
84
  llm_params = {"name": text_embedder_interface}
85
+ # if api_key_name:
86
+ # llm_params["api_key"] = os.getenv(api_key_name)
87
  llm = get_llm_engine(**llm_params)
88
  text_embedder = TextEmbedder(llm=llm, model=text_embedder_model_name_or_path)
89
 
 
105
 
106
  @output_on_top
107
  @op("LynxScribe Image Describer")
108
+ @mem.cache
109
  def ls_image_describer(
110
  *,
111
  llm_interface: str = "openai",
112
  llm_visual_model: str = "gpt-4o",
113
+ llm_prompt_path: str = "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
114
  llm_prompt_name: str = "cot_picture_descriptor",
115
+ # api_key_name: str = "OPENAI_API_KEY",
116
  ):
117
  """
118
  Returns with an image describer instance.
 
120
  """
121
 
122
  llm_params = {"name": llm_interface}
123
+ # if api_key_name:
124
+ # llm_params["api_key"] = os.getenv(api_key_name)
125
  llm = get_llm_engine(**llm_params)
126
 
127
  prompt_base = load_config(llm_prompt_path)[llm_prompt_name]
 
137
 
138
  @ops.input_position(image_describer="bottom", rag_graph="bottom")
139
  @op("LynxScribe Image RAG Builder")
140
+ @mem.cache
141
  async def ls_image_rag_builder(
142
  image_urls,
143
  image_describer,