Add Transformers.js example code
Browse filesNote: Requires Transformers.js >= v3.1.0
README.md
CHANGED
@@ -351,6 +351,66 @@ query_embeddings = model.encode(
|
|
351 |
```
|
352 |
</details>
|
353 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
354 |
<details>
|
355 |
<summary>via the <a href="https://onnxruntime.ai/">ONNX Runtime</a></summary>
|
356 |
|
|
|
351 |
```
|
352 |
</details>
|
353 |
|
354 |
+
<details>
|
355 |
+
<summary>via <a href="https://huggingface.co/docs/transformers.js/en/index">transformers.js</a></summary>
|
356 |
+
|
357 |
+
> [!NOTE]
|
358 |
+
> JinaCLIP was added in Transformers.js v3.1.0, so make sure you're using a compatible version!
|
359 |
+
> See the [release notes](https://github.com/huggingface/transformers.js/releases/tag/3.1.0) for more information.
|
360 |
+
|
361 |
+
If you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@huggingface/transformers) using:
|
362 |
+
```bash
|
363 |
+
npm i @huggingface/transformers
|
364 |
+
```
|
365 |
+
|
366 |
+
**Example:** Compute text and/or image embeddings with `jinaai/jina-clip-v2`:
|
367 |
+
```js
|
368 |
+
import { AutoModel, AutoProcessor, RawImage, matmul } from "@huggingface/transformers";
|
369 |
+
|
370 |
+
// Load processor and model
|
371 |
+
const model_id = "jinaai/jina-clip-v2";
|
372 |
+
const processor = await AutoProcessor.from_pretrained(model_id);
|
373 |
+
const model = await AutoModel.from_pretrained(model_id, { dtype: "q4" /* e.g., "fp16", "q8", or "q4" */ });
|
374 |
+
|
375 |
+
// Prepare inputs
|
376 |
+
const urls = ["https://i.ibb.co/nQNGqL0/beach1.jpg", "https://i.ibb.co/r5w8hG8/beach2.jpg"];
|
377 |
+
const images = await Promise.all(urls.map(url => RawImage.read(url)));
|
378 |
+
const sentences = [
|
379 |
+
"غروب جميل على الشاطئ", // Arabic
|
380 |
+
"海滩上美丽的日落", // Chinese
|
381 |
+
"Un beau coucher de soleil sur la plage", // French
|
382 |
+
"Ein wunderschöner Sonnenuntergang am Strand", // German
|
383 |
+
"Ένα όμορφο ηλιοβασίλεμα πάνω από την παραλία", // Greek
|
384 |
+
"समुद्र तट पर एक खूबसूरत सूर्यास्त", // Hindi
|
385 |
+
"Un bellissimo tramonto sulla spiaggia", // Italian
|
386 |
+
"浜辺に沈む美しい夕日", // Japanese
|
387 |
+
"해변 위로 아름다운 일몰", // Korean
|
388 |
+
];
|
389 |
+
|
390 |
+
// Encode text and images
|
391 |
+
const inputs = await processor(sentences, images, { padding: true, truncation: true });
|
392 |
+
const { l2norm_text_embeddings, l2norm_image_embeddings } = await model(inputs);
|
393 |
+
|
394 |
+
// Encode query (text-only)
|
395 |
+
const query_prefix = "Represent the query for retrieving evidence documents: ";
|
396 |
+
const query_inputs = await processor(query_prefix + "beautiful sunset over the beach");
|
397 |
+
const { l2norm_text_embeddings: query_embeddings } = await model(query_inputs);
|
398 |
+
|
399 |
+
// Compute text-image similarity scores
|
400 |
+
const text_to_image_scores = await matmul(query_embeddings, l2norm_image_embeddings.transpose(1, 0));
|
401 |
+
console.log("text-image similarity scores", text_to_image_scores.tolist()[0]); // [0.29530206322669983, 0.3183615803718567]
|
402 |
+
|
403 |
+
// Compute image-image similarity scores
|
404 |
+
const image_to_image_score = await matmul(l2norm_image_embeddings[0], l2norm_image_embeddings[1]);
|
405 |
+
console.log("image-image similarity score", image_to_image_score.item()); // 0.9344457387924194
|
406 |
+
|
407 |
+
// Compute text-text similarity scores
|
408 |
+
const text_to_text_scores = await matmul(query_embeddings, l2norm_text_embeddings.transpose(1, 0));
|
409 |
+
console.log("text-text similarity scores", text_to_text_scores.tolist()[0]); // [0.5566609501838684, 0.7028406858444214, 0.582255482673645, 0.6648036241531372, 0.5462006330490112, 0.6791588068008423, 0.6192430257797241, 0.6258729100227356, 0.6453716158866882]
|
410 |
+
```
|
411 |
+
</details>
|
412 |
+
|
413 |
+
|
414 |
<details>
|
415 |
<summary>via the <a href="https://onnxruntime.ai/">ONNX Runtime</a></summary>
|
416 |
|