Update README.md

README.md

@@ -1,3 +1,104 @@
 ---
 library_name: transformers
----
+---
+
+---
+
+## How to Use the *ferret-gemma* Model
+
+Please download and save `builder.py`, `conversation.py` locally.
+
+### Basic Text Generation
+```python
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+# load the model and tokenizer
+model_name = "jadechoghari/ferret-gemma"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
+
+# give input text
+input_text = "The United States of America is a country situated on earth"
+
+# tokenize the input text
+inputs = tokenizer(input_text, return_tensors="pt", padding=True).to("cuda" if torch.cuda.is_available() else "cpu")
+
+model = model.to("cuda" if torch.cuda.is_available() else "cpu")
+
+output = model.generate(inputs['input_ids'], max_length=50, num_return_sequences=1)
+
+# decode and print the output
+generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
+print(generated_text)
+```
+
+### Image and Text Generation
+```python
+import torch
+from PIL import Image
+from conversation import conv_templates
+from builder import load_pretrained_model  # custom model loader
+
+# load model and tokenizer, then preprocess an image
+def infer_single_prompt(image_path, prompt, model_path):
+    img = Image.open(image_path).convert('RGB')
+    tokenizer, model, image_processor, _ = load_pretrained_model(model_path, None, "ferret_gemma")
+    image_tensor = image_processor.preprocess(img, return_tensors='pt', size=(336, 336))['pixel_values'][0].unsqueeze(0).half()
+
+    # prepare prompt
+    conv = conv_templates["ferret_gemma_instruct"].copy()
+    conv.append_message(conv.roles[0], f"Image and prompt: {prompt}")
+    input_ids = tokenizer(conv.get_prompt(), return_tensors='pt')['input_ids'].cuda()
+
+    image_tensor = image_tensor.cuda()
+
+    # generate text output
+    with torch.inference_mode():
+        output_ids = model.generate(input_ids, images=image_tensor, max_new_tokens=1024)
+    
+    # decode the output
+    output_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    return output_text.strip()
+
+# Usage
+result = infer_single_prompt("image.jpg", "Describe the contents of the image.", "jadechoghari/ferret-gemma")
+print(result)
+```
+
+### Text, Image, and Bounding Box
+```python
+import torch
+from PIL import Image
+from functools import partial
+from builder import load_pretrained_model
+
+# generates a bounding box mask
+def generate_mask_for_feature(coor, img_w, img_h):
+    coor_mask = torch.zeros((img_w, img_h))
+    coor_mask[coor[0]:coor[2]+1, coor[1]:coor[3]+1] = 1
+    return coor_mask
+
+def infer_with_bounding_box(image_path, prompt, model_path, region):
+    img = Image.open(image_path).convert('RGB')
+    tokenizer, model, image_processor, _ = load_pretrained_model(model_path, None, "ferret_gemma")
+    image_tensor = image_processor.preprocess(img, return_tensors='pt', size=(336, 336))['pixel_values'][0].unsqueeze(0).half().cuda()
+
+    input_ids = tokenizer(f"Image and prompt: {prompt}", return_tensors='pt')['input_ids'].cuda()
+
+    # create region mask
+    mask = generate_mask_for_feature(region, *img.size).unsqueeze(0).half().cuda()
+
+    # generate output with region mask
+    with torch.inference_mode():
+        model.orig_forward = model.forward
+        model.forward = partial(model.orig_forward, region_masks=[[mask]])
+        output_ids = model.generate(input_ids, images=image_tensor, max_new_tokens=1024)
+    
+    output_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    return output_text.strip()
+
+# Usage
+result = infer_with_bounding_box("image.jpg", "Describe the contents of the box.", "jadechoghari/ferret-gemma", (50, 50, 200, 200))
+print(result)
+```