Update visualization

app_bis.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import re
+
 import time
 from io import BytesIO
 
@@ -9,6 +10,7 @@ import requests
 import torch
 import transformers
 from accelerate.utils import get_max_memory
+
 from joblib import Parallel, delayed
 from PIL import Image
 from transformers import AutoTokenizer
@@ -697,17 +699,17 @@ with gr.Blocks() as demo:
                 converted into real newline characters.
                 See examples and additional details below.""")
 
-    # gr.HTML("<h3 align='center'>Help to write prompts:🙌</h3><br>Put the urls to the images inside the image tokens, it will be converted into the real image tokens. Put <fake_token_around_image> before and after each image token WITHOUT space. The texts \\n will be converted into real newline characters. See examples and additional details below.")
-    # gr.Markdown(MSG_MAIN)
-    # with gr.Row():
-    # with gr.Column():
+    #gr.HTML("<h3 align='center'>Help to write prompts:🙌</h3><br>Put the urls to the images inside the image tokens, it will be converted into the real image tokens. Put <fake_token_around_image> before and after each image token WITHOUT space. The texts \\n will be converted into real newline characters. See examples and additional details below.")
+    #gr.Markdown(MSG_MAIN)
+    #with gr.Row():
+    #with gr.Column():
     gr.Markdown("## Input")
     with gr.Row():
         if not IS_MAIN_SPACE:
             images = gr.File(label="Images", file_count="multiple")
         prompt = gr.Textbox(label="Prompt", placeholder="Enter the prompt here", lines=5)
 
-    # gr.Markdown("## Common parameters to all decoding strategy")
+    #gr.Markdown("## Common parameters to all decoding strategy")
     with gr.Row():
         with gr.Accordion("Common parameters to all decoding strategy", open=False, elem_id="common_params"):
             temperature = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=1.0, label="Softmax temperature")
@@ -749,7 +751,7 @@ with gr.Blocks() as demo:
                 label="Stop generation when an image token, a bos or a eos token is generated", value=False
             )
 
-        # gr.Markdown("## Decoding strategy and its specific parameters")
+        #gr.Markdown("## Decoding strategy and its specific parameters")
         with gr.Accordion("Decoding strategy and its specific parameters", open=False, elem_id="decoding_params"):
             decoding_strategy = gr.Dropdown(
                 ["greedy", "beam_search", "beam_sampling", "sampling_top_k", "sampling_top_p", "contrastive_sampling"],
@@ -791,7 +793,7 @@ with gr.Blocks() as demo:
 
     submit = gr.Button(label="Generate")
 
-    # with gr.Column():
+    #with gr.Column():
     with gr.Row():
         if IS_MAIN_SPACE:
             outputs = [

app_dialogue.py

@@ -32,12 +32,7 @@ EOS_TOKENS = "</s>;User"
 import logging
 
 from accelerate.utils import get_max_memory
-from transformers import (
-    AutoTokenizer,
-    AutoProcessor,
-    AutoConfig,
-    AutoModelForCausalLM,
-)
+from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
 
 
 TOKENIZER_FAST = True
@@ -61,9 +56,7 @@ def load_processor_tokenizer_model(model_name):
     )
     # tokenizer.padding_side = "left" -> we don't need that, do we?
 
-    config = AutoConfig.from_pretrained(
-        model_name, use_auth_token=os.getenv("HF_AUTH_TOKEN", True)
-    )
+    config = AutoConfig.from_pretrained(model_name, use_auth_token=os.getenv("HF_AUTH_TOKEN", True))
     max_memory_map = get_max_memory()
 
     for key in max_memory_map.keys():
@@ -125,8 +118,7 @@ def model_generation(
         [split_prompt_into_list(prompt)],
         eval_mode=True,
         truncation=True,
-        max_length=MAX_SEQ_LEN
-        - 512,  # TODO: replace the 512 value with `max_new_tokens`
+        max_length=MAX_SEQ_LEN - 512,  # TODO: replace the 512 value with `max_new_tokens`
         padding=True,
     )
     for k, v in input_args.items():
@@ -154,9 +146,7 @@ def model_generation(
         if len(eos_tokens) > 0:
             eos_token_ids = []
             for eos_token in eos_tokens:
-                tokenized_eos_token = tokenizer(
-                    eos_token, add_special_tokens=False
-                ).input_ids
+                tokenized_eos_token = tokenizer(eos_token, add_special_tokens=False).input_ids
                 if len(tokenized_eos_token) > 1:
                     raise ValueError(
                         f"eos_tokens should be one token, here {eos_token} is {len(tokenized_eos_token)} tokens:"
@@ -214,17 +204,13 @@ def model_generation(
 
     tokens = tokenizer.convert_ids_to_tokens(generated_tokens[0])
     decoded_skip_special_tokens = repr(
-        tokenizer.batch_decode(
-            generated_tokens, skip_special_tokens=hide_special_tokens
-        )[0]
+        tokenizer.batch_decode(generated_tokens, skip_special_tokens=hide_special_tokens)[0]
     )
 
     actual_generated_tokens = generated_tokens[:, input_args["input_ids"].shape[-1] :]
     first_end_token = len(actual_generated_tokens[0])
     actual_generated_tokens = actual_generated_tokens[:, :first_end_token]
-    generated_text = tokenizer.batch_decode(
-        actual_generated_tokens, skip_special_tokens=hide_special_tokens
-    )[0]
+    generated_text = tokenizer.batch_decode(actual_generated_tokens, skip_special_tokens=hide_special_tokens)[0]
 
     logger.info(
         "Result: \n"
@@ -267,9 +253,7 @@ with gr.Blocks(title="IDEFICS", theme=gr.themes.Base()) as demo:
                     show_label=False,
                     container=False,
                 )
-            processor, tokenizer, model = load_processor_tokenizer_model(
-                model_selector.value
-            )
+            processor, tokenizer, model = load_processor_tokenizer_model(model_selector.value)
 
             imagebox = gr.Image(
                 type="pil",
@@ -282,7 +266,41 @@ with gr.Blocks(title="IDEFICS", theme=gr.themes.Base()) as demo:
                 ),
             )
 
-            with gr.Accordion("Parameters", open=False, visible=True) as parameter_row:
+            with gr.Accordion("Generation parameters", open=False, visible=True) as parameter_row:
+                max_new_tokens = gr.Slider(
+                    minimum=0,
+                    maximum=2048,
+                    value=512,
+                    step=1,
+                    interactive=True,
+                    label="Maximum number of new tokens to generate",
+                )
+                min_length = gr.Slider(
+                    minimum=0,
+                    maximum=50,
+                    value=0,
+                    step=1,
+                    interactive=True,
+                    label="Minimum number of new tokens to generate",
+                )
+                repetition_penalty = gr.Slider(
+                    minimum=0.0,
+                    maximum=5.0,
+                    value=1.0,
+                    step=0.1,
+                    interactive=True,
+                    label="Repetition penalty",
+                    info="1.0 means no penalty",
+                )
+                no_repeat_ngram_size = gr.Slider(
+                    minimum=0,
+                    maximum=10,
+                    value=0,
+                    step=1,
+                    interactive=True,
+                    label="N-gram repetition threshold",
+                    info="If set to int > 0, all ngrams of that size can only occur once.",
+                )
                 decoding_strategy = gr.Radio(
                     [
                         "greedy",
@@ -293,70 +311,81 @@ with gr.Blocks(title="IDEFICS", theme=gr.themes.Base()) as demo:
                     ],
                     value="greedy",
                     label="Decoding strategy",
+                    interactive=True,
                 )
-                num_beams = top_k = gr.Slider(
+                temperature = gr.Slider(
+                    minimum=0.0,
+                    maximum=5.0,
+                    value=1.0,
+                    step=0.1,
+                    interactive=True,
+                    label="Sampling temperature",
+                )
+                num_beams = gr.Slider(
                     minimum=0,
                     maximum=20,
                     value=3.0,
                     step=1.0,
                     interactive=True,
+                    visible=False,
                     label="Number of beams",
+                    info="Only used if `decoding_strategy` is `beam_search` or `beam_sampling`.",
                 )
-                temperature = gr.Slider(
-                    minimum=0.0,
-                    maximum=1.0,
-                    value=0.2,
-                    step=0.1,
-                    interactive=True,
-                    label="Temperature",
+                decoding_strategy.change(
+                    fn=lambda selection: gr.Slider.update(visible=(selection in ["beam_search", "beam_sampling"])),
+                    inputs=decoding_strategy,
+                    outputs=num_beams,
                 )
                 top_p = gr.Slider(
                     minimum=0.0,
                     maximum=1.0,
                     value=0.95,
-                    step=0.1,
+                    step=0.01,
                     interactive=True,
+                    visible=False,
                     label="Top P",
+                    info=(
+                        "If set to float < 1, only the smallest set of most probable tokens with probabilities that"
+                        " add up to top_p or higher are kept for generation."
+                    ),
                 )
-                top_k = gr.Slider(
-                    minimum=0.0,
-                    maximum=100.0,
-                    value=50.0,
-                    step=1.0,
-                    interactive=True,
-                    label="Top K",
+                decoding_strategy.change(
+                    fn=lambda selection: gr.Slider.update(visible=(selection in ["sampling_top_p"])),
+                    inputs=decoding_strategy,
+                    outputs=top_p,
                 )
-                max_new_tokens = gr.Slider(
+                top_k = gr.Slider(
                     minimum=0,
-                    maximum=2048,
-                    value=512,
-                    step=64,
+                    maximum=500,
+                    value=50,
+                    step=1,
                     interactive=True,
-                    label="Max output tokens",
-                )
-                repetition_penalty = gr.Slider(
-                    minimum=0.0,
-                    maximum=5.0,
-                    value=1.0,
-                    step=0.1,
-                    interactive=True,
-                    label="Repetition penalty",
+                    visible=False,
+                    label="Top K",
+                    info="The number of highest probability vocabulary tokens to keep for top-k-filtering.",
                 )
-                min_length = gr.Slider(
-                    minimum=0.0,
-                    maximum=50.0,
-                    value=0.0,
-                    step=1.0,
-                    interactive=True,
-                    label="Minimum generation length",
+                decoding_strategy.change(
+                    fn=lambda selection: gr.Slider.update(visible=(selection in ["sampling_top_k"])),
+                    inputs=decoding_strategy,
+                    outputs=top_k,
                 )
                 length_penalty = gr.Slider(
-                    minimum=0.0,
-                    maximum=5.0,
+                    minimum=-1000.0,
+                    maximum=1000.0,
                     value=1.0,
                     step=0.1,
                     interactive=True,
+                    visible=False,
                     label="Length penalty",
+                    info=(
+                        "length_penalty > 0.0 promotes longer sequences, while length_penalty < 0.0 encourages shorter"
+                        " sequences. Only used if `decoding_strategy` is `beam_search` or `beam_sampling`."
+                    ),
+                )
+                decoding_strategy.change(
+                    fn=lambda selection: gr.Slider.update(visible=(selection in ["beam_search", "beam_sampling"])),
+                    inputs=decoding_strategy,
+                    outputs=length_penalty,
                 )
                 penalty_alpha = gr.Slider(
                     minimum=0.0,
@@ -364,15 +393,14 @@ with gr.Blocks(title="IDEFICS", theme=gr.themes.Base()) as demo:
                     value=0.95,
                     step=0.05,
                     interactive=True,
+                    visible=False,
                     label="Penalty alpha",
+                    info="Only used if `decoding_strategy` is `contrastive_sampling`.",
                 )
-                no_repeat_ngram_size = gr.Slider(
-                    minimum=0.0,
-                    maximum=10.0,
-                    value=0.0,
-                    step=1.0,
-                    interactive=True,
-                    label="No repeat ngram size",
+                decoding_strategy.change(
+                    fn=lambda selection: gr.Slider.update(visible=(selection in ["contrastive_sampling"])),
+                    inputs=decoding_strategy,
+                    outputs=penalty_alpha,
                 )
 
         with gr.Column(scale=6):
@@ -380,7 +408,7 @@ with gr.Blocks(title="IDEFICS", theme=gr.themes.Base()) as demo:
                 elem_id="chatbot",
                 label="Idefics Chatbot",
                 visible=True,
-                height=750,
+                height=550,
                 value=[
                     [
                         (
@@ -430,15 +458,11 @@ with gr.Blocks(title="IDEFICS", theme=gr.themes.Base()) as demo:
     #     inputs = [chatbot]
     # )
 
-    def format_prompt_with_history_and_system_conditioning(
-        current_user_prompt, history
-    ):
+    def format_prompt_with_history_and_system_conditioning(current_user_prompt, history):
         resulting_text = SYSTEM_PROMPT
         for turn in history:
             user_utterance, assistant_utterance = turn
-            resulting_text += (
-                f"\nUser: {user_utterance}</s>\nAssistant: {assistant_utterance}</s>"
-            )
+            resulting_text += f"\nUser: {user_utterance}</s>\nAssistant: {assistant_utterance}</s>"
         resulting_text += f"\nUser: {current_user_prompt}</s>\nAssistant:"
         return resulting_text
 

elon_musk.md

@@ -20,4 +20,4 @@ Bot: Yes, it is possible. If we continue to pollute the environment and destroy
 User: I think this is a lost cause. <fake_token_around_image> <image> <fake_token_around_image>  Like Elon Musk, I think we should instead all move to another planet now.
 Bot: El ón Mus ́k is the CEO of SpaceX and Tesla Motors. Space X is developing a reusable rocket system that could be used to transport people and cargo to Mars. Mus k believes that colonizing Mars is essential for the survival of humanity, because it would provide a backup plan in case something goes wrong on earth. However, many people are skeptical about the feasibility of such a project. Some argue that it will take too long and cost too much money to establish a colony on Mars, while others say that the risks are too great. Nevertheless, Mus K remains optimistic about his vision for Mars colonization. "I think there is good reason to believe that we can build a self-sustaining city on mars," he said in an interview with Time magazine. “It would be quite an adventure.”
 ----
-The project is still in development. I'm currently working on improving the accuracy of image recognition and adding more features to it.</s>
+The project is still in development. I'm currently working on improving the accuracy of image recognition and adding more features to it.</s>

hello.py

@@ -0,0 +1,73 @@
+[
+    """The following is a conversation between a highly knowledgeable and intelligent AI assistant, called Assistant, and a human user, called User. In the following interactions, User and Assistant will converse in natural language, and Assistant will do its best to answer User’s questions. Assistant was built to be respectful, polite and inclusive. It knows a lot, and always tells the truth. When prompted with an image, it does not make up facts.\n\nThe conversation begins:\nUser:"""
+    """https://i1.pickpik.com/photos/515/906/355/kittens-cats-pet-cute-preview.jpg""",
+    """Describe this image.</s>\n""",
+    """Assistant: Five kittens are standing together in the center of the photograph. From the left to right, there is one orange kitte, two kittens white and black stripes, and two brown kittens with an orange head. They are in bright green grass and it looks like they are moving forward.</s>\n""",
+    """User:How about this image?""",
+    "https://cdn.pixabay.com/photo/2017/09/25/13/12/puppy-2785074_1280.jpg""",
+    """Can you describe it too?</s>\n"""
+    """Assistant: A dog is lying on the floor, looking at the camera. It is looking directly at you. The dog has a white body and brown patches on its face and ears. Its eyes are dark. Its nose is black, and it has long, floppy ears, white paws, long fur, big eyes.</s>\n""",
+    """User: What kind of breed is it?</s>\n""",
+    """Assistant: It looks like a Cavalier King Charles Spaniel. These breeds are known for their gentle, affectionate nature. Their long ears and big, brown eyes give them a sweet, expressive face. Cavaliers are small dogs weighing between 13 and 20 pounds. They have a silky, medium-length coat that comes in various colors, including black and tan, tricolor, red and white, or black.</s>\n""",
+    """User: What can you tell me about this breed of dogs?</s>\n"""
+    """Assistant: One specific characteristic of the Cavalier King Charles Spaniel is its friendly and affectionate nature. This breed is known for being extremely sociable and forming strong bonds with their owners. They are often described as "velcro dogs" because they love to be close to their human companions, whether it's sitting on their laps, cuddling on the couch, or simply following them around the house.</s>\n""",
+    """User: ghjkhjabnufs</s>\n""",
+    """Assistant: That doesn’t seem to be a word. Could you ask me another way?</s>\n""",
+    """User: Do you like Cavalier King Charles Spaniel?</s>\n""",
+    """Assistant: I do not have personal opinions as I’m just a computer program. However, cavaliers are known for being excellent family pets due to their gentle and patient demeanor, making them great with children and other pets. Their affectionate nature also makes them ideal therapy dogs, as they can provide comfort and emotional support to those in need.</s>\n""",
+    """User: How many dogs do you see in this image?""",
+    """https://i.dailymail.co.uk/i/pix/2011/07/01/article-2010308-0CD22A8300000578-496_634x414.jpg""",
+    """</s>\n""",
+    """Assistant: There is no dogs in this image. The picture shows a tennis player jumping to volley the ball.</s>\n""",
+    """User: What color are the cat's eyes?</s>\n""",
+    """Assistant:"""
+]
+
+SYSTEM_PROMPT = """The following is a conversation between a highly knowledgeable and intelligent AI assistant, called Assistant, and a human user, called User. In the following interactions, User and Assistant will converse in natural language, and Assistant will do its best to answer User’s questions. Assistant was built to be respectful, polite and inclusive. It knows a lot, and always tells the truth. When prompted with an image, it does not make up facts.
+
+The conversation begins:
+User:<fake_token_around_image><image:https://i1.pickpik.com/photos/515/906/355/kittens-cats-pet-cute-preview.jpg><fake_token_around_image>Describe this image.</s>
+Assistant: Five kittens are standing together in the center of the photograph. From the left to right, there is one orange kitte, two kittens white and black stripes, and two brown kittens with an orange head. They are in bright green grass and it looks like they are moving forward.</s>"""
+
+prompts = [
+    """The following is a conversation between a highly knowledgeable and intelligent AI assistant, called Assistant, and a human user, called User. In the following interactions, User and Assistant will converse in natural language, and Assistant will do its best to answer User’s questions. Assistant was built to be respectful, polite and inclusive. It knows a lot, and always tells the truth. When prompted with an image, it does not make up facts.\n\nThe conversation begins:\nUser:""",
+    """https://i1.pickpik.com/photos/515/906/355/kittens-cats-pet-cute-preview.jpg""",
+    """Describe this image.</s>\nAssistant: Five kittens are standing together in the center of the photograph. From the left to right, there is one orange kitte, two kittens white and black stripes, and two brown kittens with an orange head. They are in bright green grass and it looks like they are moving forward.</s>"""
+]
+
+SYSTEM_PROMPT = """HELLO
+AUREVOIR
+User:<fake_token_around_image><image:AAA><fake_token_around_image><image:BBB><fake_token_around_image>BOJOUR</s>
+Assistant: SPECIFIC</s>"""
+
+def split_prompt_into_list(prompt_str):
+    prompt_splitted = prompt_str.split("<fake_token_around_image>")
+    prompt_list = []
+    for ps in prompt_splitted:
+        if ps.startswith("<image:"):
+            prompt_list.append(ps[7:-1])
+        else:
+            prompt_list.append(ps)
+    return prompt_list
+
+prompts = split_prompt_into_list(formated_prompt)
+i = processor(prompts, eval_mode=True)
+
+from transformers import AutoTokenizer, AutoProcessor
+checkpoint = "HuggingFaceM4/idefics-9b-instruct"
+tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+processor = AutoProcessor.from_pretrained(checkpoint)
+
+prompts = [
+    [
+        "User:",
+        # "https://hips.hearstapps.com/hmg-prod/images/cute-photos-of-cats-in-grass-1593184777.jpg",
+        # "https://hips.hearstapps.com/hmg-prod/images/cute-photos-of-cats-in-grass-1593184777.jpg",
+        "Describe this image.\nAssistant: An image of two kittens in grass.\n",
+        "User:",
+        # "https://hips.hearstapps.com/hmg-prod/images/dog-puns-1581708208.jpg",
+        "Describe this image.\nAssistant:",
+    ],
+]
+inputs = processor(prompts, eval_mode=True)
+tokenizer.decode(inputs["input_ids"][0])

requirements.txt

@@ -1,6 +1,6 @@
 --extra-index-url https://download.pytorch.org/whl/cu113
 torch
-transformers @ git+https://github.com/huggingface/transformers@0e35f259e454fee5c9a7925a525507164dd0685f
+transformers @ git+https://github.com/huggingface/transformers.git@786092a35e18154cacad62c30fe92bac2c27a1e1
 requests
 pillow
 torchvision