Upload folder using huggingface_hub

README.md

@@ -239,7 +239,7 @@ tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast
 
 # set the max number of tiles in `max_num`
 pixel_values = load_image('./examples/image1.jpg', max_num=12).to(torch.bfloat16).cuda()
-generation_config = dict(max_new_tokens=1024, do_sample=False)
+generation_config = dict(max_new_tokens=1024, do_sample=True)
 
 # pure-text conversation (纯文本对话)
 question = 'Hello, who are you?'
@@ -391,7 +391,7 @@ for new_text in streamer:
 
 ## Finetune
 
-SWIFT from ModelScope community has supported the fine-tuning (Image/Video) of InternVL, please check [this link](https://github.com/modelscope/swift/blob/main/docs/source_en/Multi-Modal/internvl-best-practice.md) for more details.
+Many repositories now support fine-tuning of the InternVL series models, including [InternVL](https://github.com/OpenGVLab/InternVL), [SWIFT](https://github.com/modelscope/ms-swift), [XTurner](https://github.com/InternLM/xtuner), and others. Please refer to their documentation for more details on fine-tuning.
 
 ## Deployment
 
@@ -400,7 +400,7 @@ SWIFT from ModelScope community has supported the fine-tuning (Image/Video) of I
 LMDeploy is a toolkit for compressing, deploying, and serving LLM, developed by the MMRazor and MMDeploy teams.
 
 ```sh
-pip install lmdeploy
+pip install lmdeploy==0.5.3
 ```
 
 LMDeploy abstracts the complex inference process of multi-modal Vision-Language Models (VLM) into an easy-to-use pipeline, similar to the Large Language Model (LLM) inference pipeline.
@@ -408,14 +408,12 @@ LMDeploy abstracts the complex inference process of multi-modal Vision-Language
 #### A 'Hello, world' example
 
 ```python
-from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig
+from lmdeploy import pipeline, TurbomindEngineConfig
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
 image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg')
-chat_template_config = ChatTemplateConfig('internvl-phi3')
-pipe = pipeline(model, chat_template_config=chat_template_config,
-                backend_config=PytorchEngineConfig(session_len=8192))
+pipe = pipeline(model, backend_config=TurbomindEngineConfig(session_len=8192))
 response = pipe(('describe this image', image))
 print(response.text)
 ```
@@ -429,14 +427,12 @@ When dealing with multiple images, you can put them all in one list. Keep in min
 > Warning: Due to the scarcity of multi-image conversation data, the performance on multi-image tasks may be unstable, and it may require multiple attempts to achieve satisfactory results.
 
 ```python
-from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig
+from lmdeploy import pipeline, TurbomindEngineConfig
 from lmdeploy.vl import load_image
 from lmdeploy.vl.constants import IMAGE_TOKEN
 
 model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
-chat_template_config = ChatTemplateConfig('internvl-phi3')
-pipe = pipeline(model, chat_template_config=chat_template_config,
-                backend_config=PytorchEngineConfig(session_len=8192))
+pipe = pipeline(model, backend_config=TurbomindEngineConfig(session_len=8192))
 
 image_urls=[
     'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg',
@@ -454,13 +450,11 @@ print(response.text)
 Conducting inference with batch prompts is quite straightforward; just place them within a list structure:
 
 ```python
-from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig
+from lmdeploy import pipeline, TurbomindEngineConfig
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
-chat_template_config = ChatTemplateConfig('internvl-phi3')
-pipe = pipeline(model, chat_template_config=chat_template_config,
-                backend_config=PytorchEngineConfig(session_len=8192))
+pipe = pipeline(model, backend_config=TurbomindEngineConfig(session_len=8192))
 
 image_urls=[
     "https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg",
@@ -476,13 +470,11 @@ print(response)
 There are two ways to do the multi-turn conversations with the pipeline. One is to construct messages according to the format of OpenAI and use above introduced method, the other is to use the `pipeline.chat` interface.
 
 ```python
-from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig, GenerationConfig
+from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
-chat_template_config = ChatTemplateConfig('internvl-phi3')
-pipe = pipeline(model, chat_template_config=chat_template_config,
-                backend_config=PytorchEngineConfig(session_len=8192))
+pipe = pipeline(model, backend_config=TurbomindEngineConfig(session_len=8192))
 
 image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg')
 gen_config = GenerationConfig(top_k=40, top_p=0.8, temperature=0.8)
@@ -497,7 +489,7 @@ print(sess.response.text)
 LMDeploy's `api_server` enables models to be easily packed into services with a single command. The provided RESTful APIs are compatible with OpenAI's interfaces. Below are an example of service startup:
 
 ```shell
-lmdeploy serve api_server OpenGVLab/Mini-InternVL-Chat-4B-V1-5 --backend pytorch --server-port 23333
+lmdeploy serve api_server OpenGVLab/Mini-InternVL-Chat-4B-V1-5 --backend turbomind --server-port 23333
 ```
 
 To use the OpenAI-style interface, you need to install OpenAI:

modeling_intern_vit.py

@@ -20,18 +20,12 @@ from transformers.utils import logging
 from .configuration_intern_vit import InternVisionConfig
 
 try:
-    try:  # v1
-        from flash_attn.flash_attn_interface import \
-            flash_attn_unpadded_qkvpacked_func
-    except:  # v2
-        from flash_attn.flash_attn_interface import \
-            flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func
-
     from flash_attn.bert_padding import pad_input, unpad_input
-
+    from flash_attn.flash_attn_interface import \
+        flash_attn_varlen_qkvpacked_func
     has_flash_attn = True
 except:
-    print('FlashAttention is not installed.')
+    print('FlashAttention2 is not installed.')
     has_flash_attn = False
 
 logger = logging.get_logger(__name__)
@@ -74,7 +68,7 @@ class FlashAttention(nn.Module):
                 max_s = seqlen
                 cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
                                           device=qkv.device)
-                output = flash_attn_unpadded_qkvpacked_func(
+                output = flash_attn_varlen_qkvpacked_func(
                     qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
                     softmax_scale=self.softmax_scale, causal=causal
                 )
@@ -84,7 +78,7 @@ class FlashAttention(nn.Module):
                 x = rearrange(qkv, 'b s three h d -> b s (three h d)')
                 x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
                 x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
-                output_unpad = flash_attn_unpadded_qkvpacked_func(
+                output_unpad = flash_attn_varlen_qkvpacked_func(
                     x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
                     softmax_scale=self.softmax_scale, causal=causal
                 )
@@ -93,7 +87,7 @@ class FlashAttention(nn.Module):
                                    'b s (h d) -> b s h d', h=nheads)
         else:
             assert max_s is not None
-            output = flash_attn_unpadded_qkvpacked_func(
+            output = flash_attn_varlen_qkvpacked_func(
                 qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
                 softmax_scale=self.softmax_scale, causal=causal
             )