Upload folder using huggingface_hub

README.md

@@ -2,7 +2,7 @@
 license: llama3
 pipeline_tag: image-text-to-text
 library_name: transformers
-base_model: 
+base_model:
   - OpenGVLab/InternViT-6B-448px-V1-5
   - NousResearch/Hermes-2-Theta-Llama-3-70B
 base_model_relation: merge

modeling_internvl_chat.py

@@ -36,7 +36,7 @@ class InternVLChatModel(PreTrainedModel):
     _supports_flash_attn_2 = True
     _no_split_modules = ['InternVisionModel', 'LlamaDecoderLayer']
 
-    def __init__(self, config: InternVLChatConfig, vision_model=None, language_model=None):
+    def __init__(self, config: InternVLChatConfig, vision_model=None, language_model=None, use_flash_attn=True):
         super().__init__(config)
 
         assert version_cmp(transformers.__version__, '4.36.2', 'ge')
@@ -48,6 +48,8 @@ class InternVLChatModel(PreTrainedModel):
         self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
         self.downsample_ratio = config.downsample_ratio
         self.ps_version = config.ps_version
+        config.vision_config.use_flash_attn = True if use_flash_attn else False
+        config.llm_config._attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
 
         logger.info(f'num_image_token: {self.num_image_token}')
         logger.info(f'ps_version: {self.ps_version}')