[WIP] Upload folder using huggingface_hub (multi-commit 4ff51e9f8311962175ab7efd2bdb581fab5ea17e24aec24703649d2fc01e907f)

README.md

@@ -49,15 +49,6 @@ More details can be found in our paper.
 If you use LibMoE, please cite it using this BibTeX:
 
 ```
-@misc{nguyen2024libmoelibrarycomprehensivebenchmarking,
-      title={LIBMoE: A Library for comprehensive benchmarking Mixture of Experts in Large Language Models}, 
-      author={Nam V. Nguyen and Thong T. Doan and Luong Tran and Van Nguyen and Quang Pham},
-      year={2024},
-      eprint={2411.00918},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL},
-      url={https://arxiv.org/abs/2411.00918}, 
-}
 ```
 
 

pft/added_tokens.json

@@ -1,13 +0,0 @@
-{
-  "<|assistant|>": 32001,
-  "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|placeholder1|>": 32002,
-  "<|placeholder2|>": 32003,
-  "<|placeholder3|>": 32004,
-  "<|placeholder4|>": 32005,
-  "<|placeholder5|>": 32008,
-  "<|placeholder6|>": 32009,
-  "<|system|>": 32006,
-  "<|user|>": 32010
-}

pft/clip.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5b1f13272c4ab8444bc5f83478f1d15155b8e702c8a8aa900cd9d5d775aa3985
-size 824993767

pft/config.json

@@ -1,66 +0,0 @@
-{
-  "_name_or_path": "/cm/archive/thongdt4/toolkitmoe/checkpoints/transformers_checkpoints/phi-3-mini-4k-instruct",
-  "architectures": [
-    "LlavaPhiForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "configuration_phi3.Phi3Config",
-    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
-  },
-  "balance_loss_coef": 0.0,
-  "bos_token_id": 1,
-  "clip_smoe": false,
-  "dropout": false,
-  "embd_pdrop": 0.0,
-  "eos_token_id": 32000,
-  "freeze_mm_mlp_adapter": false,
-  "hidden_act": "silu",
-  "hidden_size": 3072,
-  "image_aspect_ratio": "pad",
-  "initializer_range": 0.02,
-  "intermediate_size": 8192,
-  "local_rank": 0,
-  "max_position_embeddings": 4096,
-  "mlp_smoe": false,
-  "mm_hidden_size": 1152,
-  "mm_patch_merge_type": "flat",
-  "mm_projector_lr": null,
-  "mm_projector_type": "mlp2x_gelu",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "google/siglip-so400m-patch14-224",
-  "model_type": "llava_phi",
-  "moe_name": "smoe",
-  "num_attention_heads": 32,
-  "num_experts": 1,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "num_layers": 3,
-  "num_selected": 1,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "resid_pdrop": 0.0,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
-  "router_z_loss_coef": 0.0,
-  "scales": [
-    1,
-    3
-  ],
-  "sliding_window": 2047,
-  "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 4096,
-  "tokenizer_padding_side": "right",
-  "torch_dtype": "bfloat16",
-  "training": true,
-  "transformers_version": "4.43.2",
-  "tune_mm_mlp_adapter": false,
-  "use_cache": true,
-  "use_mm_proj": true,
-  "vocab_size": 32064
-}

pft/generation_config.json

@@ -1,12 +0,0 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 1,
-  "do_sample": true,
-  "eos_token_id": [
-    32000,
-    32001,
-    32007
-  ],
-  "pad_token_id": 32000,
-  "transformers_version": "4.43.2"
-}

pft/mm_projector.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0e72e505ed3caaa80a5ca6abf7c96dda3c412c513581b6a779f770ec93532a5c
-size 33044536

pft/model-00001-of-00002.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a71c80ca9162a80052aeae0c4118b94c172ff2689c661cc40ce1d5eec6d4abd3
-size 4972489328

pft/model-00002-of-00002.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:640ff76f658c238dec128bfeb1715c7fcdb3686a7f0218a445de898c096370ff
-size 3527678240

pft/model.safetensors.index.json

@@ -1,641 +0,0 @@
-{
-  "metadata": {
-    "total_size": 8500081632
-  },
-  "weight_map": {
-    "lm_head.weight": "model-00002-of-00002.safetensors",
-    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
-    "model.mm_projector.0.bias": "model-00002-of-00002.safetensors",
-    "model.mm_projector.0.weight": "model-00002-of-00002.safetensors",
-    "model.mm_projector.2.bias": "model-00002-of-00002.safetensors",
-    "model.mm_projector.2.weight": "model-00002-of-00002.safetensors",
-    "model.norm.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.embeddings.patch_embedding.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
-    "model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00002-of-00002.safetensors"
-  }
-}

pft/special_tokens_map.json

@@ -1,24 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<unk>",
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

pft/tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723

pft/tokenizer_config.json

@@ -1,132 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "add_prefix_space": true,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": false
-    },
-    "32000": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<|assistant|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<|placeholder1|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<|placeholder2|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<|placeholder3|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32005": {
-      "content": "<|placeholder4|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32006": {
-      "content": "<|system|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32007": {
-      "content": "<|end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32008": {
-      "content": "<|placeholder5|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32009": {
-      "content": "<|placeholder6|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32010": {
-      "content": "<|user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "legacy": false,
-  "model_max_length": 4096,
-  "pad_token": "<unk>",
-  "padding_side": "right",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}

pft/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f1e11108b9ed715af3ef653385c30fabeab6a4f375c5a8908dd3b347ecff7b7f
-size 7416

sft/hyperrouter/added_tokens.json

@@ -1,13 +0,0 @@
-{
-  "<|assistant|>": 32001,
-  "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|placeholder1|>": 32002,
-  "<|placeholder2|>": 32003,
-  "<|placeholder3|>": 32004,
-  "<|placeholder4|>": 32005,
-  "<|placeholder5|>": 32008,
-  "<|placeholder6|>": 32009,
-  "<|system|>": 32006,
-  "<|user|>": 32010
-}

sft/hyperrouter/config.json

@@ -1,68 +0,0 @@
-{
-  "_name_or_path": "/cm/archive/thongdt4/toolkitmoe/checkpoints/phi3mini-siglip224/pft",
-  "architectures": [
-    "LlavaPhiForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "configuration_phi3.Phi3Config",
-    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
-  },
-  "balance_loss_coef": 0.1,
-  "bos_token_id": 1,
-  "clip_smoe": true,
-  "dropout": false,
-  "embd_pdrop": 0.0,
-  "eos_token_id": 32000,
-  "freeze_mm_mlp_adapter": false,
-  "hidden_act": "silu",
-  "hidden_size": 3072,
-  "image_aspect_ratio": "pad",
-  "initializer_range": 0.02,
-  "intermediate_size": 8192,
-  "local_rank": 0,
-  "max_position_embeddings": 4096,
-  "mlp_smoe": true,
-  "mm_hidden_size": 1152,
-  "mm_patch_merge_type": "flat",
-  "mm_projector_lr": null,
-  "mm_projector_type": "moe",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "google/siglip-so400m-patch14-224",
-  "model_type": "llava_phi",
-  "moe_name": "hyperrouter",
-  "num_attention_heads": 32,
-  "num_experts": 4,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "num_layers": 3,
-  "num_selected": 2,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "resid_pdrop": 0.0,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
-  "router_z_loss_coef": 0.01,
-  "scales": [
-    1,
-    3
-  ],
-  "sliding_window": 2047,
-  "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 2048,
-  "tokenizer_padding_side": "right",
-  "topk_max": 2,
-  "topk_min": 1,
-  "torch_dtype": "bfloat16",
-  "training": true,
-  "transformers_version": "4.43.2",
-  "tune_mm_mlp_adapter": false,
-  "use_cache": true,
-  "use_mm_proj": true,
-  "vocab_size": 32064
-}

sft/hyperrouter/generation_config.json

@@ -1,12 +0,0 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 1,
-  "do_sample": true,
-  "eos_token_id": [
-    32000,
-    32001,
-    32007
-  ],
-  "pad_token_id": 32000,
-  "transformers_version": "4.43.2"
-}

sft/hyperrouter/model-00001-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9b4d0844c655591f1968d2ce825d5e82df6c438775c090daee14b7412ccc7055
-size 4972489328

sft/hyperrouter/model-00002-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7ee48788c3759ffa9c2c4ed88ce6ee26a64e9e3c0e1386318292139ce2c500cf
-size 4995022432

sft/hyperrouter/model-00003-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:73a616bad6c479415ca0994b6c542dc194b7ded1ce11a763ca4163704544dd86
-size 342468696

sft/hyperrouter/special_tokens_map.json

@@ -1,24 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<unk>",
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

sft/hyperrouter/tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723

sft/hyperrouter/tokenizer_config.json

@@ -1,132 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "add_prefix_space": true,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": false
-    },
-    "32000": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<|assistant|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<|placeholder1|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<|placeholder2|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<|placeholder3|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32005": {
-      "content": "<|placeholder4|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32006": {
-      "content": "<|system|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32007": {
-      "content": "<|end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32008": {
-      "content": "<|placeholder5|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32009": {
-      "content": "<|placeholder6|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32010": {
-      "content": "<|user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "legacy": false,
-  "model_max_length": 2048,
-  "pad_token": "<unk>",
-  "padding_side": "right",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}

sft/hyperrouter/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c09427a177e86daca87c380071eb69bb649199ac37ec8b4066e07e5d8e69e1bb
-size 7352

sft/smoe/added_tokens.json

@@ -1,13 +0,0 @@
-{
-  "<|assistant|>": 32001,
-  "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|placeholder1|>": 32002,
-  "<|placeholder2|>": 32003,
-  "<|placeholder3|>": 32004,
-  "<|placeholder4|>": 32005,
-  "<|placeholder5|>": 32008,
-  "<|placeholder6|>": 32009,
-  "<|system|>": 32006,
-  "<|user|>": 32010
-}

sft/smoe/config.json

@@ -1,66 +0,0 @@
-{
-  "_name_or_path": "/cm/archive/thongdt4/toolkitmoe/checkpoints/phi3mini-siglip224/pft",
-  "architectures": [
-    "LlavaPhiForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "configuration_phi3.Phi3Config",
-    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
-  },
-  "balance_loss_coef": 0.1,
-  "bos_token_id": 1,
-  "clip_smoe": true,
-  "dropout": false,
-  "embd_pdrop": 0.0,
-  "eos_token_id": 32000,
-  "freeze_mm_mlp_adapter": false,
-  "hidden_act": "silu",
-  "hidden_size": 3072,
-  "image_aspect_ratio": "pad",
-  "initializer_range": 0.02,
-  "intermediate_size": 8192,
-  "local_rank": 0,
-  "max_position_embeddings": 4096,
-  "mlp_smoe": true,
-  "mm_hidden_size": 1152,
-  "mm_patch_merge_type": "flat",
-  "mm_projector_lr": null,
-  "mm_projector_type": "moe",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "google/siglip-so400m-patch14-224",
-  "model_type": "llava_phi",
-  "moe_name": "smoe",
-  "num_attention_heads": 32,
-  "num_experts": 4,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "num_layers": 3,
-  "num_selected": 2,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "resid_pdrop": 0.0,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
-  "router_z_loss_coef": 0.01,
-  "scales": [
-    1,
-    3
-  ],
-  "sliding_window": 2047,
-  "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 2048,
-  "tokenizer_padding_side": "right",
-  "torch_dtype": "bfloat16",
-  "training": true,
-  "transformers_version": "4.43.2",
-  "tune_mm_mlp_adapter": false,
-  "use_cache": true,
-  "use_mm_proj": true,
-  "vocab_size": 32064
-}

sft/smoe/generation_config.json

@@ -1,12 +0,0 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 1,
-  "do_sample": true,
-  "eos_token_id": [
-    32000,
-    32001,
-    32007
-  ],
-  "pad_token_id": 32000,
-  "transformers_version": "4.43.2"
-}

sft/smoe/model-00001-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:69feaff0e4b709ada48a0f2e32927ef45315c40a21da51238da2c77e5ed7a093
-size 4972489328

sft/smoe/model-00002-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:10e2c7781c5462f9ee1bc3167c625026078097aa3b5151085a195e18a6302acd
-size 4985529648

sft/smoe/model-00003-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e4138152dfc9641daf414984226e43a0d470aab2c4bab6bf7d9888918cdeee2e
-size 248943552

sft/smoe/special_tokens_map.json

@@ -1,24 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<unk>",
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

sft/smoe/tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723

sft/smoe/tokenizer_config.json

@@ -1,132 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "add_prefix_space": true,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": false
-    },
-    "32000": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<|assistant|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<|placeholder1|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<|placeholder2|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<|placeholder3|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32005": {
-      "content": "<|placeholder4|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32006": {
-      "content": "<|system|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32007": {
-      "content": "<|end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32008": {
-      "content": "<|placeholder5|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32009": {
-      "content": "<|placeholder6|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32010": {
-      "content": "<|user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "legacy": false,
-  "model_max_length": 2048,
-  "pad_token": "<unk>",
-  "padding_side": "right",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}

sft/smoe/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:61756c07bfb3f317144941ced65db861389324ed6960061a4c7838ceba32c266
-size 8120

sft/smoe_cosinegating/added_tokens.json

@@ -1,13 +0,0 @@
-{
-  "<|assistant|>": 32001,
-  "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|placeholder1|>": 32002,
-  "<|placeholder2|>": 32003,
-  "<|placeholder3|>": 32004,
-  "<|placeholder4|>": 32005,
-  "<|placeholder5|>": 32008,
-  "<|placeholder6|>": 32009,
-  "<|system|>": 32006,
-  "<|user|>": 32010
-}

sft/smoe_cosinegating/config.json

@@ -1,66 +0,0 @@
-{
-  "_name_or_path": "/cm/archive/thongdt4/toolkitmoe/checkpoints/phi3mini-siglip224/pft",
-  "architectures": [
-    "LlavaPhiForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "configuration_phi3.Phi3Config",
-    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
-  },
-  "balance_loss_coef": 0.1,
-  "bos_token_id": 1,
-  "clip_smoe": true,
-  "dropout": false,
-  "embd_pdrop": 0.0,
-  "eos_token_id": 32000,
-  "freeze_mm_mlp_adapter": false,
-  "hidden_act": "silu",
-  "hidden_size": 3072,
-  "image_aspect_ratio": "pad",
-  "initializer_range": 0.02,
-  "intermediate_size": 8192,
-  "local_rank": 0,
-  "max_position_embeddings": 4096,
-  "mlp_smoe": true,
-  "mm_hidden_size": 1152,
-  "mm_patch_merge_type": "flat",
-  "mm_projector_lr": null,
-  "mm_projector_type": "moe",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "google/siglip-so400m-patch14-224",
-  "model_type": "llava_phi",
-  "moe_name": "smoe_cosinegating",
-  "num_attention_heads": 32,
-  "num_experts": 4,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "num_layers": 3,
-  "num_selected": 2,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "resid_pdrop": 0.0,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
-  "router_z_loss_coef": 0.01,
-  "scales": [
-    1,
-    3
-  ],
-  "sliding_window": 2047,
-  "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 2048,
-  "tokenizer_padding_side": "right",
-  "torch_dtype": "bfloat16",
-  "training": true,
-  "transformers_version": "4.43.2",
-  "tune_mm_mlp_adapter": false,
-  "use_cache": true,
-  "use_mm_proj": true,
-  "vocab_size": 32064
-}

sft/smoe_cosinegating/generation_config.json

@@ -1,12 +0,0 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 1,
-  "do_sample": true,
-  "eos_token_id": [
-    32000,
-    32001,
-    32007
-  ],
-  "pad_token_id": 32000,
-  "transformers_version": "4.43.2"
-}

sft/smoe_cosinegating/model-00001-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:35ad0d676b7b860b1a5754ae2e4fc3170a859b67bdcc12b80359e39e7c9354bc
-size 4972489328

sft/smoe_cosinegating/model-00002-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cd56c83d2cd92280052b2df45a7f84808db1a6f67adf8eaa18c9ae1d54c4b748
-size 4985533608

sft/smoe_cosinegating/model-00003-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7911dd07a7b1f5f1a951c1de6fce45f76e9c8f60131be654a505124e2669bb07
-size 248943664

sft/smoe_cosinegating/special_tokens_map.json

@@ -1,24 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<unk>",
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

sft/smoe_cosinegating/tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723

sft/smoe_cosinegating/tokenizer_config.json

@@ -1,132 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "add_prefix_space": true,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": false
-    },
-    "32000": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<|assistant|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<|placeholder1|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<|placeholder2|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<|placeholder3|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32005": {
-      "content": "<|placeholder4|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32006": {
-      "content": "<|system|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32007": {
-      "content": "<|end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32008": {
-      "content": "<|placeholder5|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32009": {
-      "content": "<|placeholder6|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32010": {
-      "content": "<|user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "legacy": false,
-  "model_max_length": 2048,
-  "pad_token": "<unk>",
-  "padding_side": "right",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}

sft/smoe_cosinegating/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:60cec9748462c83f1ad285ec43d62e04acb5bd54d2e44a1d755485e7a9d09ea8
-size 8184