Upload folder using huggingface_hub

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "IbuNai/Mixtral-8x7B-v0.1",
+  "_name_or_path": "stabilityai/japanese-stablelm-base-gamma-7b",
   "architectures": [
     "MixtralForCausalLM"
   ],
@@ -10,7 +10,7 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 32768,
+  "max_position_embeddings": 4096,
   "model_type": "mixtral",
   "num_attention_heads": 32,
   "num_experts_per_tok": 2,
@@ -19,12 +19,12 @@
   "num_local_experts": 4,
   "output_router_logits": false,
   "rms_norm_eps": 1e-05,
-  "rope_theta": 1000000.0,
-  "router_aux_loss_coef": 0.02,
-  "sliding_window": 4096,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.37.0.dev0",
+  "transformers_version": "4.36.2",
   "use_cache": true,
   "vocab_size": 32000
 }

mergekit_moe_config.yml

@@ -0,0 +1,15 @@
+base_model: stabilityai/japanese-stablelm-base-gamma-7b
+gate_mode: random
+experts:
+  - source_model: stabilityai/japanese-stablelm-base-gamma-7b
+    positive_prompts:
+      - ""      
+  - source_model: stabilityai/japanese-stablelm-base-gamma-7b
+    positive_prompts:
+      - ""      
+  - source_model: stabilityai/japanese-stablelm-base-gamma-7b
+    positive_prompts:
+      - ""      
+  - source_model: stabilityai/japanese-stablelm-base-gamma-7b
+    positive_prompts:
+      - ""      

model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64797d5b1966d01b09c88276af83c1f7ed48ef8b931aab2987b8369fe6bed18b
+size 9919813704

model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f880eded26eb7956b3eedae60fef15de0648cff4bcc068b715cb85b4515917c
+size 9982454720

model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ae7e7967a196d90784eb7359797339ef4560841eaf23f035fba3073e5da721c
+size 9982454752

model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:996ab3f39c3d8e6391998ed7f66aef6de26427c0e312ae8fce07ca5f6e031061
+size 9982454720

model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09d31b98bc2e8cfb0aa5598ec43ebcb08653b93ed0fd33e02ee024f350dcd77e
+size 8440279464

special_tokens_map.json

@@ -13,6 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
+  "pad_token": "<s>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer_config.json

@@ -33,10 +33,10 @@
   "eos_token": "</s>",
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "pad_token": "<s>",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "use_default_system_prompt": true
 }