Training in progress, epoch 0

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "mistralai/Mixtral-8x7B-v0.1",
+  "base_model_name_or_path": "rishiraj/CatPPT-base",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -19,14 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "w3",
-    "o_proj",
-    "w2",
-    "gate",
+    "down_proj",
     "q_proj",
     "k_proj",
-    "w1"
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24b84449af861f32cfcd5913ae1065f9da3d0065caba8879e896a55a64e6d7dc
-size 1938079368
+oid sha256:6bc8ba1f57661247e8a42a6aebac223cd38e9d5fb7f135054ed7ea6f42e210f1
+size 335605144

config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "mistralai/Mixtral-8x7B-v0.1",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.1",
+  "use_cache": true,
+  "vocab_size": 32002
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.38.1"
+}

model-00001-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:294ce339b5b9a31e21c2071622fb7c12c151882566e7c0b32c424b46282513fb
+size 1993356879

model-00002-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f757999c765587bdac8091e928ce717a4001d125fa7fec1a38413820fd8be25
+size 1982178655

model-00003-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b0d6d86daacab33aa2758541f9fe01bdcf8d0fa318672d8684e075616c4d1e8
+size 1973565157

model-00004-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cdda72f3d8e401c20af56174b02b8db150b2dc4c05c6b41c100f10180ee99ba
+size 1973565191

model-00005-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b2bd724c9e535f94975d66ded598935be506af0a9cc6801cf6e4f505b8ddd02
+size 1982179112

model-00006-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a756c170a31bd5bab9bfe2ea890e12183aafe3f5a30ca229987deeb8c2eade4
+size 1973565628

model-00007-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c2bd273d16132f5ceec910190b6b0bd22466a5f2005ef8a64c89b91e9d3a08c
+size 1973565666

model-00008-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7deda8e9b898f612042a3de94ed2a429e533772d2fcc061c72d496eb40dfc93
+size 1982179112

model-00009-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78330c51cb6ee836bd61f88320be0af2dac90d10e332855dbeb3ee99bf2ee20f
+size 1973565626

model-00010-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b05a33449cdf2526bf91290483f372f9855e9f767164ad44dfc24e79ffa7d25c
+size 1973565642

model-00011-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2f4a0c4370b48ee0f7effe7774002bf15e69ff9dea0be389879b71b3c546542
+size 1982179109

model-00012-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cea7022f0cc0ef0d9fa84485d7835ab0d57b3f303fd10db14bf0853d28b6548a
+size 1973565628

model-00013-of-00013.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44d28fb0ad5620b6280b409786ebda7ab1979d4e9dd9a88437a6df57a2ea3056
+size 746889808

runs/Mar21_16-03-45_llm-a100-40/events.out.tfevents.1711037188.llm-a100-40.13908.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77ecb51bbace8f3652f70c97d5c814ae2300b8013e7e67129ee5247407d71c24
+size 5460

tokenizer_config.json

@@ -58,5 +58,5 @@
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "use_default_system_prompt": true
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efc90214823a2395511441e64e7cea4b523e5e278cd13f4982c2faaacc5ae9e5
+oid sha256:2980b72d84e275381465951148dc0d834faf30bc04f7a6c252e5082253001d33
 size 4856