Pragades/Phi-3 instruct(Interview) Eager Attention!

README.md

@@ -47,6 +47,7 @@ The following hyperparameters were used during training:
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 3
+- mixed_precision_training: Native AMP
 
 ### Training results
 

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj",
-    "gate_proj",
-    "down_proj",
     "k_proj",
+    "o_proj",
+    "down_proj",
+    "gate_proj",
     "up_proj",
-    "o_proj"
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6983742a0b589265d0ab643d9a7b32172e04f863d75c853ee0a297424ab0baa1
-size 9456424
+oid sha256:c3a8b63c65e2485ddb4dd0fc96e4e403cc777ce8d4553e6ee9998776b0db3d33
+size 18893616

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 512,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddb0404b9bb67a8f86d70201e8facac4c84b58083f9fa1d86dacc87468f49c34
+oid sha256:e41cb99d80f82182557a2d873531e071d236a15ce674b31c262a79295830b743
 size 4987