tcarwash/tinyllama-instruct

README.md

@@ -5,7 +5,7 @@ tags:
 - trl
 - sft
 - generated_from_trainer
-base_model: tinyllama/tinyllama-1.1b-chat-v1.0
+base_model: tinyllama/tinyllama-1.1b-intermediate-step-1431k-3t
 datasets:
 - generator
 model-index:
@@ -18,9 +18,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # tinyllama_instruct
 
-This model is a fine-tuned version of [tinyllama/tinyllama-1.1b-chat-v1.0](https://huggingface.co/tinyllama/tinyllama-1.1b-chat-v1.0) on the generator dataset.
+This model is a fine-tuned version of [tinyllama/tinyllama-1.1b-intermediate-step-1431k-3t](https://huggingface.co/tinyllama/tinyllama-1.1b-intermediate-step-1431k-3t) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.2206
+- Loss: 1.3012
 
 ## Model description
 
@@ -46,15 +46,16 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: constant
 - lr_scheduler_warmup_steps: 0.03
-- num_epochs: 2
-- mixed_precision_training: Native AMP
+- num_epochs: 4
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss |
-|:-------------:|:-----:|:----:|:---------------:|
-| 1.3509        | 1.0   | 1069 | 1.2261          |
-| 1.3601        | 2.0   | 2138 | 1.2206          |
+| Training Loss | Epoch | Step  | Validation Loss |
+|:-------------:|:-----:|:-----:|:---------------:|
+| 1.4385        | 1.0   | 4263  | 1.2766          |
+| 1.4772        | 2.0   | 8526  | 1.2743          |
+| 1.0998        | 3.0   | 12789 | 1.2837          |
+| 1.3263        | 4.0   | 17052 | 1.3012          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,8 +20,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj"
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa83878e9ed23aea2d73dc55616f82325803e26ec7abc6627a79f7ca02cd64ba
-size 36073064
+oid sha256:dcb0045517d11e5dc96444a144c9f7e188f081679dbbc1a52764bdf7ffcf551b
+size 36058104

runs/May03_22-17-03_3d3bf14e01d0/events.out.tfevents.1714774626.3d3bf14e01d0.265.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c31b464bc4985979d24f62e32be6a14e720f0490f1a9309ce6b603f77b5e9919
+size 6817

runs/May03_22-17-50_3d3bf14e01d0/events.out.tfevents.1714774673.3d3bf14e01d0.265.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4f4576169022536120fecf08e569c80760734d771e95461d32916ee0343197a
+size 366170

tokenizer_config.json

@@ -28,11 +28,10 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": false,
-  "model_max_length": 2048,
+  "model_max_length": 1000000000000000019884624838656,
   "pad_token": "</s>",
   "padding_side": "right",
   "sp_model_kwargs": {},

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d92afdb66ecaab798d9aed611224a18b199a1621cd3dc2a1d006ba3466f40d24
+oid sha256:db0cad0f34b276b350f7482750bf5f1c4fe718452bce3cf3888dce72e6f5b0e0
 size 4984