Model save

README.md

@@ -0,0 +1,69 @@
+---
+license: mit
+library_name: peft
+tags:
+- trl
+- sft
+- generated_from_trainer
+base_model: microsoft/phi-2
+datasets:
+- generator
+model-index:
+- name: phi-2-sft-lora-chat
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# phi-2-sft-lora-chat
+
+This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.2272
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 48
+- total_eval_batch_size: 12
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 0.02
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.2627        | 0.02  | 111  | 1.2272          |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:706b9114fd6f16d11df20816efdccbe4aaa733bc7fc2434429bc9ff023182178
+oid sha256:d65e9b75d99d8a42ff66359a3ae4406271efb1d4107ca56e601092c07016c5bc
 size 335579632

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 0.02,
+    "eval_loss": 1.2271578311920166,
+    "eval_runtime": 1802.0046,
+    "eval_samples": 23110,
+    "eval_samples_per_second": 16.275,
+    "eval_steps_per_second": 1.356,
+    "train_loss": 1.2791897563246992,
+    "train_runtime": 2444.0756,
+    "train_samples": 207865,
+    "train_samples_per_second": 2.168,
+    "train_steps_per_second": 0.045
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.02,
+    "eval_loss": 1.2271578311920166,
+    "eval_runtime": 1802.0046,
+    "eval_samples": 23110,
+    "eval_samples_per_second": 16.275,
+    "eval_steps_per_second": 1.356
+}

runs/May18_23-14-07_gpu4-119-5/events.out.tfevents.1716038098.gpu4-119-5.2598658.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae1537b5ecaa6c99e78b41a2b546e11602973c01c500ccc9af9bf01ef6e15b4b
-size 7958
+oid sha256:4210c56ac3b8a174da30a2d4922bcccc7f16a777012647eaaaf18497e69821b7
+size 8880

runs/May18_23-14-07_gpu4-119-5/events.out.tfevents.1716042344.gpu4-119-5.2598658.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:392d9adb2eaa719969370551b24d2e67839b936011be9464f17910cbdaf75705
+size 354

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.02,
+    "train_loss": 1.2791897563246992,
+    "train_runtime": 2444.0756,
+    "train_samples": 207865,
+    "train_samples_per_second": 2.168,
+    "train_steps_per_second": 0.045
+}

trainer_state.json

@@ -0,0 +1,176 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.020107784973506633,
+  "eval_steps": 500,
+  "global_step": 111,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 1.2849,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 1.3474,
+      "step": 5
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001666666666666667,
+      "loss": 1.3589,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019954719225730847,
+      "loss": 1.3078,
+      "step": 15
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019679487013963564,
+      "loss": 1.2688,
+      "step": 20
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019161084574320696,
+      "loss": 1.3346,
+      "step": 25
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00018412535328311814,
+      "loss": 1.2897,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0001745264449675755,
+      "loss": 1.2717,
+      "step": 35
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00016305526670845226,
+      "loss": 1.2734,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 1.2862,
+      "step": 45
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00013568862215918717,
+      "loss": 1.2708,
+      "step": 50
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00012048066680651908,
+      "loss": 1.2435,
+      "step": 55
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00010475819158237425,
+      "loss": 1.2824,
+      "step": 60
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.891618000989891e-05,
+      "loss": 1.2778,
+      "step": 65
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.335261863099651e-05,
+      "loss": 1.2511,
+      "step": 70
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.845849869981137e-05,
+      "loss": 1.2728,
+      "step": 75
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.4607993613388976e-05,
+      "loss": 1.2866,
+      "step": 80
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 3.21490588442868e-05,
+      "loss": 1.2377,
+      "step": 85
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 2.139469052572127e-05,
+      "loss": 1.2673,
+      "step": 90
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2615062293021507e-05,
+      "loss": 1.2778,
+      "step": 95
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 6.030737921409169e-06,
+      "loss": 1.2666,
+      "step": 100
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.8071302737293295e-06,
+      "loss": 1.2382,
+      "step": 105
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 5.0345761681491746e-08,
+      "loss": 1.2627,
+      "step": 110
+    },
+    {
+      "epoch": 0.02,
+      "eval_loss": 1.2271578311920166,
+      "eval_runtime": 1802.5494,
+      "eval_samples_per_second": 16.27,
+      "eval_steps_per_second": 1.356,
+      "step": 111
+    },
+    {
+      "epoch": 0.02,
+      "step": 111,
+      "total_flos": 79541494087680.0,
+      "train_loss": 1.2791897563246992,
+      "train_runtime": 2444.0756,
+      "train_samples_per_second": 2.168,
+      "train_steps_per_second": 0.045
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 111,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 79541494087680.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}