|
--- |
|
base_model: Medixspace/Mistral-Nemo-Instruct-2407-freeze-with-TNM |
|
library_name: peft |
|
--- |
|
|
|
Checkpoint: /media/mlazure4775847383/storage/LM_saves_Mistral/Mistral-Nemo-Instruct-2407/lora/train_2024-11-05-09-59-46/checkpoint-2103 |
|
|
|
NB! The tokenizer in this repo has been changed for the base model's tokenizer. This is needed for predibase run. The original tokenizer is in storage. |
|
|
|
Training args: |
|
``` |
|
bf16: true |
|
cutoff_len: 25000 |
|
dataset: train_data_checklist,val_data_checklist |
|
dataset_dir: data |
|
ddp_timeout: 180000000 |
|
deepspeed: cache/ds_z3_offload_config.json |
|
do_train: true |
|
finetuning_type: lora |
|
flash_attn: fa2 |
|
gradient_accumulation_steps: 8 |
|
include_num_input_tokens_seen: true |
|
learning_rate: 5.0e-05 |
|
logging_steps: 250 |
|
lora_alpha: 16 |
|
lora_dropout: 0 |
|
lora_rank: 8 |
|
lora_target: all |
|
lr_scheduler_type: cosine |
|
max_grad_norm: 1.0 |
|
max_samples: 100000 |
|
model_name_or_path: Medixspace/Mistral-Nemo-Instruct-2407-freeze-with-TNM |
|
num_train_epochs: 3.0 |
|
optim: adamw_torch |
|
output_dir: saves/Mistral-Nemo-Instruct-2407/lora/train_2024-11-05-09-59-46 |
|
packing: false |
|
per_device_train_batch_size: 1 |
|
plot_loss: true |
|
preprocessing_num_workers: 16 |
|
report_to: none |
|
save_steps: 500 |
|
stage: sft |
|
template: mistral |
|
warmup_steps: 0 |
|
``` |
|
|
|
Best inference args: |
|
``` |
|
temperature=0.9 |
|
top-p=0.5 |
|
``` |
