Hugging Face's logo

ausboss
/
llygma-13b-lora

Model card Files Community
1
llygma-13b-lora
File size: 680 Bytes 
9bd7284
f3e0ea6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9bd7284
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
```
training_arguments = transformers.TrainingArguments(
    per_device_train_batch_size=4,
    gradient_accumulation_steps=BATCH_SIZE // MICRO_BATCH_SIZE,
    warmup_steps=100,
    max_steps=300,
    learning_rate=3e-4,
    fp16=True,
    logging_steps=10,
    optim="adamw_torch",
    evaluation_strategy="steps",
    save_strategy="steps",
    eval_steps=50,
    save_steps=50,
    save_total_limit=3,
    load_best_model_at_end=True,
)

LORA_R = 8
LORA_ALPHA = 16
LORA_DROPOUT= 0
LORA_TARGET_MODULES = [
    "q_proj",
    "v_proj",
]

BATCH_SIZE = 128
MICRO_BATCH_SIZE = 4
GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
LEARNING_RATE = 3e-4
TRAIN_STEPS = 300
```