Create README.md

README.md

@@ -0,0 +1,31 @@
+training_arguments = transformers.TrainingArguments(
+    per_device_train_batch_size=4,
+    gradient_accumulation_steps=BATCH_SIZE // MICRO_BATCH_SIZE,
+    warmup_steps=100,
+    max_steps=300,
+    learning_rate=3e-4,
+    fp16=True,
+    logging_steps=10,
+    optim="adamw_torch",
+    evaluation_strategy="steps",
+    save_strategy="steps",
+    eval_steps=50,
+    save_steps=50,
+    save_total_limit=3,
+    load_best_model_at_end=True,
+)
+
+LORA_R = 8
+LORA_ALPHA = 16
+LORA_DROPOUT= 0
+LORA_TARGET_MODULES = [
+    "q_proj",
+    "v_proj",
+]
+
+BATCH_SIZE = 128
+MICRO_BATCH_SIZE = 4
+GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
+LEARNING_RATE = 3e-4
+TRAIN_STEPS = 300
+OUTPUT_DIR = "llygmaV2-13B"