training cofigs

training_configs/optimizer_dadapt.json

@@ -0,0 +1,53 @@
+{
+    "doc": {
+        "base": "base optimizer configuration for unet and text encoder",
+        "text_encoder_overrides": "text encoder config overrides",
+        "text_encoder_lr_scale": "if LR not set on text encoder, sets the Lr to a multiple of the Base LR. for example, if base `lr` is 2e-6 and `text_encoder_lr_scale` is 0.5, the text encoder's LR will be set to `1e-6`.",
+        "-----------------": "-----------------",
+        "optimizer": "adamw, adamw8bit, lion, dadapt_adam, dadapt_lion",
+        "optimizer_desc": "'adamw' in standard 32bit, 'adamw8bit' is bitsandbytes, 'lion' is EvoLved Sign Momentum, 'dadapt_...' are D-Adaptation methods",
+        "lr": "learning rate, if null will use CLI or main JSON config value",
+        "lr_scheduler": "'constant' or 'cosine'",
+        "lr_warmup_steps": "number of steps to warmup LR to target LR, if null will use CLI or default a value based on max epochs",
+        "lr_decay_steps": "number of steps to decay LR to zero for cosine, if null will use CLI or default a value based on max epochs",
+        "betas": "exponential decay rates for the moment estimates",
+        "epsilon": "value added to denominator for numerical stability, unused for lion, also used as d0 for dadaptation",
+        "weight_decay": "weight decay (L2 penalty)",
+        "d0": "for dadaptation only, scale of initial steps (def: 1e-6)",
+        "decouple": "for dadapt_adam only, whether to decouple the learning rates of the two distributions, suggested true",
+        "momentum": "for dadapt_sgd only, the momentum factor",
+        "------------------": "-----------------",
+        "freeze_embeddings": "whether to freeze the text embeddings",
+        "freeze_front_n_layers": "if not null, freeze the front N layers of the text encoder (you can pass eg -2 to leave only the last 2 layers unfrozen)",
+        "freeze_final_layer_norm": "whether to freeze the text encoder's final layer norm"
+    },
+    "base": {
+        "optimizer": "dadapt_adam",
+        "lr": 1e-1,
+        "lr_scheduler": "constant",
+        "lr_decay_steps": null,
+        "lr_warmup_steps": null,
+        "betas": [0.9, 0.999],
+        "epsilon": 1e-8,
+        "weight_decay": 0.80,
+        "d0": 1e-6,
+        "decouple": true
+    },
+    "text_encoder_overrides": {
+        "optimizer": "dadapt_adam",
+        "lr": 1e-1,
+        "lr_scheduler": "constant",
+        "lr_decay_steps": null,
+        "lr_warmup_steps": null,
+        "betas": [0.9, 0.999],
+        "epsilon": 1e-8,
+        "weight_decay": 0.80,
+        "d0": 1e-6,
+        "decouple": true
+    },
+    "text_encoder_freezing": {
+        "freeze_embeddings": true,
+        "freeze_front_n_layers": -3,
+        "freeze_final_layer_norm": false
+    }
+}

training_configs/train.json

@@ -0,0 +1,43 @@
+{
+  "batch_size": 4,
+  "ckpt_every_n_minutes": null,
+  "clip_grad_norm": null,
+  "clip_skip": 0,
+  "cond_dropout": 0.05,
+  "data_root": "input/vodka_v4", 
+  "disable_amp": false,
+  "disable_textenc_training": false,
+  "disable_xformers": false,
+  "flip_p": 0.0,
+  "gpuid": 0,
+  "gradient_checkpointing": true,
+  "grad_accum": 1,
+  "logdir": "logs",
+  "log_step": 1,
+  "lowvram": false,
+  "lr_decay_steps": 0,
+  "lr_scheduler": "constant",
+  "lr_warmup_steps": null,
+  "max_epochs": 100,
+  "notebook": false,
+  "optimizer_config": "optimizer_dadapt.json",
+  "project_name": "vodka_v4_3",
+  "resolution": 512,
+  "resume_ckpt": "sd_v1-5_vae",
+  "run_name": null,
+  "sample_prompts": "sample_prompts.txt",
+  "sample_steps": 10000,
+  "save_ckpt_dir": null,
+  "save_ckpts_from_n_epochs": 0,
+  "save_every_n_epochs": 25,
+  "save_optimizer": false,
+  "scale_lr": false,
+  "seed": 555,
+  "shuffle_tags": false,
+  "validation_config": "validation_default.json",
+  "wandb": false,
+  "write_schedule": false,
+  "rated_dataset": false,
+  "rated_dataset_target_dropout_percent": 50,
+  "zero_frequency_noise_ratio": 0.00
+}

training_configs/validation_default.json

@@ -0,0 +1,24 @@
+{
+    "documentation": {
+        "validate_training": "If true, validate the training using a separate set of image/caption pairs, and log the results as `loss/val`. The curve will trend downwards as the model trains, then flatten and start to trend upwards as effective training finishes and the model begins to overfit the training data. Very useful for preventing overfitting, for checking if your learning rate is too low or too high, and for deciding when to stop training.",
+        "val_split_mode": "Either 'automatic' or 'manual', ignored if validate_training is false. 'automatic' val_split_mode picks a random subset of the training set (the number of items is controlled by auto_split_proportion) and removes them from training to use as a validation set. 'manual' val_split_mode lets you provide your own folder of validation items (images+captions), specified using 'manual_data_root'.",
+        "auto_split_proportion": "For 'automatic' val_split_mode, how much of the train dataset that should be removed to use for validation. Typical values are 0.15-0.2 (15-20% of the total dataset). Higher is more accurate but slower.",
+        "manual_data_root": "For 'manual' val_split_mode, the path to a folder containing validation items.",
+        "extra_manual_datasets": "Dictionary of 'name':'path' pairs defining additional validation datasets to load and log. eg { 'santa_suit': '/path/to/captioned_santa_suit_images', 'flamingo_suit': '/path/to/flamingo_suit_images' }",
+        "stabilize_training_loss": "If true, stabilize the train loss curves for `loss/epoch` and `loss/log step` by re-calculating training loss with a fixed random seed, and log the results as `loss/train-stabilized`. This more clearly shows the training progress, but it is not enough alone to tell you if you're overfitting.",
+        "stabilize_split_proportion": "For stabilize_training_loss, the proportion of the train dataset to overlap for stabilizing the train loss graph. Typical values are 0.15-0.2 (15-20% of the total dataset). Higher is more accurate but slower.",
+        "every_n_epochs": "How often to run validation (1=every epoch, 2=every second epoch; 0.5=twice per epoch, 0.33=three times per epoch, etc.).",
+        "seed": "The seed to use when running validation and stabilization passes.",
+        "use_relative_loss": "logs val/loss as negative relative to first pre-train val/loss value"
+    },
+    "validate_training": true,
+    "val_split_mode": "automatic",
+    "auto_split_proportion": 0.15,
+    "manual_data_root": null,
+    "extra_manual_datasets" : {},
+    "stabilize_training_loss": false,
+    "stabilize_split_proportion": 0.15,
+    "every_n_epochs": 1,
+    "seed": 555,
+    "use_relative_loss": false
+}