Upload decoder_config.json

decoder_config.json

@@ -4,12 +4,24 @@
     "decoder": {
         "unets": [
             {
-                "dim": 64,
+                "dim": 416,
+                "cond_dim": 512,
                 "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
                 "channels": 3,
-                "dim_mults": [1, 2, 3, 4]
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true
             }
         ],
+        "clip":{
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
         "image_sizes": [64],
         "channels": 3,
         "timesteps": 1000,
@@ -18,13 +30,13 @@
         "learned_variance": true
     },
     "data": {
-        "webdataset_base_url": "pipe:s3cmd get s3://s-datasets/laion5b/laion2B-data/{}.tar -",
-        "embeddings_url": "s3://s-laion5b/embeddings/vit-l-14/laion2B-en/img_emb_reordered_tiny/",
-        "num_workers": 4,
-        "batch_size": 24,
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -", 
+        "embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
+        "num_workers": 12,
+        "batch_size": 22,
         "start_shard": 0,
-        "end_shard": 49,
-        "shard_width": 6,
+        "end_shard": 5247,
+        "shard_width": 5,
         "index_width": 4,
         "splits": {
             "train": 0.75,
@@ -43,17 +55,20 @@
         }
     },
     "train": {
-        "epochs": 100,
-        "lr":5e-5,
-        "wd": 0.1,
+        "epochs": 1000,
+        "lr":1e-4,
+        "wd": 0.01,
         "max_grad_norm": 0.5,
-        "save_every_n_samples": 300,
+        "save_every_n_samples": 200000,
         "n_sample_images": 10,
         "device": "cuda:0",
-        "epoch_samples": 500,
-        "validation_samples": 500,
+        "epoch_samples": 2000000,
+        "validation_samples": 100000,
         "use_ema": true,
         "ema_beta": 0.99,
+        "save_all": false,
+        "save_latest": true,
+        "save_best": true,
         "unet_training_mask": [true]
     },
     "evaluate": {
@@ -67,20 +82,22 @@
         }
     },
     "tracker": {
+        "data_path": ".tracker-data",
         "overwrite_data_path": true,
 
         "log": {
             "log_type": "wandb",
 
             "wandb_entity": "veldrovive",
-            "wandb_project": "trackers_tests",
+            "wandb_project": "trackers-test",
+            "wandb_run_name": "nousr config",
 
             "verbose": true
         },
 
         "load": {
-            "load_from": null,
-            "file_path": "/opt/dalle2_generator/test_models/latest.pth"
+            "load_from": "local",
+            "file_path": "/home/aidan/DALLE2-pytorch/latest.pth"
         },
 
         "save": [{
@@ -90,11 +107,11 @@
             "save_to": "huggingface",
             "huggingface_repo": "Veldrovive/test_model",
 
-            "save_all": true,
+            "save_all": false,
             "save_latest": true,
             "save_best": true,
 
-            "save_type": "checkpoint"
+            "save_type": "model"
         }]
     }
-}
+}