rom1504 commited on
Commit
3493484
1 Parent(s): 64f01e3

move decoder_config.json 32 gpus to the right place

Browse files
decoder/small_32gpus/decoder_config.json ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "seed": 1,
3
+
4
+ "decoder": {
5
+ "unets": [
6
+ {
7
+ "dim": 256,
8
+ "cond_dim": 512,
9
+ "image_embed_dim": 768,
10
+ "text_embed_dim": 768,
11
+ "cond_on_text_encodings": true,
12
+ "channels": 3,
13
+ "dim_mults": [1, 2, 3, 4],
14
+ "num_resnet_blocks": 4,
15
+ "attn_heads": 8,
16
+ "attn_dim_head": 64,
17
+ "sparse_attn": true,
18
+ "memory_efficient": true,
19
+ "self_attn": [false, true, true, true]
20
+ }
21
+ ],
22
+ "clip": {
23
+ "make": "openai",
24
+ "model": "ViT-L/14"
25
+ },
26
+ "image_sizes": [64],
27
+ "channels": 3,
28
+ "timesteps": 1000,
29
+ "loss_type": "l2",
30
+ "beta_schedule": ["cosine"],
31
+ "learned_variance": true
32
+ },
33
+ "data": {
34
+ "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -",
35
+ "embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
36
+ "num_workers": 12,
37
+ "batch_size": 60,
38
+ "start_shard": 0,
39
+ "end_shard": 5247,
40
+ "shard_width": 5,
41
+ "index_width": 4,
42
+ "splits": {
43
+ "train": 0.75,
44
+ "val": 0.15,
45
+ "test": 0.1
46
+ },
47
+ "shuffle_train": false,
48
+ "resample_train": true,
49
+ "preprocessing": {
50
+ "RandomResizedCrop": {
51
+ "size": [64, 64],
52
+ "scale": [0.75, 1.0],
53
+ "ratio": [1.0, 1.0]
54
+ },
55
+ "ToTensor": true
56
+ }
57
+ },
58
+ "train": {
59
+ "epochs": 1000,
60
+ "lr":1e-4,
61
+ "wd": 0.01,
62
+ "max_grad_norm": 0.5,
63
+ "save_every_n_samples": 2000000,
64
+ "n_sample_images": 10,
65
+ "device": "cuda:0",
66
+ "epoch_samples": 40000000,
67
+ "validation_samples": 100000,
68
+ "use_ema": true,
69
+ "ema_beta": 0.99,
70
+ "save_all": false,
71
+ "save_latest": true,
72
+ "save_best": true,
73
+ "unet_training_mask": [true]
74
+ },
75
+ "evaluate": {
76
+ "n_evaluation_samples": 30,
77
+ "FID": {
78
+ "feature": 64
79
+ },
80
+ "LPIPS": {
81
+ "net_type": "vgg",
82
+ "reduction": "mean"
83
+ }
84
+ },
85
+ "tracker": {
86
+ "data_path": ".tracker-data-2",
87
+ "overwrite_data_path": true,
88
+
89
+ "log": {
90
+ "log_type": "wandb",
91
+
92
+ "wandb_entity": "rom1504",
93
+ "wandb_project": "dalle2_train_decoder",
94
+ "wandb_resume": false,
95
+
96
+ "verbose": true
97
+ },
98
+
99
+ "load": {
100
+ "resume": true,
101
+ "load_from": "url",
102
+ "url": "https://api.wandb.ai/files/rom1504/dalle2_train_decoder/3tmnv289/latest.pth"
103
+ },
104
+
105
+ "save": [{
106
+ "save_to": "wandb"
107
+ },
108
+ {
109
+ "save_to": "huggingface",
110
+ "huggingface_repo": "laion/DALLE2-PyTorch",
111
+ "save_meta_to": "decoder/small_32gpus/",
112
+
113
+ "save_latest_to": "decoder/small_32gpus/latest.pth",
114
+
115
+ "save_type": "model"
116
+ }]
117
+ }
118
+ }