fix: v1 model

classifier.py

@@ -0,0 +1,15 @@
+from transformers import TrOCRProcessor, VisionEncoderDecoderModel
+from PIL import Image
+import requests
+
+url = './dataset/val/0003.png'
+image = Image.open(url).convert("RGB")
+
+processor = TrOCRProcessor.from_pretrained('./model/checkpoint-80')
+model = VisionEncoderDecoderModel.from_pretrained('./model/checkpoint-80').to("cuda")
+pixel_values = processor(images=image, return_tensors="pt").pixel_values.to("cuda")
+
+generated_ids = model.generate(pixel_values)
+generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+
+print(generated_text)

model/checkpoint-500/trainer_state.json

@@ -1,321 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 100.0,
-  "eval_steps": 500,
-  "global_step": 500,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 2.0,
-      "learning_rate": 9.8e-05,
-      "loss": 8.2056,
-      "step": 10
-    },
-    {
-      "epoch": 4.0,
-      "learning_rate": 9.6e-05,
-      "loss": 4.4086,
-      "step": 20
-    },
-    {
-      "epoch": 6.0,
-      "learning_rate": 9.4e-05,
-      "loss": 3.1851,
-      "step": 30
-    },
-    {
-      "epoch": 8.0,
-      "learning_rate": 9.200000000000001e-05,
-      "loss": 3.2542,
-      "step": 40
-    },
-    {
-      "epoch": 10.0,
-      "learning_rate": 9e-05,
-      "loss": 2.6913,
-      "step": 50
-    },
-    {
-      "epoch": 12.0,
-      "learning_rate": 8.800000000000001e-05,
-      "loss": 2.3765,
-      "step": 60
-    },
-    {
-      "epoch": 14.0,
-      "learning_rate": 8.6e-05,
-      "loss": 2.2853,
-      "step": 70
-    },
-    {
-      "epoch": 16.0,
-      "learning_rate": 8.4e-05,
-      "loss": 2.3182,
-      "step": 80
-    },
-    {
-      "epoch": 18.0,
-      "learning_rate": 8.2e-05,
-      "loss": 2.156,
-      "step": 90
-    },
-    {
-      "epoch": 20.0,
-      "learning_rate": 8e-05,
-      "loss": 1.9019,
-      "step": 100
-    },
-    {
-      "epoch": 22.0,
-      "learning_rate": 7.800000000000001e-05,
-      "loss": 1.8288,
-      "step": 110
-    },
-    {
-      "epoch": 24.0,
-      "learning_rate": 7.6e-05,
-      "loss": 1.7968,
-      "step": 120
-    },
-    {
-      "epoch": 26.0,
-      "learning_rate": 7.4e-05,
-      "loss": 1.6366,
-      "step": 130
-    },
-    {
-      "epoch": 28.0,
-      "learning_rate": 7.2e-05,
-      "loss": 1.5084,
-      "step": 140
-    },
-    {
-      "epoch": 30.0,
-      "learning_rate": 7e-05,
-      "loss": 1.3425,
-      "step": 150
-    },
-    {
-      "epoch": 32.0,
-      "learning_rate": 6.800000000000001e-05,
-      "loss": 1.3157,
-      "step": 160
-    },
-    {
-      "epoch": 34.0,
-      "learning_rate": 6.6e-05,
-      "loss": 1.1184,
-      "step": 170
-    },
-    {
-      "epoch": 36.0,
-      "learning_rate": 6.400000000000001e-05,
-      "loss": 0.8982,
-      "step": 180
-    },
-    {
-      "epoch": 38.0,
-      "learning_rate": 6.2e-05,
-      "loss": 0.7471,
-      "step": 190
-    },
-    {
-      "epoch": 40.0,
-      "learning_rate": 6e-05,
-      "loss": 0.7546,
-      "step": 200
-    },
-    {
-      "epoch": 42.0,
-      "learning_rate": 5.8e-05,
-      "loss": 0.5103,
-      "step": 210
-    },
-    {
-      "epoch": 44.0,
-      "learning_rate": 5.6000000000000006e-05,
-      "loss": 0.4532,
-      "step": 220
-    },
-    {
-      "epoch": 46.0,
-      "learning_rate": 5.4000000000000005e-05,
-      "loss": 0.4687,
-      "step": 230
-    },
-    {
-      "epoch": 48.0,
-      "learning_rate": 5.2000000000000004e-05,
-      "loss": 0.4073,
-      "step": 240
-    },
-    {
-      "epoch": 50.0,
-      "learning_rate": 5e-05,
-      "loss": 0.5488,
-      "step": 250
-    },
-    {
-      "epoch": 52.0,
-      "learning_rate": 4.8e-05,
-      "loss": 0.5888,
-      "step": 260
-    },
-    {
-      "epoch": 54.0,
-      "learning_rate": 4.600000000000001e-05,
-      "loss": 0.3194,
-      "step": 270
-    },
-    {
-      "epoch": 56.0,
-      "learning_rate": 4.4000000000000006e-05,
-      "loss": 0.47,
-      "step": 280
-    },
-    {
-      "epoch": 58.0,
-      "learning_rate": 4.2e-05,
-      "loss": 0.2736,
-      "step": 290
-    },
-    {
-      "epoch": 60.0,
-      "learning_rate": 4e-05,
-      "loss": 0.2798,
-      "step": 300
-    },
-    {
-      "epoch": 62.0,
-      "learning_rate": 3.8e-05,
-      "loss": 0.322,
-      "step": 310
-    },
-    {
-      "epoch": 64.0,
-      "learning_rate": 3.6e-05,
-      "loss": 0.1707,
-      "step": 320
-    },
-    {
-      "epoch": 66.0,
-      "learning_rate": 3.4000000000000007e-05,
-      "loss": 0.1222,
-      "step": 330
-    },
-    {
-      "epoch": 68.0,
-      "learning_rate": 3.2000000000000005e-05,
-      "loss": 0.1246,
-      "step": 340
-    },
-    {
-      "epoch": 70.0,
-      "learning_rate": 3e-05,
-      "loss": 0.1404,
-      "step": 350
-    },
-    {
-      "epoch": 72.0,
-      "learning_rate": 2.8000000000000003e-05,
-      "loss": 0.1098,
-      "step": 360
-    },
-    {
-      "epoch": 74.0,
-      "learning_rate": 2.6000000000000002e-05,
-      "loss": 0.1441,
-      "step": 370
-    },
-    {
-      "epoch": 76.0,
-      "learning_rate": 2.4e-05,
-      "loss": 0.1531,
-      "step": 380
-    },
-    {
-      "epoch": 78.0,
-      "learning_rate": 2.2000000000000003e-05,
-      "loss": 0.1241,
-      "step": 390
-    },
-    {
-      "epoch": 80.0,
-      "learning_rate": 2e-05,
-      "loss": 0.103,
-      "step": 400
-    },
-    {
-      "epoch": 82.0,
-      "learning_rate": 1.8e-05,
-      "loss": 0.0907,
-      "step": 410
-    },
-    {
-      "epoch": 84.0,
-      "learning_rate": 1.6000000000000003e-05,
-      "loss": 0.0909,
-      "step": 420
-    },
-    {
-      "epoch": 86.0,
-      "learning_rate": 1.4000000000000001e-05,
-      "loss": 0.0874,
-      "step": 430
-    },
-    {
-      "epoch": 88.0,
-      "learning_rate": 1.2e-05,
-      "loss": 0.0757,
-      "step": 440
-    },
-    {
-      "epoch": 90.0,
-      "learning_rate": 1e-05,
-      "loss": 0.0753,
-      "step": 450
-    },
-    {
-      "epoch": 92.0,
-      "learning_rate": 8.000000000000001e-06,
-      "loss": 0.0763,
-      "step": 460
-    },
-    {
-      "epoch": 94.0,
-      "learning_rate": 6e-06,
-      "loss": 0.0714,
-      "step": 470
-    },
-    {
-      "epoch": 96.0,
-      "learning_rate": 4.000000000000001e-06,
-      "loss": 0.0736,
-      "step": 480
-    },
-    {
-      "epoch": 98.0,
-      "learning_rate": 2.0000000000000003e-06,
-      "loss": 0.0601,
-      "step": 490
-    },
-    {
-      "epoch": 100.0,
-      "learning_rate": 0.0,
-      "loss": 0.0731,
-      "step": 500
-    }
-  ],
-  "logging_steps": 10,
-  "max_steps": 500,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 100,
-  "save_steps": 500,
-  "total_flos": 1.4217418628923392e+18,
-  "train_batch_size": 2,
-  "trial_name": null,
-  "trial_params": null
-}

model/{checkpoint-500 → checkpoint-80}/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b40685900fc767af31aef4a3a62d0f2fc964f910e9d5d6eb3a6accc9c83324f2
+oid sha256:8ce666f30135fe9e723b0337705aa538819616ab1c0c45d7c0f9f549314a0dce
 size 1335747032

model/{checkpoint-500 → checkpoint-80}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:277ee31979f928e16dacf35727b361540431efc2179066178de8aff98fade57e
+oid sha256:8568a62b60ef9d306370cc762c375ed8fd56ebb7d39410683030cda2df3a92bc
 size 2667050412

model/checkpoint-80/preprocessor_config.json

@@ -0,0 +1,17 @@
+{
+  "do_normalize": true,
+  "do_resize": true,
+  "image_processor_type": "ViTImageProcessor",
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "resample": 2,
+  "size": 384
+}

model/{checkpoint-500 → checkpoint-80}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae815a4262cf019aa02c8291a4c9c2a2b22f9c4534ccef44ebbef7835c2c5e48
+oid sha256:3a8b95444924300e71db054e45eaa19e41cd683c6b7cd4716def561162febc69
 size 14244

model/{checkpoint-500 → checkpoint-80}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d02c16cc82c2dd3c41a58b5d09ae955539e7b9165124433ee976a0bb0323a2ee
+oid sha256:e04a01946a1a22d7f67e161ee04b6430e62bf4eaaeb4428ef0cbe16393ea0cff
 size 1064

model/checkpoint-80/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}

model/checkpoint-80/tokenizer_config.json

@@ -0,0 +1 @@
+{"errors": "replace", "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "roberta-large", "tokenizer_class": "RobertaTokenizer"}

model/checkpoint-80/trainer_state.json

@@ -0,0 +1,213 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 17.77777777777778,
+  "eval_steps": 500,
+  "global_step": 80,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.89,
+      "eval_loss": 11.255721092224121,
+      "eval_runtime": 3.4824,
+      "eval_samples_per_second": 1.436,
+      "eval_steps_per_second": 0.287,
+      "step": 4
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 4.6637139320373535,
+      "eval_runtime": 1.651,
+      "eval_samples_per_second": 3.029,
+      "eval_steps_per_second": 0.606,
+      "step": 9
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 8.75e-05,
+      "loss": 9.1378,
+      "step": 10
+    },
+    {
+      "epoch": 2.89,
+      "eval_loss": 3.9639782905578613,
+      "eval_runtime": 3.628,
+      "eval_samples_per_second": 1.378,
+      "eval_steps_per_second": 0.276,
+      "step": 13
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 3.660576343536377,
+      "eval_runtime": 1.6465,
+      "eval_samples_per_second": 3.037,
+      "eval_steps_per_second": 0.607,
+      "step": 18
+    },
+    {
+      "epoch": 4.44,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 3.7563,
+      "step": 20
+    },
+    {
+      "epoch": 4.89,
+      "eval_loss": 3.774608612060547,
+      "eval_runtime": 3.6221,
+      "eval_samples_per_second": 1.38,
+      "eval_steps_per_second": 0.276,
+      "step": 22
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 4.067192077636719,
+      "eval_runtime": 1.6587,
+      "eval_samples_per_second": 3.014,
+      "eval_steps_per_second": 0.603,
+      "step": 27
+    },
+    {
+      "epoch": 6.67,
+      "learning_rate": 6.25e-05,
+      "loss": 2.8464,
+      "step": 30
+    },
+    {
+      "epoch": 6.89,
+      "eval_loss": 4.304504871368408,
+      "eval_runtime": 3.619,
+      "eval_samples_per_second": 1.382,
+      "eval_steps_per_second": 0.276,
+      "step": 31
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 3.801119327545166,
+      "eval_runtime": 1.8745,
+      "eval_samples_per_second": 2.667,
+      "eval_steps_per_second": 0.533,
+      "step": 36
+    },
+    {
+      "epoch": 8.89,
+      "learning_rate": 5e-05,
+      "loss": 2.462,
+      "step": 40
+    },
+    {
+      "epoch": 8.89,
+      "eval_loss": 4.001826763153076,
+      "eval_runtime": 3.9564,
+      "eval_samples_per_second": 1.264,
+      "eval_steps_per_second": 0.253,
+      "step": 40
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 4.09613037109375,
+      "eval_runtime": 1.772,
+      "eval_samples_per_second": 2.822,
+      "eval_steps_per_second": 0.564,
+      "step": 45
+    },
+    {
+      "epoch": 10.89,
+      "eval_loss": 4.627002239227295,
+      "eval_runtime": 4.0617,
+      "eval_samples_per_second": 1.231,
+      "eval_steps_per_second": 0.246,
+      "step": 49
+    },
+    {
+      "epoch": 11.11,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 2.1133,
+      "step": 50
+    },
+    {
+      "epoch": 12.0,
+      "eval_loss": 4.324659824371338,
+      "eval_runtime": 1.8005,
+      "eval_samples_per_second": 2.777,
+      "eval_steps_per_second": 0.555,
+      "step": 54
+    },
+    {
+      "epoch": 12.89,
+      "eval_loss": 4.851120471954346,
+      "eval_runtime": 3.967,
+      "eval_samples_per_second": 1.26,
+      "eval_steps_per_second": 0.252,
+      "step": 58
+    },
+    {
+      "epoch": 13.33,
+      "learning_rate": 2.5e-05,
+      "loss": 1.7555,
+      "step": 60
+    },
+    {
+      "epoch": 14.0,
+      "eval_loss": 4.627060413360596,
+      "eval_runtime": 1.8189,
+      "eval_samples_per_second": 2.749,
+      "eval_steps_per_second": 0.55,
+      "step": 63
+    },
+    {
+      "epoch": 14.89,
+      "eval_loss": 4.949428081512451,
+      "eval_runtime": 3.9565,
+      "eval_samples_per_second": 1.264,
+      "eval_steps_per_second": 0.253,
+      "step": 67
+    },
+    {
+      "epoch": 15.56,
+      "learning_rate": 1.25e-05,
+      "loss": 1.2005,
+      "step": 70
+    },
+    {
+      "epoch": 16.0,
+      "eval_loss": 5.440162658691406,
+      "eval_runtime": 1.803,
+      "eval_samples_per_second": 2.773,
+      "eval_steps_per_second": 0.555,
+      "step": 72
+    },
+    {
+      "epoch": 16.89,
+      "eval_loss": 5.508076190948486,
+      "eval_runtime": 4.004,
+      "eval_samples_per_second": 1.249,
+      "eval_steps_per_second": 0.25,
+      "step": 76
+    },
+    {
+      "epoch": 17.78,
+      "learning_rate": 0.0,
+      "loss": 0.8324,
+      "step": 80
+    },
+    {
+      "epoch": 17.78,
+      "eval_loss": 5.481626510620117,
+      "eval_runtime": 0.8414,
+      "eval_samples_per_second": 5.942,
+      "eval_steps_per_second": 1.188,
+      "step": 80
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 80,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "total_flos": 2.267304128717783e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

model/{checkpoint-500 → checkpoint-80}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04deb0d096a36c5f7da938e19ec51b9bd341c0c139090e278562f90413d3087e
+oid sha256:9d72d8dc11b5be144d27c80846224147801c5764c4f0d0a5b47575c67d145b15
 size 4664

requirements.txt

@@ -1,6 +1,7 @@
-transformers
-torch # pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
-pandas
-pillow
-scikit-learn
-accelerate
+transformers~=4.37.2
+torch~=2.2.0+cu121
+pandas~=2.2.0
+pillow~=10.2.0
+scikit-learn~=1.4.1.post1
+accelerate
+requests~=2.31.0

train.py

@@ -16,7 +16,7 @@ class HandwrittenMathDataset(Dataset):
     """
     def __init__(self, annotations_file, img_dir, processor, subset="train"):
         self.img_labels = pd.read_csv(annotations_file)
-        self.train_data, self.test_data = train_test_split(self.img_labels, test_size=0.1, random_state=42)
+        self.train_data, self.test_data = train_test_split(self.img_labels, test_size=0.2, random_state=42)
         self.data = self.train_data if subset == "train" else self.test_data
         self.img_dir = img_dir
         self.processor = processor
@@ -62,15 +62,16 @@ def main():
     training_args = TrainingArguments(
         output_dir='./model',
         per_device_train_batch_size=2,
-        num_train_epochs=100,
+        num_train_epochs=20,
         logging_dir='./training_logs',
         logging_steps=10,
         save_strategy="epoch",
         save_total_limit=1,
-        weight_decay=0.01,
+        weight_decay=0.1,
         learning_rate=1e-4,
         gradient_checkpointing=True,
-        gradient_accumulation_steps=2
+        gradient_accumulation_steps=2,
+        evaluation_strategy="epoch"
     )
 
     trainer = Trainer(