Add normalization steps, fix som bugs, add tfboard tracker

.gitattributes

@@ -14,3 +14,4 @@
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -33,9 +33,9 @@ python create_config.py --name_or_path gpt2-medium --params '{"vocab_size": 4200
 
 Steps:
 
-- [ ] Remove stretched words such as ســــــــــلام
+- [x] Remove stretched words such as ســــــــــلام
 
-- [ ] Remove links, user-mentioning (such as @jane_doe)
+- [x] Remove links, user-mentioning (such as @jane_doe)
 
 - [ ] Remove Telegram, Instagram advertisements, or posts (a whole record)
 

src/data_utils.py

@@ -2,7 +2,6 @@ from hazm import word_tokenize
 from hazm import sent_tokenize
 import re
 import six
-import string
 
 from normalizer import normalize
 
@@ -13,15 +12,15 @@ def filter_by_lang_regex(text, ratio=0.7, regex="0-9۰۱۲۳۴۵۶۷۸۹ءآئا
     candidate_text = re.sub(r"[^" + regex + "]+", " ", six.ensure_str(text)).replace(" ", "")
     text = text.replace(" ", "")
 
-    return True if (len(candidate_text) / len(text)) > ratio else False
+    return (len(candidate_text) / len(text)) > ratio
 
 
 def filter_by_num_tokens(text, gt=64):
-    return True if len(word_tokenize(text)) > gt else False
+    return len(word_tokenize(text)) > gt
 
 
 def filter_by_num_sents(text, gt=2):
-    return True if len(sent_tokenize(text)) > gt else False
+    return len(sent_tokenize(text)) > gt
 
 
 def normalizer(text, do_lowercase=False):
@@ -31,6 +30,3 @@ def normalizer(text, do_lowercase=False):
         text = text.lower()
 
     return text
-
-
-

src/requirements.txt

@@ -3,4 +3,5 @@ jax>=0.2.8
 jaxlib>=0.1.59
 flax>=0.3.4
 optax>=0.0.8
-hazm
+hazm
+tensorboard