New tokenizer with cleaned data

train_tokenizer.py

@@ -3,7 +3,7 @@ from tokenizers import trainers, Tokenizer, normalizers, ByteLevelBPETokenizer
 
 # load dataset
 # dataset = load_dataset("mc4", "sw", split="train")
-dataset = load_dataset("text", "sw", split="train", data_files={"train": ["/home/shared/clean_swahili/train.txt"]})
+dataset = load_dataset("text", "sw", split="train", data_files={"train": ["/home/shared/clean_swahili/train_v1.4.txt"]})
 
 # Instantiate tokenizer
 tokenizer = ByteLevelBPETokenizer()