infinitejoy's picture
add tokenizer
ae5cbc4
raw
history blame contribute delete
771 Bytes
{"e": 1, "ँ": 2, "ं": 3, "ः": 4, "अ": 5, "आ": 6, "इ": 7, "ई": 8, "उ": 9, "ऊ": 10, "ऋ": 11, "ए": 12, "ऐ": 13, "ऑ": 14, "ओ": 15, "औ": 16, "क": 17, "ख": 18, "ग": 19, "घ": 20, "च": 21, "छ": 22, "ज": 23, "झ": 24, "ञ": 25, "ट": 26, "ठ": 27, "ड": 28, "ढ": 29, "ण": 30, "त": 31, "थ": 32, "द": 33, "ध": 34, "न": 35, "प": 36, "फ": 37, "ब": 38, "भ": 39, "म": 40, "य": 41, "र": 42, "ल": 43, "व": 44, "श": 45, "ष": 46, "स": 47, "ह": 48, "़": 49, "ा": 50, "ि": 51, "ी": 52, "ु": 53, "ू": 54, "ृ": 55, "ॅ": 56, "े": 57, "ै": 58, "ॉ": 59, "ो": 60, "ौ": 61, "्": 62, "क़": 63, "ग़": 64, "ज़": 65, "ड़": 66, "ढ़": 67, "’": 68, "|": 0, "[UNK]": 69, "[PAD]": 70}