infinitejoy's picture
add tokenizer
54423b1
raw
history blame
504 Bytes
{"'": 1, "a": 2, "e": 3, "g": 4, "h": 5, "m": 6, "n": 7, "o": 8, "r": 9, "t": 10, "v": 11, "«": 12, "»": 13, "́": 14, "ΐ": 15, "ά": 16, "έ": 17, "ή": 18, "ί": 19, "α": 20, "β": 21, "γ": 22, "δ": 23, "ε": 24, "ζ": 25, "η": 26, "θ": 27, "ι": 28, "κ": 29, "λ": 30, "μ": 31, "ν": 32, "ξ": 33, "ο": 34, "π": 35, "ρ": 36, "ς": 37, "σ": 38, "τ": 39, "υ": 40, "φ": 41, "χ": 42, "ψ": 43, "ω": 44, "ϊ": 45, "ϋ": 46, "ό": 47, "ύ": 48, "ώ": 49, "|": 0, "[UNK]": 50, "[PAD]": 51}