infinitejoy's picture
add tokenizer
fd859f4
raw
history blame
524 Bytes
{"'": 1, "a": 2, "e": 3, "g": 4, "h": 5, "m": 6, "n": 7, "o": 8, "r": 9, "t": 10, "v": 11, "«": 12, "´": 13, "·": 14, "»": 15, "́": 16, "ΐ": 17, "ά": 18, "έ": 19, "ή": 20, "ί": 21, "α": 22, "β": 23, "γ": 24, "δ": 25, "ε": 26, "ζ": 27, "η": 28, "θ": 29, "ι": 30, "κ": 31, "λ": 32, "μ": 33, "ν": 34, "ξ": 35, "ο": 36, "π": 37, "ρ": 38, "ς": 39, "σ": 40, "τ": 41, "υ": 42, "φ": 43, "χ": 44, "ψ": 45, "ω": 46, "ϊ": 47, "ϋ": 48, "ό": 49, "ύ": 50, "ώ": 51, "|": 0, "[UNK]": 52, "[PAD]": 53}