shaurya0512's picture
Upload tokenizer
012fd15 verified
raw
history blame contribute delete
840 Bytes
{
"(": 1,
")": 2,
"[PAD]": 69,
"[UNK]": 68,
"_": 3,
"|": 0,
"،": 4,
"؛": 5,
"؟": 6,
"ء": 7,
"آ": 8,
"أ": 9,
"ؤ": 10,
"إ": 11,
"ئ": 12,
"ا": 13,
"ب": 14,
"ة": 15,
"ت": 16,
"ث": 17,
"ج": 18,
"ح": 19,
"خ": 20,
"د": 21,
"ذ": 22,
"ر": 23,
"ز": 24,
"س": 25,
"ش": 26,
"ص": 27,
"ض": 28,
"ط": 29,
"ظ": 30,
"ع": 31,
"غ": 32,
"ـ": 33,
"ف": 34,
"ق": 35,
"ك": 36,
"ل": 37,
"م": 38,
"ن": 39,
"ه": 40,
"و": 41,
"ى": 42,
"ي": 43,
"ً": 44,
"ٌ": 45,
"ٍ": 46,
"َ": 47,
"ُ": 48,
"ِ": 49,
"ّ": 50,
"ْ": 51,
"ٰ": 52,
"چ": 53,
"ڨ": 54,
"ک": 55,
"ھ": 56,
"ی": 57,
"ۖ": 58,
"ۗ": 59,
"ۘ": 60,
"ۚ": 61,
"ۛ": 62,
"—": 63,
"…": 64,
"☭": 65,
"ﺃ": 66,
"ﻻ": 67
}