update tokenizer

merges.txt

@@ -101742,3 +101742,403 @@ In nov
 éª Ħ
 Ġpur plish
 Ġíķĺ 기ëıĦ
+Ġapprox im
+æ¢ħ åħĭ
+åı¥ åŃIJ
+ĠConc orde
+ĠCost ume
+ãĥŁãĥ¥ãĥ¼ ãĤº
+ãģ®éķ· ãģĦ
+á¹ Ľ
+ĠEg gs
+Ġì¡´ìŀ¬ íķĺì§Ģ
+Ġfract uring
+奨 åѦéĩij
+Ġreprin ts
+ĠGA BA
+ĠSamar itan
+Ġdy ed
+Ġannih ilation
+ĠíĶ¼ìķĦ ëħ¸
+5 88
+ag ency
+ap ala
+æĺ¯ åı¯
+æ¨ ģ
+Ġinter collegiate
+ash ir
+16 99
+ĠCor man
+éĿŀ æŃ£å¼ı
+éĺ² æ°´
+ü d
+åħ³ èĬĤ
+ä¼Ŀ ãģĪãģŁ
+æģ© ç§ij
+ĠëĤ´ ìĦ¸
+Ġrein vent
+ĠBan at
+ìĥī ìľ¼ë¡ľ
+Ty pically
+ĠìłĦìŁģ ìĹIJ
+èľĤ èľľ
+éĻįä½İ äºĨ
+Architect ural
+ãģ«æ¯Ķãģ¹ ãĤĭãģ¨
+ãĥ¬ãĥĥ ãĤº
+zoll ern
+ãĤĴ 強調
+ĠK ia
+åľ¨ åħ¨åľĭ
+åIJĮ ä¸Ģ个
+åĮĹ æ¢Ŀ
+åĭķ æ¤įçī©
+Ġinc rim
+Ġmon ot
+æģ ¼
+ãĤĪãģĨ ãģł
+çĶº é§ħ
+客 ãģ®
+åī¯ èŃ°éķ·
+åī¯ ä¼ļéķ¿
+ç¥ĸ åľĭ
+ĠMont clair
+ĠëIJĺ ëıĮ
+ê²½ ìĿ´
+BO L
+å·¥åł´ ãģ¯
+íĻĺ ìĿĦ
+説æĺİ ãģĻãĤĭ
+ĠBrun el
+åIJĦå¼ı åIJĦ
+ĠP ML
+ãĤĤ åĩº
+cl oud
+act yl
+ins urgency
+ãĥĸ ãĥ©ãĤ¤ãĥĪ
+ìĸ´ ë²Ħ
+å°į æµģ
+ĠAm t
+ĠëĮĢ ìĿĺ
+ëIJĺ ê²Į
+Ġsl iced
+Ġ27 6
+ãĥ¦ ãĥª
+ä»ĭ ä¹İ
+ठª
+æĬľ ãģĦãģŁ
+éł» 度
+ãĥĪãĤ¥ ãĥ¼
+Ġmal ware
+ĠLear y
+ĠKin ney
+ĠMons ignor
+UEFA ãĥģãĥ£ãĥ³ãĥĶãĤªãĥ³ãĤºãĥªãĥ¼ãĤ°
+ãĤªãĥķãĤ£ ãĤ·ãĥ£ãĥ«
+Ġpunct uated
+Ġsplic ing
+Ê Ĵ
+ĠF ires
+ĠK ari
+ib bon
+ãĥIJ ãĥĥãĥī
+ĠCon sole
+sc ot
+çħ ¬
+åħ« 个
+iqu i
+é¾Ļ ç§ij
+ĠRos well
+å¼Ĺ èIJĬ
+AN T
+åѸçĶŁ çļĦ
+æľįåĬ¡ çļĦ
+ãĥĻãĤ¹ãĥĪ ãĤ¢ãĥ«ãĥIJãĥł
+Ġdelight ful
+ĠCasual ty
+5 14
+C ul
+M ills
+ãĤ IJ
+it ur
+ĠR au
+ĠF ass
+ãĤ· ãĥį
+åĪ¶ ãģĹ
+ç´Ħ æŦ
+å°Ĩ 被
+ç«Ļ 廳
+Ġfurther ing
+Al ways
+ĠRoyal ists
+æ¾ Ģ
+ĠSa ber
+ĠMcC ar
+ĠBor ges
+ĠìŀĦ 무
+Ġì²Ń ëĤĺëĿ¼
+罰 款
+ĠBog art
+ãĥIJãĥĥãĤ¯ ãĤ¢ãĥĥãĥĹ
+修建 äºĨ
+ĠCret an
+Ġbrib ed
+ï¹ IJ
+çĭ¹ çªĦ
+m RNA
+p ick
+Ġc ren
+Ġp rick
+ĠS re
+ĠM ert
+çļĦ æĮĩ示
+æĸ Ľ
+ĠSt ig
+ãĤĤ æĭħå½ĵ
+ãģĻãĤĭ ä¸Ģæĸ¹
+ãģĭãĤī èĦ±
+Ġ7 27
+Ġrest less
+éķ¿ å´İ
+åħµ è£ħ
+ãĥį ãĥIJãĥĢ
+å°ĩ æѤ
+ç´ł 質
+å¡Ķ çī¹
+ä¸ĢèĪ¬ ç¤¾åĽ£æ³ķ人
+åıĸãĤĬ åħ¥ãĤĮ
+æľŁéĸĵ ä¸Ń
+éľĩ æĴ¼
+ĠChild ers
+ì²ľ ë§Į
+ãģ®ä¸Ĭ æĺĩ
+æľĢè¿ij ãģ®
+ĠíĸĪ ëĬĶëį°
+Ġê²Ģ íĨł
+尽管 å¦ĤæѤ
+Ġpink ish
+ãĤĦãĤĬ åıĸãĤĬ
+Su ica
+ĠìĿ¼ìľ¼ ì¼°ëĭ¤
+Ġìĭ¸ ìĽĢ
+ĠGut ierrez
+Ġgel atin
+俾 æĸ¯
+Ġc oven
+ãģ® 模æ§ĺ
+ãģ« çµ±ä¸Ģ
+çļĦ æĹł
+ĠG TP
+ĠIn oue
+éĩ Ĺ
+Ġ2 ½
+æĻĤ ãĢħ
+15 93
+16 65
+女 éĥİ
+å°Ĩ æѤ
+ãĤ¬ ãĥ«ãĥĪ
+Ġaff ections
+ĠNor wegians
+Ġmaterial ism
+ĠKe ats
+éĢ± ãģ«
+ï¼Ł ï¼ī
+iti ate
+ë°Ľ ëĬĶ
+å¿į åıĹ
+Ġgrid s
+ĠØ ³
+ĠìľłëŁ½ ìĿĺ
+ãĥģãĥ£ãĥ¼ãĥĪ ãģ§ãģ¯
+grav ity
+Cham ber
+辿 ãĤĬ
+pract ice
+Ġailer ons
+R ash
+T ir
+n ist
+ãģ® æĶ¯æĮģ
+ine ers
+ĠHe els
+ãĤ¤ ãĥ´
+ex pected
+Ġinf usion
+augh an
+ze g
+æĻĤ代 ãģĭãĤīãģ®
+ĠëıĦ ì¿ł
+æĬĺ å°Ħ
+åİĭ åĪ¶
+ĠTan zan
+äºĪå®ļ ãģ§ãģĤãĤĭ
+Mc K
+hu ang
+ĠìĦ¸ê³Ħ ìĿĺ
+ĠAqu atics
+ĠConn aught
+Ġtens or
+義大åĪ© èªŀ
+è»įãĤĴ çİĩãģĦãģ¦
+ãģ«å¯¾å¿ľ ãģĹãģ¦ãģĦãĤĭ
+éĺªç¥ŀ ãĤ¿ãĤ¤ãĤ¬ãĥ¼ãĤ¹
+Ġexhaust ive
+ĠPos ition
+Ġfond ness
+ĠShiv aji
+ĠOdys se
+ĠGhe orghe
+BAS IC
+oclet ian
+0 25
+N in
+ĠT ough
+ĠL iss
+Ġover heard
+å¤ļ éģĶ
+åIJİ ä¸º
+æĹ¥æľ¬ è»įãģ¯
+ãģĺ ãĤĬ
+Ġhum ility
+念 éłŃ
+延 平
+çļ® åį¡
+ä½³ ä½ľ
+59 8
+ë°ľ ìĿĦ
+èĻ½çĦ¶ åľ¨
+éĽĸçĦ¶ åľ¨
+Ġë²ł ìĿ´ìĬ¤
+åłħ 決
+çł´å£Ĭ ãģķãĤĮ
+ãģ£ãģ ·
+æľĪåı· ãģ¾ãģ§
+éİĮåĢī å¹ķåºľ
+Ġìłľê³µ íķľëĭ¤
+è¶ķ åĪ°
+ĠHick ory
+Ġlawn s
+Ġtric ycle
+Ġ그곳 ìĹIJìĦľ
+Ġcush ion
+ĠMoir a
+" '
+Ġt ad
+Ġh u
+ĠS urgical
+ĠT ama
+ĠB eb
+çļĦ æĦı
+ãģ¯ åŁºæľ¬çļĦãģ«
+ĠF enerbahçe
+ãģ§ ä½ľãĤīãĤĮãģŁ
+os ki
+ãĥĪ ãģ¯
+are ts
+åĴĮ è°IJ
+Ġkn itting
+Ġass urances
+éŁ ľ
+åİŁ çĶ»
+æ°´ æľ¨
+Ġcre sted
+該 年
+ĠMon c
+Ġless en
+Ġ27 8
+Ġnarrow ing
+Ġnom ads
+Ġë³µ 무
+éĢĤ çĶ¨
+ĠVel asco
+IL L
+Ġgal ley
+ĠChal uk
+Ġdesp ised
+YP E
+ä¸ĩç¾İåħĥ çļĦ
+Ġpedag ogical
+ãģ® 次çĶ·
+Ġst umbled
+ĠO M
+man if
+own ing
+ĠWest brook
+ĠìłĦ ë¶Ģ
+ili ary
+Ġì¶ ¤
+Ġconduct ive
+åIJĪä½ľ 社
+Ev angel
+礦 æ¥Ń
+éĢ£éļĬ éķ·
+ĠCort ez
+æľĪåł´æīĢ ã��§ãģ¯
+ĠTill man
+T itan
+d un
+Ġ ÑĤ
+ver gne
+if lora
+amp hetamine
+Ġover ran
+æĶ¿ æķĻ
+æĸ° é«ĺ
+è¦ģ ãģĻãĤĭ
+ann ounced
+æĿ± 欧
+äºĮ æĺ¯
+Ġdef lect
+è»Ĭ ãģ§ãģĤãĤĭ
+Ġconst ants
+åıĤ æĪĺ
+åıĤ è©£
+ĠBar ak
+復 åİŁ
+ĠMag ical
+Ġëĭ¤ 리
+æīį åı¯
+åĦªåĭĿ ãĤĴ飾ãģ£ãģŁ
+çł² æŀ¶
+ĠOs wego
+é«ĺæł¡ ãģ§ãģ¯
+ĠShel f
+Jose f
+ĠThunder birds
+Ġfeather weight
+çĽĪ åĪ©
+ĠCompos ite
+æ³ķèĺŃåħĭ ç¦ı
+ro ix
+ãģ® çĻºè¡Į
+çļĦ çIJĨ论
+çļĦ çĬ¶æĢģ
+ĠP uma
+次 ãģ®ãĤĪãģĨãģª
+ç§ij ç¶Ńå¥ĩ
+Ġav ian
+ãģĿãĤĮ 以éĻį
+æĿİ å¤§
+ãĤ¢ãĥ« ãĥŀ
+Ġran ches
+Ġdepend encies
+Ġbill board
+缺 é»ŀ
+ĠCzech s
+çī©çIJĨ ç³»
+ĠRang oon
+ãģ®ãĤ¨ ãĥįãĥ«
+Fl ood
+ĠQual ification
+å·Ŀå´İ å¸Ĥ
+௠ģ
+Ġannot ated
+Ġardu ous
+I ch
+s occer
+ĠC Äĥ
+om ology
+çļĦ çµĦç¹Ķ
+åĽ ij

tokenizer.json

@@ -102040,7 +102040,407 @@
       "åħ©åľ°": 101996,
       "éªĦ": 101997,
       "Ġpurplish": 101998,
-      "Ġíķĺ기ëıĦ": 101999
+      "Ġíķĺ기ëıĦ": 101999,
+      "Ġapproxim": 102000,
+      "æ¢ħåħĭ": 102001,
+      "åı¥åŃIJ": 102002,
+      "ĠConcorde": 102003,
+      "ĠCostume": 102004,
+      "ãĥŁãĥ¥ãĥ¼ãĤº": 102005,
+      "ãģ®éķ·ãģĦ": 102006,
+      "á¹Ľ": 102007,
+      "ĠEggs": 102008,
+      "Ġì¡´ìŀ¬íķĺì§Ģ": 102009,
+      "Ġfracturing": 102010,
+      "奨åѦéĩij": 102011,
+      "Ġreprints": 102012,
+      "ĠGABA": 102013,
+      "ĠSamaritan": 102014,
+      "Ġdyed": 102015,
+      "Ġannihilation": 102016,
+      "ĠíĶ¼ìķĦëħ¸": 102017,
+      "588": 102018,
+      "agency": 102019,
+      "apala": 102020,
+      "æĺ¯åı¯": 102021,
+      "æ¨ģ": 102022,
+      "Ġintercollegiate": 102023,
+      "ashir": 102024,
+      "1699": 102025,
+      "ĠCorman": 102026,
+      "éĿŀæŃ£å¼ı": 102027,
+      "éĺ²æ°´": 102028,
+      "üd": 102029,
+      "åħ³èĬĤ": 102030,
+      "ä¼ĿãģĪãģŁ": 102031,
+      "æģ©ç§ij": 102032,
+      "ĠëĤ´ìĦ¸": 102033,
+      "Ġreinvent": 102034,
+      "ĠBanat": 102035,
+      "ìĥīìľ¼ë¡ľ": 102036,
+      "Typically": 102037,
+      "ĠìłĦìŁģìĹIJ": 102038,
+      "èľĤèľľ": 102039,
+      "éĻįä½İäºĨ": 102040,
+      "Architectural": 102041,
+      "ãģ«æ¯Ķãģ¹ãĤĭãģ¨": 102042,
+      "ãĥ¬ãĥĥãĤº": 102043,
+      "zollern": 102044,
+      "ãĤĴ強調": 102045,
+      "ĠKia": 102046,
+      "åľ¨åħ¨åľĭ": 102047,
+      "åIJĮä¸Ģ个": 102048,
+      "åĮĹæ¢Ŀ": 102049,
+      "åĭķæ¤įçī©": 102050,
+      "Ġincrim": 102051,
+      "Ġmonot": 102052,
+      "æģ¼": 102053,
+      "ãĤĪãģĨãģł": 102054,
+      "çĶºé§ħ": 102055,
+      "客ãģ®": 102056,
+      "åī¯èŃ°éķ·": 102057,
+      "åī¯ä¼ļéķ¿": 102058,
+      "ç¥ĸåľĭ": 102059,
+      "ĠMontclair": 102060,
+      "ĠëIJĺëıĮ": 102061,
+      "ê²½ìĿ´": 102062,
+      "BOL": 102063,
+      "å·¥åł´ãģ¯": 102064,
+      "íĻĺìĿĦ": 102065,
+      "説æĺİãģĻãĤĭ": 102066,
+      "ĠBrunel": 102067,
+      "åIJĦå¼ıåIJĦ": 102068,
+      "ĠPML": 102069,
+      "ãĤĤåĩº": 102070,
+      "cloud": 102071,
+      "actyl": 102072,
+      "insurgency": 102073,
+      "ãĥĸãĥ©ãĤ¤ãĥĪ": 102074,
+      "ìĸ´ë²Ħ": 102075,
+      "å°įæµģ": 102076,
+      "ĠAmt": 102077,
+      "ĠëĮĢìĿĺ": 102078,
+      "ëIJĺê²Į": 102079,
+      "Ġsliced": 102080,
+      "Ġ276": 102081,
+      "ãĥ¦ãĥª": 102082,
+      "ä»ĭä¹İ": 102083,
+      "प": 102084,
+      "æĬľãģĦãģŁ": 102085,
+      "éł»åº¦": 102086,
+      "ãĥĪãĤ¥ãĥ¼": 102087,
+      "Ġmalware": 102088,
+      "ĠLeary": 102089,
+      "ĠKinney": 102090,
+      "ĠMonsignor": 102091,
+      "UEFAãĥģãĥ£ãĥ³ãĥĶãĤªãĥ³ãĤºãĥªãĥ¼ãĤ°": 102092,
+      "ãĤªãĥķãĤ£ãĤ·ãĥ£ãĥ«": 102093,
+      "Ġpunctuated": 102094,
+      "Ġsplicing": 102095,
+      "ÊĴ": 102096,
+      "ĠFires": 102097,
+      "ĠKari": 102098,
+      "ibbon": 102099,
+      "ãĥIJãĥĥãĥī": 102100,
+      "ĠConsole": 102101,
+      "scot": 102102,
+      "çħ¬": 102103,
+      "åħ«ä¸ª": 102104,
+      "iqui": 102105,
+      "é¾Ļç§ij": 102106,
+      "ĠRoswell": 102107,
+      "å¼ĹèIJĬ": 102108,
+      "ANT": 102109,
+      "åѸçĶŁçļĦ": 102110,
+      "æľįåĬ¡çļĦ": 102111,
+      "ãĥĻãĤ¹ãĥĪãĤ¢ãĥ«ãĥIJãĥł": 102112,
+      "Ġdelightful": 102113,
+      "ĠCasualty": 102114,
+      "514": 102115,
+      "Cul": 102116,
+      "Mills": 102117,
+      "ãĤIJ": 102118,
+      "itur": 102119,
+      "ĠRau": 102120,
+      "ĠFass": 102121,
+      "ãĤ·ãĥį": 102122,
+      "åĪ¶ãģĹ": 102123,
+      "ç´ĦæŦ": 102124,
+      "å°Ĩ被": 102125,
+      "ç«Ļ廳": 102126,
+      "Ġfurthering": 102127,
+      "Always": 102128,
+      "ĠRoyalists": 102129,
+      "æ¾Ģ": 102130,
+      "ĠSaber": 102131,
+      "ĠMcCar": 102132,
+      "ĠBorges": 102133,
+      "ĠìŀĦ무": 102134,
+      "Ġì²ŃëĤĺëĿ¼": 102135,
+      "罰款": 102136,
+      "ĠBogart": 102137,
+      "ãĥIJãĥĥãĤ¯ãĤ¢ãĥĥãĥĹ": 102138,
+      "修建äºĨ": 102139,
+      "ĠCretan": 102140,
+      "Ġbribed": 102141,
+      "ï¹IJ": 102142,
+      "çĭ¹çªĦ": 102143,
+      "mRNA": 102144,
+      "pick": 102145,
+      "Ġcren": 102146,
+      "Ġprick": 102147,
+      "ĠSre": 102148,
+      "ĠMert": 102149,
+      "çļĦæĮĩ示": 102150,
+      "æĸĽ": 102151,
+      "ĠStig": 102152,
+      "ãĤĤæĭħå½ĵ": 102153,
+      "ãģĻãĤĭä¸Ģæĸ¹": 102154,
+      "ãģĭãĤīèĦ±": 102155,
+      "Ġ727": 102156,
+      "Ġrestless": 102157,
+      "éķ¿å´İ": 102158,
+      "åħµè£ħ": 102159,
+      "ãĥįãĥIJãĥĢ": 102160,
+      "å°ĩæѤ": 102161,
+      "ç´łè³ª": 102162,
+      "å¡Ķçī¹": 102163,
+      "ä¸ĢèĪ¬ç¤¾åĽ£æ³ķ人": 102164,
+      "åıĸãĤĬåħ¥ãĤĮ": 102165,
+      "æľŁéĸĵä¸Ń": 102166,
+      "éľĩæĴ¼": 102167,
+      "ĠChilders": 102168,
+      "ì²ľë§Į": 102169,
+      "ãģ®ä¸Ĭæĺĩ": 102170,
+      "æľĢè¿ijãģ®": 102171,
+      "ĠíĸĪëĬĶëį°": 102172,
+      "Ġê²ĢíĨł": 102173,
+      "尽管å¦ĤæѤ": 102174,
+      "Ġpinkish": 102175,
+      "ãĤĦãĤĬåıĸãĤĬ": 102176,
+      "Suica": 102177,
+      "ĠìĿ¼ìľ¼ì¼°ëĭ¤": 102178,
+      "Ġìĭ¸ìĽĢ": 102179,
+      "ĠGutierrez": 102180,
+      "Ġgelatin": 102181,
+      "俾æĸ¯": 102182,
+      "Ġcoven": 102183,
+      "ãģ®æ¨¡æ§ĺ": 102184,
+      "ãģ«çµ±ä¸Ģ": 102185,
+      "çļĦæĹł": 102186,
+      "ĠGTP": 102187,
+      "ĠInoue": 102188,
+      "éĩĹ": 102189,
+      "Ġ2½": 102190,
+      "æĻĤãĢħ": 102191,
+      "1593": 102192,
+      "1665": 102193,
+      "女éĥİ": 102194,
+      "å°ĨæѤ": 102195,
+      "ãĤ¬ãĥ«ãĥĪ": 102196,
+      "Ġaffections": 102197,
+      "ĠNorwegians": 102198,
+      "Ġmaterialism": 102199,
+      "ĠKeats": 102200,
+      "éĢ±ãģ«": 102201,
+      "ï¼Łï¼ī": 102202,
+      "itiate": 102203,
+      "ë°ĽëĬĶ": 102204,
+      "å¿įåıĹ": 102205,
+      "Ġgrids": 102206,
+      "Ġس": 102207,
+      "ĠìľłëŁ½ìĿĺ": 102208,
+      "ãĥģãĥ£ãĥ¼ãĥĪãģ§ãģ¯": 102209,
+      "gravity": 102210,
+      "Chamber": 102211,
+      "辿ãĤĬ": 102212,
+      "practice": 102213,
+      "Ġailerons": 102214,
+      "Rash": 102215,
+      "Tir": 102216,
+      "nist": 102217,
+      "ãģ®æĶ¯æĮģ": 102218,
+      "ineers": 102219,
+      "ĠHeels": 102220,
+      "ãĤ¤ãĥ´": 102221,
+      "expected": 102222,
+      "Ġinfusion": 102223,
+      "aughan": 102224,
+      "zeg": 102225,
+      "æĻĤ代ãģĭãĤīãģ®": 102226,
+      "ĠëıĦì¿ł": 102227,
+      "æĬĺå°Ħ": 102228,
+      "åİĭåĪ¶": 102229,
+      "ĠTanzan": 102230,
+      "äºĪå®ļãģ§ãģĤãĤĭ": 102231,
+      "McK": 102232,
+      "huang": 102233,
+      "ĠìĦ¸ê³ĦìĿĺ": 102234,
+      "ĠAquatics": 102235,
+      "ĠConnaught": 102236,
+      "Ġtensor": 102237,
+      "義大åĪ©èªŀ": 102238,
+      "è»įãĤĴçİĩãģĦãģ¦": 102239,
+      "ãģ«å¯¾å¿ľãģĹãģ¦ãģĦãĤĭ": 102240,
+      "éĺªç¥ŀãĤ¿ãĤ¤ãĤ¬ãĥ¼ãĤ¹": 102241,
+      "Ġexhaustive": 102242,
+      "ĠPosition": 102243,
+      "Ġfondness": 102244,
+      "ĠShivaji": 102245,
+      "ĠOdysse": 102246,
+      "ĠGheorghe": 102247,
+      "BASIC": 102248,
+      "ocletian": 102249,
+      "025": 102250,
+      "Nin": 102251,
+      "ĠTough": 102252,
+      "ĠLiss": 102253,
+      "Ġoverheard": 102254,
+      "å¤ļéģĶ": 102255,
+      "åIJİ为": 102256,
+      "æĹ¥æľ¬è»įãģ¯": 102257,
+      "ãģĺãĤĬ": 102258,
+      "Ġhumility": 102259,
+      "念éłŃ": 102260,
+      "延平": 102261,
+      "çļ®åį¡": 102262,
+      "ä½³ä½ľ": 102263,
+      "598": 102264,
+      "ë°ľìĿĦ": 102265,
+      "èĻ½çĦ¶åľ¨": 102266,
+      "éĽĸçĦ¶åľ¨": 102267,
+      "Ġë²łìĿ´ìĬ¤": 102268,
+      "åłħ決": 102269,
+      "çł´å£ĬãģķãĤĮ": 102270,
+      "ãģ£ãģ·": 102271,
+      "æľĪåı·ãģ¾ãģ§": 102272,
+      "éİĮåĢīå¹ķåºľ": 102273,
+      "Ġìłľê³µíķľëĭ¤": 102274,
+      "è¶ķåĪ°": 102275,
+      "ĠHickory": 102276,
+      "Ġlawns": 102277,
+      "Ġtricycle": 102278,
+      "Ġ그곳ìĹIJìĦľ": 102279,
+      "Ġcushion": 102280,
+      "ĠMoira": 102281,
+      "\"'": 102282,
+      "Ġtad": 102283,
+      "Ġhu": 102284,
+      "ĠSurgical": 102285,
+      "ĠTama": 102286,
+      "ĠBeb": 102287,
+      "çļĦæĦı": 102288,
+      "ãģ¯åŁºæľ¬çļĦãģ«": 102289,
+      "ĠFenerbahçe": 102290,
+      "ãģ§ä½ľãĤīãĤĮãģŁ": 102291,
+      "oski": 102292,
+      "ãĥĪãģ¯": 102293,
+      "arets": 102294,
+      "åĴĮè°IJ": 102295,
+      "Ġknitting": 102296,
+      "Ġassurances": 102297,
+      "飾": 102298,
+      "åİŁçĶ»": 102299,
+      "æ°´æľ¨": 102300,
+      "Ġcrested": 102301,
+      "該年": 102302,
+      "ĠMonc": 102303,
+      "Ġlessen": 102304,
+      "Ġ278": 102305,
+      "Ġnarrowing": 102306,
+      "Ġnomads": 102307,
+      "Ġ복무": 102308,
+      "éĢĤçĶ¨": 102309,
+      "ĠVelasco": 102310,
+      "ILL": 102311,
+      "Ġgalley": 102312,
+      "ĠChaluk": 102313,
+      "Ġdespised": 102314,
+      "YPE": 102315,
+      "ä¸ĩç¾İåħĥçļĦ": 102316,
+      "Ġpedagogical": 102317,
+      "ãģ®æ¬¡çĶ·": 102318,
+      "Ġstumbled": 102319,
+      "ĠOM": 102320,
+      "manif": 102321,
+      "owning": 102322,
+      "ĠWestbrook": 102323,
+      "ĠìłĦë¶Ģ": 102324,
+      "iliary": 102325,
+      "Ġ춤": 102326,
+      "Ġconductive": 102327,
+      "åIJĪä½ľç¤¾": 102328,
+      "Evangel": 102329,
+      "礦æ¥Ń": 102330,
+      "éĢ£éļĬéķ·": 102331,
+      "ĠCortez": 102332,
+      "æľĪåł´æīĢãģ§ãģ¯": 102333,
+      "ĠTillman": 102334,
+      "Titan": 102335,
+      "dun": 102336,
+      "ĠÑĤ": 102337,
+      "vergne": 102338,
+      "iflora": 102339,
+      "amphetamine": 102340,
+      "Ġoverran": 102341,
+      "æĶ¿æķĻ": 102342,
+      "æĸ°é«ĺ": 102343,
+      "è¦ģãģĻãĤĭ": 102344,
+      "announced": 102345,
+      "æĿ±æ¬§": 102346,
+      "äºĮæĺ¯": 102347,
+      "Ġdeflect": 102348,
+      "è»Ĭãģ§ãģĤãĤĭ": 102349,
+      "Ġconstants": 102350,
+      "åıĤæĪĺ": 102351,
+      "åıĤè©£": 102352,
+      "ĠBarak": 102353,
+      "復åİŁ": 102354,
+      "ĠMagical": 102355,
+      "Ġëĭ¤ë¦¬": 102356,
+      "æīįåı¯": 102357,
+      "åĦªåĭĿãĤĴ飾ãģ£ãģŁ": 102358,
+      "çł²æŀ¶": 102359,
+      "ĠOswego": 102360,
+      "é«ĺæł¡ãģ§ãģ¯": 102361,
+      "ĠShelf": 102362,
+      "Josef": 102363,
+      "ĠThunderbirds": 102364,
+      "Ġfeatherweight": 102365,
+      "çĽĪåĪ©": 102366,
+      "ĠComposite": 102367,
+      "æ³ķèĺŃåħĭç¦ı": 102368,
+      "roix": 102369,
+      "ãģ®çĻºè¡Į": 102370,
+      "çļĦçIJĨ论": 102371,
+      "çļĦçĬ¶æĢģ": 102372,
+      "ĠPuma": 102373,
+      "次ãģ®ãĤĪãģĨãģª": 102374,
+      "ç§ijç¶Ńå¥ĩ": 102375,
+      "Ġavian": 102376,
+      "ãģĿãĤĮ以éĻį": 102377,
+      "æĿİ大": 102378,
+      "ãĤ¢ãĥ«ãĥŀ": 102379,
+      "Ġranches": 102380,
+      "Ġdependencies": 102381,
+      "Ġbillboard": 102382,
+      "缺é»ŀ": 102383,
+      "ĠCzechs": 102384,
+      "çī©çIJĨç³»": 102385,
+      "ĠRangoon": 102386,
+      "ãģ®ãĤ¨ãĥįãĥ«": 102387,
+      "Flood": 102388,
+      "ĠQualification": 102389,
+      "å·Ŀå´İå¸Ĥ": 102390,
+      "à¯ģ": 102391,
+      "Ġannotated": 102392,
+      "Ġarduous": 102393,
+      "Ich": 102394,
+      "soccer": 102395,
+      "ĠCÄĥ": 102396,
+      "omology": 102397,
+      "çļĦçµĦç¹Ķ": 102398,
+      "åĽij": 102399
     },
     "merges": [
       "Ġ t",
@@ -203785,7 +204185,407 @@
       "åħ© åľ°",
       "éª Ħ",
       "Ġpur plish",
-      "Ġíķĺ 기ëıĦ"
+      "Ġíķĺ 기ëıĦ",
+      "Ġapprox im",
+      "æ¢ħ åħĭ",
+      "åı¥ åŃIJ",
+      "ĠConc orde",
+      "ĠCost ume",
+      "ãĥŁãĥ¥ãĥ¼ ãĤº",
+      "ãģ®éķ· ãģĦ",
+      "á¹ Ľ",
+      "ĠEg gs",
+      "Ġì¡´ìŀ¬ íķĺì§Ģ",
+      "Ġfract uring",
+      "奨 åѦéĩij",
+      "Ġreprin ts",
+      "ĠGA BA",
+      "ĠSamar itan",
+      "Ġdy ed",
+      "Ġannih ilation",
+      "ĠíĶ¼ìķĦ ëħ¸",
+      "5 88",
+      "ag ency",
+      "ap ala",
+      "æĺ¯ åı¯",
+      "æ¨ ģ",
+      "Ġinter collegiate",
+      "ash ir",
+      "16 99",
+      "ĠCor man",
+      "éĿŀ æŃ£å¼ı",
+      "éĺ² æ°´",
+      "ü d",
+      "åħ³ èĬĤ",
+      "ä¼Ŀ ãģĪãģŁ",
+      "æģ© ç§ij",
+      "ĠëĤ´ ìĦ¸",
+      "Ġrein vent",
+      "ĠBan at",
+      "ìĥī ìľ¼ë¡ľ",
+      "Ty pically",
+      "ĠìłĦìŁģ ìĹIJ",
+      "èľĤ èľľ",
+      "éĻįä½İ äºĨ",
+      "Architect ural",
+      "ãģ«æ¯Ķãģ¹ ãĤĭãģ¨",
+      "ãĥ¬ãĥĥ ãĤº",
+      "zoll ern",
+      "ãĤĴ 強調",
+      "ĠK ia",
+      "åľ¨ åħ¨åľĭ",
+      "åIJĮ ä¸Ģ个",
+      "åĮĹ æ¢Ŀ",
+      "åĭķ æ¤įçī©",
+      "Ġinc rim",
+      "Ġmon ot",
+      "æģ ¼",
+      "ãĤĪãģĨ ãģł",
+      "çĶº é§ħ",
+      "客 ãģ®",
+      "åī¯ èŃ°éķ·",
+      "åī¯ ä¼ļéķ¿",
+      "ç¥ĸ åľĭ",
+      "ĠMont clair",
+      "ĠëIJĺ ëıĮ",
+      "ê²½ ìĿ´",
+      "BO L",
+      "å·¥åł´ ãģ¯",
+      "íĻĺ ìĿĦ",
+      "説æĺİ ãģĻãĤĭ",
+      "ĠBrun el",
+      "åIJĦå¼ı åIJĦ",
+      "ĠP ML",
+      "ãĤĤ åĩº",
+      "cl oud",
+      "act yl",
+      "ins urgency",
+      "ãĥĸ ãĥ©ãĤ¤ãĥĪ",
+      "ìĸ´ ë²Ħ",
+      "å°į æµģ",
+      "ĠAm t",
+      "ĠëĮĢ ìĿĺ",
+      "ëIJĺ ê²Į",
+      "Ġsl iced",
+      "Ġ27 6",
+      "ãĥ¦ ãĥª",
+      "ä»ĭ ä¹İ",
+      "ठª",
+      "æĬľ ãģĦãģŁ",
+      "éł» 度",
+      "ãĥĪãĤ¥ ãĥ¼",
+      "Ġmal ware",
+      "ĠLear y",
+      "ĠKin ney",
+      "ĠMons ignor",
+      "UEFA ãĥģãĥ£ãĥ³ãĥĶãĤªãĥ³ãĤºãĥªãĥ¼ãĤ°",
+      "ãĤªãĥķãĤ£ ãĤ·ãĥ£ãĥ«",
+      "Ġpunct uated",
+      "Ġsplic ing",
+      "Ê Ĵ",
+      "ĠF ires",
+      "ĠK ari",
+      "ib bon",
+      "ãĥIJ ãĥĥãĥī",
+      "ĠCon sole",
+      "sc ot",
+      "çħ ¬",
+      "åħ« 个",
+      "iqu i",
+      "é¾Ļ ç§ij",
+      "ĠRos well",
+      "å¼Ĺ èIJĬ",
+      "AN T",
+      "åѸçĶŁ çļĦ",
+      "æľįåĬ¡ çļĦ",
+      "ãĥĻãĤ¹ãĥĪ ãĤ¢ãĥ«ãĥIJãĥł",
+      "Ġdelight ful",
+      "ĠCasual ty",
+      "5 14",
+      "C ul",
+      "M ills",
+      "ãĤ IJ",
+      "it ur",
+      "ĠR au",
+      "ĠF ass",
+      "ãĤ· ãĥį",
+      "åĪ¶ ãģĹ",
+      "ç´Ħ æŦ",
+      "å°Ĩ 被",
+      "ç«Ļ 廳",
+      "Ġfurther ing",
+      "Al ways",
+      "ĠRoyal ists",
+      "æ¾ Ģ",
+      "ĠSa ber",
+      "ĠMcC ar",
+      "ĠBor ges",
+      "ĠìŀĦ 무",
+      "Ġì²Ń ëĤĺëĿ¼",
+      "罰 款",
+      "ĠBog art",
+      "ãĥIJãĥĥãĤ¯ ãĤ¢ãĥĥãĥĹ",
+      "修建 äºĨ",
+      "ĠCret an",
+      "Ġbrib ed",
+      "ï¹ IJ",
+      "çĭ¹ çªĦ",
+      "m RNA",
+      "p ick",
+      "Ġc ren",
+      "Ġp rick",
+      "ĠS re",
+      "ĠM ert",
+      "çļĦ æĮĩ示",
+      "æĸ Ľ",
+      "ĠSt ig",
+      "ãĤĤ æĭħå½ĵ",
+      "ãģĻãĤĭ ä¸Ģæĸ¹",
+      "ãģĭãĤī èĦ±",
+      "Ġ7 27",
+      "Ġrest less",
+      "éķ¿ å´İ",
+      "åħµ è£ħ",
+      "ãĥį ãĥIJãĥĢ",
+      "å°ĩ æѤ",
+      "ç´ł 質",
+      "å¡Ķ çī¹",
+      "ä¸ĢèĪ¬ ç¤¾åĽ£æ³ķ人",
+      "åıĸãĤĬ åħ¥ãĤĮ",
+      "æľŁéĸĵ ä¸Ń",
+      "éľĩ æĴ¼",
+      "ĠChild ers",
+      "ì²ľ ë§Į",
+      "ãģ®ä¸Ĭ æĺĩ",
+      "æľĢè¿ij ãģ®",
+      "ĠíĸĪ ëĬĶëį°",
+      "Ġê²Ģ íĨł",
+      "尽管 å¦ĤæѤ",
+      "Ġpink ish",
+      "ãĤĦãĤĬ åıĸãĤĬ",
+      "Su ica",
+      "ĠìĿ¼ìľ¼ ì¼°ëĭ¤",
+      "Ġìĭ¸ ìĽĢ",
+      "ĠGut ierrez",
+      "Ġgel atin",
+      "俾 æĸ¯",
+      "Ġc oven",
+      "ãģ® 模æ§ĺ",
+      "ãģ« çµ±ä¸Ģ",
+      "çļĦ æĹł",
+      "ĠG TP",
+      "ĠIn oue",
+      "éĩ Ĺ",
+      "Ġ2 ½",
+      "æĻĤ ãĢħ",
+      "15 93",
+      "16 65",
+      "女 éĥİ",
+      "å°Ĩ æѤ",
+      "ãĤ¬ ãĥ«ãĥĪ",
+      "Ġaff ections",
+      "ĠNor wegians",
+      "Ġmaterial ism",
+      "ĠKe ats",
+      "éĢ± ãģ«",
+      "ï¼Ł ï¼ī",
+      "iti ate",
+      "ë°Ľ ëĬĶ",
+      "å¿į åıĹ",
+      "Ġgrid s",
+      "ĠØ ³",
+      "ĠìľłëŁ½ ìĿĺ",
+      "ãĥģãĥ£ãĥ¼ãĥĪ ãģ§ãģ¯",
+      "grav ity",
+      "Cham ber",
+      "辿 ãĤĬ",
+      "pract ice",
+      "Ġailer ons",
+      "R ash",
+      "T ir",
+      "n ist",
+      "ãģ® æĶ¯æĮģ",
+      "ine ers",
+      "ĠHe els",
+      "ãĤ¤ ãĥ´",
+      "ex pected",
+      "Ġinf usion",
+      "augh an",
+      "ze g",
+      "æĻĤ代 ãģĭãĤīãģ®",
+      "ĠëıĦ ì¿ł",
+      "æĬĺ å°Ħ",
+      "åİĭ åĪ¶",
+      "ĠTan zan",
+      "äºĪå®ļ ãģ§ãģĤãĤĭ",
+      "Mc K",
+      "hu ang",
+      "ĠìĦ¸ê³Ħ ìĿĺ",
+      "ĠAqu atics",
+      "ĠConn aught",
+      "Ġtens or",
+      "義大åĪ© èªŀ",
+      "è»įãĤĴ çİĩãģĦãģ¦",
+      "ãģ«å¯¾å¿ľ ãģĹãģ¦ãģĦãĤĭ",
+      "éĺªç¥ŀ ãĤ¿ãĤ¤ãĤ¬ãĥ¼ãĤ¹",
+      "Ġexhaust ive",
+      "ĠPos ition",
+      "Ġfond ness",
+      "ĠShiv aji",
+      "ĠOdys se",
+      "ĠGhe orghe",
+      "BAS IC",
+      "oclet ian",
+      "0 25",
+      "N in",
+      "ĠT ough",
+      "ĠL iss",
+      "Ġover heard",
+      "å¤ļ éģĶ",
+      "åIJİ ä¸º",
+      "æĹ¥æľ¬ è»įãģ¯",
+      "ãģĺ ãĤĬ",
+      "Ġhum ility",
+      "念 éłŃ",
+      "延 平",
+      "çļ® åį¡",
+      "ä½³ ä½ľ",
+      "59 8",
+      "ë°ľ ìĿĦ",
+      "èĻ½çĦ¶ åľ¨",
+      "éĽĸçĦ¶ åľ¨",
+      "Ġë²ł ìĿ´ìĬ¤",
+      "åłħ 決",
+      "çł´å£Ĭ ãģķãĤĮ",
+      "ãģ£ãģ ·",
+      "æľĪåı· ãģ¾ãģ§",
+      "éİĮåĢī å¹ķåºľ",
+      "Ġìłľê³µ íķľëĭ¤",
+      "è¶ķ åĪ°",
+      "ĠHick ory",
+      "Ġlawn s",
+      "Ġtric ycle",
+      "Ġ그곳 ìĹIJìĦľ",
+      "Ġcush ion",
+      "ĠMoir a",
+      "\" '",
+      "Ġt ad",
+      "Ġh u",
+      "ĠS urgical",
+      "ĠT ama",
+      "ĠB eb",
+      "çļĦ æĦı",
+      "ãģ¯ åŁºæľ¬çļĦãģ«",
+      "ĠF enerbahçe",
+      "ãģ§ ä½ľãĤīãĤĮãģŁ",
+      "os ki",
+      "ãĥĪ ãģ¯",
+      "are ts",
+      "åĴĮ è°IJ",
+      "Ġkn itting",
+      "Ġass urances",
+      "éŁ ľ",
+      "åİŁ çĶ»",
+      "æ°´ æľ¨",
+      "Ġcre sted",
+      "該 年",
+      "ĠMon c",
+      "Ġless en",
+      "Ġ27 8",
+      "Ġnarrow ing",
+      "Ġnom ads",
+      "Ġë³µ 무",
+      "éĢĤ çĶ¨",
+      "ĠVel asco",
+      "IL L",
+      "Ġgal ley",
+      "ĠChal uk",
+      "Ġdesp ised",
+      "YP E",
+      "ä¸ĩç¾İ��ħĥ çļĦ",
+      "Ġpedag ogical",
+      "ãģ® 次çĶ·",
+      "Ġst umbled",
+      "ĠO M",
+      "man if",
+      "own ing",
+      "ĠWest brook",
+      "ĠìłĦ ë¶Ģ",
+      "ili ary",
+      "Ġì¶ ¤",
+      "Ġconduct ive",
+      "åIJĪä½ľ 社",
+      "Ev angel",
+      "礦 æ¥Ń",
+      "éĢ£éļĬ éķ·",
+      "ĠCort ez",
+      "æľĪåł´æīĢ ãģ§ãģ¯",
+      "ĠTill man",
+      "T itan",
+      "d un",
+      "Ġ ÑĤ",
+      "ver gne",
+      "if lora",
+      "amp hetamine",
+      "Ġover ran",
+      "æĶ¿ æķĻ",
+      "æĸ° é«ĺ",
+      "è¦ģ ãģĻãĤĭ",
+      "ann ounced",
+      "æĿ± 欧",
+      "äºĮ æĺ¯",
+      "Ġdef lect",
+      "è»Ĭ ãģ§ãģĤãĤĭ",
+      "Ġconst ants",
+      "åıĤ æĪĺ",
+      "åıĤ è©£",
+      "ĠBar ak",
+      "復 åİŁ",
+      "ĠMag ical",
+      "Ġëĭ¤ 리",
+      "æīį åı¯",
+      "åĦªåĭĿ ãĤĴ飾ãģ£ãģŁ",
+      "çł² æŀ¶",
+      "ĠOs wego",
+      "é«ĺæł¡ ãģ§ãģ¯",
+      "ĠShel f",
+      "Jose f",
+      "ĠThunder birds",
+      "Ġfeather weight",
+      "çĽĪ åĪ©",
+      "ĠCompos ite",
+      "æ³ķèĺŃåħĭ ç¦ı",
+      "ro ix",
+      "ãģ® çĻºè¡Į",
+      "çļĦ çIJĨ论",
+      "çļĦ çĬ¶æĢģ",
+      "ĠP uma",
+      "次 ãģ®ãĤĪãģĨãģª",
+      "ç§ij ç¶Ńå¥ĩ",
+      "Ġav ian",
+      "ãģĿãĤĮ 以éĻį",
+      "æĿİ å¤§",
+      "ãĤ¢ãĥ« ãĥŀ",
+      "Ġran ches",
+      "Ġdepend encies",
+      "Ġbill board",
+      "缺 é»ŀ",
+      "ĠCzech s",
+      "çī©çIJĨ ç³»",
+      "ĠRang oon",
+      "ãģ®ãĤ¨ ãĥįãĥ«",
+      "Fl ood",
+      "ĠQual ification",
+      "å·Ŀå´İ å¸Ĥ",
+      "௠ģ",
+      "Ġannot ated",
+      "Ġardu ous",
+      "I ch",
+      "s occer",
+      "ĠC Äĥ",
+      "om ology",
+      "çļĦ çµĦç¹Ķ",
+      "åĽ ij"
     ]
   }
 }

tokenizer_config.json

@@ -4,6 +4,6 @@
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
   "model_max_length": 2048,
-  "tokenizer_class": "ShamiTokenizer",
+  "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"
 }

train_tokenizer.py

@@ -23,6 +23,6 @@ def training_dataset_iterator():
             yield line['text']
 
 # tokenizer.train(training_files, trainer)
-tokenizer = old_tokenizer.train_new_from_iterator(training_dataset_iterator(), 102000, total_line)
+tokenizer = old_tokenizer.train_new_from_iterator(training_dataset_iterator(), 102400, total_line)
 
 tokenizer.save_pretrained("tokenizer-shami")