Hugging Face's logo

KoichiYasuoka
/
Swallow-7b-plus-upos

Token Classification
Transformers
PyTorch
Japanese
llama
japanese
pos
text-generation-inference
Inference Endpoints
Model card Files Community
KoichiYasuoka commited on
Commit
8ebd785
•
1 Parent(s): f156762

model improved

Browse files
Files changed (8) hide show
  1. maker.sh +1 -4
  2. pytorch_model-00001-of-00006.bin +1 -1
  3. pytorch_model-00002-of-00006.bin +1 -1
  4. pytorch_model-00003-of-00006.bin +1 -1
  5. pytorch_model-00004-of-00006.bin +1 -1
  6. pytorch_model-00005-of-00006.bin +1 -1
  7. pytorch_model-00006-of-00006.bin +1 -1
  8. tokenizer.json +571 -583
maker.sh CHANGED
@@ -20,7 +20,7 @@ with open("ja_gsd_modern.conllu","r",encoding="utf-8") as r:
20
  if unicodedata.name(c).startswith("CJK "):
21
  cjk.append(c)
22
  cjk=list(set(cjk))
23
- tkz=LlamaTokenizerFast.from_pretrained(src,cls_token="<s>",sep_token="<s>",mask_token="<unk>",pad_token="</s>")
24
  c={i:j[2:] for i,j in zip(cjk,tkz(cjk)["input_ids"]) if len(j)>3}
25
  d=json.loads(tkz.backend_tokenizer.to_str())
26
  for i,j in enumerate(c,len(tkz)):
@@ -49,7 +49,6 @@ cat << 'EOF' > $TMPB
49
  src="exSwallow-7b-plus-hf"
50
  tgt="KoichiYasuoka/Swallow-7b-plus-upos"
51
  from transformers import LlamaTokenizerFast,LlamaForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
52
- from tokenizers.normalizers import Replace
53
 
54
  class UPOSFileDataset(object):
55
  def __init__(self,conllu,tokenizer):
@@ -124,8 +123,6 @@ class UPOSFileDataset(object):
124
  return {"input_ids":ids,"labels":[self.label2id[t] for t in upos]}
125
 
126
  tkz=LlamaTokenizerFast.from_pretrained(src)
127
- tkz.backend_tokenizer.normalizer=Replace(" ","\u2581")
128
- tkz.backend_tokenizer.model.byte_fallback=False
129
  trainDS=UPOSFileDataset("ja_gsd_modern.conllu",tkz)
130
  lid=trainDS.label2id
131
  cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
 
20
  if unicodedata.name(c).startswith("CJK "):
21
  cjk.append(c)
22
  cjk=list(set(cjk))
23
+ tkz=LlamaTokenizerFast.from_pretrained(src,cls_token="<s>",sep_token="<s>",mask_token="<unk>",pad_token="</s>",add_prefix_space=False)
24
  c={i:j[2:] for i,j in zip(cjk,tkz(cjk)["input_ids"]) if len(j)>3}
25
  d=json.loads(tkz.backend_tokenizer.to_str())
26
  for i,j in enumerate(c,len(tkz)):
 
49
  src="exSwallow-7b-plus-hf"
50
  tgt="KoichiYasuoka/Swallow-7b-plus-upos"
51
  from transformers import LlamaTokenizerFast,LlamaForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 
52
 
53
  class UPOSFileDataset(object):
54
  def __init__(self,conllu,tokenizer):
 
123
  return {"input_ids":ids,"labels":[self.label2id[t] for t in upos]}
124
 
125
  tkz=LlamaTokenizerFast.from_pretrained(src)
 
 
126
  trainDS=UPOSFileDataset("ja_gsd_modern.conllu",tkz)
127
  lid=trainDS.label2id
128
  cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
pytorch_model-00001-of-00006.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:15b7ecb40dc994c01d584b8f6061aafb937adb91d3ea88d190874393ef152b77
3
  size 4965712452
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebcdd54fc40ee21fbb12f0cbdda730d69dbc2238ec9e0da54274c03356295826
3
  size 4965712452
pytorch_model-00002-of-00006.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7ee7cc71f1d19b1403f8f7fdefee9c2fdc7db7fb2d793fb1676bd9440335cc78
3
  size 4924328556
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:138fa90e9c53f35d096aa6c6e4c6b1ad6150d1855bd85085ea66a6eb3bb98d76
3
  size 4924328556
pytorch_model-00003-of-00006.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:504a5638d62594b8c0025a56083668458291dcd1137455a403efa35093b7c15c
3
  size 4857219294
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f26c1ab5ff0f7baf08fb529ddd0bc73ae65ab229e9055e0db3f2f3a5e8ac33e6
3
  size 4857219294
pytorch_model-00004-of-00006.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a1e030bf6ba8296546621ba35acdc1486c0695b500141aa20f242bbc514002a2
3
  size 4857219294
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d64ad917668a7f15b3c14ec088830f1ff95fee281108bb27f643f1a757dc053
3
  size 4857219294
pytorch_model-00005-of-00006.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c068f31f763a9235aeef0f60407db1fb5ffde971ede030ccf9787eb7490bb8d8
3
  size 4857219294
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7dba45b149767756ff62683c886ade833657b032109a21596cb2e6216d2077a
3
  size 4857219294
pytorch_model-00006-of-00006.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4fb6cbefff2a23a4edc4414e8b48e6d7d336f9cb4e3338fe21b55273bd827dff
3
  size 2161173694
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c295abca414ff159a210437b077e791d22acd601d8ca7f5e1935e103a27422d
3
  size 2161173694
tokenizer.json CHANGED
@@ -31,17 +31,11 @@
31
  "special": true
32
  }
33
  ],
34
- "normalizer": {
35
- "type": "Replace",
36
- "pattern": {
37
- "String": " "
38
- },
39
- "content": "▁"
40
- },
41
  "pre_tokenizer": {
42
  "type": "Metaspace",
43
  "replacement": "▁",
44
- "prepend_scheme": "first",
45
  "split": false
46
  },
47
  "post_processor": {
@@ -113,12 +107,6 @@
113
  },
114
  {
115
  "type": "Fuse"
116
- },
117
- {
118
- "type": "Strip",
119
- "content": " ",
120
- "start": 1,
121
- "stop": 0
122
  }
123
  ]
124
  },
@@ -129,7 +117,7 @@
129
  "continuing_subword_prefix": null,
130
  "end_of_word_suffix": null,
131
  "fuse_unk": true,
132
- "byte_fallback": false,
133
  "ignore_merges": false,
134
  "vocab": {
135
  "<unk>": 0,
@@ -43308,574 +43296,574 @@
43308
  "趙": 43173,
43309
  "弛": 43174,
43310
  "徊": 43175,
43311
- "戲": 43176,
43312
- "舍": 43177,
43313
- "猾": 43178,
43314
- "專": 43179,
43315
- "爲": 43180,
43316
- "黠": 43181,
43317
- "爰": 43182,
43318
- "橙": 43183,
43319
- "鰭": 43184,
43320
- "榜": 43185,
43321
- "摧": 43186,
43322
- "飜": 43187,
43323
- "關": 43188,
43324
- "朧": 43189,
43325
- "畢": 43190,
43326
- "汝": 43191,
43327
- "挈": 43192,
43328
- "吻": 43193,
43329
- "懺": 43194,
43330
- "僻": 43195,
43331
- "毋": 43196,
43332
- "盡": 43197,
43333
- "羣": 43198,
43334
- "嫡": 43199,
43335
- "煥": 43200,
43336
- "鐸": 43201,
43337
- "忿": 43202,
43338
- "參": 43203,
43339
- "欵": 43204,
43340
- "聘": 43205,
43341
- "攝": 43206,
43342
- "纂": 43207,
43343
- "扣": 43208,
43344
- "險": 43209,
43345
- "儘": 43210,
43346
- "尙": 43211,
43347
- "臥": 43212,
43348
- "淇": 43213,
43349
- "籾": 43214,
43350
- "豫": 43215,
43351
- "龕": 43216,
43352
- "佩": 43217,
43353
- "鍾": 43218,
43354
- "歟": 43219,
43355
- "饜": 43220,
43356
- "權": 43221,
43357
- "雖": 43222,
43358
- "狡": 43223,
43359
- "翹": 43224,
43360
- "吏": 43225,
43361
- "繼": 43226,
43362
- "鄙": 43227,
43363
- "儕": 43228,
43364
- "袈": 43229,
43365
- "轍": 43230,
43366
- "袁": 43231,
43367
- "囮": 43232,
43368
- "謭": 43233,
43369
- "皺": 43234,
43370
- "梃": 43235,
43371
- "餘": 43236,
43372
- "彝": 43237,
43373
- "岨": 43238,
43374
- "舒": 43239,
43375
- "埸": 43240,
43376
- "碍": 43241,
43377
- "鸞": 43242,
43378
- "胤": 43243,
43379
- "屡": 43244,
43380
- "崗": 43245,
43381
- "靜": 43246,
43382
- "禀": 43247,
43383
- "聯": 43248,
43384
- "皓": 43249,
43385
- "騷": 43250,
43386
- "擔": 43251,
43387
- "逼": 43252,
43388
- "盜": 43253,
43389
- "譯": 43254,
43390
- "駢": 43255,
43391
- "勅": 43256,
43392
- "錐": 43257,
43393
- "僞": 43258,
43394
- "厭": 43259,
43395
- "丞": 43260,
43396
- "臂": 43261,
43397
- "憮": 43262,
43398
- "轉": 43263,
43399
- "蜻": 43264,
43400
- "竊": 43265,
43401
- "攘": 43266,
43402
- "內": 43267,
43403
- "鄭": 43268,
43404
- "嚴": 43269,
43405
- "靱": 43270,
43406
- "縣": 43271,
43407
- "茫": 43272,
43408
- "俟": 43273,
43409
- "菑": 43274,
43410
- "隋": 43275,
43411
- "倩": 43276,
43412
- "冰": 43277,
43413
- "輕": 43278,
43414
- "盍": 43279,
43415
- "豈": 43280,
43416
- "據": 43281,
43417
- "實": 43282,
43418
- "紂": 43283,
43419
- "滊": 43284,
43420
- "墮": 43285,
43421
- "臘": 43286,
43422
- "濶": 43287,
43423
- "歸": 43288,
43424
- "驗": 43289,
43425
- "寔": 43290,
43426
- "脩": 43291,
43427
- "糀": 43292,
43428
- "禎": 43293,
43429
- "藥": 43294,
43430
- "絜": 43295,
43431
- "羞": 43296,
43432
- "寃": 43297,
43433
- "亨": 43298,
43434
- "濤": 43299,
43435
- "恤": 43300,
43436
- "鑒": 43301,
43437
- "朕": 43302,
43438
- "嚼": 43303,
43439
- "拔": 43304,
43440
- "蓁": 43305,
43441
- "臾": 43306,
43442
- "滿": 43307,
43443
- "蹈": 43308,
43444
- "擇": 43309,
43445
- "兒": 43310,
43446
- "兩": 43311,
43447
- "嬰": 43312,
43448
- "揠": 43313,
43449
- "脹": 43314,
43450
- "總": 43315,
43451
- "詔": 43316,
43452
- "恊": 43317,
43453
- "肚": 43318,
43454
- "悖": 43319,
43455
- "沮": 43320,
43456
- "續": 43321,
43457
- "媢": 43322,
43458
- "駸": 43323,
43459
- "慘": 43324,
43460
- "賂": 43325,
43461
- "礦": 43326,
43462
- "哨": 43327,
43463
- "蠻": 43328,
43464
- "舅": 43329,
43465
- "纉": 43330,
43466
- "傅": 43331,
43467
- "埠": 43332,
43468
- "鵬": 43333,
43469
- "儁": 43334,
43470
- "篆": 43335,
43471
- "勵": 43336,
43472
- "體": 43337,
43473
- "隱": 43338,
43474
- "盈": 43339,
43475
- "竭": 43340,
43476
- "澳": 43341,
43477
- "呵": 43342,
43478
- "耘": 43343,
43479
- "閻": 43344,
43480
- "衷": 43345,
43481
- "胥": 43346,
43482
- "蛾": 43347,
43483
- "諺": 43348,
43484
- "薛": 43349,
43485
- "戊": 43350,
43486
- "咎": 43351,
43487
- "剝": 43352,
43488
- "處": 43353,
43489
- "孛": 43354,
43490
- "逈": 43355,
43491
- "游": 43356,
43492
- "尤": 43357,
43493
- "遑": 43358,
43494
- "雜": 43359,
43495
- "嘲": 43360,
43496
- "憚": 43361,
43497
- "釀": 43362,
43498
- "葢": 43363,
43499
- "唖": 43364,
43500
- "傳": 43365,
43501
- "黽": 43366,
43502
- "騁": 43367,
43503
- "頽": 43368,
43504
- "殉": 43369,
43505
- "榮": 43370,
43506
- "頒": 43371,
43507
- "蘊": 43372,
43508
- "澁": 43373,
43509
- "汀": 43374,
43510
- "畫": 43375,
43511
- "趨": 43376,
43512
- "峻": 43377,
43513
- "饒": 43378,
43514
- "菉": 43379,
43515
- "宍": 43380,
43516
- "諮": 43381,
43517
- "證": 43382,
43518
- "玻": 43383,
43519
- "麿": 43384,
43520
- "嚮": 43385,
43521
- "窘": 43386,
43522
- "碇": 43387,
43523
- "鬢": 43388,
43524
- "緡": 43389,
43525
- "蘐": 43390,
43526
- "贖": 43391,
43527
- "誣": 43392,
43528
- "獨": 43393,
43529
- "醉": 43394,
43530
- "單": 43395,
43531
- "丙": 43396,
43532
- "戌": 43397,
43533
- "覽": 43398,
43534
- "燮": 43399,
43535
- "撿": 43400,
43536
- "允": 43401,
43537
- "逬": 43402,
43538
- "痍": 43403,
43539
- "恣": 43404,
43540
- "厘": 43405,
43541
- "緝": 43406,
43542
- "稱": 43407,
43543
- "斤": 43408,
43544
- "羸": 43409,
43545
- "酉": 43410,
43546
- "惠": 43411,
43547
- "蹄": 43412,
43548
- "胖": 43413,
43549
- "錮": 43414,
43550
- "璽": 43415,
43551
- "愈": 43416,
43552
- "斷": 43417,
43553
- "戮": 43418,
43554
- "竝": 43419,
43555
- "彥": 43420,
43556
- "棍": 43421,
43557
- "菰": 43422,
43558
- "亥": 43423,
43559
- "懥": 43424,
43560
- "攬": 43425,
43561
- "殲": 43426,
43562
- "殘": 43427,
43563
- "駑": 43428,
43564
- "俄": 43429,
43565
- "恕": 43430,
43566
- "茲": 43431,
43567
- "悌": 43432,
43568
- "蜀": 43433,
43569
- "舷": 43434,
43570
- "筭": 43435,
43571
- "揖": 43436,
43572
- "姦": 43437,
43573
- "堡": 43438,
43574
- "聲": 43439,
43575
- "歎": 43440,
43576
- "廢": 43441,
43577
- "肆": 43442,
43578
- "數": 43443,
43579
- "滌": 43444,
43580
- "鳶": 43445,
43581
- "沃": 43446,
43582
- "隨": 43447,
43583
- "决": 43448,
43584
- "屬": 43449,
43585
- "蛉": 43450,
43586
- "靡": 43451,
43587
- "桀": 43452,
43588
- "愧": 43453,
43589
- "謨": 43454,
43590
- "儼": 43455,
43591
- "瑟": 43456,
43592
- "勞": 43457,
43593
- "虧": 43458,
43594
- "匈": 43459,
43595
- "歔": 43460,
43596
- "恙": 43461,
43597
- "啜": 43462,
43598
- "匍": 43463,
43599
- "逓": 43464,
43600
- "鎭": 43465,
43601
- "漸": 43466,
43602
- "岌": 43467,
43603
- "懼": 43468,
43604
- "覺": 43469,
43605
- "迭": 43470,
43606
- "穆": 43471,
43607
- "逕": 43472,
43608
- "繭": 43473,
43609
- "蓉": 43474,
43610
- "詭": 43475,
43611
- "酋": 43476,
43612
- "儡": 43477,
43613
- "將": 43478,
43614
- "隧": 43479,
43615
- "爭": 43480,
43616
- "辟": 43481,
43617
- "痘": 43482,
43618
- "宋": 43483,
43619
- "揆": 43484,
43620
- "誥": 43485,
43621
- "槙": 43486,
43622
- "濟": 43487,
43623
- "辭": 43488,
43624
- "埴": 43489,
43625
- "耆": 43490,
43626
- "顯": 43491,
43627
- "孟": 43492,
43628
- "諧": 43493,
43629
- "搖": 43494,
43630
- "恂": 43495,
43631
- "爨": 43496,
43632
- "弐": 43497,
43633
- "壤": 43498,
43634
- "諷": 43499,
43635
- "臀": 43500,
43636
- "遲": 43501,
43637
- "陷": 43502,
43638
- "號": 43503,
43639
- "辨": 43504,
43640
- "聽": 43505,
43641
- "堯": 43506,
43642
- "亂": 43507,
43643
- "烟": 43508,
43644
- "碩": 43509,
43645
- "遁": 43510,
43646
- "瞻": 43511,
43647
- "瀑": 43512,
43648
- "倹": 43513,
43649
- "讀": 43514,
43650
- "缺": 43515,
43651
- "竄": 43516,
43652
- "矩": 43517,
43653
- "庸": 43518,
43654
- "赫": 43519,
43655
- "扈": 43520,
43656
- "瑣": 43521,
43657
- "毆": 43522,
43658
- "柘": 43523,
43659
- "猗": 43524,
43660
- "駭": 43525,
43661
- "糾": 43526,
43662
- "裔": 43527,
43663
- "圓": 43528,
43664
- "尸": 43529,
43665
- "楔": 43530,
43666
- "賤": 43531,
43667
- "頴": 43532,
43668
- "禊": 43533,
43669
- "墾": 43534,
43670
- "惟": 43535,
43671
- "籌": 43536,
43672
- "珊": 43537,
43673
- "壓": 43538,
43674
- "寶": 43539,
43675
- "從": 43540,
43676
- "諛": 43541,
43677
- "憊": 43542,
43678
- "咤": 43543,
43679
- "窩": 43544,
43680
- "忒": 43545,
43681
- "咀": 43546,
43682
- "貶": 43547,
43683
- "魯": 43548,
43684
- "舜": 43549,
43685
- "愼": 43550,
43686
- "啻": 43551,
43687
- "綬": 43552,
43688
- "灼": 43553,
43689
- "芒": 43554,
43690
- "撹": 43555,
43691
- "楷": 43556,
43692
- "毫": 43557,
43693
- "苅": 43558,
43694
- "鉤": 43559,
43695
- "楯": 43560,
43696
- "馭": 43561,
43697
- "綮": 43562,
43698
- "戾": 43563,
43699
- "勸": 43564,
43700
- "渺": 43565,
43701
- "矜": 43566,
43702
- "敎": 43567,
43703
- "擧": 43568,
43704
- "欷": 43569,
43705
- "賈": 43570,
43706
- "筵": 43571,
43707
- "廟": 43572,
43708
- "豕": 43573,
43709
- "彗": 43574,
43710
- "諟": 43575,
43711
- "樣": 43576,
43712
- "貽": 43577,
43713
- "蛯": 43578,
43714
- "譬": 43579,
43715
- "鵠": 43580,
43716
- "皈": 43581,
43717
- "跋": 43582,
43718
- "疆": 43583,
43719
- "皷": 43584,
43720
- "恰": 43585,
43721
- "襄": 43586,
43722
- "亞": 43587,
43723
- "洽": 43588,
43724
- "躓": 43589,
43725
- "團": 43590,
43726
- "膠": 43591,
43727
- "麾": 43592,
43728
- "斥": 43593,
43729
- "艱": 43594,
43730
- "敖": 43595,
43731
- "篩": 43596,
43732
- "糢": 43597,
43733
- "熙": 43598,
43734
- "拷": 43599,
43735
- "𠮟": 43600,
43736
- "當": 43601,
43737
- "圖": 43602,
43738
- "浹": 43603,
43739
- "辜": 43604,
43740
- "熾": 43605,
43741
- "覊": 43606,
43742
- "蕃": 43607,
43743
- "僩": 43608,
43744
- "驅": 43609,
43745
- "宥": 43610,
43746
- "簪": 43611,
43747
- "變": 43612,
43748
- "乘": 43613,
43749
- "諠": 43614,
43750
- "蔗": 43615,
43751
- "僥": 43616,
43752
- "狸": 43617,
43753
- "巖": 43618,
43754
- "甍": 43619,
43755
- "醨": 43620,
43756
- "巍": 43621,
43757
- "翰": 43622,
43758
- "闢": 43623,
43759
- "擕": 43624,
43760
- "抔": 43625,
43761
- "裴": 43626,
43762
- "頗": 43627,
43763
- "禮": 43628,
43764
- "雞": 43629,
43765
- "雉": 43630,
43766
- "毘": 43631,
43767
- "僇": 43632,
43768
- "陋": 43633,
43769
- "儒": 43634,
43770
- "讓": 43635,
43771
- "闍": 43636,
43772
- "虞": 43637,
43773
- "伍": 43638,
43774
- "沒": 43639,
43775
- "婿": 43640,
43776
- "聊": 43641,
43777
- "劾": 43642,
43778
- "檣": 43643,
43779
- "嗚": 43644,
43780
- "盖": 43645,
43781
- "淫": 43646,
43782
- "勒": 43647,
43783
- "竢": 43648,
43784
- "淺": 43649,
43785
- "聚": 43650,
43786
- "僨": 43651,
43787
- "欣": 43652,
43788
- "闡": 43653,
43789
- "亦": 43654,
43790
- "壯": 43655,
43791
- "奧": 43656,
43792
- "傀": 43657,
43793
- "耻": 43658,
43794
- "碎": 43659,
43795
- "寵": 43660,
43796
- "觀": 43661,
43797
- "搆": 43662,
43798
- "甞": 43663,
43799
- "驕": 43664,
43800
- "陪": 43665,
43801
- "慄": 43666,
43802
- "巌": 43667,
43803
- "與": 43668,
43804
- "尹": 43669,
43805
- "揜": 43670,
43806
- "畧": 43671,
43807
- "竟": 43672,
43808
- "徽": 43673,
43809
- "魏": 43674,
43810
- "醗": 43675,
43811
- "頰": 43676,
43812
- "拮": 43677,
43813
- "罷": 43678,
43814
- "况": 43679,
43815
- "惡": 43680,
43816
- "恪": 43681,
43817
- "闊": 43682,
43818
- "倖": 43683,
43819
- "諫": 43684,
43820
- "肅": 43685,
43821
- "憾": 43686,
43822
- "價": 43687,
43823
- "巓": 43688,
43824
- "蹊": 43689,
43825
- "醵": 43690,
43826
- "槓": 43691,
43827
- "葦": 43692,
43828
- "飮": 43693,
43829
- "獻": 43694,
43830
- "賣": 43695,
43831
- "夭": 43696,
43832
- "犂": 43697,
43833
- "禽": 43698,
43834
- "塡": 43699,
43835
- "謁": 43700,
43836
- "羹": 43701,
43837
- "壹": 43702,
43838
- "苟": 43703,
43839
- "誦": 43704,
43840
- "舊": 43705,
43841
- "觸": 43706,
43842
- "妾": 43707,
43843
- "膽": 43708,
43844
- "鐵": 43709,
43845
- "邑": 43710,
43846
- "乎": 43711,
43847
- "發": 43712,
43848
- "殷": 43713,
43849
- "恆": 43714,
43850
- "貮": 43715,
43851
- "罕": 43716,
43852
- "掩": 43717,
43853
- "傚": 43718,
43854
- "歐": 43719,
43855
- "縱": 43720,
43856
- "廠": 43721,
43857
- "畏": 43722,
43858
- "彭": 43723,
43859
- "虔": 43724,
43860
- "帥": 43725,
43861
- "鏃": 43726,
43862
- "憬": 43727,
43863
- "鋒": 43728,
43864
- "諂": 43729,
43865
- "悉": 43730,
43866
- "歡": 43731,
43867
- "裨": 43732,
43868
- "斂": 43733,
43869
- "營": 43734,
43870
- "諱": 43735,
43871
- "假": 43736,
43872
- "稍": 43737,
43873
- "擾": 43738,
43874
- "饗": 43739,
43875
- "黨": 43740,
43876
- "對": 43741,
43877
- "匐": 43742,
43878
- "獸": 43743
43879
  },
43880
  "merges": [
43881
  "▁ t",
 
31
  "special": true
32
  }
33
  ],
34
+ "normalizer": null,
 
 
 
 
 
 
35
  "pre_tokenizer": {
36
  "type": "Metaspace",
37
  "replacement": "▁",
38
+ "prepend_scheme": "never",
39
  "split": false
40
  },
41
  "post_processor": {
 
107
  },
108
  {
109
  "type": "Fuse"
 
 
 
 
 
 
110
  }
111
  ]
112
  },
 
117
  "continuing_subword_prefix": null,
118
  "end_of_word_suffix": null,
119
  "fuse_unk": true,
120
+ "byte_fallback": true,
121
  "ignore_merges": false,
122
  "vocab": {
123
  "<unk>": 0,
 
43296
  "趙": 43173,
43297
  "弛": 43174,
43298
  "徊": 43175,
43299
+ "逕": 43176,
43300
+ "撹": 43177,
43301
+ "碎": 43178,
43302
+ "逼": 43179,
43303
+ "鳶": 43180,
43304
+ "毘": 43181,
43305
+ "參": 43182,
43306
+ "攬": 43183,
43307
+ "夭": 43184,
43308
+ "罷": 43185,
43309
+ "贖": 43186,
43310
+ "劾": 43187,
43311
+ "儁": 43188,
43312
+ "禎": 43189,
43313
+ "闍": 43190,
43314
+ "覺": 43191,
43315
+ "汝": 43192,
43316
+ "畧": 43193,
43317
+ "儒": 43194,
43318
+ "埸": 43195,
43319
+ "藥": 43196,
43320
+ "愼": 43197,
43321
+ "賣": 43198,
43322
+ "剝": 43199,
43323
+ "斂": 43200,
43324
+ "緝": 43201,
43325
+ "恆": 43202,
43326
+ "遑": 43203,
43327
+ "號": 43204,
43328
+ "咎": 43205,
43329
+ "譯": 43206,
43330
+ "堯": 43207,
43331
+ "葢": 43208,
43332
+ "錮": 43209,
43333
+ "飮": 43210,
43334
+ "匍": 43211,
43335
+ "漸": 43212,
43336
+ "歔": 43213,
43337
+ "貽": 43214,
43338
+ "巌": 43215,
43339
+ "嚮": 43216,
43340
+ "陪": 43217,
43341
+ "專": 43218,
43342
+ "壓": 43219,
43343
+ "膠": 43220,
43344
+ "瀑": 43221,
43345
+ "尹": 43222,
43346
+ "險": 43223,
43347
+ "鐵": 43224,
43348
+ "濟": 43225,
43349
+ "單": 43226,
43350
+ "滿": 43227,
43351
+ "竭": 43228,
43352
+ "沃": 43229,
43353
+ "綮": 43230,
43354
+ "顯": 43231,
43355
+ "澁": 43232,
43356
+ "榜": 43233,
43357
+ "麾": 43234,
43358
+ "蛾": 43235,
43359
+ "欣": 43236,
43360
+ "袁": 43237,
43361
+ "諧": 43238,
43362
+ "扈": 43239,
43363
+ "駑": 43240,
43364
+ "罕": 43241,
43365
+ "媢": 43242,
43366
+ "艱": 43243,
43367
+ "疆": 43244,
43368
+ "樣": 43245,
43369
+ "歟": 43246,
43370
+ "儘": 43247,
43371
+ "宋": 43248,
43372
+ "賂": 43249,
43373
+ "洽": 43250,
43374
+ "虧": 43251,
43375
+ "薛": 43252,
43376
+ "耆": 43253,
43377
+ "駭": 43254,
43378
+ "權": 43255,
43379
+ "慘": 43256,
43380
+ "賈": 43257,
43381
+ "鄙": 43258,
43382
+ "矜": 43259,
43383
+ "汀": 43260,
43384
+ "羹": 43261,
43385
+ "敎": 43262,
43386
+ "肚": 43263,
43387
+ "揠": 43264,
43388
+ "鐸": 43265,
43389
+ "蘐": 43266,
43390
+ "奧": 43267,
43391
+ "畫": 43268,
43392
+ "吏": 43269,
43393
+ "竟": 43270,
43394
+ "茲": 43271,
43395
+ "壹": 43272,
43396
+ "瞻": 43273,
43397
+ "靜": 43274,
43398
+ "翹": 43275,
43399
+ "饒": 43276,
43400
+ "廠": 43277,
43401
+ "絜": 43278,
43402
+ "脩": 43279,
43403
+ "雉": 43280,
43404
+ "膽": 43281,
43405
+ "蓉": 43282,
43406
+ "靱": 43283,
43407
+ "蹈": 43284,
43408
+ "總": 43285,
43409
+ "蠻": 43286,
43410
+ "陋": 43287,
43411
+ "囮": 43288,
43412
+ "從": 43289,
43413
+ "恕": 43290,
43414
+ "實": 43291,
43415
+ "內": 43292,
43416
+ "盖": 43293,
43417
+ "竄": 43294,
43418
+ "畏": 43295,
43419
+ "雖": 43296,
43420
+ "瑣": 43297,
43421
+ "俟": 43298,
43422
+ "獻": 43299,
43423
+ "愧": 43300,
43424
+ "餘": 43301,
43425
+ "畢": 43302,
43426
+ "耘": 43303,
43427
+ "恙": 43304,
43428
+ "謨": 43305,
43429
+ "赫": 43306,
43430
+ "鉤": 43307,
43431
+ "舍": 43308,
43432
+ "謁": 43309,
43433
+ "戊": 43310,
43434
+ "鬢": 43311,
43435
+ "蜀": 43312,
43436
+ "婿": 43313,
43437
+ "儕": 43314,
43438
+ "淇": 43315,
43439
+ "巓": 43316,
43440
+ "恤": 43317,
43441
+ "團": 43318,
43442
+ "愈": 43319,
43443
+ "丙": 43320,
43444
+ "鏃": 43321,
43445
+ "簪": 43322,
43446
+ "僩": 43323,
43447
+ "彗": 43324,
43448
+ "讀": 43325,
43449
+ "豈": 43326,
43450
+ "誥": 43327,
43451
+ "僞": 43328,
43452
+ "礦": 43329,
43453
+ "勒": 43330,
43454
+ "苅": 43331,
43455
+ "發": 43332,
43456
+ "闢": 43333,
43457
+ "鄭": 43334,
43458
+ "酉": 43335,
43459
+ "僻": 43336,
43460
+ "倩": 43337,
43461
+ "驗": 43338,
43462
+ "胖": 43339,
43463
+ "岨": 43340,
43464
+ "兒": 43341,
43465
+ "蜻": 43342,
43466
+ "舜": 43343,
43467
+ "搆": 43344,
43468
+ "圓": 43345,
43469
+ "勸": 43346,
43470
+ "尸": 43347,
43471
+ "盡": 43348,
43472
+ "恣": 43349,
43473
+ "盜": 43350,
43474
+ "聊": 43351,
43475
+ "烟": 43352,
43476
+ "勵": 43353,
43477
+ "臘": 43354,
43478
+ "竝": 43355,
43479
+ "諺": 43356,
43480
+ "戌": 43357,
43481
+ "碍": 43358,
43482
+ "亦": 43359,
43483
+ "勞": 43360,
43484
+ "賤": 43361,
43485
+ "殘": 43362,
43486
+ "舒": 43363,
43487
+ "緡": 43364,
43488
+ "糾": 43365,
43489
+ "擾": 43366,
43490
+ "穆": 43367,
43491
+ "崗": 43368,
43492
+ "寵": 43369,
43493
+ "謭": 43370,
43494
+ "儡": 43371,
43495
+ "嬰": 43372,
43496
+ "騷": 43373,
43497
+ "戾": 43374,
43498
+ "籾": 43375,
43499
+ "輕": 43376,
43500
+ "圖": 43377,
43501
+ "黨": 43378,
43502
+ "棍": 43379,
43503
+ "殉": 43380,
43504
+ "僇": 43381,
43505
+ "淺": 43382,
43506
+ "璽": 43383,
43507
+ "與": 43384,
43508
+ "煥": 43385,
43509
+ "摧": 43386,
43510
+ "痘": 43387,
43511
+ "欷": 43388,
43512
+ "鰭": 43389,
43513
+ "淫": 43390,
43514
+ "跋": 43391,
43515
+ "襄": 43392,
43516
+ "酋": 43393,
43517
+ "碩": 43394,
43518
+ "苟": 43395,
43519
+ "梃": 43396,
43520
+ "關": 43397,
43521
+ "狸": 43398,
43522
+ "饜": 43399,
43523
+ "稍": 43400,
43524
+ "燮": 43401,
43525
+ "歡": 43402,
43526
+ "辨": 43403,
43527
+ "孛": 43404,
43528
+ "堡": 43405,
43529
+ "楷": 43406,
43530
+ "擕": 43407,
43531
+ "諂": 43408,
43532
+ "傅": 43409,
43533
+ "倖": 43410,
43534
+ "灼": 43411,
43535
+ "迭": 43412,
43536
+ "詔": 43413,
43537
+ "殷": 43414,
43538
+ "呵": 43415,
43539
+ "忿": 43416,
43540
+ "傀": 43417,
43541
+ "墾": 43418,
43542
+ "巖": 43419,
43543
+ "滊": 43420,
43544
+ "俄": 43421,
43545
+ "竢": 43422,
43546
+ "歎": 43423,
43547
+ "啻": 43424,
43548
+ "庸": 43425,
43549
+ "糢": 43426,
43550
+ "鍾": 43427,
43551
+ "爨": 43428,
43552
+ "拮": 43429,
43553
+ "揆": 43430,
43554
+ "乎": 43431,
43555
+ "槙": 43432,
43556
+ "逈": 43433,
43557
+ "騁": 43434,
43558
+ "裴": 43435,
43559
+ "屡": 43436,
43560
+ "筵": 43437,
43561
+ "趨": 43438,
43562
+ "皈": 43439,
43563
+ "徽": 43440,
43564
+ "蛉": 43441,
43565
+ "頗": 43442,
43566
+ "蕃": 43443,
43567
+ "寔": 43444,
43568
+ "憚": 43445,
43569
+ "辟": 43446,
43570
+ "龕": 43447,
43571
+ "臾": 43448,
43572
+ "壤": 43449,
43573
+ "舅": 43450,
43574
+ "黽": 43451,
43575
+ "恂": 43452,
43576
+ "帥": 43453,
43577
+ "舷": 43454,
43578
+ "擔": 43455,
43579
+ "攝": 43456,
43580
+ "廟": 43457,
43581
+ "袈": 43458,
43582
+ "醵": 43459,
43583
+ "駸": 43460,
43584
+ "聯": 43461,
43585
+ "鸞": 43462,
43586
+ "頽": 43463,
43587
+ "脹": 43464,
43588
+ "醗": 43465,
43589
+ "禀": 43466,
43590
+ "戲": 43467,
43591
+ "恪": 43468,
43592
+ "覽": 43469,
43593
+ "懼": 43470,
43594
+ "隋": 43471,
43595
+ "遁": 43472,
43596
+ "爭": 43473,
43597
+ "瑟": 43474,
43598
+ "宍": 43475,
43599
+ "峻": 43476,
43600
+ "殲": 43477,
43601
+ "黠": 43478,
43602
+ "厭": 43479,
43603
+ "歸": 43480,
43604
+ "蛯": 43481,
43605
+ "憬": 43482,
43606
+ "營": 43483,
43607
+ "朧": 43484,
43608
+ "塡": 43485,
43609
+ "當": 43486,
43610
+ "抔": 43487,
43611
+ "缺": 43488,
43612
+ "尤": 43489,
43613
+ "熙": 43490,
43614
+ "爰": 43491,
43615
+ "肅": 43492,
43616
+ "禽": 43493,
43617
+ "懺": 43494,
43618
+ "豫": 43495,
43619
+ "馭": 43496,
43620
+ "猾": 43497,
43621
+ "廢": 43498,
43622
+ "躓": 43499,
43623
+ "甍": 43500,
43624
+ "虞": 43501,
43625
+ "邑": 43502,
43626
+ "窘": 43503,
43627
+ "葦": 43504,
43628
+ "驅": 43505,
43629
+ "羣": 43506,
43630
+ "闊": 43507,
43631
+ "惠": 43508,
43632
+ "芒": 43509,
43633
+ "屬": 43510,
43634
+ "岌": 43511,
43635
+ "魯": 43512,
43636
+ "錐": 43513,
43637
+ "醨": 43514,
43638
+ "朕": 43515,
43639
+ "乘": 43516,
43640
+ "濤": 43517,
43641
+ "矩": 43518,
43642
+ "隱": 43519,
43643
+ "嘲": 43520,
43644
+ "欵": 43521,
43645
+ "柘": 43522,
43646
+ "熾": 43523,
43647
+ "醉": 43524,
43648
+ "伍": 43525,
43649
+ "紂": 43526,
43650
+ "覊": 43527,
43651
+ "麿": 43528,
43652
+ "毫": 43529,
43653
+ "寶": 43530,
43654
+ "裨": 43531,
43655
+ "亨": 43532,
43656
+ "轉": 43533,
43657
+ "盍": 43534,
43658
+ "碇": 43535,
43659
+ "皺": 43536,
43660
+ "唖": 43537,
43661
+ "閻": 43538,
43662
+ "蓁": 43539,
43663
+ "虔": 43540,
43664
+ "丞": 43541,
43665
+ "橙": 43542,
43666
+ "變": 43543,
43667
+ "臥": 43544,
43668
+ "渺": 43545,
43669
+ "蘊": 43546,
43670
+ "恰": 43547,
43671
+ "纉": 43548,
43672
+ "亂": 43549,
43673
+ "諱": 43550,
43674
+ "亞": 43551,
43675
+ "啜": 43552,
43676
+ "桀": 43553,
43677
+ "諟": 43554,
43678
+ "楔": 43555,
43679
+ "恊": 43556,
43680
+ "冰": 43557,
43681
+ "斥": 43558,
43682
+ "驕": 43559,
43683
+ "儼": 43560,
43684
+ "體": 43561,
43685
+ "挈": 43562,
43686
+ "駢": 43563,
43687
+ "皓": 43564,
43688
+ "誣": 43565,
43689
+ "頒": 43566,
43690
+ "鎭": 43567,
43691
+ "蹄": 43568,
43692
+ "續": 43569,
43693
+ "狡": 43570,
43694
+ "魏": 43571,
43695
+ "觀": 43572,
43696
+ "沮": 43573,
43697
+ "逓": 43574,
43698
+ "詭": 43575,
43699
+ "懥": 43576,
43700
+ "處": 43577,
43701
+ "盈": 43578,
43702
+ "窩": 43579,
43703
+ "匐": 43580,
43704
+ "慄": 43581,
43705
+ "遲": 43582,
43706
+ "傚": 43583,
43707
+ "搖": 43584,
43708
+ "鵬": 43585,
43709
+ "惟": 43586,
43710
+ "菑": 43587,
43711
+ "綬": 43588,
43712
+ "臀": 43589,
43713
+ "籌": 43590,
43714
+ "憮": 43591,
43715
+ "貮": 43592,
43716
+ "縱": 43593,
43717
+ "聚": 43594,
43718
+ "釀": 43595,
43719
+ "轍": 43596,
43720
+ "兩": 43597,
43721
+ "嚼": 43598,
43722
+ "豕": 43599,
43723
+ "闡": 43600,
43724
+ "禮": 43601,
43725
+ "戮": 43602,
43726
+ "埠": 43603,
43727
+ "雜": 43604,
43728
+ "拔": 43605,
43729
+ "吻": 43606,
43730
+ "獨": 43607,
43731
+ "扣": 43608,
43732
+ "將": 43609,
43733
+ "聽": 43610,
43734
+ "諫": 43611,
43735
+ "諠": 43612,
43736
+ "繭": 43613,
43737
+ "尙": 43614,
43738
+ "彭": 43615,
43739
+ "决": 43616,
43740
+ "咤": 43617,
43741
+ "飜": 43618,
43742
+ "證": 43619,
43743
+ "繼": 43620,
43744
+ "嗚": 43621,
43745
+ "鑒": 43622,
43746
+ "彝": 43623,
43747
+ "歐": 43624,
43748
+ "玻": 43625,
43749
+ "僥": 43626,
43750
+ "諷": 43627,
43751
+ "僨": 43628,
43752
+ "浹": 43629,
43753
+ "禊": 43630,
43754
+ "允": 43631,
43755
+ "𠮟": 43632,
43756
+ "撿": 43633,
43757
+ "犂": 43634,
43758
+ "毆": 43635,
43759
+ "筭": 43636,
43760
+ "陷": 43637,
43761
+ "澳": 43638,
43762
+ "惡": 43639,
43763
+ "纂": 43640,
43764
+ "衷": 43641,
43765
+ "諛": 43642,
43766
+ "楯": 43643,
43767
+ "蹊": 43644,
43768
+ "隨": 43645,
43769
+ "亥": 43646,
43770
+ "弐": 43647,
43771
+ "嫡": 43648,
43772
+ "誦": 43649,
43773
+ "寃": 43650,
43774
+ "巍": 43651,
43775
+ "宥": 43652,
43776
+ "悉": 43653,
43777
+ "猗": 43654,
43778
+ "聘": 43655,
43779
+ "壯": 43656,
43780
+ "憊": 43657,
43781
+ "掩": 43658,
43782
+ "隧": 43659,
43783
+ "胤": 43660,
43784
+ "羞": 43661,
43785
+ "爲": 43662,
43786
+ "羸": 43663,
43787
+ "擇": 43664,
43788
+ "觸": 43665,
43789
+ "糀": 43666,
43790
+ "翰": 43667,
43791
+ "游": 43668,
43792
+ "辜": 43669,
43793
+ "忒": 43670,
43794
+ "厘": 43671,
43795
+ "數": 43672,
43796
+ "濶": 43673,
43797
+ "傳": 43674,
43798
+ "鵠": 43675,
43799
+ "篆": 43676,
43800
+ "譬": 43677,
43801
+ "哨": 43678,
43802
+ "斷": 43679,
43803
+ "嚴": 43680,
43804
+ "肆": 43681,
43805
+ "珊": 43682,
43806
+ "悌": 43683,
43807
+ "孟": 43684,
43808
+ "稱": 43685,
43809
+ "沒": 43686,
43810
+ "墮": 43687,
43811
+ "獸": 43688,
43812
+ "妾": 43689,
43813
+ "揖": 43690,
43814
+ "擧": 43691,
43815
+ "悖": 43692,
43816
+ "况": 43693,
43817
+ "篩": 43694,
43818
+ "滌": 43695,
43819
+ "勅": 43696,
43820
+ "姦": 43697,
43821
+ "舊": 43698,
43822
+ "痍": 43699,
43823
+ "頰": 43700,
43824
+ "聲": 43701,
43825
+ "裔": 43702,
43826
+ "毋": 43703,
43827
+ "諮": 43704,
43828
+ "咀": 43705,
43829
+ "耻": 43706,
43830
+ "對": 43707,
43831
+ "蔗": 43708,
43832
+ "假": 43709,
43833
+ "彥": 43710,
43834
+ "雞": 43711,
43835
+ "拷": 43712,
43836
+ "揜": 43713,
43837
+ "甞": 43714,
43838
+ "逬": 43715,
43839
+ "榮": 43716,
43840
+ "讓": 43717,
43841
+ "竊": 43718,
43842
+ "辭": 43719,
43843
+ "斤": 43720,
43844
+ "檣": 43721,
43845
+ "茫": 43722,
43846
+ "憾": 43723,
43847
+ "埴": 43724,
43848
+ "鋒": 43725,
43849
+ "饗": 43726,
43850
+ "敖": 43727,
43851
+ "靡": 43728,
43852
+ "倹": 43729,
43853
+ "槓": 43730,
43854
+ "價": 43731,
43855
+ "攘": 43732,
43856
+ "胥": 43733,
43857
+ "臂": 43734,
43858
+ "菉": 43735,
43859
+ "菰": 43736,
43860
+ "佩": 43737,
43861
+ "匈": 43738,
43862
+ "貶": 43739,
43863
+ "縣": 43740,
43864
+ "據": 43741,
43865
+ "皷": 43742,
43866
+ "頴": 43743
43867
  },
43868
  "merges": [
43869
  "▁ t",