REPROD_dpo_helpfulhelpful_human_subset-1_modelgemma2b_maxsteps10000_bz8_lr5e-06
/
tokenizer_config.json
{ | |
"add_bos_token": true, | |
"add_eos_token": false, | |
"added_tokens_decoder": { | |
"0": { | |
"content": "<pad>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"1": { | |
"content": "<eos>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"2": { | |
"content": "<bos>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"3": { | |
"content": "<unk>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"4": { | |
"content": "<mask>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"5": { | |
"content": "<2mass>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"6": { | |
"content": "[@BOS@]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"7": { | |
"content": "<unused0>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"8": { | |
"content": "<unused1>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"9": { | |
"content": "<unused2>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"10": { | |
"content": "<unused3>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"11": { | |
"content": "<unused4>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"12": { | |
"content": "<unused5>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"13": { | |
"content": "<unused6>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"14": { | |
"content": "<unused7>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"15": { | |
"content": "<unused8>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"16": { | |
"content": "<unused9>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"17": { | |
"content": "<unused10>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"18": { | |
"content": "<unused11>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"19": { | |
"content": "<unused12>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"20": { | |
"content": "<unused13>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"21": { | |
"content": "<unused14>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"22": { | |
"content": "<unused15>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"23": { | |
"content": "<unused16>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"24": { | |
"content": "<unused17>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"25": { | |
"content": "<unused18>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"26": { | |
"content": "<unused19>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"27": { | |
"content": "<unused20>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"28": { | |
"content": "<unused21>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"29": { | |
"content": "<unused22>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"30": { | |
"content": "<unused23>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"31": { | |
"content": "<unused24>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"32": { | |
"content": "<unused25>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"33": { | |
"content": "<unused26>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"34": { | |
"content": "<unused27>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"35": { | |
"content": "<unused28>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"36": { | |
"content": "<unused29>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"37": { | |
"content": "<unused30>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"38": { | |
"content": "<unused31>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"39": { | |
"content": "<unused32>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"40": { | |
"content": "<unused33>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"41": { | |
"content": "<unused34>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"42": { | |
"content": "<unused35>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"43": { | |
"content": "<unused36>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"44": { | |
"content": "<unused37>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"45": { | |
"content": "<unused38>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"46": { | |
"content": "<unused39>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"47": { | |
"content": "<unused40>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"48": { | |
"content": "<unused41>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"49": { | |
"content": "<unused42>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"50": { | |
"content": "<unused43>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"51": { | |
"content": "<unused44>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"52": { | |
"content": "<unused45>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"53": { | |
"content": "<unused46>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"54": { | |
"content": "<unused47>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"55": { | |
"content": "<unused48>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"56": { | |
"content": "<unused49>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"57": { | |
"content": "<unused50>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"58": { | |
"content": "<unused51>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"59": { | |
"content": "<unused52>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"60": { | |
"content": "<unused53>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"61": { | |
"content": "<unused54>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"62": { | |
"content": "<unused55>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"63": { | |
"content": "<unused56>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"64": { | |
"content": "<unused57>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"65": { | |
"content": "<unused58>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"66": { | |
"content": "<unused59>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"67": { | |
"content": "<unused60>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"68": { | |
"content": "<unused61>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"69": { | |
"content": "<unused62>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"70": { | |
"content": "<unused63>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"71": { | |
"content": "<unused64>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"72": { | |
"content": "<unused65>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"73": { | |
"content": "<unused66>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"74": { | |
"content": "<unused67>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"75": { | |
"content": "<unused68>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"76": { | |
"content": "<unused69>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"77": { | |
"content": "<unused70>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"78": { | |
"content": "<unused71>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"79": { | |
"content": "<unused72>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"80": { | |
"content": "<unused73>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"81": { | |
"content": "<unused74>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"82": { | |
"content": "<unused75>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"83": { | |
"content": "<unused76>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"84": { | |
"content": "<unused77>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"85": { | |
"content": "<unused78>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"86": { | |
"content": "<unused79>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"87": { | |
"content": "<unused80>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"88": { | |
"content": "<unused81>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"89": { | |
"content": "<unused82>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"90": { | |
"content": "<unused83>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"91": { | |
"content": "<unused84>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"92": { | |
"content": "<unused85>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"93": { | |
"content": "<unused86>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"94": { | |
"content": "<unused87>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"95": { | |
"content": "<unused88>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"96": { | |
"content": "<unused89>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"97": { | |
"content": "<unused90>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"98": { | |
"content": "<unused91>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"99": { | |
"content": "<unused92>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"100": { | |
"content": "<unused93>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"101": { | |
"content": "<unused94>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"102": { | |
"content": "<unused95>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"103": { | |
"content": "<unused96>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"104": { | |
"content": "<unused97>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"105": { | |
"content": "<unused98>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"106": { | |
"content": "<start_of_turn>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"107": { | |
"content": "<end_of_turn>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"108": { | |
"content": "\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"109": { | |
"content": "\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"110": { | |
"content": "\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"111": { | |
"content": "\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"112": { | |
"content": "\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"113": { | |
"content": "\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"114": { | |
"content": "\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"115": { | |
"content": "\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"116": { | |
"content": "\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"117": { | |
"content": "\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"118": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"119": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"120": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"121": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"122": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"123": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"124": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"125": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"126": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"127": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"128": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"129": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"130": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"131": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"132": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"133": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"134": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"135": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"136": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"137": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"138": { | |
"content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"139": { | |
"content": "▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"140": { | |
"content": "▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"141": { | |
"content": "▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"142": { | |
"content": "▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"143": { | |
"content": "▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"144": { | |
"content": "▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"145": { | |
"content": "▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"146": { | |
"content": "▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"147": { | |
"content": "▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"148": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"149": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"150": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"151": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"152": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"153": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"154": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"155": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"156": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"157": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"158": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"159": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"160": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"161": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"162": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"163": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"164": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"165": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"166": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"167": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"168": { | |
"content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"169": { | |
"content": "<table>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"170": { | |
"content": "<caption>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"171": { | |
"content": "<thead>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"172": { | |
"content": "<tbody>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"173": { | |
"content": "<tfoot>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"174": { | |
"content": "<tr>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"175": { | |
"content": "<th>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"176": { | |
"content": "<td>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"177": { | |
"content": "</table>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"178": { | |
"content": "</caption>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"179": { | |
"content": "</thead>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"180": { | |
"content": "</tbody>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"181": { | |
"content": "</tfoot>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"182": { | |
"content": "</tr>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"183": { | |
"content": "</th>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"184": { | |
"content": "</td>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"185": { | |
"content": "<h1>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"186": { | |
"content": "<h2>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"187": { | |
"content": "<h3>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"188": { | |
"content": "<h4>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"189": { | |
"content": "<h5>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"190": { | |
"content": "<h6>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"191": { | |
"content": "<blockquote>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"192": { | |
"content": "</h1>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"193": { | |
"content": "</h2>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"194": { | |
"content": "</h3>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"195": { | |
"content": "</h4>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"196": { | |
"content": "</h5>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"197": { | |
"content": "</h6>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"198": { | |
"content": "</blockquote>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"199": { | |
"content": "<strong>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"200": { | |
"content": "<em>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"201": { | |
"content": "<b>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"202": { | |
"content": "<i>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"203": { | |
"content": "<u>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"204": { | |
"content": "<s>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"205": { | |
"content": "<sub>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"206": { | |
"content": "<sup>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"207": { | |
"content": "<code>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"208": { | |
"content": "</strong>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"209": { | |
"content": "</em>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"210": { | |
"content": "</b>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"211": { | |
"content": "</i>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"212": { | |
"content": "</u>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"213": { | |
"content": "</s>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"214": { | |
"content": "</sub>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"215": { | |
"content": "</sup>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
}, | |
"216": { | |
"content": "</code>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": false | |
} | |
}, | |
"additional_special_tokens": [ | |
"<start_of_turn>", | |
"<end_of_turn>" | |
], | |
"bos_token": "<bos>", | |
"clean_up_tokenization_spaces": false, | |
"eos_token": "<eos>", | |
"model_max_length": 1000000000000000019884624838656, | |
"pad_token": "<pad>", | |
"sp_model_kwargs": {}, | |
"spaces_between_special_tokens": false, | |
"tokenizer_class": "GemmaTokenizer", | |
"unk_token": "<unk>", | |
"use_default_system_prompt": false | |
} | |