File size: 317 Bytes
74b17e0 |
1 2 3 4 5 6 7 8 9 10 11 |
from transformers import PhiForCausalLM, AutoTokenizer
from . import register_llm
@register_llm('phi')
def return_phiclass():
def tokenizer_and_post_load(tokenizer):
tokenizer.pad_token = tokenizer.unk_token
return tokenizer
return (PhiForCausalLM, (AutoTokenizer, tokenizer_and_post_load))
|