File size: 336 Bytes
74b17e0 |
1 2 3 4 5 6 7 8 9 10 11 12 |
from typing import Any
from .template import TemplateFactory
class TextPreprocess:
def __init__(self, tokenizer, version):
self.tokenizer = tokenizer
self.template = TemplateFactory(version)()
def __call__(self, messages, mode='train'):
return self.template.encode(messages, self.tokenizer, mode) |