File size: 200 Bytes
de4ade4 |
1 2 3 4 5 6 7 8 9 |
# Copyright 2022 MosaicML LLM Foundry authors
# SPDX-License-Identifier: Apache-2.0
from llmfoundry.tokenizers.tiktoken import TiktokenTokenizerWrapper
__all__ = [
'TiktokenTokenizerWrapper',
]
|