| # Copyright 2022 MosaicML LLM Foundry authors | |
| # SPDX-License-Identifier: Apache-2.0 | |
| from llmfoundry.tokenizers.tiktoken import TiktokenTokenizerWrapper | |
| __all__ = [ | |
| 'TiktokenTokenizerWrapper', | |
| ] | |
| # Copyright 2022 MosaicML LLM Foundry authors | |
| # SPDX-License-Identifier: Apache-2.0 | |
| from llmfoundry.tokenizers.tiktoken import TiktokenTokenizerWrapper | |
| __all__ = [ | |
| 'TiktokenTokenizerWrapper', | |
| ] | |