from s3prl.dataio.encoder.tokenizer import CharacterTokenizer, default_phoneme_tokenizer


def test_tokenizer():
    char_tokenizer = CharacterTokenizer()
    phone_tokenizer = default_phoneme_tokenizer()

    char_text = "HELLO WORLD"
    char_text_enc = char_tokenizer.encode(char_text)
    char_text_dec = char_tokenizer.decode(char_text_enc)

    assert isinstance(char_text_enc, list)
    assert char_text == char_text_dec