test/utils/test_preprocess.py

import pytest  # noqa E402

from neollm.utils.preprocess import dict2json, optimize_token


def test_dict2json() -> None:
    assert dict2json({"a": 1, "b": 2.1}) == '{\n  "a": 1,\n  "b": 2.1\n}'
    assert dict2json({"あいうえお": False, "漢字": None}) == '{\n  "あいうえお": false,\n  "漢字": null\n}'


def test_optimize_token() -> None:
    assert optimize_token("\na\n\n\nb \n") == "a\n\nb"