twNLP-app/src/configs/ckip.py

import pickle
import asyncio
import aiofiles
from pathlib import Path


pkg_path = Path("__file__").resolve().parent / "src"
ckip_path = pkg_path / "models" / "ckip"


async def write_drivers(nlp_model: str) -> None:
    """The write drivers function writes the ckip drivers to pickle files asynchronously.
    Args:
        nlp_model (str): the nlp model name
    """
    from ckip_transformers.nlp import (
        CkipWordSegmenter,
        CkipPosTagger,
        CkipNerChunker,
    )

    drivers = (
        CkipWordSegmenter(model=nlp_model),
        CkipPosTagger(model=nlp_model),
        CkipNerChunker(model=nlp_model),
    )

    driver_path = ckip_path / f"{nlp_model}_drivers.pickle"
    async with aiofiles.open(driver_path, mode="wb") as file:
        result = pickle.dumps(drivers)
        await file.write(result)
        print(f"{nlp_model}_drivers.pickle done!")


async def download_ckip_drivers(ckip_nlp_models):
    await asyncio.gather(*list(map(write_drivers, ckip_nlp_models)))