twNLP-app/src/configs/ckip.py

import pickle
import asyncio
import aiofiles
from pathlib import Path


pkg_path = Path("__file__").resolve().parent / "src"
print('pkg_path = {}'.format(pkg_path))
ckip_path = pkg_path / "models" / "ckip"
print('ckip_path = {}'.format(ckip_path))
pkg_path.parent.mkdir(parents=True, exist_ok=True)
ckip_path.parent.mkdir(parents=True, exist_ok=True)
ckip_path.mkdir(parents=True, exist_ok=True)
permission_tester = ckip_path / f"drivers_permission_tester.txt"
with open(permission_tester, 'w') as f_test: pass  # check permission


async def write_drivers(nlp_model: str) -> None:
    """The write drivers function writes the ckip drivers to pickle files asynchronously.
    Args:
        nlp_model (str): the nlp model name
    """
    driver_path = ckip_path / f"{nlp_model}_drivers.pickle"
    with open(driver_path, 'w') as f_test: pass  # check permission
    from ckip_transformers.nlp import (
        CkipWordSegmenter,
        CkipPosTagger,
        CkipNerChunker,
    )

    drivers = (
        CkipWordSegmenter(model=nlp_model),
        CkipPosTagger(model=nlp_model),
        CkipNerChunker(model=nlp_model),
    )

    driver_path = ckip_path / f"{nlp_model}_drivers.pickle"
    async with aiofiles.open(driver_path, mode="wb") as file:
        result = pickle.dumps(drivers)
        await file.write(result)
        print(f"{nlp_model}_drivers.pickle done!")


async def download_ckip_drivers(ckip_nlp_models):
    await asyncio.gather(*list(map(write_drivers, ckip_nlp_models)))