utils/sentence_cutter.py

import nltk
import jieba
import sudachipy
import langid
nltk.download('punkt')
langid.set_languages(['en', 'zh', 'ja'])

def split_text_into_sentences(text):
    if langid.classify(text)[0] == "en":
        sentences = nltk.tokenize.sent_tokenize(text)

        return sentences
    elif langid.classify(text)[0] == "zh":
        sentences = []
        segs = jieba.cut(text, cut_all=False)
        segs = list(segs)
        start = 0
        for i, seg in enumerate(segs):
            if seg in ["。", "！", "？", "……"]:
                sentences.append("".join(segs[start:i + 1]))
                start = i + 1
        if start < len(segs):
            sentences.append("".join(segs[start:]))

        return sentences
    elif langid.classify(text)[0] == "ja":
        sentences = []
        tokenizer = sudachipy.Dictionary().create()
        tokens = tokenizer.tokenize(text)
        current_sentence = ""

        for token in tokens:
            current_sentence += token.surface()
            if token.part_of_speech()[0] == "補助記号" and token.part_of_speech()[1] == "句点":
                sentences.append(current_sentence)
                current_sentence = ""

        if current_sentence:
            sentences.append(current_sentence)

        return sentences

    raise RuntimeError("It is impossible to reach here.")