function/tc_v2.py

from tensorRT import inference
import re
from collections import Counter
from vncorenlp import VnCoreNLP
from nltk.tokenize import sent_tokenize
import torch
import datetime
from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
import json
from . import utils
import time
from summary import text_summary, get_summary_bert
from function.clean_text import normalize_text
from .summary_with_llm import summary_with_llama
from .translate import translate_text_multi_layer
from scipy.spatial import distance
import copy
from .sentence_embbeding import embbeded_zh, embbeded_en, embedded_bge


# from . import detect_time as dt

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
use_cuda = torch.cuda.is_available()
print(torch.cuda.is_available())

# annotator = VnCoreNLP('vncorenlp/VnCoreNLP-1.1.1.jar', port=9191, annotators="wseg,pos", max_heap_size='-Xmx2g')


def detect_postaging(text_in):
    word_segmented_text = annotator.annotate(text_in)
    lst_k = []
    for se in word_segmented_text["sentences"]:
        for kw in se:
            if kw["posTag"] in ("Np", "Ny", "N"):
                if kw["posTag"] == "N" and "_" not in kw["form"]:
                    continue
                lst_k.append(kw["form"].replace("_", " "))
    return list(set(lst_k))

def clean_text(text_in):
    doc = re.sub('<.*?>', '', text_in)
    doc = re.sub('(function).*}', ' ', doc)
    # link
    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)

    doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
    doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
    doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
    doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
    doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
    doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
    doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
    doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
    doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
    # escape sequence
    doc = re.sub('\n', ' ', doc)
    doc = re.sub('\t', ' ', doc)
    doc = re.sub('\r', ' ', doc)

    doc = normalize_text(doc)
    return doc


def data_cleaning(docs):
    res = []
    for d in docs:
        if 'message' in d:
            # css and js
            doc = re.sub('<.*?>', '', d['message'])
            doc = re.sub('(function).*}', ' ', doc)

            # link
            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)

            doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
            doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
            doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
            doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
            doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
            doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
            doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
            doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
            doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
            # escape sequence
            doc = re.sub('\n', ' ', doc)
            doc = re.sub('\t', ' ', doc)
            doc = re.sub('\r', ' ', doc)

            d['message'] = doc
            res.append(d)
    return res


def segment(docs, lang="vi"):
    segmented_docs = []
    for d in docs:
        # print(d)
        # if len(d.get('message', "")) > 8000 or len(d.get('message', "")) < 100:
        if len(d.get('message', "")) > 8000:
            continue
        if 'snippet' not in d:
            continue
        try:
            if lang == "vi":
                snippet = d.get('snippet', "")
                segmented_snippet = ""
                segmented_sentences_snippet = annotator.tokenize(snippet)
                for sentence in segmented_sentences_snippet:
                    segmented_snippet += ' ' + ' '.join(sentence)
                segmented_snippet = segmented_snippet.replace('\xa0', '')
                d['segmented_snippet'] = segmented_snippet
            segmented_docs.append(d)
        except Exception:
            pass
    return segmented_docs


def timestamp_to_date(timestamp):
    return datetime.datetime.fromtimestamp(timestamp).strftime('%d/%m/%Y')


def re_ranking(result_topic, vectors_prompt, sorted_field):
    lst_score = []
    lst_ids = []
    lst_top = []
    try:
        for k in result_topic:
            lst_ids.append(k)
            if  not sorted_field.strip():
                    lst_top.append(len(result_topic[k]))
            else:
                lst_top.append(result_topic[k][0]['max_score'])
            vector_center = result_topic[k][0]["vector"]
            max_score = 11.0
            for vec in vectors_prompt:
                score = distance.cosine(np.array(vec), np.array(vector_center))
                if score < max_score:
                    max_score = score
            lst_score.append(max_score)
            result_topic[k][0]["similarity_score"] = max_score
            for d in result_topic[k]:
                d["similarity_score"] = max_score
            del result_topic[k][0]["vector"]
        idx = np.argsort(np.array(lst_score))
    except Exception as ve:
        return [], lst_ids, lst_top
    return idx, lst_ids, lst_top

def post_processing(response, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster = 50, delete_message=True, prompt="", hash_str: str= "", vectors_prompt: list = []):
    print(f'[INFO] sorted_field: {sorted_field}')
    MAX_DOC_PER_CLUSTER = max_doc_per_cluster

    lst_ids = []
    lst_top = []
    lst_res = []
    idx = []
    if prompt:
        idx, lst_ids, lst_top = re_ranking(response, vectors_prompt, sorted_field)
        print("idx_prompt: ", idx)
    if len(prompt) == 0 or len(idx) == 0:
        for i in response:
            lst_ids.append(i)
            if  not sorted_field.strip():
                lst_top.append(len(response[i]))
            else:
                lst_top.append(response[i][0]['max_score'])
        idx = np.argsort(np.array(lst_top))[::-1] 
        print("idx_not_prompt: ", idx)
    if top_cluster == -1:
        top_cluster = len(idx)
    for i in idx[: top_cluster]:
        ik = lst_ids[i]
        if top_sentence == -1:
            top_sentence = len(response[ik])
        lst_check_title = []
        lst_check_not_title = []
        i_c_t = 0
        for resss in response[ik]:
            r_title = resss.get("title", "")
            if r_title and not r_title.endswith("..."):
                lst_check_title.append(resss)
                i_c_t += 1
            else:
                lst_check_not_title.append(resss)
            if i_c_t == top_sentence:
                break
        if i_c_t == top_sentence:
            lst_res.append(lst_check_title)
        else:
            lst_check_title.extend(lst_check_not_title)
            lst_res.append(lst_check_title[:top_sentence])
        #lst_res.append(response[ik][:top_sentence])
    dict_res = {}
    for i in range(len(lst_res)):
        dict_res[str(i + 1)] = lst_res[i][:MAX_DOC_PER_CLUSTER]
        for j in range(min(len(dict_res[str(i + 1)]), 3)):
            dict_res[str(i + 1)][0]["title_summarize"].append(dict_res[str(i + 1)][j].get("snippet", ""))
        # t11 = time.time()
        summary_text = get_summary_bert(dict_res[str(i + 1)][0].get("message", ""), dict_res[str(i + 1)][0].get("lang", "vi"), topn=topn_summary, title=dict_res[str(i + 1)][0].get("title", ""), snippet=dict_res[str(i + 1)][0].get("snippet", ""))
        # print("time_summary: ", time.time() - t11)
        if len(summary_text) < 10:
            summary_text = dict_res[str(i + 1)][0].get("snippet", "")
            if len(summary_text) < 10:
                summary_text = dict_res[str(i + 1)][0].get("title", "")
        summary_text = utils.remove_image_keyword(summary_text)
        # if prompt:
        #     if dict_res[str(i + 1)][0].get("message", ""):
        #         src_lang = dict_res[str(i + 1)][0].get("lang", "")
        #         print("src_lang: ", src_lang)
        #         print("summary_text: ", summary_text)
        #         summary_text = translate_text_multi_layer(src_lang, "vi", summary_text)
        #         text_tran = translate_text_multi_layer(src_lang, "vi", dict_res[str(i + 1)][0].get("message", ""))
        #         ans_from_llama = summary_with_llama(prompt, text_tran, "vi", version="vi-llama", max_word_per_context=1000)
        #         print("ans_from_llama: ", ans_from_llama)
        #     summary_text = summary_text + "$$$$\n" + ans_from_llama
        #     print("summary_text: ", summary_text, len(summary_text))
        dict_res[str(i + 1)][0]["content_summary"] = summary_text
        dict_res[str(i + 1)][0]["num_of_post"] = len(lst_res[i])
        kew_phares = []
        dict_res[str(i + 1)][0]["topic_keywords"] = kew_phares
        
        # print("delete_message: ", delete_message)
        if delete_message:
            for j in range(len(dict_res[str(i + 1)])):
                if "message" in dict_res[str(i + 1)][j]:
                    del dict_res[str(i + 1)][j]["message"]
    
    with open(f"log_llm/topic_result_after_postprocessing/{hash_str}.json", "w") as f:
        dict_log_pos = {}
        for k in dict_res:
            dict_log_pos[k] = copy.deepcopy(dict_res[k])
            for d in dict_log_pos[k]:
                if "message" in d:
                    del d["message"]
                if "vector" in d:
                    del d["vector"]
        json.dump(dict_log_pos, f, ensure_ascii= False)
    return dict_res


def get_lang(docs):
    lang_vi = 0
    lang_en = 0
    dict_lang = {}
    for d in docs:
        lang = d.get("lang", "")
        if lang not in dict_lang:
            dict_lang[lang] = 0
        dict_lang[lang] += 1
        # if d.get("lang", "") == "vi":
        #     lang_vi += 1
        # else:
        #     lang_en += 1
    lst_lang = []
    lst_cnt = []
    for k in dict_lang:
        lst_lang.append(k)
        lst_cnt.append(dict_lang[k])
    idx_max = np.argsort(np.array(lst_cnt))[::-1][0]
    lang = lst_lang[int(idx_max)]
    
    if lang.startswith("zh_"):
        lang = "zh"
    print("lang: ", lang, lst_cnt[int(idx_max)])
    return lang


def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster=50, 

                        delete_message=True, prompt="", type_cluster:str = "single", hash_str: str= "", id_topic=""):
    # global model, model_en
    with open("data/topic_name.txt") as f:
        dict_topic_name = json.load(f)
    topic_name_relevant = dict_topic_name.get(id_topic , "")
    docs = docs[:30000]
    lang = get_lang(docs)
    if type_cluster == "complete" and lang == "zh":
        distance_threshold = 0.4
    if type_cluster == "complete" and lang == "en":
        distance_threshold = 0.4
        # type_cluster = "single"

    result = {}
    cluster_score = {}
    cluster_real_vectors = {}
    # docs = segment(docs, lang=lang)

    t1 = time.time()
    if len(docs) < 1:
        return result
    elif len(docs) == 1:
        return {
            "0": docs
        }
    vec_prompt = []
    prompt_strips = []
    # prompt = ""
    if topic_name_relevant:
        prompt_split = topic_name_relevant.split("#####")
        for prom in prompt_split:
            sys_p = prom.strip().split("$$$$")
            if len(sys_p) == 1:
                prompt_strips.append(prom.strip())
            else:
                prompt_strips.append(sys_p[1].strip())
        if lang == "zh":
            vec_prompt = embbeded_zh(prompt_split)
        elif lang == "en":
            vec_prompt = embbeded_en(prompt_split)
        else:
            vec_prompt = inference.encode(prompt_split, lang=lang)
    if lang == "zh":
        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
        vectors = embbeded_zh(features)
        # vectors = embedded_bge(features)
        if len(vectors) == 0:
            print(f"[WARNING] Embedded {lang}: {len(vectors)} / {len(features)}")
            vectors = inference.encode(features, lang=lang)
        # vectors = model.encode(features, show_progress_bar=False)
    elif lang == "en":
        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
        vectors = embbeded_en(features)
        # vectors = embedded_bge(features)
        if len(vectors) == 0:
            print(f"[WARNING] Embedded {lang}: {len(vectors)} / {len(features)}")
            vectors = inference.encode(features, lang=lang)
    else:
        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
        # vectors = embedded_bge(features)
        # if len(vectors) == 0:
        #     print(f"[WARNING] Embedded {lang}: {len(vectors)} / {len(features)}")
        vectors = inference.encode(features, lang=lang)
        # vectors = model_en.encode(features, show_progress_bar=False)
    clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
                                        linkage=type_cluster, distance_threshold=distance_threshold)
    clusteror.fit(vectors)
    matrix_vec = np.stack(vectors, axis=0)
    print(f"Time encode + clustering: {time.time() - t1} {clusteror.n_clusters_}")
    for i in range(clusteror.n_clusters_):
        result[str(i + 1)] = []
        cluster_score[str(i + 1)] = 0
        ids = clusteror.labels_ # == i
        # cluster_real_vectors[str(i + 1)] = re_clustering(ids, matrix_vec, distance_threshold, max_doc_per_cluster)

    for i in range(len(clusteror.labels_)):
        cluster_no = clusteror.labels_[i]
        # if any((cluster_real_vectors[str(cluster_no+1)][:] == vectors[i]).all(1)):
        if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
            response_doc = {}
            response_doc = docs[i]
            score = response_doc.get('score', 0)
            if not docs[i].get('message','').strip():
                continue 
            if score > cluster_score[str(cluster_no + 1)]:
                cluster_score[str(cluster_no + 1)] = score
            if 'domain' in docs[i]:
                response_doc['domain'] = docs[i]['domain']
            if 'url' in docs[i]:
                response_doc['url'] = docs[i]['url']
            if 'title' in docs[i]:
                response_doc['title'] = clean_text(docs[i]['title'])
            if 'snippet' in docs[i]:
                response_doc['snippet'] = clean_text(docs[i]['snippet'])
            if 'created_time' in docs[i]:
                response_doc['created_time'] = docs[i]['created_time']
            if "sentiment" in docs[i]:
                response_doc['sentiment'] = docs[i]['sentiment']
            if 'message' in docs[i]:
                title = docs[i].get('title','')
                snippet = docs[i].get('snippet','')
                message = docs[i].get('message','')
                if title.strip():
                    split_mess = message.split(title)
                    if len(split_mess) > 1:
                        message = title.join(split_mess[1:])
                if snippet.strip():
                    split_mess = message.split(snippet)
                    if len(split_mess) > 1:
                        message = snippet.join(split_mess[1:])

                response_doc['message'] = clean_text(message)
            if 'id' in docs[i]:
                response_doc['id'] = docs[i]['id']
            # response_doc['score'] = 0.0
            response_doc['title_summarize'] = []
            response_doc['content_summary'] = ""
            response_doc['total_facebook_viral'] = 0
            response_doc["vector"] = np.array(vectors[i]).tolist()
            result[str(cluster_no + 1)].append(response_doc)
    empty_clus_ids = []
    for x in result:
        result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
        if len( result[x]) > 0:
            if len(result[x]) > 1:
                result[x] = check_duplicate_title_domain(result[x])
            result[x][0]['num_docs'] = len(result[x])
            result[x][0]['max_score'] = cluster_score[x]
        else:
            empty_clus_ids.append(x)
    
    for x in empty_clus_ids:
        result.pop(x,None)
    # result = dict(sorted(result.items(), key=lambda i: -len(i[1])))[:top_cluster]
    with open(f"log_llm/topic_result_before_postprocessing/{hash_str}.json", "w") as f:
        dict_log = {}
        for k in result:
            dict_log[k] = copy.deepcopy(result[k])
            for d in dict_log[k]:
                if "message" in d:
                    del d["message"]
                if "vector" in d:
                    del d["vector"]
        json.dump(dict_log, f, ensure_ascii= False)
    return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field, max_doc_per_cluster=max_doc_per_cluster, delete_message=delete_message,
                            prompt=topic_name_relevant, hash_str=hash_str, vectors_prompt=vec_prompt)

def check_duplicate_title_domain(docs):
    lst_title_domain = [f"{d.get('domain', '')} {d.get('title','')}" for d in docs]
    for i in range(1,len(lst_title_domain) -1):
        for j in range(i+1,len(lst_title_domain)):
            if lst_title_domain[j] == lst_title_domain[i]:
                lst_title_domain[j] = 'dup'
    lst_filter_docs = [docs[i] for i,x in enumerate(lst_title_domain) if x != 'dup']
    return lst_filter_docs
def convert_date(text):
    text = text.replace(".", "/")
    text = text.replace("-", "/")
    return text


def check_keyword(sentence):
    keyword = ['sáng', 'trưa', 'chiều', 'tối', 'đến', 'hôm', 'ngày', 'tới']
    for k in keyword:
        if k in sentence:
            return True
    return False


def extract_events_and_time(docs, publish_date):
    def standardize(date_str):
        return date_str.replace('.', '/').replace('-', '/')

    def add_0(date_str):

        date_str = date_str.split('/')
        res = []
        for o in date_str:
            o = re.sub('\s+', '', o)
            if len(o) < 2:
                o = '0' + o
            res.append(o)
        date_str = '/'.join(res)
        return date_str

    def get_date_list(reg, sentence):
        find_object = re.finditer(reg, sentence)
        date_list = [x.group() for x in find_object]
        return date_list

    year = publish_date.split('/')[2]

    # dd/mm/yyyy
    reg_exp_1 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
    # #mm/yyyy
    # reg_exp_5 = '(\D|^)(?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
    # dd/mm
    reg_exp_2 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])(\D|$)'

    # ngày  dd tháng mm năm yyyy
    reg_exp_3 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}\s*(năm)\s*\d{4}'
    # ngày dd tháng mm
    reg_exp_4 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}'

    result = []
    for d in docs:
        text = d['message']
        for sentence in sent_tokenize(text):
            lower_sentence = sentence.lower()
            c = re.search(reg_exp_3, sentence.lower())
            d = re.search(reg_exp_4, sentence.lower())
            # e = re.search(reg_exp_5, sentence.lower())
            a = re.search(reg_exp_1, sentence)
            b = re.search(reg_exp_2, sentence)
            #
            if (a or b or c or d) and check_keyword(lower_sentence):
                date_list = get_date_list(reg_exp_1, lower_sentence)
                date_entity = ''
                if date_list:
                    date_entity = add_0(standardize(date_list[0]))
                elif get_date_list(reg_exp_2, lower_sentence):
                    date_list = get_date_list(reg_exp_2, lower_sentence)
                    date_entity = add_0(standardize(date_list[0]) + '/' + year)
                elif get_date_list(reg_exp_3, lower_sentence):
                    date_list = get_date_list(reg_exp_3, lower_sentence)

                    date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
                    date_entity = re.sub('\s+', ' ', date_entity)
                    date_entity = date_entity.replace(' ', '/')
                    date_entity = add_0(date_entity)
                else:
                    date_list = get_date_list(reg_exp_4, lower_sentence)
                    if date_list != []:
                        date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
                        date_entity = re.sub('\s+', ' ', date_entity)
                        date_entity = date_entity.replace(' ', '/')
                        date_entity = date_entity + '/' + year
                        date_entity = add_0(date_entity)
                result.append((sentence, date_entity))
    return result

def find_index_nearest_vector(cluster, vectors):
    # Compute the centroid of the cluster
    centroid = np.mean(cluster, axis=0, keepdims=True)
    
    # Calculate the Euclidean distance between each vector and the centroid
    distances = cosine_similarity(centroid, vectors)
    
    # Find the index of the vector with the minimum distance
    nearest_index = np.argmin(distances, axis=1)
    
    
    return nearest_index

def re_clustering(ids, vectors, distance_threshold, max_doc_per_cluster):
    sub_vectors = vectors[ids]

    try:
        if sub_vectors.shape[0] < 2:
            return sub_vectors
        sub_clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
                        linkage='complete', distance_threshold=0.12)
        sub_clusteror.fit(sub_vectors)
        dict_cluster = {id_clus: sub_vectors[sub_clusteror.labels_ == id_clus] for id_clus in range(sub_clusteror.n_clusters_)}
        dict_num_vec = {id_clus: v.shape[0] for id_clus, v in dict_cluster.items()}

        max_num_cluster = max(dict_num_vec, key=dict_num_vec.get)
        other_vectors = sub_vectors[sub_clusteror.labels_ != max_num_cluster]

        # if other_vectors.shape[0]:
        #     while dict_num_vec[max_num_cluster] < max_doc_per_cluster:
        #         tmp_index_vec = find_index_nearest_vector(dict_cluster[max_num_cluster], other_vectors)
        #         dict_cluster[max_num_cluster] = np.vstack((dict_cluster[max_num_cluster], other_vectors[tmp_index_vec]))
        #         dict_num_vec[max_num_cluster] += 1
        #         if other_vectors.shape[0] != 1:
        #             other_vectors = np.delete(other_vectors, tmp_index_vec, axis=0)
        #         else:
        #             break
        cosine_scores = cosine_similarity(dict_cluster[max_num_cluster], dict_cluster[max_num_cluster])
        with open("/home/vietle/topic-clustering/log_score.txt", "a") as f:
            f.write(str(cosine_scores) + "\n")
        return dict_cluster[max_num_cluster]
    except Exception as e:
        with open("/home/vietle/topic-clustering/log_clustering_diemtin/log_cluster_second.txt", "a") as f:
            f.write(str(e)+"$$"+json.dumps({"ids": ids.tolist(), "vectors": vectors.tolist()}))
        return sub_vectors