multi_language.py

import os
import functools
import os
import pickle
import time

CACHE_FOLDER = "gpt_log"

if not os.path.exists(CACHE_FOLDER):
    os.makedirs(CACHE_FOLDER)


def lru_file_cache(maxsize=128, ttl=None, filename=None):
    """
    Decorator that caches a function's return value after being called with given arguments. 
    It uses a Least Recently Used (LRU) cache strategy to limit the size of the cache.
    maxsize: Maximum size of the cache. Defaults to 128.
    ttl: Time-to-Live of the cache. If a value hasn't been accessed for `ttl` seconds, it will be evicted from the cache.
    filename: Name of the file to store the cache in. If not supplied, the function name + ".cache" will be used.
    """
    cache_path = os.path.join(CACHE_FOLDER, f"{filename}.cache") if filename is not None else None

    def decorator_function(func):
        cache = {}
        _cache_info = {
            "hits": 0,
            "misses": 0,
            "maxsize": maxsize,
            "currsize": 0,
            "ttl": ttl,
            "filename": cache_path,
        }

        @functools.wraps(func)
        def wrapper_function(*args, **kwargs):
            key = str((args, frozenset(kwargs)))
            if key in cache:
                if _cache_info["ttl"] is None or (cache[key][1] + _cache_info["ttl"]) >= time.time():
                    _cache_info["hits"] += 1
                    print(f'Warning, reading cache, last read {(time.time()-cache[key][1])//60} minutes ago'); time.sleep(2)
                    cache[key][1] = time.time()
                    return cache[key][0]
                else:
                    del cache[key]

            result = func(*args, **kwargs)
            cache[key] = [result, time.time()]
            _cache_info["misses"] += 1
            _cache_info["currsize"] += 1

            if _cache_info["currsize"] > _cache_info["maxsize"]:
                oldest_key = None
                for k in cache:
                    if oldest_key is None:
                        oldest_key = k
                    elif cache[k][1] < cache[oldest_key][1]:
                        oldest_key = k
                del cache[oldest_key]
                _cache_info["currsize"] -= 1

            if cache_path is not None:
                with open(cache_path, "wb") as f:
                    pickle.dump(cache, f)

            return result

        def cache_info():
            return _cache_info

        wrapper_function.cache_info = cache_info

        if cache_path is not None and os.path.exists(cache_path):
            with open(cache_path, "rb") as f:
                cache = pickle.load(f)
            _cache_info["currsize"] = len(cache)

        return wrapper_function

    return decorator_function



def extract_chinese_characters(file_path):
    with open(file_path, 'r', encoding='utf-8') as f:
        content = f.read()
        chinese_characters = [] 
        sentence = {'file':file_path, 'begin':-1, 'end':-1, 'word': ""} 
        for index, char in enumerate(content):
            if 0x4e00 <= ord(char) <= 0x9fff:
                sentence['word'] += char
                if sentence['begin'] == -1: sentence['begin'] = index
                sentence['end'] = index
            else:
                if len(sentence['word'])>0:
                    chinese_characters.append(sentence)
                    sentence = {'file':file_path, 'begin':-1, 'end':-1, 'word': ""} 
        return chinese_characters

def extract_chinese_characters_from_directory(directory_path):
    chinese_characters = []
    for root, dirs, files in os.walk(directory_path):
        for file in files:
            if file.endswith('.py'):
                file_path = os.path.join(root, file)
                chinese_characters.extend(extract_chinese_characters(file_path))
    return chinese_characters

directory_path = './'
chinese_characters = extract_chinese_characters_from_directory(directory_path)
word_to_translate = {}
for d in chinese_characters:
    word_to_translate[d['word']] = "TRANS"

def break_dictionary(d, n):
    items = list(d.items())
    num_dicts = (len(items) + n - 1) // n
    return [{k: v for k, v in items[i*n:(i+1)*n]} for i in range(num_dicts)]

N_EACH_REQ = 50
word_to_translate_split = break_dictionary(word_to_translate, N_EACH_REQ)
LANG = "English"

@lru_file_cache(maxsize=10, ttl=1e40, filename="translation_cache")
def trans(words):
    # from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
    # from toolbox import get_conf, ChatBotWithCookies
    # proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, API_KEY = \
    #     get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'API_KEY')
    # llm_kwargs = {
    #     'api_key': API_KEY,
    #     'llm_model': LLM_MODEL,
    #     'top_p':1.0, 
    #     'max_length': None,
    #     'temperature':0.0,
    # }
    # plugin_kwargs = {}
    # chatbot = ChatBotWithCookies(llm_kwargs)
    # history = []
    # for gpt_say in request_gpt_model_in_new_thread_with_ui_alive(
    #     inputs=words, inputs_show_user=words, 
    #     llm_kwargs=llm_kwargs, chatbot=chatbot, history=[], 
    #     sys_prompt=f"Translate following words to {LANG}, replace `TRANS` with translated result."
    # ):
    #     gpt_say = gpt_say[1][0][1]
    # return gpt_say
    return '{}'

translated_result = {}
for d in word_to_translate_split:
    res = trans(str(d))
    try:
        # convert translated result back to python dictionary
        res_dict = eval(res)
    except:
        print('Unexpected output.')
    translated_result.update(res_dict)

print('All Chinese characters:', chinese_characters)


# =================== create copy =====================
def copy_source_code():
    """
    一键更新协议：备份和下载
    """
    from toolbox import get_conf
    import shutil
    import os
    import requests
    import zipfile
    try: shutil.rmtree(f'./multi-language/{LANG}/')
    except: pass
    os.makedirs(f'./multi-language', exist_ok=True)
    backup_dir = f'./multi-language/{LANG}/'
    shutil.copytree('./', backup_dir, ignore=lambda x, y: ['multi-language', 'gpt_log', '.git', 'private_upload'])
copy_source_code()


for d in chinese_characters:
    d['file'] = f'./multi-language/{LANG}/' + d['file']
    if d['word'] in translated_result:
        d['trans'] = translated_result[d['word']]
    else:
        d['trans'] = None

chinese_characters = sorted(chinese_characters, key=lambda x: len(x['word']), reverse=True)
for d in chinese_characters:
    if d['trans'] is None:
        continue
    


    with open(d['file'], 'r', encoding='utf-8') as f:
        content = f.read()
    
    content.replace(d['word'], d['trans'])
    substring = d['trans']
    substring_start_index = content.find(substring)
    substring_end_index = substring_start_index + len(substring) - 1
    if content[substring_start_index].isalpha() or content[substring_start_index].isdigit():
        content = content[:substring_start_index+1]