Spaces:
Running
Running
| """ | |
| Translate this project to other languages (experimental, please open an issue if there is any bug) | |
| Usage: | |
| 1. modify config.py, set your LLM_MODEL and API_KEY(s) to provide access to OPENAI (or any other LLM model provider) | |
| 2. modify LANG (below ↓) | |
| LANG = "English" | |
| 3. modify TransPrompt (below ↓) | |
| TransPrompt = f"Replace each json value `#` with translated results in English, e.g., \"原始文本\":\"TranslatedText\". Keep Json format. Do not answer #." | |
| 4. Run `python multi_language.py`. | |
| Note: You need to run it multiple times to increase translation coverage because GPT makes mistakes sometimes. | |
| (You can also run `CACHE_ONLY=True python multi_language.py` to use cached translation mapping) | |
| 5. Find the translated program in `multi-language\English\*` | |
| P.S. | |
| - The translation mapping will be stored in `docs/translation_xxxx.json`, you can revised mistaken translation there. | |
| - If you would like to share your `docs/translation_xxxx.json`, (so that everyone can use the cached & revised translation mapping), please open a Pull Request | |
| - If there is any translation error in `docs/translation_xxxx.json`, please open a Pull Request | |
| - Welcome any Pull Request, regardless of language | |
| """ | |
| import os | |
| import json | |
| import functools | |
| import re | |
| import pickle | |
| import time | |
| from toolbox import get_conf | |
| CACHE_ONLY = os.environ.get('CACHE_ONLY', False) | |
| CACHE_FOLDER = get_conf('PATH_LOGGING') | |
| blacklist = ['multi-language', CACHE_FOLDER, '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv'] | |
| # LANG = "TraditionalChinese" | |
| # TransPrompt = f"Replace each json value `#` with translated results in Traditional Chinese, e.g., \"原始文本\":\"翻譯後文字\". Keep Json format. Do not answer #." | |
| # LANG = "Japanese" | |
| # TransPrompt = f"Replace each json value `#` with translated results in Japanese, e.g., \"原始文本\":\"テキストの翻訳\". Keep Json format. Do not answer #." | |
| LANG = "English" | |
| TransPrompt = f"Replace each json value `#` with translated results in English, e.g., \"原始文本\":\"TranslatedText\". Keep Json format. Do not answer #." | |
| if not os.path.exists(CACHE_FOLDER): | |
| os.makedirs(CACHE_FOLDER) | |
| def lru_file_cache(maxsize=128, ttl=None, filename=None): | |
| """ | |
| Decorator that caches a function's return value after being called with given arguments. | |
| It uses a Least Recently Used (LRU) cache strategy to limit the size of the cache. | |
| maxsize: Maximum size of the cache. Defaults to 128. | |
| ttl: Time-to-Live of the cache. If a value hasn't been accessed for `ttl` seconds, it will be evicted from the cache. | |
| filename: Name of the file to store the cache in. If not supplied, the function name + ".cache" will be used. | |
| """ | |
| cache_path = os.path.join(CACHE_FOLDER, f"{filename}.cache") if filename is not None else None | |
| def decorator_function(func): | |
| cache = {} | |
| _cache_info = { | |
| "hits": 0, | |
| "misses": 0, | |
| "maxsize": maxsize, | |
| "currsize": 0, | |
| "ttl": ttl, | |
| "filename": cache_path, | |
| } | |
| def wrapper_function(*args, **kwargs): | |
| key = str((args, frozenset(kwargs))) | |
| if key in cache: | |
| if _cache_info["ttl"] is None or (cache[key][1] + _cache_info["ttl"]) >= time.time(): | |
| _cache_info["hits"] += 1 | |
| print(f'Warning, reading cache, last read {(time.time()-cache[key][1])//60} minutes ago'); time.sleep(2) | |
| cache[key][1] = time.time() | |
| return cache[key][0] | |
| else: | |
| del cache[key] | |
| result = func(*args, **kwargs) | |
| cache[key] = [result, time.time()] | |
| _cache_info["misses"] += 1 | |
| _cache_info["currsize"] += 1 | |
| if _cache_info["currsize"] > _cache_info["maxsize"]: | |
| oldest_key = None | |
| for k in cache: | |
| if oldest_key is None: | |
| oldest_key = k | |
| elif cache[k][1] < cache[oldest_key][1]: | |
| oldest_key = k | |
| del cache[oldest_key] | |
| _cache_info["currsize"] -= 1 | |
| if cache_path is not None: | |
| with open(cache_path, "wb") as f: | |
| pickle.dump(cache, f) | |
| return result | |
| def cache_info(): | |
| return _cache_info | |
| wrapper_function.cache_info = cache_info | |
| if cache_path is not None and os.path.exists(cache_path): | |
| with open(cache_path, "rb") as f: | |
| cache = pickle.load(f) | |
| _cache_info["currsize"] = len(cache) | |
| return wrapper_function | |
| return decorator_function | |
| def contains_chinese(string): | |
| """ | |
| Returns True if the given string contains Chinese characters, False otherwise. | |
| """ | |
| chinese_regex = re.compile(u'[\u4e00-\u9fff]+') | |
| return chinese_regex.search(string) is not None | |
| def split_list(lst, n_each_req): | |
| """ | |
| Split a list into smaller lists, each with a maximum number of elements. | |
| :param lst: the list to split | |
| :param n_each_req: the maximum number of elements in each sub-list | |
| :return: a list of sub-lists | |
| """ | |
| result = [] | |
| for i in range(0, len(lst), n_each_req): | |
| result.append(lst[i:i + n_each_req]) | |
| return result | |
| def map_to_json(map, language): | |
| dict_ = read_map_from_json(language) | |
| dict_.update(map) | |
| with open(f'docs/translate_{language.lower()}.json', 'w', encoding='utf8') as f: | |
| json.dump(dict_, f, indent=4, ensure_ascii=False) | |
| def read_map_from_json(language): | |
| if os.path.exists(f'docs/translate_{language.lower()}.json'): | |
| with open(f'docs/translate_{language.lower()}.json', 'r', encoding='utf8') as f: | |
| res = json.load(f) | |
| res = {k:v for k, v in res.items() if v is not None and contains_chinese(k)} | |
| return res | |
| return {} | |
| def advanced_split(splitted_string, spliter, include_spliter=False): | |
| splitted_string_tmp = [] | |
| for string_ in splitted_string: | |
| if spliter in string_: | |
| splitted = string_.split(spliter) | |
| for i, s in enumerate(splitted): | |
| if include_spliter: | |
| if i != len(splitted)-1: | |
| splitted[i] += spliter | |
| splitted[i] = splitted[i].strip() | |
| for i in reversed(range(len(splitted))): | |
| if not contains_chinese(splitted[i]): | |
| splitted.pop(i) | |
| splitted_string_tmp.extend(splitted) | |
| else: | |
| splitted_string_tmp.append(string_) | |
| splitted_string = splitted_string_tmp | |
| return splitted_string_tmp | |
| cached_translation = {} | |
| cached_translation = read_map_from_json(language=LANG) | |
| def trans(word_to_translate, language, special=False): | |
| if len(word_to_translate) == 0: return {} | |
| from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency | |
| from toolbox import get_conf, ChatBotWithCookies | |
| proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, API_KEY = \ | |
| get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'API_KEY') | |
| llm_kwargs = { | |
| 'api_key': API_KEY, | |
| 'llm_model': LLM_MODEL, | |
| 'top_p':1.0, | |
| 'max_length': None, | |
| 'temperature':0.4, | |
| } | |
| import random | |
| N_EACH_REQ = random.randint(16, 32) | |
| word_to_translate_split = split_list(word_to_translate, N_EACH_REQ) | |
| inputs_array = [str(s) for s in word_to_translate_split] | |
| inputs_show_user_array = inputs_array | |
| history_array = [[] for _ in inputs_array] | |
| if special: # to English using CamelCase Naming Convention | |
| sys_prompt_array = [f"Translate following names to English with CamelCase naming convention. Keep original format" for _ in inputs_array] | |
| else: | |
| sys_prompt_array = [f"Translate following sentences to {LANG}. E.g., You should translate sentences to the following format ['translation of sentence 1', 'translation of sentence 2']. Do NOT answer with Chinese!" for _ in inputs_array] | |
| chatbot = ChatBotWithCookies(llm_kwargs) | |
| gpt_say_generator = request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( | |
| inputs_array, | |
| inputs_show_user_array, | |
| llm_kwargs, | |
| chatbot, | |
| history_array, | |
| sys_prompt_array, | |
| ) | |
| while True: | |
| try: | |
| gpt_say = next(gpt_say_generator) | |
| print(gpt_say[1][0][1]) | |
| except StopIteration as e: | |
| result = e.value | |
| break | |
| translated_result = {} | |
| for i, r in enumerate(result): | |
| if i%2 == 1: | |
| try: | |
| res_before_trans = eval(result[i-1]) | |
| res_after_trans = eval(result[i]) | |
| if len(res_before_trans) != len(res_after_trans): | |
| raise RuntimeError | |
| for a,b in zip(res_before_trans, res_after_trans): | |
| translated_result[a] = b | |
| except: | |
| # try: | |
| # res_before_trans = word_to_translate_split[(i-1)//2] | |
| # res_after_trans = [s for s in result[i].split("', '")] | |
| # for a,b in zip(res_before_trans, res_after_trans): | |
| # translated_result[a] = b | |
| # except: | |
| print('GPT answers with unexpected format, some words may not be translated, but you can try again later to increase translation coverage.') | |
| res_before_trans = eval(result[i-1]) | |
| for a in res_before_trans: | |
| translated_result[a] = None | |
| return translated_result | |
| def trans_json(word_to_translate, language, special=False): | |
| if len(word_to_translate) == 0: return {} | |
| from crazy_functions.crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency | |
| from toolbox import get_conf, ChatBotWithCookies | |
| proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, API_KEY = \ | |
| get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'API_KEY') | |
| llm_kwargs = { | |
| 'api_key': API_KEY, | |
| 'llm_model': LLM_MODEL, | |
| 'top_p':1.0, | |
| 'max_length': None, | |
| 'temperature':0.1, | |
| } | |
| import random | |
| N_EACH_REQ = random.randint(16, 32) | |
| random.shuffle(word_to_translate) | |
| word_to_translate_split = split_list(word_to_translate, N_EACH_REQ) | |
| inputs_array = [{k:"#" for k in s} for s in word_to_translate_split] | |
| inputs_array = [ json.dumps(i, ensure_ascii=False) for i in inputs_array] | |
| inputs_show_user_array = inputs_array | |
| history_array = [[] for _ in inputs_array] | |
| sys_prompt_array = [TransPrompt for _ in inputs_array] | |
| chatbot = ChatBotWithCookies(llm_kwargs) | |
| gpt_say_generator = request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( | |
| inputs_array, | |
| inputs_show_user_array, | |
| llm_kwargs, | |
| chatbot, | |
| history_array, | |
| sys_prompt_array, | |
| ) | |
| while True: | |
| try: | |
| gpt_say = next(gpt_say_generator) | |
| print(gpt_say[1][0][1]) | |
| except StopIteration as e: | |
| result = e.value | |
| break | |
| translated_result = {} | |
| for i, r in enumerate(result): | |
| if i%2 == 1: | |
| try: | |
| translated_result.update(json.loads(result[i])) | |
| except: | |
| print(result[i]) | |
| print(result) | |
| return translated_result | |
| def step_1_core_key_translate(): | |
| LANG_STD = 'std' | |
| def extract_chinese_characters(file_path): | |
| syntax = [] | |
| with open(file_path, 'r', encoding='utf-8') as f: | |
| content = f.read() | |
| import ast | |
| root = ast.parse(content) | |
| for node in ast.walk(root): | |
| if isinstance(node, ast.Name): | |
| if contains_chinese(node.id): syntax.append(node.id) | |
| if isinstance(node, ast.Import): | |
| for n in node.names: | |
| if contains_chinese(n.name): syntax.append(n.name) | |
| elif isinstance(node, ast.ImportFrom): | |
| for n in node.names: | |
| if contains_chinese(n.name): syntax.append(n.name) | |
| # if node.module is None: print(node.module) | |
| for k in node.module.split('.'): | |
| if contains_chinese(k): syntax.append(k) | |
| return syntax | |
| def extract_chinese_characters_from_directory(directory_path): | |
| chinese_characters = [] | |
| for root, dirs, files in os.walk(directory_path): | |
| if any([b in root for b in blacklist]): | |
| continue | |
| print(files) | |
| for file in files: | |
| if file.endswith('.py'): | |
| file_path = os.path.join(root, file) | |
| chinese_characters.extend(extract_chinese_characters(file_path)) | |
| return chinese_characters | |
| directory_path = './' | |
| chinese_core_names = extract_chinese_characters_from_directory(directory_path) | |
| chinese_core_keys = [name for name in chinese_core_names] | |
| chinese_core_keys_norepeat = [] | |
| for d in chinese_core_keys: | |
| if d not in chinese_core_keys_norepeat: chinese_core_keys_norepeat.append(d) | |
| need_translate = [] | |
| cached_translation = read_map_from_json(language=LANG_STD) | |
| cached_translation_keys = list(cached_translation.keys()) | |
| for d in chinese_core_keys_norepeat: | |
| if d not in cached_translation_keys: | |
| need_translate.append(d) | |
| if CACHE_ONLY: | |
| need_translate_mapping = {} | |
| else: | |
| need_translate_mapping = trans(need_translate, language=LANG_STD, special=True) | |
| map_to_json(need_translate_mapping, language=LANG_STD) | |
| cached_translation = read_map_from_json(language=LANG_STD) | |
| cached_translation = dict(sorted(cached_translation.items(), key=lambda x: -len(x[0]))) | |
| chinese_core_keys_norepeat_mapping = {} | |
| for k in chinese_core_keys_norepeat: | |
| chinese_core_keys_norepeat_mapping.update({k:cached_translation[k]}) | |
| chinese_core_keys_norepeat_mapping = dict(sorted(chinese_core_keys_norepeat_mapping.items(), key=lambda x: -len(x[0]))) | |
| # =============================================== | |
| # copy | |
| # =============================================== | |
| def copy_source_code(): | |
| from toolbox import get_conf | |
| import shutil | |
| import os | |
| try: shutil.rmtree(f'./multi-language/{LANG}/') | |
| except: pass | |
| os.makedirs(f'./multi-language', exist_ok=True) | |
| backup_dir = f'./multi-language/{LANG}/' | |
| shutil.copytree('./', backup_dir, ignore=lambda x, y: blacklist) | |
| copy_source_code() | |
| # =============================================== | |
| # primary key replace | |
| # =============================================== | |
| directory_path = f'./multi-language/{LANG}/' | |
| for root, dirs, files in os.walk(directory_path): | |
| for file in files: | |
| if file.endswith('.py'): | |
| file_path = os.path.join(root, file) | |
| syntax = [] | |
| # read again | |
| with open(file_path, 'r', encoding='utf-8') as f: | |
| content = f.read() | |
| for k, v in chinese_core_keys_norepeat_mapping.items(): | |
| content = content.replace(k, v) | |
| with open(file_path, 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| def step_2_core_key_translate(): | |
| # ================================================================================================= | |
| # step2 | |
| # ================================================================================================= | |
| def load_string(strings, string_input): | |
| string_ = string_input.strip().strip(',').strip().strip('.').strip() | |
| if string_.startswith('[Local Message]'): | |
| string_ = string_.replace('[Local Message]', '') | |
| string_ = string_.strip().strip(',').strip().strip('.').strip() | |
| splitted_string = [string_] | |
| # -------------------------------------- | |
| splitted_string = advanced_split(splitted_string, spliter="，", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="。", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="）", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="（", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="(", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter=")", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="<", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter=">", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="[", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="]", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="【", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="】", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="？", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="：", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter=":", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter=",", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="#", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="\n", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter=";", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="`", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter=" ", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="- ", include_spliter=False) | |
| splitted_string = advanced_split(splitted_string, spliter="---", include_spliter=False) | |
| # -------------------------------------- | |
| for j, s in enumerate(splitted_string): # .com | |
| if '.com' in s: continue | |
| if '\'' in s: continue | |
| if '\"' in s: continue | |
| strings.append([s,0]) | |
| def get_strings(node): | |
| strings = [] | |
| # recursively traverse the AST | |
| for child in ast.iter_child_nodes(node): | |
| node = child | |
| if isinstance(child, ast.Str): | |
| if contains_chinese(child.s): | |
| load_string(strings=strings, string_input=child.s) | |
| elif isinstance(child, ast.AST): | |
| strings.extend(get_strings(child)) | |
| return strings | |
| string_literals = [] | |
| directory_path = f'./multi-language/{LANG}/' | |
| for root, dirs, files in os.walk(directory_path): | |
| for file in files: | |
| if file.endswith('.py'): | |
| file_path = os.path.join(root, file) | |
| syntax = [] | |
| with open(file_path, 'r', encoding='utf-8') as f: | |
| content = f.read() | |
| # comments | |
| comments_arr = [] | |
| for code_sp in content.splitlines(): | |
| comments = re.findall(r'#.*$', code_sp) | |
| for comment in comments: | |
| load_string(strings=comments_arr, string_input=comment) | |
| string_literals.extend(comments_arr) | |
| # strings | |
| import ast | |
| tree = ast.parse(content) | |
| res = get_strings(tree, ) | |
| string_literals.extend(res) | |
| [print(s) for s in string_literals] | |
| chinese_literal_names = [] | |
| chinese_literal_names_norepeat = [] | |
| for string, offset in string_literals: | |
| chinese_literal_names.append(string) | |
| chinese_literal_names_norepeat = [] | |
| for d in chinese_literal_names: | |
| if d not in chinese_literal_names_norepeat: chinese_literal_names_norepeat.append(d) | |
| need_translate = [] | |
| cached_translation = read_map_from_json(language=LANG) | |
| cached_translation_keys = list(cached_translation.keys()) | |
| for d in chinese_literal_names_norepeat: | |
| if d not in cached_translation_keys: | |
| need_translate.append(d) | |
| if CACHE_ONLY: | |
| up = {} | |
| else: | |
| up = trans_json(need_translate, language=LANG, special=False) | |
| map_to_json(up, language=LANG) | |
| cached_translation = read_map_from_json(language=LANG) | |
| LANG_STD = 'std' | |
| cached_translation.update(read_map_from_json(language=LANG_STD)) | |
| cached_translation = dict(sorted(cached_translation.items(), key=lambda x: -len(x[0]))) | |
| # =============================================== | |
| # literal key replace | |
| # =============================================== | |
| directory_path = f'./multi-language/{LANG}/' | |
| for root, dirs, files in os.walk(directory_path): | |
| for file in files: | |
| if file.endswith('.py'): | |
| file_path = os.path.join(root, file) | |
| syntax = [] | |
| # read again | |
| with open(file_path, 'r', encoding='utf-8') as f: | |
| content = f.read() | |
| for k, v in cached_translation.items(): | |
| if v is None: continue | |
| if '"' in v: | |
| v = v.replace('"', "`") | |
| if '\'' in v: | |
| v = v.replace('\'', "`") | |
| content = content.replace(k, v) | |
| with open(file_path, 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| if file.strip('.py') in cached_translation: | |
| file_new = cached_translation[file.strip('.py')] + '.py' | |
| file_path_new = os.path.join(root, file_new) | |
| with open(file_path_new, 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| os.remove(file_path) | |
| step_1_core_key_translate() | |
| step_2_core_key_translate() | |
| print('Finished, checkout generated results at ./multi-language/') |