File size: 3,830 Bytes
3bfe3dd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 |
import re
import string
import requests
from langchain.callbacks import get_openai_callback
from langchain_anthropic import ChatAnthropic
def get_content(filepath: str) -> str:
url = string.Template(
"https://raw.githubusercontent.com/huggingface/" "transformers/main/$filepath"
).safe_substitute(filepath=filepath)
response = requests.get(url)
if response.status_code == 200:
content = response.text
return content
else:
raise ValueError("Failed to retrieve content from the URL.", url)
def preprocess_content(content: str) -> str:
# Extract text to translate from document
## ignore top license comment
to_translate = content[content.find("#") :]
## remove code blocks from text
to_translate = re.sub(r"```.*?```", "", to_translate, flags=re.DOTALL)
## remove markdown tables from text
to_translate = re.sub(r"^\|.*\|$\n?", "", to_translate, flags=re.MULTILINE)
## remove empty lines from text
to_translate = re.sub(r"\n\n+", "\n\n", to_translate)
return to_translate
def get_full_prompt(language: str, to_translate: str) -> str:
prompt = string.Template(
"What do these sentences about Hugging Face Transformers "
"(a machine learning library) mean in $language? "
"Please do not translate the word after a 🤗 emoji "
"as it is a product name. Output only the translated markdown result "
"without any explanations or introductions.\n\n```md"
).safe_substitute(language=language)
return "\n".join([prompt, to_translate.strip(), "```"])
def split_markdown_sections(markdown: str) -> list:
# Find all titles using regular expressions
return re.split(r"^(#+\s+)(.*)$", markdown, flags=re.MULTILINE)[1:]
# format is like [level, title, content, level, title, content, ...]
def get_anchors(divided: list) -> list:
anchors = []
# from https://github.com/huggingface/doc-builder/blob/01b262bae90d66e1150cdbf58c83c02733ed4366/src/doc_builder/build_doc.py#L300-L302
for title in divided[1::3]:
anchor = re.sub(r"[^a-z0-9\s]+", "", title.lower())
anchor = re.sub(r"\s{2,}", " ", anchor.strip()).replace(" ", "-")
anchors.append(f"[[{anchor}]]")
return anchors
def make_scaffold(content: str, to_translate: str) -> string.Template:
scaffold = content
for i, text in enumerate(to_translate.split("\n\n")):
scaffold = scaffold.replace(text, f"$hf_i18n_placeholder{i}", 1)
return string.Template(scaffold)
def fill_scaffold(content: str, to_translate: str, translated: str) -> str:
scaffold = make_scaffold(content, to_translate)
divided = split_markdown_sections(to_translate)
anchors = get_anchors(divided)
translated = split_markdown_sections(translated)
translated[1::3] = [
f"{korean_title} {anchors[i]}"
for i, korean_title in enumerate(translated[1::3])
]
translated = "".join(
["".join(translated[i * 3 : i * 3 + 3]) for i in range(len(translated) // 3)]
).split("\n\n")
if newlines := scaffold.template.count("$hf_i18n_placeholder") - len(translated):
return str(
[
f"Please {'recover' if newlines > 0 else 'remove'} "
f"{abs(newlines)} incorrectly inserted double newlines."
]
)
translated_doc = scaffold.safe_substitute(
{f"hf_i18n_placeholder{i}": text for i, text in enumerate(translated)}
)
return translated_doc
def llm_translate(to_translate: str) -> tuple[str, str]:
with get_openai_callback() as cb:
model = ChatAnthropic(
model="claude-sonnet-4-20250514", max_tokens=64000, streaming=True
)
ai_message = model.invoke(to_translate)
print("cb:", cb)
return cb, ai_message.content
|