File size: 2,289 Bytes
e01620b 161967b e01620b af7959d 161967b af7959d 161967b af7959d 161967b e01620b d008f82 e01620b d008f82 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
from difflib import Differ
from typing import Tuple, Iterable
import logging
def compute_diff(text1, text2) -> Iterable[Tuple[str, str | None]]:
d = Differ()
pairs = [
(token[2:], token[0] if token[0] != " " else None)
for token in d.compare(text1, text2)
]
return _postprocess_compute_diff(pairs)
def _postprocess_compute_diff(pairs: Iterable[Tuple[str, str | None]]) -> Iterable[Tuple[str, str | None]]:
"""Whitespace deletions add additions are missed by the diff component."""
for idx, (char, flag) in enumerate(pairs):
if char == " " and flag in ["+", "-"]:
if idx > 0 and idx < len(pairs):
if pairs[idx - 1][1] == flag or pairs[idx + 1][1] == flag:
yield (" ", flag)
else:
yield ("^", "+") if flag == "+" else ("#", "-")
else:
yield (char, flag)
def get_logger():
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
return logging.getLogger(__name__)
LANGS = {
"All": None,
"English": "en",
"French": "fr",
"German": "de",
"Spanish": "es",
"Italian": "it",
"Dutch": "nl",
"Polish": "pl",
"Portuguese": "pt",
"Swedish": "sv",
"Bulgarian": "bg",
"Romanian": "ro",
"Finnish": "fi",
"Russian": "ru",
"Norwegian Bokmål": "nb",
"Czech": "cs",
"Thai": "th",
"Danish": "da",
"Croatian": "hr",
"Hungarian": "hu",
"Arabic": "ar",
"Greek": "el",
"Japanese": "ja",
"Catalan": "ca",
"Serbian": "sr",
"Slovenian": "sl",
"Slovak": "sk",
"Turkish": "tr",
"Lithuanian": "lt",
"Chinese": "zh",
"Estonian": "et",
"Latvian": "lv",
"Undefined": "xx",
"Ukrainian": "uk",
"Indonesian": "id",
"Hebrew": "he",
"Vietnamese": "vi",
"Icelandic": "is",
"Latin": "la",
"Korean": "ko",
"Albanian": "sq",
"Georgian": "ka",
"Malay": "ms",
"Bosnian": "bs",
"Persian": "fa",
"Bengali": "bn",
"Galician": "gl",
"Kazakh": "kk",
"Macedonian": "mk",
"Norwegian Nynorsk": "nn",
"Hindi": "hi",
"Afar": "aa",
"Uzbek": "uz",
"Somali": "so",
"Afrikaans": "af"
}
|