Spaces:
Sleeping
Sleeping
File size: 2,923 Bytes
8778cfe |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
PTB_UNESCAPE_MAPPING = {
"«": '"',
"»": '"',
"‘": "'",
"’": "'",
"“": '"',
"”": '"',
"„": '"',
"‹": "'",
"›": "'",
"\u2013": "--", # en dash
"\u2014": "--", # em dash
}
NO_SPACE_BEFORE = {"-RRB-", "-RCB-", "-RSB-", "''"} | set("%.,!?:;")
NO_SPACE_AFTER = {"-LRB-", "-LCB-", "-LSB-", "``", "`"} | set("$#")
NO_SPACE_BEFORE_TOKENS_ENGLISH = {"'", "'s", "'ll", "'re", "'d", "'m", "'ve"}
PTB_DASH_ESCAPED = {"-RRB-", "-RCB-", "-RSB-", "-LRB-", "-LCB-", "-LSB-", "--"}
def ptb_unescape(words):
cleaned_words = []
for word in words:
word = PTB_UNESCAPE_MAPPING.get(word, word)
# This un-escaping for / and * was not yet added for the
# parser version in https://arxiv.org/abs/1812.11760v1
# and related model releases (e.g. benepar_en2)
word = word.replace("\\/", "/").replace("\\*", "*")
# Mid-token punctuation occurs in biomedical text
word = word.replace("-LSB-", "[").replace("-RSB-", "]")
word = word.replace("-LRB-", "(").replace("-RRB-", ")")
word = word.replace("-LCB-", "{").replace("-RCB-", "}")
word = word.replace("``", '"').replace("`", "'").replace("''", '"')
cleaned_words.append(word)
return cleaned_words
def guess_space_after_non_english(escaped_words):
sp_after = [True for _ in escaped_words]
for i, word in enumerate(escaped_words):
if i > 0 and (
(
word.startswith("-")
and not any(word.startswith(x) for x in PTB_DASH_ESCAPED)
)
or any(word.startswith(x) for x in NO_SPACE_BEFORE)
or word == "'"
):
sp_after[i - 1] = False
if (
word.endswith("-") and not any(word.endswith(x) for x in PTB_DASH_ESCAPED)
) or any(word.endswith(x) for x in NO_SPACE_AFTER):
sp_after[i] = False
return sp_after
def guess_space_after(escaped_words, for_english=True):
if not for_english:
return guess_space_after_non_english(escaped_words)
sp_after = [True for _ in escaped_words]
for i, word in enumerate(escaped_words):
if word.lower() == "n't" and i > 0:
sp_after[i - 1] = False
elif word.lower() == "not" and i > 0 and escaped_words[i - 1].lower() == "can":
sp_after[i - 1] = False
if i > 0 and (
(
word.startswith("-")
and not any(word.startswith(x) for x in PTB_DASH_ESCAPED)
)
or any(word.startswith(x) for x in NO_SPACE_BEFORE)
or word.lower() in NO_SPACE_BEFORE_TOKENS_ENGLISH
):
sp_after[i - 1] = False
if (
word.endswith("-") and not any(word.endswith(x) for x in PTB_DASH_ESCAPED)
) or any(word.endswith(x) for x in NO_SPACE_AFTER):
sp_after[i] = False
return sp_after
|