|
""" |
|
This module contains various rule-based components aiming to improve on baseline lemmatization tools. |
|
""" |
|
|
|
import re |
|
from typing import List, Callable |
|
|
|
from spacy.lang.hu import Hungarian |
|
from spacy.pipeline import Pipe |
|
from spacy.tokens import Token |
|
from spacy.tokens.doc import Doc |
|
|
|
|
|
@Hungarian.component( |
|
"lemma_case_smoother", |
|
assigns=["token.lemma"], |
|
requires=["token.lemma", "token.pos"], |
|
) |
|
def lemma_case_smoother(doc: Doc) -> Doc: |
|
"""Smooth lemma casing by POS. |
|
|
|
DEPRECATED: This is not needed anymore, as the lemmatizer is now case-insensitive. |
|
|
|
Args: |
|
doc (Doc): Input document. |
|
|
|
Returns: |
|
Doc: Output document. |
|
""" |
|
for token in doc: |
|
if token.is_sent_start and token.tag_ != "PROPN": |
|
token.lemma_ = token.lemma_.lower() |
|
|
|
return doc |
|
|
|
|
|
class LemmaSmoother(Pipe): |
|
"""Smooths lemma by fixing common errors of the edit-tree lemmatizer.""" |
|
|
|
_DATE_PATTERN = re.compile(r"(\d+)-j?[éá]?n?a?(t[őó]l)?") |
|
_NUMBER_PATTERN = re.compile(r"(\d+([-,/_.:]?(._)?\d+)*%?)") |
|
|
|
|
|
@staticmethod |
|
@Hungarian.factory("lemma_smoother", assigns=["token.lemma"], requires=["token.lemma", "token.pos"]) |
|
def create_lemma_smoother(nlp: Hungarian, name: str) -> "LemmaSmoother": |
|
return LemmaSmoother() |
|
|
|
def __call__(self, doc: Doc) -> Doc: |
|
rules: List[Callable] = [ |
|
self._remove_exclamation_marks, |
|
self._remove_question_marks, |
|
self._remove_date_suffixes, |
|
self._remove_suffix_after_numbers, |
|
] |
|
|
|
for token in doc: |
|
for rule in rules: |
|
rule(token) |
|
|
|
return doc |
|
|
|
@classmethod |
|
def _remove_exclamation_marks(cls, token: Token) -> None: |
|
"""Removes exclamation marks from the lemma. |
|
|
|
Args: |
|
token (Token): The original token. |
|
""" |
|
|
|
if "!" != token.lemma_: |
|
exclamation_mark_index = token.lemma_.find("!") |
|
if exclamation_mark_index != -1: |
|
token.lemma_ = token.lemma_[:exclamation_mark_index] |
|
|
|
@classmethod |
|
def _remove_question_marks(cls, token: Token) -> None: |
|
"""Removes question marks from the lemma. |
|
|
|
Args: |
|
token (Token): The original token. |
|
""" |
|
|
|
if "?" != token.lemma_: |
|
question_mark_index = token.lemma_.find("?") |
|
if question_mark_index != -1: |
|
token.lemma_ = token.lemma_[:question_mark_index] |
|
|
|
@classmethod |
|
def _remove_date_suffixes(cls, token: Token) -> None: |
|
"""Fixes the suffixes of dates. |
|
|
|
Args: |
|
token (Token): The original token. |
|
""" |
|
|
|
if token.pos_ == "NOUN": |
|
match = cls._DATE_PATTERN.match(token.lemma_) |
|
if match is not None: |
|
token.lemma_ = match.group(1) + "." |
|
|
|
@classmethod |
|
def _remove_suffix_after_numbers(cls, token: Token) -> None: |
|
"""Removes suffixes after numbers. |
|
|
|
Args: |
|
token (str): The original token. |
|
""" |
|
|
|
if token.pos_ == "NUM": |
|
match = cls._NUMBER_PATTERN.match(token.text) |
|
if match is not None: |
|
token.lemma_ = match.group(0) |
|
|
