| import yaml | |
| import time | |
| from typing import Dict, Union | |
| import pandas as pd | |
| import spacy | |
| import os | |
| def read_yaml_config(file_path: str) -> Dict: | |
| """ | |
| Reads a YAML configuration file and returns the loaded configuration as a dictionary. | |
| Args: | |
| file_path (str): The path to the YAML configuration file. | |
| """ | |
| with open(file_path, 'r') as file: | |
| config = yaml.safe_load(file) | |
| return config | |
| def validate_and_create_subfolders( | |
| model_name: str, | |
| parent_subfolder: str = "/Users/luis.morales/Desktop/arxiv-paper-recommender/models" | |
| ): | |
| model_subfolders = ["data", "dictionaries", "similarities_matrix", "tdidf"] | |
| if not os.path.exists(f"{parent_subfolder}/{model_name}"): | |
| os.makedirs(f"{parent_subfolder}/{model_name}") | |
| for msubfolder in model_subfolders: | |
| os.makedirs(f"{parent_subfolder}/{model_name}/{msubfolder}") | |
| def execution_time(func): | |
| """ | |
| Decorator that measures the execution time of a function and prints the elapsed time. | |
| """ | |
| def wrapper(*args, **kwargs): | |
| start_time = time.time() | |
| result = func(*args, **kwargs) | |
| end_time = time.time() | |
| execution_seconds = end_time - start_time | |
| print(f"Function '{func.__name__}' executed in {execution_seconds:.4f} seconds.") | |
| return result | |
| return wrapper | |
| def cleanData(doc: Union[pd.Series, str], nlp = spacy.load('en_core_web_sm')): | |
| """ | |
| TODO: Optimize NLP Object to only obtain stopwords, lemmas, and tokenize docs. | |
| Cleans and processes the input documents by performing various text cleaning operations. | |
| Args: | |
| doc (pd.Series): The documents to be cleaned, passed in a Series object. | |
| stemming (bool, optional): Specifies whether stemming should be applied. Defaults to False. | |
| Returns: | |
| str: The cleaned and processed document as a single string. | |
| """ | |
| doc = doc.lower() | |
| doc = nlp(doc) | |
| tokens = [tokens.lower_ for tokens in doc] | |
| tokens = [tokens for tokens in doc if (tokens.is_stop == False)] | |
| tokens = [tokens for tokens in tokens if (tokens.is_punct == False)] | |
| final_token = [token.lemma_ for token in tokens] | |
| return " ".join(final_token) | |