core/init_nlp.py

import nltk
import nltk.downloader
import spacy
from core.config import settings

def initialize_nlp():
    print("Initializing NLP resources...")
    
    # Download NLTK resources
    nltk_resources = [
        'maxent_ne_chunker',
        'words',
        'treebank',
        'maxent_treebank_pos_tagger',
        'punkt',
        'averaged_perceptron_tagger'
    ]
    
    for resource in nltk_resources:
        nltk.downloader.download(resource, quiet=True)
    
    # Load spaCy model
    spacy.load(settings.SPACY_MODEL)
    
    print("NLP resources initialized successfully.")

# Global variables to store initialized resources
nlp = None
nltk_initialized = False

def get_nlp():
    global nlp
    if nlp is None:
        nlp = spacy.load(settings.SPACY_MODEL)
    return nlp

def get_nltk():
    global nltk_initialized
    if not nltk_initialized:
        nltk.downloader.download('punkt', quiet=True)
        nltk.download('averaged_perceptron_tagger', quiet=True)
        nltk_initialized = True
    return nltk