Add application file

app.py

@@ -17,16 +17,7 @@ def main():
     logger = logging.getLogger(__name__)
     logger.info("Initializing training process...")
     import os
-    import nltk
-    
-    # Set up NLTK data directory
-    nltk_data_path = "./nltk_data"
-    os.makedirs(nltk_data_path, exist_ok=True)
-    nltk.data.path.append(nltk_data_path)
-
-    # Ensure NLTK resources are available
-    nltk.download('punkt', download_dir=nltk_data_path, quiet=True)
-    nltk.download('stopwords', download_dir=nltk_data_path, quiet=True)
+   
     
     # Set up logging
     logging.basicConfig(level=logging.INFO)

requirements.txt

@@ -7,3 +7,5 @@ tiktoken
 sentencepiece
 pandas
 numpy
+spacy
+en_core_web_sm

scripts/data_processing/data_preparation.py

@@ -3,9 +3,7 @@ import numpy as np
 from pathlib import Path
 from typing import Dict, List, Tuple
 import re
-import nltk
-from nltk.tokenize import word_tokenize, sent_tokenize
-from nltk.corpus import stopwords
+import spacy
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.model_selection import StratifiedKFold
 import torch
@@ -25,10 +23,9 @@ class AdvancedNarrativeProcessor:
         # Initialize tokenizer
         self.tokenizer = AutoTokenizer.from_pretrained(model_name)
         
-        # Initialize NLTK
-        nltk.download('punkt', quiet=True)
-        nltk.download('stopwords', quiet=True)
-        self.stopwords = set(stopwords.words('english'))
+        # Initialize SpaCy
+        self.nlp = spacy.load("en_core_web_sm")  # Download it with `python -m spacy download en_core_web_sm`
+        self.stopwords = spacy.lang.en.stop_words.STOP_WORDS
         
         # Initialize state
         self.df = None
@@ -119,9 +116,11 @@ class AdvancedNarrativeProcessor:
         return text.strip()
 
     def extract_features(self, text: str) -> Dict:
-        """Extract rich text features"""
-        words = word_tokenize(text)
-        sentences = sent_tokenize(text)
+        """Extract rich text features using SpaCy."""
+        # Process text with SpaCy
+        doc = self.nlp(text)
+        words = [token.text for token in doc if not token.is_space]
+        sentences = list(doc.sents)
         
         return {
             'length': len(words),