moved some functions to anonymize.py

.gitignore

@@ -1,2 +1,3 @@
 
 __pycache__/spacy_recognizer.cpython-310.pyc
+__pycache__/anonymize.cpython-310.pyc

anonymize.py

@@ -0,0 +1,44 @@
+from spacy_recognizer import CustomSpacyRecognizer
+from presidio_analyzer.nlp_engine import NlpEngineProvider
+from presidio_anonymizer import AnonymizerEngine
+from presidio_analyzer import AnalyzerEngine, RecognizerRegistry
+from presidio_anonymizer.entities import OperatorConfig
+import pandas as pd
+from json import JSONEncoder
+import json
+import warnings
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+warnings.filterwarnings('ignore')
+
+def prepare_analyzer(configuration):
+    """Return AnalyzerEngine."""
+
+    spacy_recognizer = CustomSpacyRecognizer()
+
+    print('Hallej')
+
+    # Create NLP engine based on configuration
+    provider = NlpEngineProvider(nlp_configuration=configuration)
+    nlp_engine = provider.create_engine()
+
+    # add rule-based recognizers
+    registry = RecognizerRegistry()
+    registry.load_predefined_recognizers(nlp_engine=nlp_engine)
+    registry.add_recognizer(spacy_recognizer)
+
+    # remove the nlp engine we passed, to use custom label mappings
+    registry.remove_recognizer("SpacyRecognizer")
+
+    analyzer = AnalyzerEngine(nlp_engine=nlp_engine,
+                              registry=registry,
+                              supported_languages=["en"])
+
+    return analyzer
+
+def generate_surrogate(name):
+    """Return appropriate surrogate name from text string"""
+    if "John" in name:
+        return "Jill"
+    else:
+        return "SURROGATE_NAME"

app.py

@@ -1,10 +1,8 @@
 
 """Streamlit app for Student Name Detection models."""
 
-from spacy_recognizer import CustomSpacyRecognizer
-from presidio_analyzer.nlp_engine import NlpEngineProvider
+from anonymize import prepare_analyzer, generate_surrogate
 from presidio_anonymizer import AnonymizerEngine
-from presidio_analyzer import AnalyzerEngine, RecognizerRegistry
 from presidio_anonymizer.entities import OperatorConfig
 import pandas as pd
 from annotated_text import annotated_text
@@ -21,27 +19,13 @@ warnings.filterwarnings('ignore')
 def analyzer_engine():
     """Return AnalyzerEngine."""
 
-    spacy_recognizer = CustomSpacyRecognizer()
-
     configuration = {
         "nlp_engine_name": "spacy",
         "models": [
             {"lang_code": "en", "model_name": "en_student_name_detector"}],
     }
 
-    # Create NLP engine based on configuration
-    provider = NlpEngineProvider(nlp_configuration=configuration)
-    nlp_engine = provider.create_engine()
-
-    registry = RecognizerRegistry()
-    # add rule-based recognizers
-    registry.load_predefined_recognizers(nlp_engine=nlp_engine)
-    registry.add_recognizer(spacy_recognizer)
-    # remove the nlp engine we passed, to use custom label mappings
-    registry.remove_recognizer("SpacyRecognizer")
-
-    analyzer = AnalyzerEngine(nlp_engine=nlp_engine,
-                              registry=registry, supported_languages=["en"])
+    analyzer = prepare_analyzer(configuration)
 
     return analyzer
 
@@ -60,13 +44,6 @@ def analyze(**kwargs):
         kwargs["entities"] = None
     return analyzer_engine().analyze(**kwargs)
 
-def generate_surrogate(name):
-    """Return appropriate surrogate name from text string"""
-    if "John" in name:
-        return "Jill"
-    else:
-        return "SURROGATE_NAME"
-
 def anonymize(text, analyze_results):
     """Anonymize identified input using Presidio Anonymizer."""
     if not text:

spacy_recognizer.py

@@ -11,41 +11,22 @@ from presidio_analyzer.predefined_recognizers.spacy_recognizer import SpacyRecog
 
 logger = logging.getLogger("presidio-analyzer")
 
-
 class CustomSpacyRecognizer(LocalRecognizer):
-
+    
     ENTITIES = [
-        "LOCATION",
-        "PERSON",
         "STUDENT",
-        "NRP",
-        "ORGANIZATION",
-        "DATE_TIME",
     ]
 
-    DEFAULT_EXPLANATION = "Identified as {} by Spacy's Named Entity Recognition"
+    DEFAULT_EXPLANATION = "Identified as {} by a Student Name Detection Model"
 
     CHECK_LABEL_GROUPS = [
-        ({"LOCATION"}, {"LOC", "LOCATION", "STREET_ADDRESS", "COORDINATE"}),
-        ({"PERSON"}, {"PER", "PERSON"}),
         ({"STUDENT"}, {"STUDENT"}),
-        ({"NRP"}, {"NORP", "NRP"}),
-        ({"ORGANIZATION"}, {"ORG"}),
-        ({"DATE_TIME"}, {"DATE_TIME"}),
     ]
 
     MODEL_LANGUAGES = {
         "en": "langdonholmes/en_student_name_detector",
     }
 
-    PRESIDIO_EQUIVALENCES = {
-        "PER": "PERSON",
-        "LOC": "LOCATION",
-        "ORG": "ORGANIZATION",
-        "NROP": "NRP",
-        "DATE_TIME": "DATE_TIME",
-    }
-
     def __init__(
         self,
         supported_language: str = "en",