merge adjacent subtokens during anonymization

app.py

@@ -225,6 +225,24 @@ def color_substrings(input_string, model_output):
 
 # Selectively anonymize entities
 def anonymize_text(input_string, model_output, selected_entities=None):
+    merged_model_output = []
+    sorted_entities = sorted(model_output, key=lambda x: x["start"])
+    if sorted_entities:
+        current = sorted_entities[0]
+        for entity in sorted_entities[1:]:
+            if (
+                entity["label"] == current["label"]
+                and input_string[current["end"] : entity["start"]].strip() == ""
+            ):
+                current["end"] = entity["end"]
+                current["word"] = input_string[current["start"] : current["end"]]
+            else:
+                merged_model_output.append(current)
+                current = entity
+        merged_model_output.append(current)
+    else:
+        merged_model_output = sorted_entities
+
     anonymized_text = ""
     last_end = 0
     colors = generate_colors(len(ner_labels))
@@ -232,7 +250,7 @@ def anonymize_text(input_string, model_output, selected_entities=None):
         label: colors[i % len(colors)] for i, label in enumerate(ner_labels)
     }
 
-    for entity in sorted(model_output, key=lambda x: x["start"]):
+    for entity in merged_model_output:
         start, end, label = entity["start"], entity["end"], entity["label"]
         anonymized_text += input_string[last_end:start]
         if selected_entities is None or label in selected_entities: