create new date extractor, optimize testing and pipelines

pages/9_Testing.py

@@ -3,7 +3,7 @@ import streamlit as st
 from src.nlp.experimental.textclassification.classify_title import train_data
 from src.nlp.playground.pipelines.event_data_extractor import EventDataExtractor
 from src.persistence.db import init_db
-from src.utils.Event import Event, DateTime, Address
+from src.utils.Event import Event, Schedule
 from src.utils.apis.googlemaps_api import GoogleMapsAPI
 from src.utils.helpers import normalize_data
 
@@ -90,7 +90,7 @@ if start_tests:
 
         dates = el.get("information", {}).get("actual", {}).get("dates", [])
         actual_event.schedule = [
-            DateTime(date.get("start_date", None), date.get("end_date", None), date.get("start_time", None),
+            Schedule(date.get("start_date", None), date.get("end_date", None), date.get("start_time", None),
                      date.get("end_time", None), date.get("admittance_time", None))
             for date in dates]
 

src/nlp/playground/pipelines/date_experimentals.py

@@ -0,0 +1,297 @@
+import os
+import re
+from datetime import datetime
+
+import joblib
+import spacy
+from dotenv import load_dotenv
+from huggingface_hub import hf_hub_download, login
+from spacy import Language
+from spacy.tokenizer import Tokenizer
+from spacy.util import compile_suffix_regex, compile_infix_regex
+
+from src.resources.TEXTS import TEXTS
+from src.utils.Event import Schedule
+from src.utils.helpers import normalize_data
+from src.utils.markdown_processing.CustomMarkdownAnalyzer.MarkdownAnalyzer import MarkdownAnalyzer
+
+load_dotenv()
+token = os.getenv("HUGGING_FACE_SPACES_TOKEN")
+login(token=token)
+
+
+placeholder = {
+  "DATE_RANGE_TIME_RANGE": "[DATE] [TIME] - [DATE] [TIME]",
+  "DATE_RANGE": "[DATE] - [DATE]",
+  "DATE_TIME_RANGE": "[DATE] [TIME] - [TIME]",
+  "TIME_RANGE": "[TIME] - [TIME]",
+  "DATE_TIME": "[DATE] [TIME]",
+  "DATE": "[DATE]",
+  "TIME": "[TIME]"
+}
+
+def convert_to_schedule(date_time, label):
+    print("Converting ", date_time, label)
+    try:
+        if label == "DATE_RANGE_TIME_RANGE":
+            return Schedule(
+                start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                end_date=datetime.strptime(date_time[2], "%d.%m.%Y").date(),
+                start_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                end_time=datetime.strptime(date_time[3], "%H:%M").time(),
+                admittance_time=None
+            )
+
+        if label == "DATE_RANGE":
+            return Schedule(
+                start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                end_date=datetime.strptime(date_time[1], "%d.%m.%Y").date(),
+                start_time=None,
+                end_time=None,
+                admittance_time=None
+            )
+
+        if label == "DATE_TIME_RANGE":
+            return Schedule(
+                start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                end_date=None,
+                start_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                end_time=datetime.strptime(date_time[2], "%H:%M").time(),
+                admittance_time=None
+            )
+
+        if label == "TIME_RANGE":
+            return Schedule(
+                start_date=None,
+                end_date=None,
+                start_time=datetime.strptime(date_time[0], "%H:%M").time(),
+                end_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                admittance_time=None
+            )
+
+        if label == "DATE_TIME":
+            return Schedule(
+                start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                end_date=None,
+                start_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                end_time=None,
+                admittance_time=None
+            )
+
+        if label == "DATE":
+            return Schedule(
+                start_date=datetime.strptime(date_time, "%d.%m.%Y").date(),
+                end_date=None,
+                start_time=None,
+                end_time=None,
+                admittance_time=None
+            )
+
+        if label == "TIME":
+            return Schedule(
+                start_date=None,
+                end_date=None,
+                start_time=datetime.strptime(date_time, "%H:%M").time(),
+                end_time=None,
+                admittance_time=None
+            )
+    except Exception as e:
+        print(e)
+        return None
+
+def _load_classifier(repo_id, model_name):
+    return joblib.load(
+        hf_hub_download(repo_id=repo_id, filename=model_name + ".pkl")
+    )
+
+def classify_date_time(date_times, label, text):
+    # Text anhand des Platzhalters [LABEL] in Segmente teilen
+    segments = text.split(f"[{label}]")
+    tokens = []
+    # print(date_times)
+    date_time_positions = []
+    for i, segment in enumerate(segments):
+        tokens.extend(segment.split())  # Segment als Token hinzufügen
+        if i < len(date_times):  # Falls noch Date-Times übrig sind
+            tokens.append(placeholder.get(label, "ERROR"))  # Date-Time als eigenes Token einfügen
+            date_time_positions.append(len(tokens)-1)
+
+    # print("TOKENS:", tokens)
+    # print(date_time_positions)
+    # print(len(date_time_positions)==len(date_times))
+
+
+    # sliding window classification
+    window_size = 5
+    event_date_total = 0
+    other_total = 0
+
+
+    schedules = []
+    for i, date_time in enumerate(date_times):
+        # Berechne den Start-Index für das Fenster
+        start = max(0, date_time_positions[i] - (window_size - 1))
+
+        # Führe Klassifikation für jedes Fenster durch
+        while start + window_size <= len(tokens):  # Solange das Fenster in den Tokens bleibt
+            window = tokens[start:start + window_size]
+            # print(window)
+
+            # Klassifikation durchführen
+            if label == "TIME":
+                time_class = time_classifier(" ".join(window))
+                # print(time_class)
+            else:
+                date_class = date_classifier(" ".join(window))
+                # print(date_class)
+
+                # Aufaddieren der Werte
+                event_date_total += date_class.get('EVENT_DATE', 0)
+                other_total += date_class.get('OTHER', 0)
+
+            # Fenster verschieben
+            start += 1
+
+        # Rückgabe der Gesamtsummen
+        if label == "TIME":
+            pass
+        else:
+            # print("Gesamtsumme EVENT_DATE:", event_date_total)
+            # print("Gesamtsumme OTHER:", other_total)
+            if event_date_total > other_total:
+
+                schedule = convert_to_schedule(date_time, label)
+                schedules.append(schedule)
+                # print(date_time)
+                # print("EVENT DATE: ", schedule)
+    return schedules
+
+try:
+    date_classifier = _load_classifier("adojode/date_classifier", "date_classifier")
+    time_classifier = _load_classifier("adojode/time_classifier", "time_classifier")
+except Exception as e:
+    print("Error loading classifier models from hugging face: ", e)
+
+
+
+def extract_schedules(text):
+    try:
+        normalized = normalize_data(text)
+        # print("*"*100)
+        # print(normalized)
+        # print("*"*100)
+        cleaned = re.sub(r"\*", " ", normalized)
+        cleaned = re.sub(r"=", " ", cleaned)
+        cleaned = re.sub(r"#", " ", cleaned)
+        cleaned = re.sub(r"(-|—|–|bis)", "-", cleaned)
+        cleaned = re.sub(r"(und|sowie)", "+", cleaned)
+        # cleaned = re.sub( r"\b(?:mo|di|mi|do|fr|sa|so|montag|dienstag|mittwoch|donnerstag|freitag|samstag|sonntag)(?:s?)\b",
+        #                  " ", cleaned, flags=re.IGNORECASE)
+
+        cleaned = re.sub(r"(von|vom|am|um|ab)", " ", cleaned, flags=re.IGNORECASE)
+        cleaned = re.sub(r",", " ", cleaned)
+        cleaned = re.sub(r"\|", " ", cleaned)
+        cleaned = re.sub(r"\s+", " ", cleaned)
+
+
+        matches = {}
+
+        # Match für das Datum und die Zeit mit einer Zeitspanne
+        date_range_time_range_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})\s*-\s*(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})"
+        match = re.findall(date_range_time_range_pattern, cleaned)
+        if match:
+            matches["DATE_RANGE_TIME_RANGE"] = match
+        # print("DATE_RANGE_TIME_RANGE matches:", matches["DATE_RANGE_TIME_RANGE"])
+        cleaned = re.sub(date_range_time_range_pattern, "[DATE_RANGE_TIME_RANGE]", cleaned)
+
+        # Match für das Datum mit einem Zeitraum ohne Zeitangabe
+        date_range_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*-\s*(\d{2}\.\d{2}\.\d{4})"
+        match = re.findall(date_range_pattern, cleaned)
+        if match:
+            matches["DATE_RANGE"] = match
+        # print("DATE_RANGE matches:", matches["DATE_RANGE"])
+        cleaned = re.sub(date_range_pattern, "[DATE_RANGE]", cleaned)
+
+        # Match für das Datum mit einer Zeitspanne ohne Start- und Enddatum
+        date_time_range_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})\s*-\s*(\d{2}:\d{2})"
+        match = re.findall(date_time_range_pattern, cleaned)
+        if match:
+            matches["DATE_TIME_RANGE"] = match
+        # print("DATE_TIME_RANGE matches:", matches["DATE_TIME_RANGE"])
+        cleaned = re.sub(date_time_range_pattern, "[DATE_TIME_RANGE]", cleaned)
+
+        # Match für eine reine Zeitspanne ohne Datum
+        time_range_pattern = r"(\d{2}:\d{2})\s*-\s*(\d{2}:\d{2})"
+        match = re.findall(time_range_pattern, cleaned)
+        if match:
+            matches["TIME_RANGE"] = match
+        # print("TIME_RANGE matches:", matches["TIME_RANGE"])
+        cleaned = re.sub(time_range_pattern, "[TIME_RANGE]", cleaned)
+
+        # Match für Datum mit Zeitangabe
+        date_time_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})"
+        match = re.findall(date_time_pattern, cleaned)
+        if match:
+            matches["DATE_TIME"] = match
+        # print("DATE_TIME matches:", matches["DATE_TIME"])
+        cleaned = re.sub(date_time_pattern, "[DATE_TIME]", cleaned)
+
+        date_pattern = r"(\d{2}\.\d{2}\.\d{4})"
+        match = re.findall(date_pattern, cleaned)
+        if match:
+            matches["DATE"] = match
+        # print("DATE matches:", matches["DATE"])
+        cleaned = re.sub(date_pattern, "[DATE]", cleaned)
+
+        time_pattern = r"(\d{2}:\d{2})"
+        match = re.findall(time_pattern, cleaned)
+        if match:
+            matches["TIME"] = match
+        # print("TIME matches:", matches["TIME"])
+        cleaned = re.sub(time_pattern, "[TIME]", cleaned)
+
+
+        event_schedules = []
+
+        # return date_time if only one found
+        if len(matches)==1:
+            key, value = next(iter(matches.items()))
+
+            event_schedules.append(convert_to_schedule(label=key,date_time=value[0]))
+            return event_schedules
+
+
+        for key, value in matches.items():
+            # print(f"{key}: {value}")
+            schedules = classify_date_time(date_times=value, label=key, text=cleaned)
+            if schedules:
+                event_schedules.extend(schedules)
+
+
+        if len(event_schedules)==1:
+            return event_schedules
+
+
+        print(event_schedules)
+        unique_schedules = []
+        for i, schedule in enumerate(event_schedules):
+            if any(schedule in other for j, other in enumerate(event_schedules) if
+                   i != j):
+                continue
+            unique_schedules.append(schedule)
+        return unique_schedules
+
+    except Exception as ex:
+        print(ex)
+
+
+# TEXTS = ["\n\nTermin für öffentliche Besichtigung\n=================================== \n\n07.01.2025\n\n * Am 07.01.2025\n* Von 18:00 bis 19:00 Uhr\n* Tasköprüstraße 10 (ehemalige Selgros-Markthalle)\n* Termin im Kalender speichern\n"]
+
+
+for text in TEXTS:
+    print(text)
+    schedules = extract_schedules(text)
+    print("*" * 100)
+    print("EXTRACTED SCHEDULES: ")
+    print(schedules)
+    print("*" * 100)

src/nlp/playground/pipelines/date_extractor_v2.py

@@ -6,7 +6,7 @@ from spacy.tokenizer import Tokenizer
 from spacy.util import compile_suffix_regex, compile_infix_regex
 import os
 from dotenv import load_dotenv
-from src.utils.Event import DateTime
+from src.utils.Event import Schedule
 from huggingface_hub import hf_hub_download
 import joblib
 from huggingface_hub import login
@@ -202,10 +202,10 @@ class ScheduleExtractor(NLPProcessor):
                                                        datetime.strptime(token.text, "%H:%M").time())
 
                 if start_date and end_date and start_time and end_time and admittance_time:
-                    date_times.append(DateTime(start_date, end_date, start_time, end_time, admittance_time))
+                    date_times.append(Schedule(start_date, end_date, start_time, end_time, admittance_time))
                     start_date = end_date = start_time = end_time = admittance_time = None
 
-            date_times.append(DateTime(start_date, end_date, start_time, end_time, admittance_time))
+            date_times.append(Schedule(start_date, end_date, start_time, end_time, admittance_time))
             date_times = self.__remove_subsets(date_times)
         return list(set(date_times))
 

src/nlp/playground/pipelines/date_extractor_v3.py

@@ -0,0 +1,288 @@
+import re
+import os
+from dotenv import load_dotenv
+
+from src.utils.Event import Schedule
+from huggingface_hub import hf_hub_download
+import joblib
+from huggingface_hub import login
+from datetime import datetime
+
+from src.utils.helpers import normalize_data
+
+load_dotenv()
+token = os.getenv("HUGGING_FACE_SPACES_TOKEN")
+login(token=token)
+
+
+
+
+
+class ScheduleExtractorV3:
+    def __init__(self):
+        try:
+            self.date_classifier = self._load_classifier("adojode/date_classifier", "date_classifier")
+            self.time_classifier = self._load_classifier("adojode/time_classifier", "time_classifier")
+        except Exception as e:
+            print("Error loading classifier models from hugging face: ", e)
+
+    def _load_classifier(self,repo_id, model_name):
+        return joblib.load(
+            hf_hub_download(repo_id=repo_id, filename=model_name + ".pkl")
+        )
+
+    def extract(self, text):
+        try:
+            normalized = normalize_data(text)
+            # print("*"*100)
+            # print(normalized)
+            # print("*"*100)
+            cleaned = re.sub(r"\*", " ", normalized)
+            cleaned = re.sub(r"=", " ", cleaned)
+            cleaned = re.sub(r"#", " ", cleaned)
+            cleaned = re.sub(r"(-|—|–|bis)", "-", cleaned)
+            cleaned = re.sub(r"(und|sowie)", "+", cleaned)
+            # cleaned = re.sub( r"\b(?:mo|di|mi|do|fr|sa|so|montag|dienstag|mittwoch|donnerstag|freitag|samstag|sonntag)(?:s?)\b",
+            #                  " ", cleaned, flags=re.IGNORECASE)
+
+            cleaned = re.sub(r"(von|vom|am|um|ab)", " ", cleaned, flags=re.IGNORECASE)
+            cleaned = re.sub(r",", " ", cleaned)
+            cleaned = re.sub(r"\|", " ", cleaned)
+            cleaned = re.sub(r"\s+", " ", cleaned)
+
+            matches = {}
+
+            # Match für das Datum und die Zeit mit einer Zeitspanne
+            date_range_time_range_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})\s*-\s*(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})"
+            match = re.findall(date_range_time_range_pattern, cleaned)
+            if match:
+                matches["DATE_RANGE_TIME_RANGE"] = match
+            # print("DATE_RANGE_TIME_RANGE matches:", matches["DATE_RANGE_TIME_RANGE"])
+            cleaned = re.sub(date_range_time_range_pattern, "[DATE_RANGE_TIME_RANGE]", cleaned)
+
+            # Match für das Datum mit einem Zeitraum ohne Zeitangabe
+            date_range_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*-\s*(\d{2}\.\d{2}\.\d{4})"
+            match = re.findall(date_range_pattern, cleaned)
+            if match:
+                matches["DATE_RANGE"] = match
+            # print("DATE_RANGE matches:", matches["DATE_RANGE"])
+            cleaned = re.sub(date_range_pattern, "[DATE_RANGE]", cleaned)
+
+            # Match für das Datum mit einer Zeitspanne ohne Start- und Enddatum
+            date_time_range_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})\s*-\s*(\d{2}:\d{2})"
+            match = re.findall(date_time_range_pattern, cleaned)
+            if match:
+                matches["DATE_TIME_RANGE"] = match
+            # print("DATE_TIME_RANGE matches:", matches["DATE_TIME_RANGE"])
+            cleaned = re.sub(date_time_range_pattern, "[DATE_TIME_RANGE]", cleaned)
+
+            # Match für eine reine Zeitspanne ohne Datum
+            time_range_pattern = r"(\d{2}:\d{2})\s*-\s*(\d{2}:\d{2})"
+            match = re.findall(time_range_pattern, cleaned)
+            if match:
+                matches["TIME_RANGE"] = match
+            # print("TIME_RANGE matches:", matches["TIME_RANGE"])
+            cleaned = re.sub(time_range_pattern, "[TIME_RANGE]", cleaned)
+
+            # Match für Datum mit Zeitangabe
+            date_time_pattern = r"(\d{2}\.\d{2}\.\d{4})\s*(\d{2}:\d{2})"
+            match = re.findall(date_time_pattern, cleaned)
+            if match:
+                matches["DATE_TIME"] = match
+            # print("DATE_TIME matches:", matches["DATE_TIME"])
+            cleaned = re.sub(date_time_pattern, "[DATE_TIME]", cleaned)
+
+            date_pattern = r"(\d{2}\.\d{2}\.\d{4})"
+            match = re.findall(date_pattern, cleaned)
+            if match:
+                matches["DATE"] = match
+            # print("DATE matches:", matches["DATE"])
+            cleaned = re.sub(date_pattern, "[DATE]", cleaned)
+
+            time_pattern = r"(\d{2}:\d{2})"
+            match = re.findall(time_pattern, cleaned)
+            if match:
+                matches["TIME"] = match
+            # print("TIME matches:", matches["TIME"])
+            cleaned = re.sub(time_pattern, "[TIME]", cleaned)
+
+            event_schedules = []
+
+            # return date_time if only one found
+            if len(matches) == 1:
+                key, value = next(iter(matches.items()))
+
+                event_schedules.append(self.convert_to_schedule(label=key, date_time=value[0]))
+                return event_schedules
+
+            for key, value in matches.items():
+                # print(f"{key}: {value}")
+                schedules = self.classify_date_time(date_times=value, label=key, text=cleaned)
+                if schedules:
+                    event_schedules.extend(schedules)
+
+            if len(event_schedules) == 1:
+                return event_schedules
+
+            # print(event_schedules)
+            unique_schedules = []
+            for i, schedule in enumerate(event_schedules):
+                if any(schedule in other for j, other in enumerate(event_schedules) if
+                       i != j):
+                    continue
+                unique_schedules.append(schedule)
+
+            if len(unique_schedules) == 2:
+                first, second = unique_schedules
+                print("Versuche Schedules zu mergen....", first,second)
+                if any(not e for e in [first.start_date, second.start_date]) and any(not e for e in [first.end_date, second.end_date]) and any(not e for e in [first.start_time, second.start_time]) and any(not e for e in [first.end_time, second.end_time]) and any(not e for e in [first.admittance_time, second.admittance_time]):
+                    merged = Schedule(
+                        start_date=first.start_date or second.start_date,
+                        end_date=first.end_date or second.end_date,
+                        start_time=first.start_time or second.start_time,
+                        end_time=first.end_time or second.end_time,
+                        admittance_time=first.admittance_time or second.admittance_time
+                    )
+                    print("Merged:", merged)
+                    return [merged]
+            return unique_schedules
+
+        except Exception as ex:
+            print(ex)
+
+    def classify_date_time(self, date_times, label, text):
+        # Text anhand des Platzhalters [LABEL] in Segmente teilen
+        segments = text.split(f"[{label}]")
+        tokens = []
+        # print(date_times)
+        date_time_positions = []
+        for i, segment in enumerate(segments):
+            tokens.extend(segment.split())  # Segment als Token hinzufügen
+            if i < len(date_times):  # Falls noch Date-Times übrig sind
+                tokens.append(placeholder.get(label, "ERROR"))  # Date-Time als eigenes Token einfügen
+                date_time_positions.append(len(tokens) - 1)
+
+
+        # sliding window classification
+        window_size = 5
+        event_date_total = 0
+        other_total = 0
+
+        schedules = []
+        for i, date_time in enumerate(date_times):
+            # Berechne den Start-Index für das Fenster
+            start = max(0, date_time_positions[i] - (window_size - 1))
+
+            # Führe Klassifikation für jedes Fenster durch
+            while start + window_size <= len(tokens):  # Solange das Fenster in den Tokens bleibt
+                window = tokens[start:start + window_size]
+                # print(window)
+
+                # Klassifikation durchführen
+                if label == "TIME":
+                    time_class = self.time_classifier(" ".join(window))
+                    # print(time_class)
+                else:
+                    date_class = self.date_classifier(" ".join(window))
+                    # print(date_class)
+
+                    # Aufaddieren der Werte
+                    event_date_total += date_class.get('EVENT_DATE', 0)
+                    other_total += date_class.get('OTHER', 0)
+
+                # Fenster verschieben
+                start += 1
+
+            # Rückgabe der Gesamtsummen
+            if label == "TIME":
+                pass
+            else:
+                # print("Gesamtsumme EVENT_DATE:", event_date_total)
+                # print("Gesamtsumme OTHER:", other_total)
+                if event_date_total > other_total:
+                    schedule = self.convert_to_schedule(date_time, label)
+                    schedules.append(schedule)
+                    # print(date_time)
+                    # print("EVENT DATE: ", schedule)
+        return schedules
+
+    def convert_to_schedule(self,date_time, label):
+        try:
+            if label == "DATE_RANGE_TIME_RANGE":
+                return Schedule(
+                    start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                    end_date=datetime.strptime(date_time[2], "%d.%m.%Y").date(),
+                    start_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                    end_time=datetime.strptime(date_time[3], "%H:%M").time(),
+                    admittance_time=None
+                )
+
+            if label == "DATE_RANGE":
+                return Schedule(
+                    start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                    end_date=datetime.strptime(date_time[1], "%d.%m.%Y").date(),
+                    start_time=None,
+                    end_time=None,
+                    admittance_time=None
+                )
+
+            if label == "DATE_TIME_RANGE":
+                return Schedule(
+                    start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                    end_date=None,
+                    start_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                    end_time=datetime.strptime(date_time[2], "%H:%M").time(),
+                    admittance_time=None
+                )
+
+            if label == "TIME_RANGE":
+                return Schedule(
+                    start_date=None,
+                    end_date=None,
+                    start_time=datetime.strptime(date_time[0], "%H:%M").time(),
+                    end_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                    admittance_time=None
+                )
+
+            if label == "DATE_TIME":
+                return Schedule(
+                    start_date=datetime.strptime(date_time[0], "%d.%m.%Y").date(),
+                    end_date=None,
+                    start_time=datetime.strptime(date_time[1], "%H:%M").time(),
+                    end_time=None,
+                    admittance_time=None
+                )
+
+            if label == "DATE":
+                return Schedule(
+                    start_date=datetime.strptime(date_time, "%d.%m.%Y").date(),
+                    end_date=None,
+                    start_time=None,
+                    end_time=None,
+                    admittance_time=None
+                )
+
+            if label == "TIME":
+                return Schedule(
+                    start_date=None,
+                    end_date=None,
+                    start_time=datetime.strptime(date_time, "%H:%M").time(),
+                    end_time=None,
+                    admittance_time=None
+                )
+        except Exception as e:
+            print(e)
+            return None
+
+placeholder = {
+    "DATE_RANGE_TIME_RANGE": "[DATE] [TIME] - [DATE] [TIME]",
+    "DATE_RANGE": "[DATE] - [DATE]",
+    "DATE_TIME_RANGE": "[DATE] [TIME] - [TIME]",
+    "TIME_RANGE": "[TIME] - [TIME]",
+    "DATE_TIME": "[DATE] [TIME]",
+    "DATE": "[DATE]",
+    "TIME": "[TIME]"
+}
+
+
+

src/nlp/playground/pipelines/event_data_extractor.py

@@ -3,6 +3,7 @@ import re
 from src.nlp.playground.ner import GlinerHandler
 from src.nlp.playground.pipelines.address_extractor import AddressExtractor
 from src.nlp.playground.pipelines.date_extractor_v2 import ScheduleExtractor
+from src.nlp.playground.pipelines.date_extractor_v3 import ScheduleExtractorV3
 from src.nlp.playground.pipelines.description_extractor import DescriptionExtractor
 from src.nlp.playground.pipelines.title_extractor import TitleExtractor
 from src.nlp.playground.textclassification import ZeroShotClassifier, CustomMode
@@ -14,7 +15,8 @@ class EventDataExtractor:
         self.title_extractor = TitleExtractor()
         self.zero_shot_classifier = ZeroShotClassifier()
         self.gliner_handler = GlinerHandler()
-        self.schedule_extractor = ScheduleExtractor()
+        # self.schedule_extractor = ScheduleExtractor()
+        self.schedule_extractor = ScheduleExtractorV3()
         self.address_extractor = AddressExtractor()
         self.description_extractor = DescriptionExtractor()
 
@@ -26,8 +28,9 @@ class EventDataExtractor:
         event.locations = self.extract_locations(data)
         event.organizers = self.extract_organizers(data)
         event.address = self.extract_address(data)
-        event.schedule =  self.extract_schedule(data)
+        event.schedule = self.extract_schedule(data)
         event.description = self.extract_description(data, event.title)
+        event.prices = self.extract_prices(data)
 
         print("Extraction process completed.")
         return event
@@ -96,9 +99,9 @@ class EventDataExtractor:
 
     def extract_prices(self, data):
         print("Extracting prices...")
-        entities = self.gliner_handler.extract_entities(data, ["PRICE"])
-
-        filtered_entities = [e["text"] for e in entities if e["text"] and re.search(r'\d', e["text"])]
+        entities = self.gliner_handler.extract_entities(data, ["Eintrittspreis der Veranstaltung"])
+        print(entities)
+        filtered_entities = [e["text"] for e in entities if e["text"] and re.search(r'\d', e["text"]) and e["score"]>=0.4]
 
         prices = [re.findall(r'\d+(?:[.,]\d+)?', price) for price in filtered_entities]
 
@@ -110,7 +113,7 @@ class EventDataExtractor:
         ))[0].label
 
         if entrance_free_category == "Eintritt frei" and not prices:
-            return ["Eintritt frei"]
+            return ["kostenlos"]
 
         return prices
 

src/nlp/playground/pipelines/testing/event_data_extractor_testing.py

@@ -1,14 +1,13 @@
 import csv
 import gc
 import time
-from collections import defaultdict
 
 import pandas as pd
 
 from src.nlp.experimental.textclassification.classify_title import train_data
 from src.nlp.playground.pipelines.event_data_extractor import EventDataExtractor
 from src.persistence.db import init_db
-from src.utils.Event import Event, DateTime
+from src.utils.Event import Event, Schedule
 from src.utils.apis.googlemaps_api import GoogleMapsAPI
 from src.utils.helpers import normalize_data
 import matplotlib.pyplot as plt
@@ -25,7 +24,7 @@ def init_db_entries():
         if all(f not in el.get("markdown", "") for f in filter_data):
             filtered_elements.append(el)
     print(f"{len(filtered_elements)} Testdatensätze in der Datenbank")
-    return filtered_elements
+    return filtered_elements[20]
 
 def event_similarity(actual, predicted):
     # Liste der Attribute, die verglichen werden
@@ -75,11 +74,11 @@ for el in elements:
     actual_event.url = el.get("url")
     print(actual_event.url)
     actual_event.title = el.get("information", {}).get("actual", {}).get("title", "")
-    actual_event.organizers = [org for org in el.get("information", {}).get("actual", {}).get("organizers", []) if
+    actual_event.organizers = [org.strip() for org in el.get("information", {}).get("actual", {}).get("organizers", []) if
                                org.strip()]
     actual_event.categories = el.get("information", {}).get("actual", {}).get("categories", [])
     actual_event.locations = [
-        loc for loc in el.get("information", {}).get("actual", {}).get("locations", []) if loc
+        loc.strip() for loc in el.get("information", {}).get("actual", {}).get("locations", []) if loc.strip()
     ]
     actual_event.prices = el.get("information", {}).get("actual", {}).get("prices", [])
     address  = el.get("information", {}).get("actual", {}).get("address")
@@ -91,7 +90,7 @@ for el in elements:
 
     dates = el.get("information", {}).get("actual", {}).get("dates", [])
     actual_event.schedule = [
-        DateTime(date.get("start_date", None), date.get("end_date", None), date.get("start_time", None),
+        Schedule(date.get("start_date", None), date.get("end_date", None), date.get("start_time", None),
                  date.get("end_time", None), date.get("admittance_time", None))
         for date in dates]
 
@@ -162,18 +161,29 @@ field_sums = {
     "address": df["address"].sum(),
     "organizers": df["organizers"].sum(),
 }
-print(df['extraction_time'])
-print(len(df))
+
+total_events = len(df)  # Gesamtanzahl der Events
+percentages = {key: (value / total_events) * 100 for key, value in field_sums.items()}  # Berechne Prozentwerte
 
 # 📊 Graphen erstellen
-plt.figure(figsize=(10, 5))
-plt.bar(field_sums.keys(), field_sums.values(), color=["blue", "orange", "green", "red", "purple"])
+plt.figure(figsize=(10, 6))
+bars = plt.bar(field_sums.keys(), field_sums.values(), color=["blue", "orange", "green", "red", "purple"])
+
+# Prozentwerte unter den Balken hinzufügen
+for bar, (key, percent) in zip(bars, percentages.items()):
+    plt.text(bar.get_x() + bar.get_width() / 2, -0.05 * total_events, f"{percent:.1f}%",
+             ha="center", va="top", fontsize=10, color="black")
 
 # 🏷️ Achsenbeschriftungen & Titel
 plt.xlabel("Event Attribute")
 plt.ylabel("Anzahl der Übereinstimmungen")
-plt.title(f"Summierte Übereinstimmungen pro Event-Attribut. Durchschittliche Verarbeitungszeit: {float(df['extraction_time'].sum()) / len(df)}")
-plt.ylim(0, len(df))  # Maximale Höhe entspricht der Anzahl der Events
+plt.title(f"Summierte Übereinstimmungen pro Event-Attribut")
+
+# 📝 Info-Box mit Anzahl der Events
+info_text = f"Getestete Events: {total_events}\nDurchschnittliche Verarbeitungszeit: {float(df['extraction_time'].sum()) / total_events:.2f}s"
+plt.text(0.5, total_events * 1.05, info_text, fontsize=12, ha="center", bbox=dict(facecolor="white", alpha=0.8))
+
+plt.ylim(0, total_events * 1.2)  # Maximale Höhe etwas erhöhen für bessere Lesbarkeit
 plt.grid(axis="y", linestyle="--", alpha=0.7)
 
 # 📈 Zeige den Graphen

src/nlp/playground/pipelines/testing/price_extractor_testing.py

@@ -6,9 +6,34 @@ from src.nlp.playground.textclassification import CustomMode, ZeroShotClassifier
 
 event_extractor = EventDataExtractor()
 
+texts = [
+    """Kunst- und Handwerksmesse 2025
+
+    📅 Datum: 12. – 14. September 2025
+    📍 Ort: Messehalle Frankfurt, Deutschland
+    
+    Die Kunst- und Handwerksmesse 2025 bringt talentierte Kunsthandwerker, Designer und kreative Köpfe aus ganz Europa zusammen. Besucher können sich auf eine Vielzahl handgefertigter Produkte freuen – von Keramik und Schmuck bis hin zu maßgeschneiderten Möbeln und Mode.
+    
+    🔹 Highlights der Messe:
+    
+    Live-Werkstätten: Erleben Sie, wie Meister ihres Fachs Kunstwerke aus Glas, Holz und Metall fertigen.
+    
+    Gourmetbereich: Probieren Sie handgemachte Schokoladenkreationen (ab 4,50 € pro Stück) und exklusive Bio-Kaffeesorten (250g-Packung für 12,99 €).
+    
+    Workshops: Nehmen Sie an einem Kalligraphie-Kurs teil (Materialkostenpauschale 15 €) oder gestalten Sie Ihre eigene Tonvase (30 € inkl. Brennkosten).
+    
+    Antiquitätenmarkt: Entdecken Sie einzigartige Sammlerstücke wie historische Postkarten (ab 3 € pro Stück) oder antike Silberlöffel (Preis je nach Gewicht und Reinheitsgrad).
+    
+    Rahmenservice: Lassen Sie Ihr vor Ort erworbenes Kunstwerk direkt rahmen (ab 25 € je nach Größe und Material).
+    
+    Ein Muss für alle, die handgefertigte Unikate und künstlerische Inspiration lieben!
+    
+    Die Teilnahme an der Messe kostet 20 € für Erwachsene, 10 € für Kinder. 
+    
+    """,
+
 
 
-texts = [
     "Das ist unser Museum! – Kinderstudio\nSo 15.12. 14:00\nZurück zur Übersicht\n\nTickets kaufen\nAngebot für: Kinder & Familien\nKosten: Kostenlos\nDauer: 1,5 Stunden\nKontakt: [email protected]\nWir treffen uns im Farblabor und entdecken zusammen spielerisch Farben und Formen. Mit unseren Lieblingsfarben ausgerüstet geht es dann ab in die Ausstellung – wer findet das erste Grashüpfergrün? Können wir zusammen ein Dreieck sein? Und hast du schon den Wal gefunden?\nDas Vermittlungsprogramm für Kinder und Familien setzt regelmäßig einen anderen Schwerpunkt. Das Thema ist momentan: Farbe ist alles!\nWeitere Veranstaltungen\n[\nSo 15.12. 14:00\nmre Blickwinkel – Rundgang Architektur](https://www.museum-re.de/de/besuch/veranstaltungskalender/mre-blickwinkel-rundgang-architektur-748/)\n[\nSo 15.12. 14:01\nmre Blickwinkel – Rundgang Architektur](https://www.museum-re.de/de/besuch/veranstaltungskalender/mre-blickwinkel-rundgang-architektur-6098/)\n[\nSo 15.12. 15:00\nRundgang Sonderausstellung – Fumihiko Maki und Maki & Associates](https://www.museum-re.de/de/besuch/veranstaltungskalender/rundgang-sonderausstellung-fumihiko-maki-und-maki-associates-3644/)\n[\nSo 15.12. 15:00\nFarbe ist alles! Rundgang Sammlung](https://www.museum-re.de/de/besuch/veranstaltungskalender/farbe-ist-alles-rundgang-sammlung-697/)\n[\nSo 15.12. 15:01\nFarbe ist alles! Rundgang Sammlung](https://www.museum-re.de/de/besuch/veranstaltungskalender/farbe-ist-alles-rundgang-sammlung-5110/)\n[\nMi 18.12. 18:00\nmre Tiefenrausch](https://www.museum-re.de/de/besuch/veranstaltungskalender/mre-tiefenrausch-6116/)",
     "Rundgang Sonderausstellung – Fumihiko Maki und Maki & Associates\nSo 15.12. 15:00\nTickets kaufen\nAngebot für: Erwachsene\nKosten: 5 €\nDauer: 1 Stunde\nKontakt: [email protected]\nLassen sie uns den „Zuckerwürfel“ einmal genauer unter die Lupe nehmen. Wir gehen auf gemeinsame Entdeckungsreise durch das Museumsgebäude und erfahren nebenbei Spannendes über die Entstehung des mre.\nEntdecken Sie Konzepte und Gestaltungselemente der Architektur von Fumihiko Maki in unserer Sonderausstellung Fumihiko Maki und Maki & Associates – Für eine menschliche Architektur und finden Sie sie am Gebäude des mre wieder.",
     "Geeignet für Kinder ab etwa 5 Jahren.\n(F) = Familienveranstaltung\nEintrittspreise:\nErwachsene: 6,50 Euro\nKinder/Ermäßigt: 3,50 Euro\nSchulvorführungen:\n2,50 Euro pro Person\nKartenreservierung:\nTelefon: 0541 323-7000\[email protected]\nBitte reservieren Sie Ihre Tickets vor Ihrem Besuch per Telefon, E-Mail oder direkt hier online. Viele Vorstellungen sind frühzeitig ausgebucht. Reservierte Karten müssen bis spätestens 15 Minuten vor Veranstaltungsbeginn an der Kasse abgeholt werden.\nTermine & Tickets:\n01.04.2025 - 14:30 Tickets",
@@ -35,5 +60,8 @@ texts = [
 
 ]
 
-for text in  texts:
-    print(event_extractor.extract_prices(text))
+for text in texts:
+    print(text)
+    print("*" * 100)
+    print("Preise: ", event_extractor.extract_prices(text))
+    print("*" * 100)

src/utils/Event.py

@@ -3,7 +3,7 @@ from textwrap import indent
 from sympy import false
 
 
-class DateTime:
+class Schedule:
     def __init__(self, start_date, end_date, start_time, end_time, admittance_time):
         self.start_date = start_date
         self.end_date = end_date
@@ -33,24 +33,41 @@ class DateTime:
 
     def __eq__(self, other):
 
-        if isinstance(other, DateTime):
-            return str(self) == str(other)
+        if isinstance(other, Schedule):
+            print("Equals?", str(self),str(other), str(self) == str(other))
+            return str(self).strip() == str(other).strip()
         else:
             return False
 
     def __hash__(self):
         return hash((self.start_date, self.end_date, self.start_time, self.end_time, self.admittance_time))
 
+    def __len__(self):
+        return len([element for element in [self.start_date, self.end_date, self.start_time, self.end_time, self.admittance_time] if element])
 
-class Address:
-    def __init__(self, street, house_number, postal_code, city):
-        self.street = street
-        self.house_number = house_number
-        self.postal_code = postal_code
-        self.city = city
-
-    def __str__(self):
-        return f"🏠 {self.street if self.street else ''} {self.house_number if self.house_number else ''}, {self.postal_code if self.postal_code else ''} {self.city if self.city else ''}"
+    def __contains__(self,item):
+        if not isinstance(item, Schedule):  # Falls other kein Schedule-Objekt ist, direkt False
+            return False
+        print("SELF: ", self)
+        print("ITEM: ", item)
+        return (
+                (self.start_date == item.start_date or self.start_date == item.end_date or item.start_date is None) and
+                (self.end_date == item.end_date or self.end_date == item.start_date or item.end_date is None) and
+                (self.start_time == item.start_time or self.start_time == self.end_time or item.start_time is None) and
+                (self.end_time == item.end_time or self.end_time == self.start_time or item.end_time is None) and
+                (self.admittance_time == item.admittance_time or item.admittance_time is None)
+        )
+
+
+# class Address:
+#     def __init__(self, street, house_number, postal_code, city):
+#         self.street = street
+#         self.house_number = house_number
+#         self.postal_code = postal_code
+#         self.city = city
+#
+#     def __str__(self):
+#         return f"🏠 {self.street if self.street else ''} {self.house_number if self.house_number else ''}, {self.postal_code if self.postal_code else ''} {self.city if self.city else ''}"
 
 
 class Event:
@@ -95,13 +112,13 @@ class Event:
 
     @schedule.setter
     def schedule(self, value):
-        if isinstance(value, list) and all(isinstance(entry, DateTime) for entry in value):
+        if isinstance(value, list) and all(isinstance(entry, Schedule) for entry in value):
             self._schedule = value
         else:
             raise ValueError("Schedule must be a list of Schedule objects")
 
     def add_schedule_entry(self, start_date, end_date, start_time, end_time, admittance_time):
-        self._schedule.append(DateTime(start_date, end_date, start_time, end_time, admittance_time))
+        self._schedule.append(Schedule(start_date, end_date, start_time, end_time, admittance_time))
 
     @property
     def address(self):

src/utils/helpers.py

@@ -2,6 +2,7 @@ from bs4 import BeautifulSoup, Comment
 import re
 from dateparser import DateDataParser
 
+
 def normalize_data(input):
     def normalize_dates(input_text):
         days = r"(?:Montag|Dienstag|Mittwoch|Donnerstag|Freitag|Samstag|Sonntag|Mo|Di|Mi|Do|Fr|Sa|So)"
@@ -12,7 +13,6 @@ def normalize_data(input):
         iso_pattern = r"(?:\d{2,4}[./-]\d{1,2}[./-]\d{1,2})"
         german_date_pattern = day_month_year_pattern + "|" + dd_mm_yyyy_pattern + "|" + iso_pattern
 
-
         compiled_pattern = re.compile(german_date_pattern, re.VERBOSE)
 
         matches = compiled_pattern.findall(input_text)
@@ -31,20 +31,20 @@ def normalize_data(input):
                 print(f"Fehler bei der Verarbeitung von '{match}': {e}")
 
         # Ersetze alle Vorkommen von '20.03. und 21.03.2025' durch '20.03.2025 und 21.03.2025'
-        german_date_pattern = r"(?<!\.)(\d{2})\.(\d{2})\.?\s*(und|\+|&|bis|bis zum|-)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})"
-        input_text = re.sub(german_date_pattern, r" \1.\2.\6 \3 \4.\5.\6 ", input_text)
+        german_date_pattern = r"(?<!\.)(\d{2})\.(\d{2})\.(\s*\d{2}:\d{2})?\s*(und|\+|&|bis|bis zum|-|—|–)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})(\s*\d{2}:\d{2})?"
+        input_text = re.sub(german_date_pattern, r" \1.\2.\7 \4 \5.\6.\7 ", input_text)
 
         # Ersetze alle Vorkommen von '20. und 21.03.2025' durch '20.03.2025 und 21.03.2025'
-        german_date_pattern = r"(?<!\d)(\d{2})\.?\s*(und|\+|&|bis|bis zum|-)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})"
-        input_text = re.sub(german_date_pattern, r" \1.\4.\5 \2 \3.\4.\5 ", input_text)
+        german_date_pattern = r"(?<!\d)(\d{2})\.(\s*\d{2}:\d{2})?\s*(und|\+|&|bis|bis zum|-|—|–)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})(\s*\d{2}:\d{2})?"
+        input_text = re.sub(german_date_pattern, r" \1.\5.\6 \2 \3 \4.\5.\6 \7 ", input_text)
 
         # Ersetze alle Vorkommen von '20.03.2025 bis/bis zum 21.03.2025' durch '20.03.2025 - 21.03.2025'
-        german_date_pattern = r"(\d{1,2})\.(\d{1,2})\.(\d{4})\.?\s*(bis|bis zum|-)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})"
-        input_text = re.sub(german_date_pattern, r" \1.\2.\3 - \5.\6.\7 ", input_text)
+        german_date_pattern = r"(\d{1,2})\.(\d{1,2})\.(\d{4})(\s*\d{2}:\d{2})?\s*(bis|bis zum|-|—|–)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})(\s*\d{2}:\d{2})?"
+        input_text = re.sub(german_date_pattern, r" \1.\2.\3 \4 - \6.\7.\8 \9 ", input_text)
 
         # Ersetze alle Vorkommen von '20.03.2025 und/& 21.03.2025' durch '20.03.2025 + 21.03.2025'
-        german_date_pattern = r"(\d{1,2})\.(\d{1,2})\.(\d{4})\.?\s*(und|\+|&)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})"
-        input_text = re.sub(german_date_pattern, r" \1.\2.\3 + \5.\6.\7 ", input_text)
+        german_date_pattern = r"(\d{1,2})\.(\d{1,2})\.(\d{4})\.?(\s*\d{2}:\d{2})?\s*(und|\+|&)\s*(\d{1,2})\.(\d{1,2})\.(\d{4})(\s*\d{2}:\d{2})?"
+        input_text = re.sub(german_date_pattern, r" \1.\2.\3 \4 + \6.\7.\8 \9 ", input_text)
         return input_text
 
     def normalize_times(input_text):
@@ -95,7 +95,8 @@ def normalize_data(input):
     normalized_data = normalize_text(normalized_data)
     return normalized_data
 
-def clean_html(html:str):
+
+def clean_html(html: str):
     soup = BeautifulSoup(html, "lxml")
     body_content = soup.body
 
@@ -117,31 +118,35 @@ def clean_html(html:str):
         cleaned_html = "\n".join(clean_html_lines)
         return cleaned_html
 
-def strip_html_to_text(html:str):
-    soup = BeautifulSoup(html,"lxml")
+
+def strip_html_to_text(html: str):
+    soup = BeautifulSoup(html, "lxml")
     return soup.get_text(separator=' ', strip=True)
 
 
-# text = """
-#     Die 18. Koblenzer Literaturtage „ganzOhr“ finden vom 22.03. bis 05.04.2025 statt. Das Programm wird im Januar 2025 veröffentlicht, der Vorverkauf startet im Februar.
-#     15. November 2024 & 13. Dezember 2024: Kunstausstellung "Der erweiterte Raum,"
-#     Der siebte Workshop Retrodigitalisierung findet am 20. und 21.03.2025 bei ZB MED –.
-#     2. März bis 21. März 2025 \n
-#     **Wann?** 05.12.2024, 19:00-21:00 **Wo?** Lesesaal im Marstallgebäude, TIB
-#     """
-#
-# expected = """
-#     Die 18. Koblenzer Literaturtage „ganzOhr“ finden vom 22.03.2025 - 05.04.2025 statt. Das Programm wird im Januar 2025 veröffentlicht, der Vorverkauf startet im Februar.
-#     15.11.2024 + 13.12.2024: Kunstausstellung "Der erweiterte Raum,"
-#     Der siebte Workshop Retrodigitalisierung findet am 20.03.2025 + 21.03.2025 bei ZB MED –.
-#     02.03.2025 - 21.03.2025 \n
-#     **Wann?** 05.12.2024, 19:00-21:00 **Wo?** Lesesaal im Marstallgebäude, TIB
-#     """
-#
+# texts = [
+#     "Die 18. Koblenzer Literaturtage „ganzOhr“ finden vom 22.03. bis 05.04.2025 statt. Das Programm wird im Januar 2025 veröffentlicht, der Vorverkauf startet im Februar.",
+#     "15. November 2024 & 13. Dezember 2024: Kunstausstellung 'Der erweiterte Raum'",
+#     "Der siebte Workshop Retrodigitalisierung findet am 20. und 21.03.2025 bei ZB MED.",
+#     "2. März bis 21. März 2025 \n"
+#     "**Wann?** 05.12.2024, 19:00-21:00 **Wo?** Lesesaal im Marstallgebäude, TIB",
+#     "22.04.25 15 Uhr bis 23.04.25 16 Uhr."
+# ]
 #
-# normalized = normalize_data(text)
+# expected_texts = [
+#     "Die 18. Koblenzer Literaturtage „ganzOhr“ finden vom 22.03.2025 - 05.04.2025 statt. Das Programm wird im Januar 2025 veröffentlicht, der Vorverkauf startet im Februar.",
+#     "15.11.2024 + 13.12.2024: Kunstausstellung 'Der erweiterte Raum'",
+#     "Der siebte Workshop Retrodigitalisierung findet am 20.03.2025 + 21.03.2025 bei ZB MED.",
+#     "02.03.2025 - 21.03.2025 \n",
+#     "**Wann?** 05.12.2024, 19:00-21:00 **Wo?** Lesesaal im Marstallgebäude, TIB",
+#     "22.04.2025 15:00 - 23.04.2025 16:00."
+#     ]
 #
-# if normalized == expected:
-#     print("Normalization successful!")
-# else:
-#     print("Normalization failed!")
+# for i, text in enumerate(texts):
+#     normalized = normalize_data(text)
+#     normalized = re.sub("\s*", " ",normalized)
+#     expected = re.sub("\s*", " ",expected_texts[i])
+#     if normalized == expected:
+#         print("Normalization successful!")
+#     else:
+#         print("Normalization failed!")