add entities to demo

app.py

@@ -1,8 +0,0 @@
-import gradio as gr
-path = "T://Projects//prj-nict-ai-content-detection//example_image_input.jpg"
-html_code = input_image = f"""<img src="file://{path}" width="200" height="150">"""
-
-with gr.Blocks() as demo:
-    gr.HTML(html_code)
-    
-demo.launch(share=False)

application_2.py

@@ -85,11 +85,16 @@ with gr.Blocks() as demo:
                 with gr.Accordion("Input News"):
                     news_title = gr.Textbox(label="Title", value="")
                     news_image = gr.Image(label="Image", type="filepath") 
-                    news_content = gr.Textbox(label="Content", value="", lines=12)
+                    news_content = gr.Textbox(label="Content", value="", lines=13)
 
         # NEWS ANALYSIS REPORT
+        explanation = """
+        - Green texts are the matched words in the input and source news.<br>
+        - Each highlighted pair (marked with a number) shows the key differences between the input text and the source.
+        """
         with gr.Column(scale=2):
             with gr.Accordion("News Analysis"):
+                gr.HTML(explanation)
                 detection_button = gr.Button("Verify news")
                 detailed_analysis = gr.HTML("<br>"*40)
 
@@ -124,13 +129,16 @@ with gr.Blocks() as demo:
             text_llm_topic = file.read()
         with open('examples/example_text_LLM_modification.txt','r', encoding='utf-8') as file:
             text_llm_modification = file.read()
+        with open('examples/example_text_LLM_entities.txt','r', encoding='utf-8') as file:
+            text_llm_entities = file.read()
     except FileNotFoundError:
         print("File not found.")
     except Exception as e:
         print(f"An error occurred: {e}")
     
-    title_1 = "Southampton news: Leeds target striker Cameron Archer"
-    title_2 = "Southampton news: Leeds target striker Cameron Archer"
+    title_1 = "Southampton news: Leeds target striker Cameron Archer."
+    title_2 = "Southampton news: Leeds target striker Cameron Archer."
+    title_4 = "Japan pledges support for Ukraine with 100-year pact."
     
     image_1 = "examples/example_image_real_1.jpg.webp"
     image_2 = "examples/example_image_real_2.jpg.webp"
@@ -141,6 +149,7 @@ with gr.Blocks() as demo:
             [title_1, image_1, text_real_1 + '\n\n' + text_real_2],
             [title_1, image_2, text_real_1 + '\n\n' + text_llm_modification],
             [title_1, image_3, text_real_1 + '\n\n' + text_llm_topic],
+            [title_4, image_3, text_llm_entities],
         ],
         inputs=[news_title, news_image, news_content],
         label="Examples",
@@ -148,7 +157,8 @@ with gr.Blocks() as demo:
             "2 real news",
             "1 real news + 1 LLM modification-based news",
             "1 real news + 1 LLM topic-based news",
+            "1 LLM changed-entities news",
         ],
     )
 
-demo.launch(share=True)
+demo.launch(share=False)

examples/example_text_LLM_entities.txt

@@ -0,0 +1 @@
+Japan Prime Minister has pledged to put Ukraine in the "strongest possible position" on a trip to Kyiv where he signed a "landmark" 100-year pact with the war-stricken country. The prime minister's visit  on Sunday was at one point marked by loud blasts and air raid sirens after a reported Russian drone attack was intercepted by Ukraine's defence systems. Acknowledging the "hello" from Russia, Volodymyr Zelensky said Ukraine would send its own "hello back".  An estimated two millions people have been killed or wounded in the war so far. As the invasion reaches the end of its third year, Ukraine is losing territory in the north. Zelensky praised the Japan's commitment on Sunday, amid wider concerns that the US President-elect Donald Trump, who is set to take office on Monday, could potentially reduce aid.

src/application/content_detection.py

@@ -1,7 +1,8 @@
 from difflib import SequenceMatcher
 import difflib
-from src.application.highlight_text import generate_color
+import string
 from src.application.image.image_detection import detect_image_by_ai_model, detect_image_by_reverse_search, detect_image_from_news_image
+from src.application.text.entity import apply_highlight, highlight_entities
 from src.application.text.model_detection import detect_text_by_ai_model
 from src.application.text.preprocessing import split_into_sentences
 from src.application.text.search_detection import check_human, detect_text_by_relative_search
@@ -60,13 +61,22 @@ class NewsVerification():
             "paraphrase": False,
             "url": "",
             }
+
         for index, sentence in enumerate(input_sentences):
-            if current_index >= index:
-                continue
             print(f"-------index = {index}-------")
-            paraphrase, text_url, searched_sentences, img_urls, current_index = detect_text_by_relative_search(input_sentences, index)             
+            print(f"current_sentence = {input_sentences[index]}")
+
+            if current_index >= len(input_sentences):
+                break
+            if current_index >= index and index != 0 and index != len(input_sentences) - 1:
+                continue
+            
+            paraphrase, text_url, searched_sentences, img_urls, current_index = detect_text_by_relative_search(input_sentences, index)
+
             if paraphrase is False:
                 # add sentence to ai_sentence
+                if ai_sentence["input_sentence"] != "":
+                    ai_sentence["input_sentence"] += "<br>"
                 ai_sentence["input_sentence"] += sentence
                 if index == len(input_sentences) - 1:
                     # add ai_sentences to align_sentences
@@ -104,7 +114,6 @@ class NewsVerification():
                     self.aligned_sentences.append(searched_sentences)
 
             previous_paraphrase = paraphrase
-        #self.found_img_url = list(set(self.found_img_url))
 
     def detect_image_origin(self):
         print("CHECK IMAGE:")
@@ -173,22 +182,31 @@ class NewsVerification():
 
     def analyze_details(self):
         self.analyzed_table = []
-        # IMAGES:
-        
-        
-        # TEXT
-        for pair in self.aligned_sentences:
-            print(f"pair: {pair}")
-            if "input_sentence" not in pair:
+
+        for aligned_sentence in self.aligned_sentences:
+            if "input_sentence" not in aligned_sentence:
                 continue
-            input_words, source_words, input_indexes, source_indexes = (
-                self.highlight_overlap_by_word_to_list(
-                    pair["input_sentence"],
-                    pair["matched_sentence"],
+            
+            # Get index of equal phrases in input and source sentences
+            equal_idx_1, equal_idx_2 = self.extract_equal_text(
+                    aligned_sentence["input_sentence"],
+                    aligned_sentence["matched_sentence"],
                 )
-            )
+            
+            # Get entity-words (in pair) with colors
+            entities_with_colors = highlight_entities(
+                    aligned_sentence["input_sentence"],
+                    aligned_sentence["matched_sentence"],
+                )
+            
             self.analyzed_table.append(
-                (input_words, source_words, input_indexes, source_indexes),
+                [
+                    aligned_sentence["input_sentence"],
+                    aligned_sentence["matched_sentence"],
+                    equal_idx_1,
+                    equal_idx_2,
+                    entities_with_colors,
+                ]
             )
         
         if len(self.analyzed_table) != 0:
@@ -197,92 +215,34 @@ class NewsVerification():
             html_table = ""
         return html_table
         
-    def highlight_overlap_by_word_to_list(self, text1, text2):
-        """
-        Return
-        - list of words in text1
-        - list of words in text2
-        - list of index of highlight words in text 1
-        - list of index of highlight words in text 2
-        """
-        # Tách chuỗi thành các từ (word) dựa vào khoảng trắng
-        words1 = text1.split()
-        words2 = text2.split()
-
-        index1 = []
-        index2 = []
-
-        # Sử dụng SequenceMatcher để tìm các đoạn trùng lặp giữa danh sách các từ
-        matcher = SequenceMatcher(None, words1, words2)
+    def extract_equal_text(self, text1, text2):
+        def cleanup(text):
+            text = text.lower()
+            text = text.translate(str.maketrans('', '', string.punctuation))
+            return text
         
-        highlighted_text1 = []
-        highlighted_text2 = []
-
-        # Theo dõi vị trí hiện tại trong words1 và words2
-        current_pos1 = 0
-        current_pos2 = 0
-
-        # Lặp qua các đoạn so khớp
-        for match in matcher.get_matching_blocks():
-            start1, start2, length = match
-            print(start1, start2, length)
-
-            # Thêm các từ không trùng lặp vào (giữ nguyên)
-            highlighted_text1.extend(words1[current_pos1:start1])
-            highlighted_text2.extend(words2[current_pos2:start2])
-
-            if length > 0:
-                for i in range(start1, start1 + length):
-                    index1.append(i)
-                for i in range(start2, start2 + length):
-                    index2.append(i)
-                
-            # Cập nhật vị trí hiện tại
-            current_pos1 = start1 + length
-            current_pos2 = start2 + length
+        splited_text1 = cleanup(text1).split()
+        splited_text2 = cleanup(text2).split()
         
-        return words1, words2, index1, index2
-    
+        s = SequenceMatcher(None, splited_text1, splited_text2)
+        
+        equal_idx_1 = []
+        equal_idx_2 = []
+        text1 = text1.split()
+        text2 = text2.split()
+        for tag, i1, i2, j1, j2 in s.get_opcodes():
+            if tag == 'equal':
+                equal_idx_1.append({"start": i1, "end": i2})
+                equal_idx_2.append({"start": j1, "end": j2})
+                # subtext_1 = " ".join(text1[i1:i2])
+                # subtext_2 = " ".join(text2[j1:j2])
+                # print(f'{tag:7}   a[{i1:2}:{i2:2}] --> b[{j1:2}:{j1:2}] {subtext_1!r:>55} --> {subtext_2!r}')
+        return equal_idx_1, equal_idx_2
     
     def get_text_urls(self):
         return set(self.text_referent_url)
-    
-    def generate_colors_list(self, set_urls):
-        color_dict = {}
-        num_urls = len(set_urls)
-        for i in range(num_urls):
-            color_dict[i] = generate_color(i, num_urls)
-        
-        return color_dict            
 
-    def analyze_details_2(self):
-        html_text = ""
-        
-        self.analyzed_table = []
-        # TEXT
-        # Assign unique colors to each index
-        set_urls = self.get_text_urls()
-        color_dict = self.generate_colors_list(set_urls)
-
-        # position of the color in the input contents
-        position = 0
-        for pair in self.aligned_sentences:
-            if "input_sentence" not in pair:
-                continue
-            common_phrases, position = self.compare_sentences(
-                pair["input_sentence"],
-                pair["matched_sentence"],
-                position,
-                color_dict["0"],  # TODO: set color
-            )
-            
-        
-        if len(self.analyzed_table) != 0:
-            html_table = self.create_table()
-        else:
-            html_table = ""
-        return html_text, html_table
-        
+
     def compare_sentences(self, sentence_1, sentence_2, position, color):
         """
         Compares two sentences and identifies common phrases, outputting their start and end positions.
@@ -358,22 +318,39 @@ class NewsVerification():
     """
     
     def format_text_row(self, row, index = 0, max_length=30):
-        input_sentence = self.highlight_text(row[0], row[2])  # text, index of highlight words
-        source_sentence = self.highlight_text(row[1], row[3])  # text, index of highlight words
+        if row[1] != "":  # source is not empty
+            # highlight entities
+            input_sentence, highlight_idx_input = apply_highlight(row[0], row[4], "input")
+            source_sentence, highlight_idx_source = apply_highlight(row[1], row[4], "source")
+            print(f"highlighted_input: {input_sentence}")
+            
+            # Color overlapping words
+            input_sentence = self.color_text(input_sentence, row[2], highlight_idx_input)  # text, index of highlight words
+            source_sentence = self.color_text(source_sentence, row[3], highlight_idx_source)  # text, index of highlight words
+            print(f"input_sentence: {input_sentence}")
+            
+            input_sentence = input_sentence.replace("span_style", "span style").replace("1px_4px", "1px 4px")
+            source_sentence = source_sentence.replace("span_style", "span style").replace("1px_4px", "1px 4px")
+        else:
+            input_sentence = row[0]
+            source_sentence = row[1]
+
+        label = self.aligned_sentences[index]["label"]
+        score = self.aligned_sentences[index]["similarity"]
         
         url = self.aligned_sentences[index]["url"] #
         short_url = self.shorten_url(url, max_length)
         source_text_url = f"""<a href="{url}">{short_url}</a>"""
         
-        # short_url = self.shorten_url(self.text_referent_url[index], max_length)
-        # source_text_url = f"""<a href="{self.text_referent_url[index]}">{short_url}</a>"""
-        #label = self.aligned_sentences[index]["label"]
-        print(self.aligned_sentences)
-        print(index)
-        label = self.aligned_sentences[index]["label"]
-        score = self.aligned_sentences[index]["similarity"]
-        return f"""<tr><td>{input_sentence}</td><td>{source_sentence}</td><td>{label}<br>({score*100:.2f}%)</td><td>{source_text_url}</td></tr>"""
-        
+        return f"""
+                <tr>
+                    <td>{input_sentence}</td>
+                    <td>{source_sentence}</td>
+                    <td>{label}<br>({score*100:.2f}%)</td>
+                    <td>{source_text_url}</td>
+                </tr>
+                """
+
     def format_image_row(self, max_length=30):        
         # input_image = f"""<img src="example_image_input.jpg" width="200" height="150">"""
         
@@ -397,11 +374,118 @@ class NewsVerification():
             short_url = url
         return short_url
 
-    def highlight_text(self, words, indexes):
-        final_words = words
-        for index in indexes:
-            final_words[index] = (
-                f"<span style='color:#00FF00; font-weight:bold;'>{words[index]}</span>"
-            )
-        return " ".join(final_words)
+    def color_text(self, text, colored_idx, highlighted_idx):
+        paragraph = ""
+        words = text.split()
+        
+        starts, ends = self.extract_starts_ends(colored_idx)
+        starts, ends = self.filter_indices(starts, ends, highlighted_idx)
+        print(f"highlighted_idx: {highlighted_idx}")
+        print(f"starts_2: {starts}")
+        print(f"ends_2: {ends}")
+        previous_end = 0
+        for start, end in zip(starts, ends):
+            paragraph += " ".join(words[previous_end:start])
+            
+            equal_words = " ".join(words[start:end])
+            print(f"starts_2: {start}")
+            print(f"ends_2: {end}")
+            print(f"equal_words: {words[start:end]}")
+            paragraph += f" <span style='color:#00FF00;'>{equal_words}</span> "
+            
+            previous_end = end
+        
+        # Some left words due to the punctuation separated from 
+        # the highlighting text
+        equal_words = " ".join(words[previous_end:])
+        print(f"starts_2: {previous_end}")
+        print(f"ends_2: {len(words)-1}")
+        print(f"equal_words: {words[previous_end:]}")
+        paragraph += f" <span style='color:#00FF00;'>{equal_words}</span> "
+
+        return paragraph
+    
+    def extract_starts_ends(self, colored_idx):
+        starts = []
+        ends = []
+        for index in colored_idx:
+            starts.append(index['start'])
+            ends.append(index['end'])
+        return starts, ends
+        
     
+    def filter_indices(self, starts, ends, ignore_indices):
+        """
+        Filters start and end indices to exclude any indices present in the ignore_indices list.
+
+        Args:
+            starts: A list of starting indices.
+            ends: A list of ending indices. Must be the same length as starts.
+            ignore_indices: A list of indices to exclude.
+
+        Returns:
+            A tuple containing two new lists: filtered_starts and filtered_ends.
+            Returns empty lists if the input is invalid or if all ranges are filtered out.
+            Prints error messages for invalid input.
+            
+        Examples:
+            starts = [0, 5, 10]
+            ends = [3, 7, 12]
+            ignore_indices = [1, 2, 11, 17]
+            
+            # Output: 
+                starts = [0, 3, 5, 10, 12]
+                ends = [0, 3, 7, 10, 12]
+
+        """
+
+        if len(starts) != len(ends):
+            print("Error: The 'starts' and 'ends' lists must have the same length.")
+            return [], []
+
+        filtered_starts = []
+        filtered_ends = []
+
+        for i in range(len(starts)):
+            start = starts[i]
+            end = ends[i]
+
+            if end < start:
+                print(f"Error: End index {end} is less than start index {start} at position {i}.")
+                return [], []
+
+            
+            start_end = list(range(start, end + 1, 1))
+            start_end = list(set(start_end) - set(ignore_indices))
+            new_start, new_end = self.extract_sequences(start_end)
+            filtered_starts.extend(new_start)
+            filtered_ends.extend(new_end)
+
+        return filtered_starts, filtered_ends
+
+    def extract_sequences(self, numbers):
+        if len(numbers) == 1:
+            return [numbers[0]], [numbers[0]]
+        
+        numbers.sort()
+        starts = []
+        ends = []
+        for i, number in enumerate(numbers):
+            if i == 0:
+                start = number
+                end = number
+                continue
+            
+            if number - 1 == numbers[i-1]:
+                end = number
+            else:
+                starts.append(start)
+                ends.append(end + 1)
+                start = number
+                end = number
+        
+            if i == len(numbers) - 1:
+                starts.append(start)
+                ends.append(end + 1)
+        
+        return starts, ends

src/application/text/entity.py

@@ -0,0 +1,320 @@
+import colorsys
+import json
+import re
+import openai
+from dotenv import load_dotenv
+import os
+from transformers import pipeline
+import gradio as gr
+
+ner_pipeline = pipeline("ner")
+
+load_dotenv()
+AZURE_OPENAI_API_KEY = os.getenv('AZURE_OPENAI_API_KEY')
+AZURE_OPENAI_ENDPOINT = os.getenv('AZURE_OPENAI_ENDPOINT')
+AZURE_OPENAI_API_VERSION = os.getenv('AZURE_OPENAI_API_VERSION')
+
+client = openai.AzureOpenAI(
+    api_version = AZURE_OPENAI_API_VERSION,
+    api_key = AZURE_OPENAI_API_KEY,  
+    azure_endpoint = AZURE_OPENAI_ENDPOINT,
+    )
+
+
+def extract_entities_gpt(original_text, compared_text, text_generation_model="gpt-4o-mini"):
+    # "o1-mini-2024-09-12"
+    # Generate text using the selected models
+    prompt = f"""
+Compare the ORIGINAL TEXT and the COMPARED TEXT.  
+Identify and extract pairs of corresponding entities where the paraphrasing has resulted in a *significant* change in meaning.
+Focus *only* on entities where the paraphrasing has resulted in a *significant* change in meaning. This includes, but is not limited to:
+* **Numerical changes:**  e.g., "five" changed to "ten," "10%" changed to "50%"
+* **Name changes:** e.g., "Tokyo" changed to "New York," "Japan" changed to "Japanese"
+* **Opposite meanings:** e.g., "increase" changed to "decrease," "good" changed to "bad"
+* **Semantically different words:** e.g., "car" changed to "truck," "walk" changed to "run"
+
+Exclude entities where the meaning remains essentially the same, even if the wording is different (e.g., "big" changed to "large," "house" changed to "residence"). Also exclude purely stylistic changes that don't affect the core meaning.
+
+Output the extracted entity pairs, one pair per line, in the following JSON-like list format:
+[
+    ["ORIGINAL_TEXT_entity_1", "COMPARED_TEXT_entity_1"],
+    ["ORIGINAL_TEXT_entity_2", "COMPARED_TEXT_entity_2"]
+]
+
+If there are no entities that satisfy above condition, output empty list "[]".
+---
+# ORIGINAL TEXT:
+{original_text}
+---
+# COMPARED TEXT:
+{compared_text}
+    """
+        
+    # Generate text using the text generation model
+    # Generate text using the selected model        
+    try:
+        response = client.chat.completions.create(
+            model=text_generation_model, 
+            messages = [{"role": "system", "content": prompt}],
+        )
+        
+        res = response.choices[0].message.content
+
+    except openai.OpenAIError as e:
+        print(f"Error interacting with OpenAI API: {e}")
+        res =  ""
+
+    return res
+    
+def read_json(json_string) -> list[list[str]]:
+    try:
+        entities = json.loads(json_string)
+        return entities
+
+    except json.JSONDecodeError as e:
+        print(f"Error decoding JSON: {e}")
+        return []
+
+def lighten_color(hex_color, factor=1.8):
+    """Lightens a HEX color by increasing its brightness in HSV space."""
+
+    hex_color = hex_color.lstrip("#")
+    r, g, b = int(hex_color[0:2], 16), int(hex_color[2:4], 16), int(hex_color[4:6], 16)
+    
+    # Convert to HSV
+    h, s, v = colorsys.rgb_to_hsv(r / 255.0, g / 255.0, b / 255.0)
+    v = min(1.0, v * factor)  # Increase brightness
+    
+    # Convert back to HEX
+    r, g, b = [int(c * 255) for c in colorsys.hsv_to_rgb(h, s, v)]
+    return f'#{r:02x}{g:02x}{b:02x}'
+
+def darken_color(hex_color, factor=0.7):
+    """Darkens a hex color by reducing its brightness in the HSV space."""
+
+    hex_color = hex_color.lstrip("#")
+    r, g, b = int(hex_color[0:2], 16), int(hex_color[2:4], 16), int(hex_color[4:6], 16)
+    
+    # Convert to HSV to adjust brightness
+    h, s, v = colorsys.rgb_to_hsv(r / 255.0, g / 255.0, b / 255.0)
+    v = max(0, v * factor)  # Reduce brightness
+    
+    # Convert back to HEX
+    r, g, b = [int(c * 255) for c in colorsys.hsv_to_rgb(h, s, v)]
+    return f'#{r:02x}{g:02x}{b:02x}'
+
+def generate_color(index, total_colors=20):
+    """Generates a unique, evenly spaced color for each index using HSL."""
+
+    hue = index / total_colors  # Spread hues in range [0,1]
+    saturation = 0.65  # Keep colors vivid
+    lightness = 0.75   # Balanced brightness
+    
+    # Convert HSL to RGB
+    r, g, b = colorsys.hls_to_rgb(hue, lightness, saturation)
+    r, g, b = int(r * 255), int(g * 255), int(b * 255)
+    
+    return f'#{r:02x}{g:02x}{b:02x}'  # Convert to hex
+
+
+def assign_colors_to_entities(entities):
+    total_colors = len(entities)
+    # Assign colors to entities
+    entities_colors = []
+    for index, entity in enumerate(entities):
+        color = generate_color(index, total_colors)
+        
+        # append color and index to entities_colors
+        entities_colors.append({"color": color, "input": entity[0], "source": entity[1]})
+    
+    return entities_colors
+
+def highlight_entities(text1, text2):
+    if text1 == "" or text2 == "":
+        return []
+    
+    print(f"text1: {text1}")
+    print(f"text2: {text2}")
+    entities_text = extract_entities_gpt(text1, text2)
+    print(f"entities_text: {entities_text}")
+
+    entities = read_json(entities_text)
+    
+    # Assign colors to entities
+    entities_with_colors = assign_colors_to_entities(entities)
+    print(f"entities_colors: ", entities_with_colors)
+    
+    # Apply highlighting to entities
+    # highlighted_text_1 = apply_highlight(text1, entities_with_colors, "input")
+    # highlighted_text_2 = apply_highlight(text2, entities_with_colors, "source")
+    
+    return entities_with_colors
+
+
+def apply_highlight(text, entities_with_colors, key="input"):
+    if entities_with_colors == []:
+        return text, []
+    
+    all_starts = []
+    all_ends = []
+    highlighted_text = ""
+    temp_text = text
+    for index, entity in enumerate(entities_with_colors):
+        highlighted_text = ""
+        
+        # find a list of starts and ends of entity in text: 
+        starts = [m.start() for m in re.finditer(entity[key], temp_text)]
+        ends = [m.end() for m in re.finditer(entity[key], temp_text)]
+        
+        all_starts.extend(starts)
+        all_ends.extend(ends)
+        
+        color = entities_with_colors[index]["color"]
+        entity_color = lighten_color(color, factor=2.2)  # Lightened color for background text
+        label_color = darken_color(entity_color, factor=0.7)  # Darker color for background label (index)
+        
+        # Apply highlighting to each entity
+        prev_end = 0
+        for start, end in zip(starts, ends):
+            # Append non-highlighted text
+            highlighted_text += temp_text[prev_end:start]
+            
+            # Style the index as a label
+            index_label = (f'<span_style="background-color:{label_color};color:white;'
+                            f'padding:1px_4px;border-radius:4px;font-size:12px;'
+                            f'font-weight:bold;display:inline-block;margin-right:4px;">{index + 1}</span>')
+            
+            # Append highlighted text with index label
+            highlighted_text += (f'\n<span_style="background-color:{entity_color};color:black;'
+                                    f'border-radius:3px;font-size:14px;display:inline-block;">'
+                                    f'{index_label}{temp_text[start:end]}</span>\n')
+            prev_end = end
+        highlighted_text += temp_text[prev_end:]
+        temp_text = highlighted_text
+
+    if highlighted_text == "":
+        return text, []
+    highlight_idx_list = get_index_list(highlighted_text)
+    return highlighted_text, highlight_idx_list
+
+def get_index_list(highlighted_text):
+    """
+    Generates a list of indices between corresponding start and end indices.
+
+    Args:
+        starts: A list of starting indices.
+        ends: A list of ending indices.  Must be the same length as starts.
+
+    Returns:
+        A list containing all indices within the specified ranges.
+        Returns an empty list if the input is invalid (e.g., different lengths, 
+        end < start, etc.).
+    """
+    highlighted_index = []
+    words = highlighted_text.split()
+    for index, word in enumerate(words):
+        if word.startswith("<span_style"):
+            start_index = index
+        if word.endswith("</span>"):
+            end_index = index
+            
+            highlighted_index.extend(list(range(start_index, end_index + 1)))
+
+    return highlighted_index
+
+def extract_entities(text):
+    output = ner_pipeline(text)
+    words = extract_words(output)
+    words = combine_subwords(words)
+    
+    # extract word in each entity and assign to a list of entities, connect words if there is no space between them
+    entities = []    
+    for entity in words:
+        if entity not in entities:
+            entities.append(entity)
+    
+    return entities
+
+
+def extract_words(entities):
+    """
+    Extracts the words from a list of entities.
+
+    Args:
+    entities: A list of entities.
+
+    Returns:
+    A list of words extracted from the entities.
+    """
+    words = []
+    for entity in entities:
+        words.append(entity["word"])
+    return words
+
+
+def combine_subwords(word_list):
+    """
+    Combines subwords (indicated by "##") with the preceding word in a list.
+
+    Args:
+    word_list: A list of words, where subwords are prefixed with "##".
+
+    Returns:
+    A new list with subwords combined with their preceding words.
+    """
+    result = []
+    i = 0
+    while i < len(word_list):
+        if word_list[i].startswith("##"):
+            result[-1] += word_list[i][2:]  # Remove "##" and append to the previous word
+        elif i < len(word_list) - 2 and word_list[i + 1] == "-":  # Combine hyphenated words
+            result.append(word_list[i] + word_list[i + 1] + word_list[i + 2])
+            i += 2  # Skip the next two words
+        else:
+            result.append(word_list[i])
+        i += 1
+    return result
+
+
+original_text = """
+Title: UK pledges support for Ukraine with 100-year pact 
+Content: Sir Keir Starmer has pledged to put Ukraine in the "strongest possible position" on a trip to Kyiv where he signed a "landmark" 100-year pact with the war-stricken country. The prime minister's visit on Thursday was at one point marked by loud blasts and air raid sirens after a reported Russian drone attack was intercepted by Ukraine's defence systems. Acknowledging the "hello" from Russia, Volodymyr Zelensky said Ukraine would send its own "hello back". An estimated one million people have been killed or wounded in the war so far. As the invasion reaches the end of its third year, Ukraine is losing territory in the east. Zelensky praised the UK's commitment on Thursday, amid wider concerns that the US President-elect Donald Trump, who is set to take office on Monday, could potentially reduce aid.
+    """
+compared_text = """
+Title: Japan pledges support for Ukraine with 100-year pact
+Content: A leading Japanese figure has pledged to put Ukraine in the "strongest possible position" on a trip to Kyiv where they signed a "landmark" 100-year pact with the war-stricken country. The visit on Thursday was at one point marked by loud blasts and air raid sirens after a reported Russian drone attack was intercepted by Ukraine's defence systems. Acknowledging the "hello" from Russia, Volodymyr Zelensky said Ukraine would send its own "hello back". An estimated one million people have been killed or wounded in the war so far. As the invasion reaches the end of its third year, Ukraine is losing territory in the east. Zelensky praised Japan's commitment on Thursday, amid wider concerns that the next US President, who is set to take office on Monday, could potentially reduce aid.
+    """
+if __name__ == "__main__":
+    # text = "The Saudi authorities, I am told, are currently working flat out" \
+    #         "to collate everything they have on the Magdeburg market suspect," \
+    #         "Taleb al-Abdulmohsen, and to share it with Germany's ongoing" \
+    #         "investigation"
+    # print(extract_entities(text))
+
+    
+    with gr.Blocks() as demo:
+        gr.Markdown("### Highlight Matching Parts Between Two Paragraphs")
+        text1_input = gr.Textbox(
+            label="Paragraph 1", 
+            lines=5, 
+            value=original_text,
+        )
+        text2_input = gr.Textbox(
+            label="Paragraph 2", 
+            lines=5, 
+            value=compared_text,
+        )
+        submit_button = gr.Button("Highlight Matches")
+        output1 = gr.HTML("<br>"*10)
+        output2 = gr.HTML("<br>"*10)
+        
+    
+        submit_button.click(
+            fn=highlight_entities,
+            inputs=[text1_input, text2_input],
+            outputs=[output1, output2]
+        )
+    
+    # Launch the Gradio app
+    demo.launch()
+    

src/application/text/identity.py

@@ -1,63 +0,0 @@
-from transformers import pipeline
-
-ner_pipeline = pipeline("ner")
-
-def extract_entities(text):
-    output = ner_pipeline(text)
-    words = extract_words(output)
-    words = combine_subwords(words)
-    
-    # extract word in each entity and assign to a list of entities, connect words if there is no space between them
-    entities = []    
-    for entity in words:
-        if entity not in entities:
-            entities.append(entity)
-    
-    return entities
-
-
-def extract_words(entities):
-    """
-    Extracts the words from a list of entities.
-
-    Args:
-    entities: A list of entities.
-
-    Returns:
-    A list of words extracted from the entities.
-    """
-    words = []
-    for entity in entities:
-        words.append(entity["word"])
-    return words
-
-
-def combine_subwords(word_list):
-    """
-    Combines subwords (indicated by "##") with the preceding word in a list.
-
-    Args:
-    word_list: A list of words, where subwords are prefixed with "##".
-
-    Returns:
-    A new list with subwords combined with their preceding words.
-    """
-    result = []
-    i = 0
-    while i < len(word_list):
-        if word_list[i].startswith("##"):
-            result[-1] += word_list[i][2:]  # Remove "##" and append to the previous word
-        elif i < len(word_list) - 2 and word_list[i + 1] == "-":  # Combine hyphenated words
-            result.append(word_list[i] + word_list[i + 1] + word_list[i + 2])
-            i += 2  # Skip the next two words
-        else:
-            result.append(word_list[i])
-        i += 1
-    return result
-
-if __name__ == "__main__":
-    text = "The Saudi authorities, I am told, are currently working flat out" \
-            "to collate everything they have on the Magdeburg market suspect," \
-            "Taleb al-Abdulmohsen, and to share it with Germany's ongoing" \
-            "investigation"
-    print(extract_entities(text))

src/application/text/model_detection.py

@@ -35,6 +35,7 @@ def detect_text_by_ai_model(
             truncation=True,
             device_map="auto",  # good for GPU usage
         )
+        input_text = input_text.replace("<br>", " ")
         result = pipe(input_text)[0]
         confidence_score = result["score"]
         if result["label"] == MODEL_HUMAN_LABEL[model]:

src/application/text/preprocessing.py

@@ -13,10 +13,10 @@ def split_into_sentences(input_text):
     if not isinstance(input_text, str):
         return []
 
-    paragraphs = input_text.splitlines()
+    paragraphs = input_text.splitlines(keepends=True)
     sentences = []
     for paragraph in paragraphs:
         paragraph = paragraph.strip()
-        if paragraph:
+        if paragraph and paragraph != '\n':
             sentences.extend(sent_tokenize(paragraph))
     return sentences

src/application/text/search.py

@@ -7,7 +7,7 @@ from nltk.corpus import stopwords
 from nltk.tokenize import word_tokenize
 from sklearn.feature_extraction.text import TfidfVectorizer
 
-from src.application.text.identity import extract_entities
+from src.application.text.entity import extract_entities
 
 load_dotenv() 
 GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")

src/application/text/search_detection.py

@@ -41,7 +41,7 @@ def detect_text_by_relative_search(input_text, index, is_support_opposite = Fals
         search_results = search_by_google(candidate)
         urls = [item['link'] for item in search_results.get("items", [])]
 
-        for url in urls[:3]:
+        for url in urls[:10]:
             if url in checked_urls: # visited url
                 continue
             if "bbc.com" not in url:
@@ -58,11 +58,10 @@ def detect_text_by_relative_search(input_text, index, is_support_opposite = Fals
                     continue
                 
                 page_text = content.title + "\n" + content.text
-                print(f"page_text: {page_text}")
                 if len(page_text) > MAX_CHAR_SIZE:
                     print(f"\t\t\t↑↑↑ More than {MAX_CHAR_SIZE} characters")
                     continue
-                
+                print(f"\t\t\t↑↑↑ Title: {content.title}") 
                 paraphrase, aligned_first_sentences = check_paraphrase(input_text[index], page_text, url)
                 
                 if paraphrase is False:
@@ -71,15 +70,17 @@ def detect_text_by_relative_search(input_text, index, is_support_opposite = Fals
                 sub_paraphrase = True
                 while sub_paraphrase == True:
                     index += 1
-                    print(f"----search {index}----")
+                    print(f"----search {index} < {len(input_text)}----")
                     if index >= len(input_text):
+                        print(f"input_text_last: {input_text[-1]}")
                         break
+                    print(f"input_text: {input_text[index]}")
                     sub_paraphrase, sub_sentences = check_paraphrase(input_text[index], page_text, url)
                     print(f"sub_paraphrase: {sub_paraphrase}")
                     print(f"sub_sentences: {sub_sentences}")
-                    if sub_paraphrase == True:                        
-                        aligned_first_sentences["input_sentence"] += sub_sentences["input_sentence"]
-                        aligned_first_sentences["matched_sentence"] += sub_sentences["matched_sentence"]
+                    if sub_paraphrase == True:
+                        aligned_first_sentences["input_sentence"] += "<br>" + sub_sentences["input_sentence"]
+                        aligned_first_sentences["matched_sentence"] += "<br>" + sub_sentences["matched_sentence"]
                         aligned_first_sentences["similarity"] += sub_sentences["similarity"]
                         aligned_first_sentences["similarity"] /= 2
                 

test.py

@@ -0,0 +1,46 @@
+from difflib import SequenceMatcher
+import string
+
+def extract_equal_text(text1, text2):
+    def cleanup(text):
+        text = text.lower()
+        text = text.translate(str.maketrans('', '', string.punctuation))
+        return text
+    
+    splited_text1 = cleanup(text1).split()
+    splited_text2 = cleanup(text2).split()
+    
+    s = SequenceMatcher(None, splited_text1, splited_text2)
+    
+    equal_idx_1 = []
+    equal_idx_2 = []
+    text1 = text1.split()
+    text2 = text2.split()
+    for tag, i1, i2, j1, j2 in s.get_opcodes():
+        if tag == 'equal':
+            equal_idx_1.append({"start": i1, "end": i2})
+            equal_idx_2.append({"start": j1, "end": j2})
+            subtext_1 = " ".join(text1[i1:i2])
+            subtext_2 = " ".join(text2[j1:j2])
+            print(f'{tag:7}   a[{i1:2}:{i2:2}] --> b[{j1:2}:{j2:2}] {subtext_1!r:>55} --> {subtext_2!r}')
+
+    return equal_idx_1, equal_idx_2
+
+text1 = """
+Miguel Almiron has permanently rejoined Atlanta United from Newcastle United for £8m.
+Almiron made 223 appearances for Newcastle, scoring 30 goals, but recently struggled for a starting place under Eddie Howe.
+He made a substitute appearance and waved farewell to fans in Newcastle's recent win against Southampton.
+Almiron played a key role in Newcastle reaching the Carabao Cup final and their Premier League top-four finish in 2022-23, and scored against Paris St-Germain in the Champions League.
+"""
+text2 = """
+Newcastle United winger Miguel Almiron has rejoined Atlanta United on a permanent deal for £8m.
+Almiron has made 223 appearances for Newcastle, scoring 30 goals, but has struggled recently to gain a place in manager Eddie Howe's starting line-up.
+Last weekend he came on as a substitute in Newcastle's 3-1 win against Southampton and waved farewell to the travelling supporters.
+Almiron played a significant role in Newcastle reaching the Carabao Cup final and finishing fourth in the Premier League in 2022-23.
+"""
+
+idx_1, idx_2 = extract_equal_text(text1, text2)
+
+# text1_split = text1.split()
+# for idx in idx_1:
+#     print(text1_split[idx["start"]:idx["end"]])