main.py

import asyncio
from flet import *
import requests
import json
import pandas as pd
import elasticsearch_serverless
import re
import os
import flet_fastapi

def remove_arabic_diacritics(text):
    diacritics_pattern = re.compile(r'[\u064B-\u065F\u0670\u06D6-\u06ED]')
    no_diacritics_text = re.sub(diacritics_pattern, '', text)
    return no_diacritics_text

diacritics = re.compile("""
    ّ    | # Tashdid
    َ    | # Fatha
    ً    | # Tanwin Fath
    ُ    | # Damma
    ٌ    | # Tanwin Damm
    ِ    | # Kasra
    ٍ    | # Tanwin Kasr
    ْ    | # Sukun
    ـ      # Tatwil/Kashida
""", re.VERBOSE)

def normalize_arabic(text):
    text = diacritics.sub('', text)
    text = text.replace('أ', 'ا')
    text = text.replace('إ', 'ا')
    text = text.replace('آ', 'ا')
    text = text.replace('ة', 'ه')
    text = text.replace('ى', 'ي')
    return text

book_selected = False
first_run = 0
p1_first_run = 0

from elasticsearch_serverless import Elasticsearch

endpoint = "https://503a98874f6241968f251209ab393a45.us-central1.gcp.cloud.es.io:443"

client = Elasticsearch(
  endpoint,
  api_key="SWZGTU5aQUJuNURpVDRSbmtZSGk6cXRSUFZDZ1lRR2k2Y3NvQW9JYjExUQ",
  request_timeout=60, max_retries=3, retry_on_timeout=True
)

async def main(page: Page):

    async def e_search(query):
        query = remove_arabic_diacritics(query)
        query = normalize_arabic(query)

        j_query = {
            "size": 250,  
            "query": {
                "match_phrase": {
                    "Text": query
                }
            }
        }

        response_search = await asyncio.to_thread(client.search, index="books_idx", body=j_query)
        unique_books = {}
        all_hits = response_search['hits']['hits']
        filtered_hits = [hit for hit in all_hits if query in hit['_source']['Text']]

        for hit in filtered_hits: 
            book = hit['_source']['Book']
            page = hit['_source']['Page']
            score = hit['_score']
            if book not in unique_books:
                unique_books[book] = {'Pages': {page: score}, 'Count': 1}
            else:
                if page not in unique_books[book]['Pages']:
                    unique_books[book]['Pages'][page] = score
                    unique_books[book]['Count'] += 1

        book_data = []
        for book, info in unique_books.items():
            pages = sorted(info['Pages'].items())
            book_data.append({'Book': book, 'Pages': [page for page, _ in pages], 'Scores': [score for _, score in pages], 'Count': info['Count']})
        
        df = pd.DataFrame(book_data)
        df = df.head(10)

        def get_top_two(row):
            sorted_row = sorted(zip(row['Pages'], row['Scores']), key=lambda x: x[1], reverse=True)
            return [page for page, score in sorted_row[:2]]
        
        try:
            df['Top Two Pages'] = df.apply(get_top_two, axis=1)
        except:
            pass

        return df, response_search

    inquiry_text = "من فضلك اكتب استفسارك."

    async def e_search_book(query, phrase_search=0):
        if phrase_search == 0:
            book_name = book_btn.text 
        else:
            book_name = phrase_search

        url_search = 'http://localhost:9202/books_01/_search'
        query = remove_arabic_diacritics(query)
        query = normalize_arabic(query)
        
        j_query = {
            "size": 50,
            "query": {
                "bool": {
                    "must": [
                        {
                            "match_phrase": {
                                "Text": query
                            }
                        }
                    ],
                    "filter": [
                        {
                            "term": {
                                "Book.keyword": book_name
                            }
                        }
                    ]
                }
            },
            "highlight": {
                "fields": {
                    "Text": {}
                }
            }
        }

        response_search = await asyncio.to_thread(client.search, index="books_idx", body=j_query)
        data = []
        for hit in response_search['hits']['hits']:
            book = hit['_source']['Book']
            page = hit['_source']['Page']
            score = hit['_score']
            text = hit['_source']['Text']
            data.append({
                "Book": book,
                "Page": page,
                "Score": score,
                "Text": text
            })

        df = pd.DataFrame(data)
        return df, response_search

    async def navigate_pages(e, page):
        print(page)
        print(df)

    async def p1_page_text_fun(e, response_search, nav="None"):
        p1_datatable_row.visible = False
        p1_page_text.visible = True    
        p1_pages_row.visible = True    

        if nav == "None":
            p1_pages_row.controls[1].controls[1].value = "رقم الصفحة \n {}".format(e.control.text)
            page_num = e.control.text 
        else:
            match = re.search(r'\d+', p1_pages_row.controls[1].controls[1].value)
            if match:
                page_number = match.group()
            page_numbers = [int(item['_source']['Page']) for item in response_search['hits']['hits']]
            page_index = page_numbers.index(int(page_number))
            page_num = page_numbers[(page_index + nav)]
            p1_pages_row.controls[1].controls[1].value = "رقم الصفحة \n {}".format(page_num)

        filtered_data = [item for item in response_search['hits']['hits'] if item['_source']['Page'] == page_num]
        highlight = filtered_data[0]['highlight']['Text']
        txt = filtered_data[0]['_source']['Text']
        
        highlight_phrases = []
        for item in highlight:
            matches = re.findall(r'<em>(.*?)</em>', item)
            highlight_phrases.extend(matches)
        
        highlight_phrases = list(set(highlight_phrases))
        for phrase in highlight_phrases:
            emphasized_phrase = f"<em>{phrase}</em>"
            highlighted_text = txt.replace(phrase, emphasized_phrase)
        
        lines = highlighted_text.split('\n')
        spans = []
        for line in lines:
            parts = re.split(r'(<em>.*?</em>)', line)
            for part in parts:
                if part.startswith('<em>') and part.endswith('</em>'):
                    word = part[4:-5]
                    spans.append(TextSpan(word, TextStyle(weight=FontWeight.BOLD, color=colors.YELLOW_600)))
                else:
                    spans.append(TextSpan(part + "\n"))
        
        p1_page_text.content.controls[0].spans = spans
        await page.update_async()

    async def p1_bookname(e):
        book_name = e.control.text
        e_search_df, response = await e_search_book(p1_query_feild.value, book_name)

        p1_res_dt.columns.clear()
        p1_res_dt.rows.clear()
        e_search_df = e_search_df[['Text', 'Score', 'Page']]
        occurrences_count = 0
        query = remove_arabic_diacritics(p1_query_feild.value)
        query = normalize_arabic(query)

        for hit in response['hits']['hits']:
            text = hit['_source']['Text']
            occurrences_count += text.count(query) 

        p1_info_table.controls = [create_table(response['hits']['hits'][0]['_source']['Book'],
                                               e_search_df.shape[0],
                                               occurrences_count,
                                               342)]
        
        translation = {"Book": "الكتاب", "Page": "الصفحه", "Score": "درجة التطابق", 'Text': "المحتوي"}

        for i in range(len(e_search_df.columns)):
            p1_res_dt.columns.append(DataColumn(Text(translation[e_search_df.columns[i]])))
        
        pages_btns = []
        for i in range(e_search_df.shape[0]):
            txt = e_search_df['Text'][i][:80].replace("\n", " ")
            p1_res_dt.rows.append(DataRow(cells=[
                DataCell(Row([Text(f"{txt}...", width=550)])),
                DataCell(Text(e_search_df['Score'][i], width=300)),
                DataCell(ElevatedButton(e_search_df['Page'][i],
                                        on_click=lambda e, name=response: asyncio.create_task(p1_page_text_fun(e, name)), width=120))
            ]))

        next_button = ElevatedButton(
            content=Row(
                controls=[
                    Text("  التالي"),
                    Icon(name=icons.NAVIGATE_NEXT, size=25),
                ],
                alignment=MainAxisAlignment.CENTER
            ),
            on_click=lambda e, name=response: asyncio.create_task(p1_page_text_fun(e, name, 1))
        )

        previous_button = ElevatedButton(
            content=Row(
                controls=[
                    Icon(name=icons.NAVIGATE_BEFORE, size=25),
                    Text("السابق  "),
                ],
                alignment=MainAxisAlignment.CENTER
            ),
            on_click=lambda e, name=response: asyncio.create_task(p1_page_text_fun(e, name, -1))
        )

        page_num_widget = Row([Text("   "), Text("رقم الصفحة \n 50", weight=FontWeight.BOLD, text_align=TextAlign.CENTER), Text("   ")])
        p1_pages_row.controls = [previous_button, page_num_widget, next_button]
        p1_pages_row.visible = False
        await page.update_async()

    def create_table(books, pages, hits, wid):
        def create_cell(content, is_header=False):
            return Container(
                content=Text(content, weight="bold" if is_header else None),
                border=border.all(1, "cyan"),
                padding=padding.all(8),
                border_radius=2,
                alignment=alignment.center,
                width=wid
            )

        header = Row(
            controls=[
                create_cell("التطابقات", is_header=True),
                create_cell("الصفحات", is_header=True),
                create_cell("الكتب", is_header=True)
            ],
            alignment="center",
            spacing=0
        )

        values = Row(
            controls=[
                create_cell(hits),
                create_cell(pages),
                create_cell(books)
            ],
            alignment="center",
            spacing=0
        )

        table = Column(
            controls=[
                header,
                values
            ],
            alignment="center",
            spacing=0
        )

        return table

    async def p1_send_button(e):
        global p1_first_run

        p1_datatable_row.visible = True
        p1_page_text.visible = False   
        p1_pages_row.visible = False

        p1_res_dt.columns.clear()
        if p1_first_run >= 1:
            p1_res_dt.rows.clear()

        p1_first_run = 1
        e_search_df, response_search = await e_search(p1_query_feild.value)
        e_search_df = e_search_df[['Top Two Pages', 'Count', 'Pages', 'Book']]

        translation = {"Book": "الكتاب", "Pages": "الصفحات", "Count": "التطابقات", 'Top Two Pages': "أعلى صفحتين متطابقتين"}
        occurrences_count = 0
        query = remove_arabic_diacritics(p1_query_feild.value)
        query = normalize_arabic(query)

        for hit in response_search['hits']['hits']:
            text = hit['_source']['Text']
            occurrences_count += text.count(query)

        p1_info_table.controls = [create_table(e_search_df.shape[0], e_search_df['Count'].sum(), occurrences_count, 342)]

        for i in range(len(e_search_df.columns)):
            p1_res_dt.columns.append(DataColumn(Text(translation[e_search_df.columns[i]])))

        for i in range(e_search_df.shape[0]):
            occurrences_count = 0
            for hit in response_search['hits']['hits']:
                if hit['_source']['Book'] == e_search_df['Book'][i]:
                    text = hit['_source']['Text']
                    occurrences_count += text.count(query)

            p1_res_dt.rows.append(DataRow(cells=[
                DataCell(Text(e_search_df['Top Two Pages'][i], width=200)),
                DataCell(Text(occurrences_count, width=120)),
                DataCell(Text(e_search_df['Count'][i], width=180)),
                DataCell(ElevatedButton(e_search_df['Book'][i], width=450, on_click=p1_bookname)),
            ]))

        await page.update_async()

    p1_res_dt = DataTable(
        columns=[DataColumn(Text())],
        border=border.all(2, "blue"),
        border_radius=10,
        column_spacing=10,
    )

    p1_info_table = Row([Text("")], alignment=MainAxisAlignment.CENTER)
    p1_datatable_row = Column([Row([p1_res_dt], alignment=MainAxisAlignment.CENTER)], alignment=MainAxisAlignment.CENTER, scroll=ScrollMode.ALWAYS, height=398)
    p1_query_feild = TextField(label="Inquiry", hint_text=inquiry_text, expand=True, rtl=True)
    p1_query_send = FloatingActionButton(icon=icons.SEND, on_click=p1_send_button)
    p1_Query_row = Row(controls=[p1_query_feild, p1_query_send])
    p1_page_text = Container(
        content=Column([Text("", rtl=True)], scroll=ScrollMode.ALWAYS),
        margin=10,
        padding=10,
        alignment=alignment.center,
        width=1050,
        height=400,
        border_radius=10,
        border=border.all(1, colors.CYAN),
    )

    page_1 = Column([p1_Query_row, p1_info_table, p1_datatable_row, Row([Text(), p1_page_text, Text()], alignment=MainAxisAlignment.CENTER),
                     Row([Text(), p1_pages_row, Text()], alignment=MainAxisAlignment.CENTER)])

    p1_datatable_row.visible = False
    p1_page_text.visible = False
    p1_pages_row.visible = False

    await page.add_async(page_1)

app = flet_fastapi.app(main)