Create app.py

app.py

@@ -0,0 +1,611 @@
+import streamlit as st
+import zipfile, shutil, time
+import os
+import hashlib
+#from streamlit_pdf_viewer import pdf_viewer
+from streamlit import runtime
+from streamlit.runtime.scriptrunner import get_script_run_ctx
+from streamlit_js_eval import streamlit_js_eval
+import secrets
+
+import threading
+from streamlit.runtime.scriptrunner import add_script_run_ctx
+#import streamlit.components.v1 as components
+from streamlit.runtime import get_instance
+
+#from pypdf import PdfReader
+import fitz
+import glob
+import logging
+
+
+def get_remote_ip() -> str:
+    """Get remote ip."""
+
+    try:
+        ctx = get_script_run_ctx()
+        if ctx is None:
+            return None
+
+        session_info = runtime.get_instance().get_client(ctx.session_id)
+        if session_info is None:
+            return None
+    except Exception as e:
+        return None
+
+    return session_info.request.remote_ip
+
+
+# colab side make dir
+def my_makedirs(path):
+    if not os.path.isdir(path):
+        os.makedirs(path)
+
+def heart_beat():
+    """
+    Heartbeat function to track whether the session is alive
+    """
+    thread = threading.Timer(interval=5, function=heart_beat)
+
+    # insert context to the current thread, needed for
+    # getting session specific attributes like st.session_state
+
+    add_script_run_ctx(thread)
+
+    # context is required to get session_id of the calling
+    # thread (which would be the script thread)
+    ctx = get_script_run_ctx()
+
+    # this is the main runtime, contains all the sessions
+    runtime = get_instance()
+
+    if runtime.is_active_session(session_id=ctx.session_id):
+        logging.info(f"{ctx.session_id} is alive.")
+        thread.start()
+    else:
+        if 'uniq' in st.session_state:
+            if os.path.isdir(f"removefolder/{st.session_state.uniq}"):
+                shutil.rmtree(f"removefolder/{st.session_state.uniq}")
+                logging.info(f"{ctx.session_id} is gone.")
+        return
+
+# JavaScript to detect browser exit
+EXIT_JS = """
+<script>
+    window.addEventListener('beforeunload', function (event) {
+        fetch('/close_session', {method: 'POST'}).then(response => {
+            return response.text();
+        }).then(data => {
+            console.log(data);
+        });
+    });
+</script>
+"""
+
+# Embed the JavaScript in the Streamlit app
+#components.html(EXIT_JS)
+streamlit_js_eval(js_expressions = EXIT_JS)
+
+def main():
+
+    if 'uniq' not in st.session_state:
+        st.session_state.uniq = secrets.token_urlsafe()
+
+    temp_dir = st.session_state.uniq
+    my_makedirs(f"removefolder/{temp_dir}")
+
+    flag = True
+    if 'count' not in st.session_state:
+        st.session_state.count = 0
+    #tempolary
+    if 'temp' not in st.session_state:
+        st.session_state.temp = 0
+
+    if 'lang' not in st.session_state:
+        st.session_state.lang = ""
+    if 'result' not in st.session_state:
+        st.session_state.result = ""
+        
+    apptitle = st.empty()
+    langs = st.empty()
+    description = st.empty()
+    obj_0 = st.empty()
+    obj_1 = st.empty()
+    
+    apptitle.header("PDF file Translator 𓆊", divider="violet")
+    langs.write('This App can translate to <`Japanese`, `English`, `French`, `Chinese (traditional)`, `Chinese (simplified)`, `Russian`, `Korean`, `Vietnamese`, `Thai`, `Catalan`, `Sinhalese`, `Nepall`>')
+    description.write("""
+It's easy to use.:black_cat: Just upload, select the language, and download the resulting .zip file.:package:
+
+After uploading a PDF file and selecting the translation language, you can wait a while.
+
+The original text, the original text and translation (a few lines of the original text followed by the translation) and the translation text will be compressed into a zip file and available for download.
+
+When you press the download button, the compressed file will be downloaded, and if you select another translation language, the process will be repeated.:leftwards_arrow_with_hook:
+
+:koala: The uploaded PDF file data will disappear <u>when you close the browser tab</u>. :eyes:
+
+**Only PDF files can be uploaded.**
+
+This translation app is useful for people who want to translate something or want to read something but cannot read it unless it is translated, and who want to quickly check the original text and the translation by comparing them in pairs. :yin_yang:
+
+:full_moon_with_face: Even if the PDF file has many pages, there is no limit to the number of pages or characters.
+
+<u>The untranslated data will be retained until the browser is closed, but once the app page is closed, the connection will be cut off and the data will be deleted.</u> :thought_balloon:
+
+####  FAQ :coffee:
+
+:baby: **Q** : Does the translated text have information about line breaks and paragraphs? :coffee:
+
+:robot_face: **A** : Line breaks and paragraphs are not reflected in the translated text.
+
+The text extracted from the original text has the same position indexed with `:::info` as the translated text.
+
+Regardless of the contents of the uploaded PDF or document, counting starts from zero and you can see which page or sentence you are in.
+The original text has an `𓃰` : elephant mark after `:::info`, which are unicode characters representing Egyptian hieroglyphics.
+
+𓃰00001-0;
+
+Similarly, the translation of the original text is followed by a `𓆏` : frog mark.
+
+𓆏00001-0;
+
+:teapot: **Tips** : If you have a text editor with a pattern replacement function,
+you can use the characters starting with
+`:::info`
+and ending with the line that contains only
+`:::`
+as a pattern to remove.
+Try searching Google for keywords such as `grep and replace`.
+
+""", unsafe_allow_html=True)
+    obj_0.success("PDF file uploader")
+#    st.markdown(f"The remote ip is `{get_remote_ip()}`")
+
+    uploaded_file = obj_1.file_uploader("UPLOAD your .pdf file", type="pdf")
+    ####
+    if uploaded_file is not None:
+        flag = False
+        st.success("PDF file translator")
+        # hashed
+        raw_filename = uploaded_file.name
+        intext_0 = f'<span style="color:LavenderBlush;background:Orchid"> {raw_filename} </span>'
+        st.write(intext_0, unsafe_allow_html=True)
+        hashed_filename = hashlib.sha1(raw_filename.encode())
+        uploadedfilename = hashed_filename.hexdigest()
+        if "uploadedfilename" not in st.session_state:
+            st.session_state.uploadedfilename = uploadedfilename
+
+        if "book" not in st.session_state:
+            #pdf_viewer(input=uploaded_file.getvalue(), width=700, height=500)
+
+            my_makedirs(
+                f"removefolder/{temp_dir}/upload_folder_{st.session_state.count}"
+            )
+
+            with open(
+                    f'removefolder/{temp_dir}/upload_folder_{st.session_state.count}/{uploadedfilename}.pdf',
+                    'wb') as file:
+                file.write(uploaded_file.getvalue())
+#            pdf_viewer(input=f'{temp_dir}/upload_folder_{st.session_state.count}/{uploadedfilename}.pdf', width=700, height=500)
+
+# read from PDF file
+            PDF = glob.glob(
+                f"removefolder/{temp_dir}/upload_folder_{st.session_state.count}/{uploadedfilename}.pdf"
+            )
+
+            #doc = PdfReader(PDF[0])
+            doc = fitz.open(PDF[0])
+            # meta = doc.metadata
+            #page_count = len(doc.pages)
+            page_count = len(doc)
+            book = []  # PDF text data pool
+            progressbar1 = st.empty()
+            my_bar1 = progressbar1.progress(0)
+            
+            from bs4 import BeautifulSoup
+            
+            for index, page in enumerate(doc):
+                #page_text = page.extract_text()
+                #page_text = page.get_text(sort=True)
+                blocks = page.get_text("xml")
+                soup = BeautifulSoup(blocks,'lxml-xml')
+                page_text2 = ""
+
+                for tag0 in soup.find_all("block"):
+                    temp_y_posi = 0.0
+                    for tag1 in tag0.find_all("line"):
+                        for tag2 in tag1.find_all("font"):
+                            for tag3 in tag2.find_all("char"):
+                                y_posi = tag3.get("y")
+                                if y_posi != temp_y_posi:
+                                    page_text2 += "\n"
+                                    temp_y_posi = y_posi
+                                page_text2 += tag3.get("c")
+                        
+                    page_text2 += "\n\n"
+                                
+
+                
+                
+            #for index, page in enumerate(doc.pages):
+            #for index, page in enumerate(doc):
+            #    #page_text = page.extract_text()
+            #    page_text = page.get_text(sort=True)
+            #    book.append((index, page_text))
+                book.append((index, page_text2))
+
+                done = int(((index + 1) / page_count) * 100)
+                my_bar1.progress(done,
+                                 text=f"Reading Page Number : {index + 1}")
+            doc.close()
+            
+            st.session_state.book = book
+            my_bar1.empty()
+            if os.path.isfile(
+                    f"removefolder/{temp_dir}/upload_folder_{st.session_state.count}/{uploadedfilename}.pdf"
+            ):
+                shutil.rmtree(
+                    f"removefolder/{temp_dir}/upload_folder_{st.session_state.count}/"
+                )
+
+        ########
+        reload_bt = st.empty()
+        if reload_bt.button("Upload another PDF file"):
+            for key in st.session_state.keys():
+                if key == "count" or key == "temp" or key == "lang":
+                    continue
+                else:
+                    del st.session_state[key]
+            shutil.rmtree(f"removefolder/{temp_dir}")
+            # page reload
+            streamlit_js_eval(js_expressions="parent.window.location.reload()")
+        st.markdown("----")
+
+        plain_text1 = " 𓃠 select target language 𓃠 "
+        var_text1 = f'##### <span style="color:green">{plain_text1}</span>'
+
+        select = st.empty()
+        select.write(var_text1, unsafe_allow_html=True)
+
+        # select language
+        st.markdown("""
+        `ja`: **Japanese**,
+        `en`: **English**,
+        `fr`: **French**,
+        `zb-TW`: **Chinese (traditional)**,
+        `zh-CN`: **Chinese (simplified)**,
+        `ru`: **Russian**,
+        `ko`: **Korean**,
+        `vi`: **Vietnamese**,
+        `th`: **Thai**,
+        `tl`: **Tagalog**,
+        `ca`: **Catalan**,
+        `si`: **Sinhalese**,
+        `ne`: **Nepall**
+        """)
+        lang_code = [
+            "select language",
+            "Japanese",
+            "English",
+            "French",
+            "Chinese traditional",
+            "Chinese simplified",
+            "Russian",
+            "Korean",
+            "Vietnamese",
+            "Thai",
+            "Tagalog",
+            "Catalan",
+            "Sinhalese",
+            "Nepall"
+        ]
+        sel = st.empty()
+        language = sel.radio(
+                    label='translate to',
+                    options=lang_code,
+                    index=0,
+                    key = f"select_lang{st.session_state.count}",
+                    horizontal=True)
+        #language = sel.selectbox(
+        #    'translate to',
+        #    lang_code,
+        #    index=0,
+        #    #placeholder = "select language",
+        #    key=f"select_lang{st.session_state.count}")
+
+        statename = f"select_lang{st.session_state.count}"
+        if "target_lang" not in st.session_state:
+            st.session_state.target_lang = "UNSELECTED"
+
+        def reset_selected_lang():
+            st.session_state[statename] = "select language"
+
+        st.button('Reset Language', on_click=reset_selected_lang)
+
+    area = st.empty()
+    if flag:
+        if "select_lang" in st.session_state:
+            if st.session_state.select_lang != "select language":
+                area2 = st.empty()
+                plain_text2 = "☟Reset Language☟"
+                empty_text = "☟              ☟"
+                var_text2 = f'<span style="color:#FF69B4">{plain_text2}</span>'
+                while flag:
+                    area2.write(var_text2, unsafe_allow_html=True)
+                    time.sleep(0.9)
+                    area2.write(empty_text)
+                    time.sleep(0.5)
+
+        while flag:
+            area.text("𓀤 upload PDF file 𓀤")
+            time.sleep(1)
+            area.text("𓀥                 𓀥")
+            time.sleep(0.8)
+    else:
+        if f"select_lang{st.session_state.count}" in st.session_state:
+            statename = f"select_lang{st.session_state.count}"
+            if st.session_state[statename] != "select language":
+                plain_text2 = "Reset Language"
+                var_text2 = f'<span style="color:gray">▲ `{plain_text2}`</span>'
+                area.write(var_text2, unsafe_allow_html=True)
+
+        obj_0.empty()
+        obj_1.empty()  # uploader hide
+
+        # pdf translator
+        #------------------------------------------
+        st.markdown("----")
+        st.success("translator")
+
+        if "book" in st.session_state:
+            book_data = st.session_state.book
+            page_count = len(book_data)
+        else:
+            page_count = 0
+
+        st.text(f"PDF total pages : {page_count}")
+
+        progressbar = st.empty()
+        my_bar = progressbar.progress(0)
+
+        #3
+        #        from google.colab import output
+        import re
+        #from googletrans import Translator
+        from deep_translator import GoogleTranslator
+
+        title_name = re.sub("\.| |%|@|\"|\'", "_", f"{uploaded_file.name}")
+
+        if st.session_state.temp != int(st.session_state.count):
+            st.session_state.lang = "init"
+            st.session_state.temp = int(st.session_state.count)
+
+        if language not in lang_code[1:]:
+            language = None
+
+        if st.session_state.lang != language and language is not None:
+            st.session_state.count += 1
+            st.session_state.result = ""
+            st.session_state.lang = language
+
+            description.empty()
+            
+            my_makedirs(
+                f"removefolder/{temp_dir}/work_{st.session_state.count}")
+
+            to = ""
+            match language:
+                case "Japanese":
+                    to = "ja"
+                case "English":
+                    to = "en"
+                case "French":
+                    to = "fr"
+                case "Chinese traditional":
+                    to = "zh-TW"
+                case "Chinese simplified":
+                    to = "zh-CN"
+                case "Russian":
+                    to = "ru"
+                case "Korean":
+                    to = "ko"
+                case "Vietnamese":
+                    to = "vi"
+                case "Thai":
+                    to = "th"
+                case "Tagalog":
+                    to = "tl"
+                case "Catalan":
+                    to = "ca"
+                case "Sinhalese":
+                    to = "si"
+                case "Nepall":
+                    to = "ne"
+                case _:
+                    to = "unknown"
+
+            st.info(f"translate to [ {language} ]")
+
+            st.session_state.target_lang = to
+
+            with st.container():
+                work_area1 = st.empty()
+                work_area2 = st.empty()
+            #--------------------------------------
+
+            for index, page in enumerate(book_data):
+                page_text = page[1]
+                #                print("\nPage Number:" + str(index))
+                done = int(((index + 1) / page_count) * 100)
+                my_bar.progress(done,
+                                text=f"Working Page Number : {index + 1}")
+                #  print(len(page_text))
+                #  text_list = [s for s in page_text.split('\n') if s]
+                page_text = re.sub('\.', '.𓂀', page_text)
+                text_list = [s for s in page_text.split('𓂀')]
+                if len(text_list) < 1:
+                    continue
+
+                limit = 0
+                temp_list = []
+                line_number = []
+
+                for n, line in enumerate(text_list):
+
+                    line2 = re.sub(r"\s+", " ", line)
+                    if line2 == "":
+                        continue
+                    temp_list.append((n, line2))
+
+                    if len(temp_list) == 14 or n == len(text_list) - 1:
+                        text_ = ""
+                        all_text_orig = ""
+                        all_text_done = ""
+                        for i, t in enumerate(temp_list):
+                            if t[1] != " ":
+                                line_number.append(t[0])
+                                text_ += '𓂀' + t[1].strip()
+                        temp_list.clear()
+
+                        text_2 = text_
+                        text_ = re.sub('𓂀', "", text_)
+                        #while (re.search('𓂀', text_2)):
+                        #    num = line_number.pop(0)
+                        #    rep_words = f"𓃐NO:{num}| "
+                        #    text_2 = text_2.replace('𓂀', rep_words, 1)
+                        line_number.clear()
+
+                        #                        print(re.sub("𓃐","\n", text_2))
+                        #ts = Translator()
+                        all_text_orig = f":::info\n𓃰{index + 1:05d}" + f"-{n}" + f";\n:::\n{text_}\n"
+
+                        for times in range(0, 5):
+
+                            try:
+                                tsd = GoogleTranslator(
+                                    source="auto",
+                                    target=to).translate(text=text_)
+                                if tsd == None:
+                                    tsd = text_
+                                #tsd = ts.translate(text_, src="en", dest="ja")
+                                #translated_text = ts.translate(line, src="en", dest="ja").text
+                                all_text_done = f":::info\n𓆏{index + 1:05d}" + f"-{n}" + f";\n:::\n{tsd}\n"
+                                #all_text_done = f"**{index:05d}" + f"-{n}" + "; " +  tsd.text + "\n"
+
+                                # all_text_orig += str(n) + "; " + tsd.pronunciation + "\n"
+                                # print(index,n, line)
+                                # print(index,n, tsd.text)
+
+                                #                                print(all_text_orig)
+                                #                                print(all_text_done + "\n")
+                                if type(all_text_orig) is str and type(
+                                        all_text_done) is str:
+
+                                    # intext_1 = f'<span style="color:DimGray;background:GhostWhite">{all_text_orig}</span>'
+                                    # work_area1.markdown(intext_1, unsafe_allow_html=True)
+                                    work_area1.write(f"{all_text_orig}")
+                                    # intext_2 = f'<span style="color:LavenderBlush;background:Gray">{all_text_done}</span>'
+                                    work_area2.write(f"{all_text_done}")
+                                    # work_area2.markdown(intext_2, unsafe_allow_html=True)
+
+                                    with open(
+                                            f"removefolder/{temp_dir}/work_{st.session_state.count}/reuseMarkdown.txt",
+                                            "a") as tempf:
+                                        tempf.write(all_text_orig + "\n\n" +
+                                                    all_text_done + "\n\n")
+
+                                    # st.session_state.result += all_text_orig + "\n\n"
+                                    # st.session_state.result += all_text_done + "\n\n"
+
+                                # print(n, tsd.pronunciation)
+                                with open(
+                                        f"removefolder/{temp_dir}/work_{st.session_state.count}/{title_name}_done.txt",
+                                        "a") as f:
+                                    f.write(all_text_orig + all_text_done +
+                                            "\n")
+                                with open(
+                                        f"removefolder/{temp_dir}/work_{st.session_state.count}/{title_name}_done_{language}.txt",
+                                        "a") as f:
+                                    f.write(all_text_done + "\n")
+
+                                break
+
+                            except Exception as e:
+                                print(e)
+                                time.sleep(3)
+                                continue
+
+                        with open(
+                                f"removefolder/{temp_dir}/work_{st.session_state.count}/{title_name}_orig.txt",
+                                "a") as f:
+                            f.write(all_text_orig + "\n")
+
+            
+            st.markdown("----")
+
+            my_makedirs(f"removefolder/{temp_dir}/download_section")
+            shutil.move(
+                f"removefolder/{temp_dir}/work_{st.session_state.count}/reuseMarkdown.txt",
+                f"removefolder/{temp_dir}/download_section/reuseMarkdown_{st.session_state.count}.txt"
+            )
+
+            shutil.make_archive(
+                f'removefolder/{temp_dir}/download_section/{st.session_state.uploadedfilename}_{st.session_state.count}',\
+                format='zip',\
+                root_dir=f'removefolder/{temp_dir}/work_{st.session_state.count}'\
+                )
+            shutil.rmtree(
+                f"removefolder/{temp_dir}/work_{st.session_state.count}")
+
+            st.balloons()
+            
+            work_area1.empty()
+            work_area2.empty()
+            
+            #--------------------------------------
+
+            st.success("Download translated text files")
+            st.write(intext_0, unsafe_allow_html=True)
+            # plain_text3 = f"[ {st.session_state.target_lang} ] : translated text files"
+            plain_text3 = f"[ {language} ] : translated text files"
+            var_text3 = f'##### <span style="color:#FF69B4">{plain_text3}</span>'
+
+            translated = st.empty()
+            translated.write(var_text3, unsafe_allow_html=True)
+
+            if os.path.isfile(
+                    f'removefolder/{temp_dir}/download_section/{st.session_state.uploadedfilename}_{st.session_state.count}.zip'
+            ):
+                with open(
+                        f"removefolder/{temp_dir}/download_section/{st.session_state.uploadedfilename}_{st.session_state.count}.zip",
+                        "rb") as fpath:
+                    btn = st.download_button(
+                        label=f"DOWNLOAD .zip file",
+                        data=fpath,
+                        file_name=
+                        f"{st.session_state.uploadedfilename}_{st.session_state.count}.zip",
+                        mime="application/zip")
+
+            plain_text4 = "download zipfile"
+            var_text4 = f'<span style="color:gray">▲ `{plain_text4}` 𓁉 </span>'
+            st.write(var_text4, unsafe_allow_html=True)
+
+            st.markdown("----")
+
+            plain_text5 = " 𓀡 results 𓁙 "
+            var_text5 = f'##### <span style="color:#20B2AA">{plain_text5}</span>'
+            st.write(var_text5, unsafe_allow_html=True)
+
+            tempf = open(
+                f"removefolder/{temp_dir}/download_section/reuseMarkdown_{st.session_state.count}.txt"
+            )
+            all_result = tempf.read()
+            tempf.close()
+            st.write(intext_0, unsafe_allow_html=True)
+            st.write(all_result, unsafe_allow_html=True)
+            # st.write(st.session_state.result, unsafe_allow_html=True)
+
+if __name__ == "__main__":
+    heart_beat()
+    main()