minor

app.py

@@ -11,143 +11,142 @@ from yomikata import utils
 from yomikata.dictionary import Dictionary
 from yomikata.utils import parse_furigana
 
-st.title("Yomikata: Disambiguate Japanese Heteronyms with a BERT model")
-
-# @st.cache
-# def add_border(html: str):
-#     WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.5rem; padding: 1rem; margin-bottom: 1.0rem; display: inline-block">{}</div>"""
-#     html = html.replace("\n", " ")
-#     return WRAPPER.format(html)
-
-
-# def get_random_sentence():
-#     from config.config import TEST_DATA_DIR
 
-#     df = pd.read_csv(Path(TEST_DATA_DIR, "test_optimized_strict_heteronyms.csv"))
-#     return df.sample(1).iloc[0].sentence
+@st.cache
+def add_border(html: str):
+    WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.5rem; padding: 1rem; margin-bottom: 1.0rem; display: inline-block">{}</div>"""
+    html = html.replace("\n", " ")
+    return WRAPPER.format(html)
 
 
-# @st.cache
-# def get_dbert_prediction_and_heteronym_list(text):
-#     from yomikata.dbert import dBert
+def get_random_sentence():
+    from config.config import TEST_DATA_DIR
 
-#     reader = dBert()
-#     return reader.furigana(text), reader.heteronyms
+    df = pd.read_csv(Path(TEST_DATA_DIR, "test_optimized_strict_heteronyms.csv"))
+    return df.sample(1).iloc[0].sentence
 
 
-# @st.cache
-# def get_stats():
-#     from config import config
-#     from yomikata.utils import load_dict
+@st.cache
+def get_dbert_prediction_and_heteronym_list(text):
+    from yomikata.dbert import dBert
 
-#     stats = load_dict(Path(config.STORES_DIR, "dbert/training_performance.json"))
+    reader = dBert()
+    return reader.furigana(text), reader.heteronyms
 
-#     global_accuracy = stats["test"]["accuracy"]
 
-#     stats = stats["test"]["heteronym_performance"]
-#     heteronyms = stats.keys()
+@st.cache
+def get_stats():
+    from config import config
+    from yomikata.utils import load_dict
 
-#     accuracy = [stats[heteronym]["accuracy"] for heteronym in heteronyms]
+    stats = load_dict(Path(config.STORES_DIR, "dbert/training_performance.json"))
 
-#     readings = [
-#         "、".join(
-#             [
-#                 "{reading} ({correct}/{n})".format(
-#                     reading=reading,
-#                     correct=stats[heteronym]["readings"][reading]["found"][reading],
-#                     n=stats[heteronym]["readings"][reading]["n"],
-#                 )
-#                 for reading in stats[heteronym]["readings"].keys()
-#                 if (
-#                     stats[heteronym]["readings"][reading]["found"][reading] != 0
-#                     or reading != "<OTHER>"
-#                 )
-#             ]
-#         )
-#         for heteronym in heteronyms
-#     ]
+    global_accuracy = stats["test"]["accuracy"]
 
-#     # if reading != '<OTHER>'
+    stats = stats["test"]["heteronym_performance"]
+    heteronyms = stats.keys()
 
-#     df = pd.DataFrame({"heteronym": heteronyms, "accuracy": accuracy, "readings": readings})
+    accuracy = [stats[heteronym]["accuracy"] for heteronym in heteronyms]
 
-#     df = df[df["readings"].str.contains("、")]
+    readings = [
+        "、".join(
+            [
+                "{reading} ({correct}/{n})".format(
+                    reading=reading,
+                    correct=stats[heteronym]["readings"][reading]["found"][reading],
+                    n=stats[heteronym]["readings"][reading]["n"],
+                )
+                for reading in stats[heteronym]["readings"].keys()
+                if (
+                    stats[heteronym]["readings"][reading]["found"][reading] != 0
+                    or reading != "<OTHER>"
+                )
+            ]
+        )
+        for heteronym in heteronyms
+    ]
 
-#     df["readings"] = df["readings"].str.replace("<OTHER>", "Other")
+    # if reading != '<OTHER>'
 
-#     df = df.rename(columns={"readings": "readings (test corr./total)"})
+    df = pd.DataFrame({"heteronym": heteronyms, "accuracy": accuracy, "readings": readings})
 
-#     df = df.sort_values("accuracy", ascending=False, ignore_index=True)
+    df = df[df["readings"].str.contains("、")]
 
-#     df.index += 1
+    df["readings"] = df["readings"].str.replace("<OTHER>", "Other")
 
-#     return global_accuracy, df
+    df = df.rename(columns={"readings": "readings (test corr./total)"})
 
+    df = df.sort_values("accuracy", ascending=False, ignore_index=True)
 
-# @st.cache
-# def furigana_to_spacy(text_with_furigana):
-#     tokens = parse_furigana(text_with_furigana)
-#     ents = []
-#     output_text = ""
-#     heteronym_count = 0
-#     for token in tokens.groups:
-#         if isinstance(token, ttlig.RubyFrag):
-#             if heteronym_count != 0:
-#                 output_text += ", "
+    df.index += 1
 
-#             ents.append(
-#                 {
-#                     "start": len(output_text),
-#                     "end": len(output_text) + len(token.text),
-#                     "label": token.furi,
-#                 }
-#             )
+    return global_accuracy, df
 
-#             output_text += token.text
-#             heteronym_count += 1
-#         else:
-#             pass
-#     return {
-#         "text": output_text,
-#         "ents": ents,
-#         "title": None,
-#     }
 
+@st.cache
+def furigana_to_spacy(text_with_furigana):
+    tokens = parse_furigana(text_with_furigana)
+    ents = []
+    output_text = ""
+    heteronym_count = 0
+    for token in tokens.groups:
+        if isinstance(token, ttlig.RubyFrag):
+            if heteronym_count != 0:
+                output_text += ", "
 
-# st.title("Yomikata: Disambiguate Japanese Heteronyms with a BERT model")
+            ents.append(
+                {
+                    "start": len(output_text),
+                    "end": len(output_text) + len(token.text),
+                    "label": token.furi,
+                }
+            )
 
-# # Input text box
-# st.markdown("Input a Japanese sentence:")
-
-# if "default_sentence" not in st.session_state:
-#     st.session_state.default_sentence = "え、{人間/にんげん}というものかい? {人間/にんげん}というものは{角/つの}の{生/は}えない、{生白/なまじろ}い{顔/かお}や{手足/てあし}をした、{何/なん}ともいわれず{気味/きみ}の{悪/わる}いものだよ。"
-
-# input_text = st.text_area(
-#     "Input a Japanese sentence:",
-#     utils.remove_furigana(st.session_state.default_sentence),
-#     label_visibility="collapsed",
-# )
+            output_text += token.text
+            heteronym_count += 1
+        else:
+            pass
+    return {
+        "text": output_text,
+        "ents": ents,
+        "title": None,
+    }
 
-# # Yomikata prediction
-# dbert_prediction, heteronyms = get_dbert_prediction_and_heteronym_list(input_text)
 
-# # spacy-style output for the predictions
-# colors = ["#85DCDF", "#DF85DC", "#DCDF85", "#85ABDF"]
-# spacy_dict = furigana_to_spacy(dbert_prediction)
-# label_colors = {
-#     reading: colors[i % len(colors)]
-#     for i, reading in enumerate(set([item["label"] for item in spacy_dict["ents"]]))
-# }
-# html = spacy.displacy.render(spacy_dict, style="ent", manual=True, options={"colors": label_colors})
+st.title("Yomikata: Disambiguate Japanese Heteronyms with a BERT model")
 
-# if len(spacy_dict["ents"]) > 0:
-#     st.markdown("**Yomikata** found and disambiguated the following heteronyms:")
-#     st.write(
-#         f"{add_border(html)}",
-#         unsafe_allow_html=True,
-#     )
-# else:
-#     st.markdown("**Yomikata** found no heteronyms in the input text.")
+# Input text box
+st.markdown("Input a Japanese sentence:")
+
+if "default_sentence" not in st.session_state:
+    st.session_state.default_sentence = "え、{人間/にんげん}というものかい? {人間/にんげん}というものは{角/つの}の{生/は}えない、{生白/なまじろ}い{顔/かお}や{手足/てあし}をした、{何/なん}ともいわれず{気味/きみ}の{悪/わる}いものだよ。"
+
+input_text = st.text_area(
+    "Input a Japanese sentence:",
+    utils.remove_furigana(st.session_state.default_sentence),
+    label_visibility="collapsed",
+)
+
+# Yomikata prediction
+dbert_prediction, heteronyms = get_dbert_prediction_and_heteronym_list(input_text)
+
+# spacy-style output for the predictions
+colors = ["#85DCDF", "#DF85DC", "#DCDF85", "#85ABDF"]
+spacy_dict = furigana_to_spacy(dbert_prediction)
+label_colors = {
+    reading: colors[i % len(colors)]
+    for i, reading in enumerate(set([item["label"] for item in spacy_dict["ents"]]))
+}
+html = spacy.displacy.render(spacy_dict, style="ent", manual=True, options={"colors": label_colors})
+
+if len(spacy_dict["ents"]) > 0:
+    st.markdown("**Yomikata** found and disambiguated the following heteronyms:")
+    st.write(
+        f"{add_border(html)}",
+        unsafe_allow_html=True,
+    )
+else:
+    st.markdown("**Yomikata** found no heteronyms in the input text.")
 
 # # Dictionary + Yomikata prediction
 # st.markdown("**Yomikata** can be coupled with a dictionary to get full furigana:")