Transformer instead of flair and use dependency image for one article as test

app.py

@@ -23,15 +23,15 @@ import spacy
 from spacy import displacy
 from spacy_streamlit import visualize_parser
 
-from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification
 from transformers import pipeline
 import os
 from transformers_interpret import SequenceClassificationExplainer
 
-
 # USE_model = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
+
 @st.experimental_singleton
 def get_sentence_embedding_model():
     return SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
@@ -43,14 +43,21 @@ def get_spacy():
     return nlp
 
 
-#TODO: might look into which one is the best here
-#TODO: might be useful to make an ml6 preloaded model for flair as this takes ridiculously long to load the first time
+# TODO: might look into which one is the best here
+# TODO: might be useful to make an ml6 preloaded model for flair as this takes ridiculously long to load the first time
 @st.experimental_singleton
-#@st.cache(suppress_st_warning=True, allow_output_mutation=True)
+# @st.cache(suppress_st_warning=True, allow_output_mutation=True)
 def get_flair_tagger():
     return SequenceTagger.load("flair/ner-english-ontonotes-fast")
 
 
+@st.experimental_singleton
+def get_transformer_pipeline():
+    tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
+    model = AutoModelForTokenClassification.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
+    return pipeline("ner", model=model, tokenizer=tokenizer, grouped_entities=True)
+
+
 # Page setup
 st.set_page_config(
     page_title="Post-processing summarization fact checker",
@@ -97,6 +104,12 @@ def fetch_dependency_specific_contents(filename: str) -> AnyStr:
     return data
 
 
+def fetch_dependency_svg(filename: str) -> AnyStr:
+    with open(f'./dependency-images/{filename.lower()}.txt', 'r') as f:
+        data = f.read()
+    return data
+
+
 def display_summary(article_name: str):
     summary_content = fetch_summary_contents(article_name)
     st.session_state.summary_output = summary_content
@@ -122,10 +135,16 @@ def get_all_entities_per_sentence(text):
             entities_this_sentence.append(str(entity))
 
         # FLAIR ENTITIES
-        sentence_entities = Sentence(str(sentence))
-        tagger.predict(sentence_entities)
-        for entity in sentence_entities.get_spans('ner'):
-            entities_this_sentence.append(entity.text)
+        # sentence_entities = Sentence(str(sentence))
+        # tagger.predict(sentence_entities)
+        # for entity in sentence_entities.get_spans('ner'):
+        #     entities_this_sentence.append(entity.text)
+
+        # XLM ENTITIES
+        entities_xlm = [entity["word"] for entity in ner_model(str(sentence))]
+        for entity in entities_xlm:
+            entities_this_sentence.append(str(entity))
+
         entities_all_sentences.append(entities_this_sentence)
 
     return entities_all_sentences
@@ -188,6 +207,7 @@ def highlight_entities(article_name: str):
 def render_dependency_parsing(text: str):
     html = render_sentence_custom(text)
     html = html.replace("\n\n", "\n")
+    # print(get_svg(html))
     st.write(get_svg(html), unsafe_allow_html=True)
 
 
@@ -275,7 +295,8 @@ currently selected article.""")
 
 nlp = get_spacy()
 sentence_embedding_model = get_sentence_embedding_model()
-tagger = get_flair_tagger()
+# tagger = get_flair_tagger()
+ner_model = get_transformer_pipeline()
 
 # GENERATING SUMMARIES PART
 st.header("Generating summaries")
@@ -309,11 +330,6 @@ else:
     st.error('**Error**: No comment to classify. Please provide a comment.',
              help="Generate summary for the given article text")
 
-if is_valid_url(article_text):
-    print("YES")
-else:
-    print("NO")
-
 
 def render_svg(svg_file):
     with open(svg_file, "r") as f:
@@ -390,17 +406,21 @@ st.markdown("However, by empirical testing, we have found that there are certain
             "dependencies that satisfy the discussed constraints. We also discuss the specific results for the "
             "currently selected article.")
 with st.spinner("Doing dependency parsing..."):
-    summary_deps = check_dependency(False)
-    article_deps = check_dependency(True)
-    total_unmatched_deps = []
-    for summ_dep in summary_deps:
-        if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
-            total_unmatched_deps.append(summ_dep)
-    # print(f'ALL UNMATCHED DEPS ARE: {total_unmatched_deps}')
-    # render_dependency_parsing(check_dependency(False))
-    if total_unmatched_deps:
-        for current_drawing_list in total_unmatched_deps:
-            render_dependency_parsing(current_drawing_list)
+    # TODO RIGHT IF FUNCTION (IF EXAMPLE AND IF INPUT UNCHANGED)
+    if selected_article == 'article11':
+        st.write(fetch_dependency_svg((selected_article)), unsafe_allow_html=True)
+    else:
+        summary_deps = check_dependency(False)
+        article_deps = check_dependency(True)
+        total_unmatched_deps = []
+        for summ_dep in summary_deps:
+            if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):
+                total_unmatched_deps.append(summ_dep)
+        # print(f'ALL UNMATCHED DEPS ARE: {total_unmatched_deps}')
+        # render_dependency_parsing(check_dependency(False))
+        if total_unmatched_deps:
+            for current_drawing_list in total_unmatched_deps:
+                render_dependency_parsing(current_drawing_list)
     dep_specific_text = fetch_dependency_specific_contents(selected_article)
     soup = BeautifulSoup(dep_specific_text, features="html.parser")
     HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem;

dependency-images/article11.txt

@@ -0,0 +1 @@
+<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem"><img src="data:image/svg+xml;base64,CiAgPHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4bWw6bGFuZz0iZW4iIGlkPSIwIiBjbGFzcz0iZGlzcGxhY3kiIHdpZHRoPSIxMjAwIiBoZWlnaHQ9Ijc1IiBkaXJlY3Rpb249Imx0ciIgc3R5bGU9Im1heC13aWR0aDogbm9uZTsgaGVpZ2h0OiA3NXB4OyBjb2xvcjogIzAwMDAwOyBiYWNrZ3JvdW5kOiAjZmZmZmZmOyBmb250LWZhbWlseTogQXJpYWw7IGRpcmVjdGlvbjogbHRyIj4KICA8dGV4dCBjbGFzcz0iZGlzcGxhY3ktdG9rZW4iIGZpbGw9ImN1cnJlbnRDb2xvciIgdGV4dC1hbmNob3I9InN0YXJ0IiB5PSI3MCI+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktd29yZCIgZmlsbD0iY3VycmVudENvbG9yIiB4PSIxMCI+VGhlIDwvdHNwYW4+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktdGFnIiBkeT0iMmVtIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjEwIj48L3RzcGFuPgogIDwvdGV4dD4KICAKICA8dGV4dCBjbGFzcz0iZGlzcGxhY3ktdG9rZW4iIGZpbGw9ImN1cnJlbnRDb2xvciIgdGV4dC1hbmNob3I9InN0YXJ0IiB5PSI3MCI+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktd29yZCIgZmlsbD0iY3VycmVudENvbG9yIiB4PSI0NiI+aGVhZHBob25lcyA8L3RzcGFuPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXRhZyIgZHk9IjJlbSIgZmlsbD0iY3VycmVudENvbG9yIiB4PSI0NiI+PC90c3Bhbj4KICA8L3RleHQ+CiAgCiAgPHRleHQgY2xhc3M9ImRpc3BsYWN5LXRva2VuIiBmaWxsPSJjdXJyZW50Q29sb3IiIHRleHQtYW5jaG9yPSJzdGFydCIgeT0iNzAiPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXdvcmQiIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iMTQzIj5zdGFydCA8L3RzcGFuPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXRhZyIgZHk9IjJlbSIgZmlsbD0iY3VycmVudENvbG9yIiB4PSIxNDMiPjwvdHNwYW4+CiAgPC90ZXh0PgogIAogIDx0ZXh0IGNsYXNzPSJkaXNwbGFjeS10b2tlbiIgZmlsbD0iY3VycmVudENvbG9yIiB0ZXh0LWFuY2hvcj0ic3RhcnQiIHk9IjcwIj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS13b3JkIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjE4MyI+YXQgPC90c3Bhbj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS10YWciIGR5PSIyZW0iIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iMTgzIj48L3RzcGFuPgogIDwvdGV4dD4KICAKICA8dGV4dCBjbGFzcz0iZGlzcGxhY3ktdG9rZW4iIGZpbGw9ImN1cnJlbnRDb2xvciIgdGV4dC1hbmNob3I9InN0YXJ0IiB5PSI3MCI+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktd29yZCIgZmlsbD0iY3VycmVudENvbG9yIiB4PSIyMDUiPiQgPC90c3Bhbj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS10YWciIGR5PSIyZW0iIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iMjA1Ij48L3RzcGFuPgogIDwvdGV4dD4KICAKICA8dGV4dCBjbGFzcz0iZGlzcGxhY3ktdG9rZW4iIGZpbGw9ImN1cnJlbnRDb2xvciIgdGV4dC1hbmNob3I9InN0YXJ0IiB5PSI3MCI+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktd29yZCIgZmlsbD0iY3VycmVudENvbG9yIiB4PSIyMjIiPjk5OSA8L3RzcGFuPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXRhZyIgZHk9IjJlbSIgZmlsbD0iY3VycmVudENvbG9yIiB4PSIyMjIiPjwvdHNwYW4+CiAgPC90ZXh0PgogIAogIDx0ZXh0IGNsYXNzPSJkaXNwbGFjeS10b2tlbiIgZmlsbD0iY3VycmVudENvbG9yIiB0ZXh0LWFuY2hvcj0ic3RhcnQiIHk9IjcwIj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS13b3JkIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjI1NyI+YW5kIDwvdHNwYW4+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktdGFnIiBkeT0iMmVtIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjI1NyI+PC90c3Bhbj4KICA8L3RleHQ+CiAgCiAgPHRleHQgY2xhc3M9ImRpc3BsYWN5LXRva2VuIiBmaWxsPSJjdXJyZW50Q29sb3IiIHRleHQtYW5jaG9yPSJzdGFydCIgeT0iNzAiPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXdvcmQiIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iMjkyIj53aWxsIDwvdHNwYW4+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktdGFnIiBkeT0iMmVtIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjI5MiI+PC90c3Bhbj4KICA8L3RleHQ+CiAgCiAgPHRleHQgY2xhc3M9ImRpc3BsYWN5LXRva2VuIiBmaWxsPSJjdXJyZW50Q29sb3IiIHRleHQtYW5jaG9yPSJzdGFydCIgeT0iNzAiPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXdvcmQiIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iMzIzIj5iZSA8L3RzcGFuPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXRhZyIgZHk9IjJlbSIgZmlsbD0iY3VycmVudENvbG9yIiB4PSIzMjMiPjwvdHNwYW4+CiAgPC90ZXh0PgogIAogIDx0ZXh0IGNsYXNzPSJkaXNwbGFjeS10b2tlbiIgZmlsbD0iY3VycmVudENvbG9yIiB0ZXh0LWFuY2hvcj0ic3RhcnQiIHk9IjcwIj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS13b3JkIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjM0OSI+YXZhaWxhYmxlIDwvdHNwYW4+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktdGFnIiBkeT0iMmVtIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjM0OSI+PC90c3Bhbj4KICA8L3RleHQ+CiAgCiAgPHRleHQgY2xhc3M9ImRpc3BsYWN5LXRva2VuIiBmaWxsPSJjdXJyZW50Q29sb3IiIHRleHQtYW5jaG9yPSJzdGFydCIgeT0iNzAiPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXdvcmQiIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iNDIxIj5zdGFydGluZyA8L3RzcGFuPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXRhZyIgZHk9IjJlbSIgZmlsbD0iY3VycmVudENvbG9yIiB4PSI0MjEiPjwvdHNwYW4+CiAgPC90ZXh0PgogIAogIDx0ZXh0IGNsYXNzPSJkaXNwbGFjeS10b2tlbiIgZmlsbD0iY3VycmVudENvbG9yIiB0ZXh0LWFuY2hvcj0ic3RhcnQiIHk9IjcwIj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS13b3JkIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjQ4MiI+dG9kYXkgPC90c3Bhbj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS10YWciIGR5PSIyZW0iIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iNDgyIj48L3RzcGFuPgogIDwvdGV4dD4KICAKICA8dGV4dCBjbGFzcz0iZGlzcGxhY3ktdG9rZW4iIGZpbGw9ImN1cnJlbnRDb2xvciIgdGV4dC1hbmNob3I9InN0YXJ0IiB5PSI3MCI+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktd29yZCIgZmlsbD0iY3VycmVudENvbG9yIiB4PSI1MzAiPmluIDwvdHNwYW4+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktdGFnIiBkeT0iMmVtIiBmaWxsPSJjdXJyZW50Q29sb3IiIHg9IjUzMCI+PC90c3Bhbj4KICA8L3RleHQ+CiAgCiAgPHRleHQgY2xhc3M9ImRpc3BsYWN5LXRva2VuIiBmaWxsPSJjdXJyZW50Q29sb3IiIHRleHQtYW5jaG9yPSJzdGFydCIgeT0iNzAiPgogICAgICA8dHNwYW4gY2xhc3M9ImRpc3BsYWN5LXdvcmQiIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iNjAxIj50aGUgPC90c3Bhbj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS10YWciIGR5PSIyZW0iIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iNjAxIj48L3RzcGFuPgogIDwvdGV4dD4KICAKICA8dGV4dCBjbGFzcz0iZGlzcGxhY3ktdG9rZW4iIGZpbGw9ImN1cnJlbnRDb2xvciIgdGV4dC1hbmNob3I9InN0YXJ0IiB5PSI3MCI+CiAgICAgIDx0c3BhbiBjbGFzcz0iZGlzcGxhY3ktd29yZCIgZmlsbD0iY3VycmVudENvbG9yIiB4PSI2MzIiPlUuUy4gPC90c3Bhbj4KICAgICAgPHRzcGFuIGNsYXNzPSJkaXNwbGFjeS10YWciIGR5PSIyZW0iIGZpbGw9ImN1cnJlbnRDb2xvciIgeD0iNjMyIj48L3RzcGFuPgogIDwvdGV4dD4KICAKICAgIDxnIGNsYXNzPSJkaXNwbGFjeS1hcnJvdyI+CiAgICAgICAgPHBhdGggY2xhc3M9ImRpc3BsYWN5LWFyYyIgaWQ9ImFycm93LTAtMCIgc3Ryb2tlLXdpZHRoPSIycHgiIGQ9Ik01NDAsNTAgQzU0MCw1IDY0Miw1IDY0Miw1MCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJyZWQiLz4KICAgICAgICA8dGV4dCBkeT0iMS4yNWVtIiBzdHlsZT0iZm9udC1zaXplOiAwLjhlbTsgbGV0dGVyLXNwYWNpbmc6IDFweCI+CiAgICAgICAgICAgIDx0ZXh0UGF0aCB4bGluazpocmVmPSIjYXJyb3ctMC0wIiBjbGFzcz0iZGlzcGxhY3ktbGFiZWwiIHN0YXJ0T2Zmc2V0PSI1MCUiIHNpZGU9InJpZ2h0IiBmaWxsPSJyZWQiIHRleHQtYW5jaG9yPSJtaWRkbGUiPnBvYmo8L3RleHRQYXRoPgogICAgICAgIDwvdGV4dD4KICAgICAgICA8cGF0aCBjbGFzcz0iZGlzcGxhY3ktYXJyb3doZWFkIiBkPSJNNjQyLDUyIEw2NDYsNDQgNjM4LDQ0IiBmaWxsPSJyZWQiLz4KICAgIDwvZz4KICAgIDwvc3ZnPgogIA==" style=""/></div>