Update app.py

app.py

@@ -3,64 +3,59 @@ from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassifica
 import gradio as gr
 from ferret import Benchmark
 
+# Sentiment
 sentiment_tokenizer = AutoTokenizer.from_pretrained("Birkir/electra-base-igc-is-sentiment-analysis")
 sentiment_model = AutoModelForSequenceClassification.from_pretrained("Birkir/electra-base-igc-is-sentiment-analysis")
-
 sentiment_bench = Benchmark(sentiment_model, sentiment_tokenizer)
 
+# Formality
+formality_tokenizer = AutoTokenizer.from_pretrained("svanhvit/formality-classification-icebert")
+formality_model = AutoModelForSequenceClassification.from_pretrained("svanhvit/formality-classification-icebert")
+formality_bench = Benchmark(formality_model, formality_tokenizer)
+
+# Toxicity
+toxicity_tokenizer = AutoTokenizer.from_pretrained("unitary/toxic-bert")
+toxicity_model = AutoModelForSequenceClassification.from_pretrained("unitary/toxic-bert")
+toxicity_bench = Benchmark(toxicity_model, toxicity_tokenizer)
+
+# Politeness
+politeness_tokenizer = AutoTokenizer.from_pretrained("Genius1237/xlm-roberta-large-tydip")
+politeness_model = AutoModelForSequenceClassification.from_pretrained("Genius1237/xlm-roberta-large-tydip")
+politeness_bench = Benchmark(politeness_model, politeness_tokenizer)
+
+# Pipelines
 translator = pipeline("translation", model="Helsinki-NLP/opus-mt-is-en")
 sentiment_classifier = pipeline("text-classification", model="Birkir/electra-base-igc-is-sentiment-analysis")
 formality_classifier = pipeline("text-classification", model="svanhvit/formality-classification-icebert")
 detoxify_pipeline = pipeline('text-classification', model='unitary/toxic-bert', tokenizer='bert-base-uncased', function_to_apply='sigmoid', top_k=None)
 politeness_classifier = pipeline("text-classification", model="Genius1237/xlm-roberta-large-tydip")
 
-def analyze_sentiment_with_influence(icelandic_text):
-    sentiment_label, sentiment_score = analyze_sentiment(icelandic_text)
-    
-    sentiment_label = sentiment_label.replace("LABEL_", "")
-    
-    explanations_sentiment = sentiment_bench.explain(icelandic_text, target=1)  # Adjust target as necessary
-    
+def replace_encoding(tokens):
+    return [token.replace('Ġ', ' ') for token in tokens]
+
+def analyze_text_with_influence(text, bench, label_conversion):
+    explanations = bench.explain(text, target=1)
     influential_words = []
-    for explanation in explanations_sentiment:
+    for explanation in explanations:
         if explanation.explainer == 'Partition SHAP':
             tokens = replace_encoding(explanation.tokens)
             token_score_pairs = zip(tokens, explanation.scores)
             influential_words.extend([(token, score) for token, score in token_score_pairs])
-    
     influential_words_str = "; ".join([f"{token} ({score:.2f})" for token, score in influential_words])
-    
-    analysis_results = (
-        f"Sentiment: Label: {sentiment_label}, Score: {round(sentiment_score, 2)}\n"
-        f"Influential Words: {influential_words_str}"
-    )
-    return analysis_results
-
-def replace_encoding(tokens):
-    return [token.replace('Ġ', ' ')
-                 .replace('ð', 'ð')
-                 .replace('é', 'é')
-                 .replace('æ', 'æ')
-                 .replace('ý', 'ý')
-                 .replace('á', 'á')
-                 .replace('ú', 'ú')
-                 .replace('ÃŃ', 'í')
-                 .replace('Ãö', 'ö')
-                 .replace('þ', 'þ')
-                 .replace('Ãģ', 'Á')
-                 .replace('Ãį', 'Ú')
-                 .replace('Ãĵ', 'Ó')
-                 .replace('ÃĨ', 'Æ')
-                 .replace('ÃIJ', 'Ð')
-                 .replace('Ãĸ', 'Ö')
-                 .replace('Ãī', 'É')
-                 .replace('Ãļ', 'ý')
-            for token in tokens[1:-1]]
+    return label_conversion, influential_words_str
 
 def translate_text(text):
     translation = translator(text, max_length=512)
     return translation[0]['translation_text']
 
+def analyze_sentiment(text):
+    sentiment_result = sentiment_classifier(text)
+    return sentiment_result[0]['label'].replace("LABEL_", ""), sentiment_result[0]['score']
+
+def analyze_formality(text):
+    formality_result = formality_classifier(text)
+    return formality_result[0]['label'], formality_result[0]['score']
+
 def analyze_toxicity(text):
     toxicity_results = detoxify_pipeline(text)
     return toxicity_results[0]
@@ -69,45 +64,40 @@ def analyze_politeness(text):
     politeness_result = politeness_classifier(text)
     return politeness_result[0]['label'], politeness_result[0]['score']
 
-def analyze_formality(text):
-    formality_result = formality_classifier(text)
-    formality_label = formality_result[0]['label']
-    formality_score = formality_result[0]['score']
-    return formality_label, formality_score
+def analyze_text(icelandic_text):
+    translated_text = translate_text(icelandic_text)
 
-def analyze_sentiment(text):
-    sentiment_result = sentiment_classifier(text)
-    sentiment_label = sentiment_result[0]['label']
-    sentiment_score = sentiment_result[0]['score']
-    return sentiment_label, sentiment_score
+    sentiment_label, sentiment_score = analyze_sentiment(icelandic_text)
+    sentiment_analysis = analyze_text_with_influence(icelandic_text, sentiment_bench, sentiment_label)
 
-def analyze_text(icelandic_text):
     formality_label, formality_score = analyze_formality(icelandic_text)
-    translated_text = translate_text(icelandic_text)
+    formality_analysis = analyze_text_with_influence(icelandic_text, formality_bench, formality_label)
+
     toxicity_results = analyze_toxicity(translated_text)
-    if isinstance(toxicity_results, list):
-        toxicity_results = toxicity_results[0]
     toxicity_label = '1' if toxicity_results['score'] >= 0.5 else '0'
-    politeness_label, politeness_score = analyze_politeness(translated_text)
-    politeness_label = '1' if politeness_label.lower() == 'polite' else '0'
-
-    sentiment_analysis_with_influence = analyze_sentiment_with_influence(icelandic_text)
-
-    analysis_results = (
-        f"Translated Text: {translated_text}\n\n"
-        f"{sentiment_analysis_with_influence}\n"  
-        f"Formality: Label: {formality_label}, Score: {round(formality_score, 2)}\n"
-        f"Toxicity: Label: {toxicity_label}, Score: {round(toxicity_results['score'], 2)}\n"
-        f"Politeness: Label: {politeness_label}, Score: {round(politeness_score, 2)}"
-    )
-    return analysis_results
+    toxicity_analysis = analyze_text_with_influence(translated_text, toxicity_bench, toxicity_label)
 
+    politeness_label, politeness_score = analyze_politeness(translated_text)
+    politeness_analysis = analyze_text_with_influence(translated_text, politeness_bench, politeness_label)
+
+    analysis_results = f"""
+    Translated Text: {translated_text}\n\n
+    Sentiment: {sentiment_analysis[0]}, Score: {round(sentiment_score, 2)}\n
+    Influential Words in Sentiment: {sentiment_analysis[1]}\n
+    Formality: {formality_analysis[0]}, Score: {round(formality_score, 2)}\n
+    Influential Words in Formality: {formality_analysis[1]}\n
+    Toxicity: {toxicity_analysis[0]}, Score: {round(toxicity_results['score'], 2)}\n
+    Influential Words in Toxicity: {toxicity_analysis[1]}\n
+    Politeness: {politeness_analysis[0]}, Score: {round(politeness_score, 2)}\n
+    Influential Words in Politeness: {politeness_analysis[1]}
+    """
+    return analysis_results.strip()
 
 demo = gr.Interface(fn=analyze_text, 
                     inputs=gr.Textbox(lines=2, placeholder="Enter Icelandic Text Here..."), 
                     outputs=gr.Textbox(label="Analysis Results"),
                     title="Icelandic Text Analysis",
-                    description="This app translates Icelandic text to English and performs sentiment, formality, toxicity, and politeness analysis.")
+                    description="This app translates Icelandic text to English and performs sentiment, formality, toxicity, and politeness analysis along with influential words analysis.")
 
 if __name__ == "__main__":
     demo.launch()