Update app.py

app.py

@@ -5,15 +5,16 @@ import logging
 import torch
 import pandas as pd
 import plotly.express as px
+from typing import Union, List
+
+from langdetect import detect, LangDetectException
 from transformers import (
     AutoTokenizer,
     AutoModelForSeq2SeqLM,
     pipeline,
     BitsAndBytesConfig,
 )
-from langdetect import detect, LangDetectException
 import evaluate
-from typing import Union, List
 
 # ────────── Logging ──────────
 logging.basicConfig(
@@ -27,9 +28,9 @@ logger = logging.getLogger(__name__)
 # ────────── Model Manager ──────────
 class ModelManager:
     """
-    Selects and loads a translation model (NLLB-200 or M2M100),
-    using 8-bit quantization only if CUDA is available.
-    Auto-detects source language and defaults target to Turkish.
+    Select & load the best translation model from a candidate list,
+    using 8-bit quant if CUDA is available, else full-precision.
+    Auto-picks Turkish target code.
     """
     def __init__(
         self,
@@ -37,81 +38,75 @@ class ModelManager:
         quantize: bool = True,
         default_tgt: str = None,
     ):
-        # Disable 8-bit if CUDA isn't available
+        # disable 8-bit if no GPU
         if quantize and not torch.cuda.is_available():
             logger.warning("CUDA unavailable; disabling 8-bit quantization")
             quantize = False
         self.quantize = quantize
 
-        self.candidates = candidates or [
+        self.candidates   = candidates or [
             "facebook/nllb-200-distilled-600M",
             "facebook/m2m100_418M",
         ]
-        self.default_tgt = default_tgt  # will auto-pick if None
-
-        self.model_name: str = None
-        self.tokenizer = None
-        self.model = None
-        self.pipeline = None
-        self.lang_codes: List[str] = []
+        self.default_tgt  = default_tgt  # will auto-pick if None
+        self.model_name   = None
+        self.tokenizer    = None
+        self.model        = None
+        self.pipeline     = None
+        self.lang_codes   = []
 
         self._select_and_load()
 
     def _select_and_load(self):
         last_err = None
-        for model_name in self.candidates:
+        for name in self.candidates:
             try:
-                # Load tokenizer
-                logger.info(f"Loading tokenizer for {model_name}")
-                tok = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+                # 1) tokenizer
+                logger.info(f"Loading tokenizer for {name}")
+                tok = AutoTokenizer.from_pretrained(name, use_fast=True)
                 if not hasattr(tok, "lang_code_to_id"):
-                    raise AttributeError(f"Tokenizer for {model_name} missing lang_code_to_id")
+                    raise AttributeError("no lang_code_to_id on tokenizer")
 
-                # Load model (with or without 8-bit)
-                logger.info(f"Loading model {model_name} (8-bit={self.quantize})")
+                # 2) model
+                logger.info(f"Loading model {name} (8-bit={self.quantize})")
                 if self.quantize:
                     bnb_cfg = BitsAndBytesConfig(load_in_8bit=True)
                     mdl = AutoModelForSeq2SeqLM.from_pretrained(
-                        model_name,
+                        name,
                         device_map="auto",
                         quantization_config=bnb_cfg,
                     )
                 else:
                     mdl = AutoModelForSeq2SeqLM.from_pretrained(
-                        model_name,
+                        name,
                         device_map="auto",
                     )
-                logger.info(f"Model {model_name} loaded successfully")
-
-                # Wrap in a translation pipeline
-                pipe = pipeline(
-                    "translation",
-                    model=mdl,
-                    tokenizer=tok,
-                )
-
-                # Store and break
-                self.model_name = model_name
-                self.tokenizer = tok
-                self.model = mdl
-                self.pipeline = pipe
+                logger.info(f"Loaded {name}")
+
+                # 3) pipeline
+                pipe = pipeline("translation", model=mdl, tokenizer=tok)
+
+                # store
+                self.model_name = name
+                self.tokenizer  = tok
+                self.model      = mdl
+                self.pipeline   = pipe
                 self.lang_codes = list(tok.lang_code_to_id.keys())
 
-                # Auto-pick Turkish target code if none specified
+                # pick Turkish code if needed
                 if not self.default_tgt:
-                    tur_codes = [c for c in self.lang_codes if c.lower().startswith("tr")]
-                    if not tur_codes:
-                        raise ValueError(f"No Turkish code found in {model_name}")
-                    self.default_tgt = tur_codes[0]
-                logger.info(f"Default target language: {self.default_tgt}")
-
+                    tur = [c for c in self.lang_codes if c.lower().startswith("tr")]
+                    if not tur:
+                        raise ValueError("No Turkish code available")
+                    self.default_tgt = tur[0]
+                logger.info(f"default_tgt = {self.default_tgt}")
                 return
 
             except Exception as e:
-                logger.warning(f"Failed to load {model_name}: {e}")
+                logger.warning(f"failed to load {name}: {e}")
                 last_err = e
 
-        raise RuntimeError(f"Could not load any model from {self.candidates}: {last_err}")
+        raise RuntimeError(f"no model loaded: {last_err}")
 
     def translate(
         self,
@@ -119,43 +114,39 @@ class ModelManager:
         src_lang: str = None,
         tgt_lang: str = None,
     ):
-        """
-        Translate `text` from src_lang → tgt_lang.
-        Auto-detects src_lang if not given.
-        """
         tgt = tgt_lang or self.default_tgt
 
-        # Auto-detect source language if missing
+        # auto-detect source
         if not src_lang:
             sample = text[0] if isinstance(text, list) else text
             try:
                 iso = detect(sample).lower()
-                candidates = [c for c in self.lang_codes if c.lower().startswith(iso)]
-                if not candidates:
+                cand = [c for c in self.lang_codes if c.lower().startswith(iso)]
+                if not cand:
                     raise LangDetectException(f"No code for ISO '{iso}'")
-                exact = [c for c in candidates if c.lower() == iso]
-                src = exact[0] if exact else candidates[0]
-                logger.info(f"Auto-detected src_lang={src}")
-            except Exception as e:
-                logger.warning(f"langdetect failed ({e}); defaulting to English")
-                eng_codes = [c for c in self.lang_codes if c.lower().startswith("en")]
-                src = eng_codes[0] if eng_codes else self.lang_codes[0]
+                # exact or first
+                exact = [c for c in cand if c.lower() == iso]
+                src = exact[0] if exact else cand[0]
+                logger.info(f"src_lang = {src}")
+            except Exception:
+                eng = [c for c in self.lang_codes if c.lower().startswith("en")]
+                src = eng[0] if eng else self.lang_codes[0]
+                logger.warning(f"defaulting src_lang = {src}")
         else:
             src = src_lang
 
         return self.pipeline(text, src_lang=src, tgt_lang=tgt)
 
     def get_info(self):
-        """Return metadata for the sidebar display."""
-        quantized = getattr(self.model, "is_loaded_in_8bit", False)
-        device = "cpu"
+        # figure out device for display
+        dev = "cpu"
         if torch.cuda.is_available() and hasattr(self.model, "device"):
-            dev = self.model.device
-            device = str(dev) if isinstance(dev, torch.device) else f"cuda:{getattr(dev, 'index', '')}"
+            d = self.model.device
+            dev = str(d) if isinstance(d, torch.device) else f"cuda:{getattr(d,'index','')}"
         return {
             "model":       self.model_name,
-            "quantized":   quantized,
-            "device":      device,
+            "quantized":   self.quantize,
+            "device":      dev,
             "default_tgt": self.default_tgt,
         }
 
@@ -163,17 +154,10 @@ class ModelManager:
 # ────────── Evaluator ──────────
 class TranslationEvaluator:
     def __init__(self):
-        self.bleu = evaluate.load("bleu")
-        try:
-            self.bertscore = evaluate.load("bertscore")
-        except Exception as e:
-            logger.error("BERTScore load error: %s", e)
-            self.bertscore = None
-        try:
-            self.comet = evaluate.load("comet", model_id="unbabel/comet-mqm-qe-da")
-        except Exception as e:
-            logger.error("COMET load error: %s", e)
-            self.comet = None
+        self.bleu      = evaluate.load("bleu")
+        self.bertscore = evaluate.load("bertscore")
+        self.comet     = evaluate.load("comet", model_id="unbabel/wmt22-comet-da")
+        logger.info("Loaded BLEU, BERTScore, COMET")
 
     def evaluate(
         self,
@@ -184,69 +168,36 @@ class TranslationEvaluator:
         results = {}
 
         # BLEU
-        try:
-            bleu_res = self.bleu.compute(
-                predictions=predictions,
-                references=[[r] for r in references],
-            )
-            results["BLEU"] = float(bleu_res.get("bleu", 0.0))
-        except Exception as e:
-            logger.error("BLEU compute error: %s", e)
-            results["BLEU"] = 0.0
-
-        # BERTScore (general)
-        if self.bertscore:
-            try:
-                bs = self.bertscore.compute(
-                    predictions=predictions, references=references, lang="xx"
-                )
-                f1 = bs.get("f1", [])
-                results["BERTScore"] = float(sum(f1)) / max(len(f1), 1)
-            except Exception as e:
-                logger.error("BERTScore compute error: %s", e)
-                results["BERTScore"] = 0.0
-        else:
-            results["BERTScore"] = 0.0
+        bleu_r = self.bleu.compute(predictions=predictions, references=[[r] for r in references])
+        results["BLEU"] = float(bleu_r.get("bleu", 0.0))
 
-        # BERTurk (Turkish)
-        if self.bertscore:
-            try:
-                bs_tr = self.bertscore.compute(
-                    predictions=predictions, references=references, lang="tr"
-                )
-                f1_tr = bs_tr.get("f1", [])
-                results["BERTurk"] = float(sum(f1_tr)) / max(len(f1_tr), 1)
-            except Exception as e:
-                logger.error("BERTurk compute error: %s", e)
-                results["BERTurk"] = 0.0
-        else:
-            results["BERTurk"] = 0.0
+        # BERTScore (xx)
+        bs = self.bertscore.compute(predictions=predictions, references=references, lang="xx")
+        f1 = bs.get("f1", [])
+        results["BERTScore"] = float(sum(f1) / len(f1)) if f1 else 0.0
+
+        # BERTurk (tr)
+        bs_tr = self.bertscore.compute(predictions=predictions, references=references, lang="tr")
+        f1t  = bs_tr.get("f1", [])
+        results["BERTurk"] = float(sum(f1t) / len(f1t)) if f1t else 0.0
 
         # COMET
-        if self.comet:
-            try:
-                cm = self.comet.compute(
-                    srcs=sources, hyps=predictions, refs=references
-                )
-                sc = cm.get("scores", None)
-                if isinstance(sc, list):
-                    results["COMET"] = float(sc[0]) if sc else 0.0
-                else:
-                    results["COMET"] = float(sc or 0.0)
-            except Exception as e:
-                logger.error("COMET compute error: %s", e)
-                results["COMET"] = 0.0
+        cm = self.comet.compute(srcs=sources, hyps=predictions, refs=references)
+        sc = cm.get("scores", None)
+        if isinstance(sc, list):
+            results["COMET"] = float(sc[0]) if sc else 0.0
         else:
-            results["COMET"] = 0.0
+            results["COMET"] = float(sc or 0.0)
 
         return results
 
 
 # ────────── Streamlit App ──────────
+
 @st.cache_resource
 def load_resources():
     mgr = ModelManager(quantize=True)
-    ev = TranslationEvaluator()
+    ev  = TranslationEvaluator()
     return mgr, ev
 
 
@@ -265,30 +216,38 @@ def process_text(
     ev: TranslationEvaluator,
     metrics: List[str],
 ):
-    out = mgr.translate(src)
-    hyp = out[0]["translation_text"] if isinstance(out, list) else out["translation_text"]
-    scores = ev.evaluate([src], [ref or ""], [hyp])
-    return {
-        "source":     src,
-        "reference":  ref,
+    # 1) translate
+    out = mgr.translate(src)  # list of dicts
+    hyp = out[0]["translation_text"]
+
+    # 2) if we have a non-blank reference → compute metrics; else all Nones
+    result = {
+        "source":    src,
+        "reference": ref or None,
         "hypothesis": hyp,
-        **{m: scores.get(m, 0.0) for m in metrics},
     }
+    if ref and ref.strip():
+        scores = ev.evaluate([src], [ref], [hyp])
+        for m in metrics:
+            result[m] = scores.get(m, 0.0)
+    else:
+        for m in metrics:
+            result[m] = None
+
+    return result
 
 
 def show_single_results(res: dict, metrics: List[str]):
     left, right = st.columns(2)
     with left:
-        st.markdown("**Source:**")
-        st.write(res["source"])
-        st.markdown("**Hypothesis (TR):**")
-        st.write(res["hypothesis"])
+        st.markdown("**Source:**");            st.write(res["source"])
+        st.markdown("**Hypothesis (TR):**");   st.write(res["hypothesis"])
         if res["reference"]:
-            st.markdown("**Reference (TR):**")
-            st.write(res["reference"])
+            st.markdown("**Reference (TR):**"); st.write(res["reference"])
     with right:
         st.markdown("### Scores")
         df = pd.DataFrame([{m: res[m] for m in metrics}])
+        df = df.replace({None: "N/A"})
         st.table(df)
 
 
@@ -305,24 +264,35 @@ def process_file(
     prog = st.progress(0)
     results = []
     total = len(df)
+
     for i in range(0, total, batch_size):
         batch = df.iloc[i : i + batch_size]
-        srcs = batch["src"].tolist()
-        refs = batch["ref_tr"].tolist()
+        srcs, refs = batch["src"].tolist(), batch["ref_tr"].tolist()
         outs = mgr.translate(srcs)
         hyps = [o["translation_text"] for o in outs]
+
         for s, r, h in zip(srcs, refs, hyps):
-            sc = ev.evaluate([s], [r], [h])
             entry = {"src": s, "ref_tr": r, "hyp_tr": h}
-            entry.update({m: sc.get(m, 0.0) for m in metrics})
+            if r and str(r).strip():
+                sc = ev.evaluate([s], [r], [h])
+                for m in metrics:
+                    entry[m] = sc.get(m, 0.0)
+            else:
+                for m in metrics:
+                    entry[m] = None
             results.append(entry)
+
         prog.progress(min(i + batch_size, total) / total)
+
     return pd.DataFrame(results)
 
 
 def show_batch_viz(df: pd.DataFrame, metrics: List[str]):
     for m in metrics:
         st.markdown(f"#### {m} Distribution")
+        if df[m].dropna().empty:
+            st.write("No reference provided, so this metric is N/A.")
+            continue
         fig = px.histogram(df, x=m)
         st.plotly_chart(fig, use_container_width=True)
 
@@ -330,20 +300,18 @@ def show_batch_viz(df: pd.DataFrame, metrics: List[str]):
 def main():
     st.set_page_config(page_title="🔤 Translation→Turkish Quality", layout="wide")
     st.title("🔤 Translation → TR Quality & COMET")
-    st.markdown(
-        "Translate any language into Turkish and evaluate with BLEU, BERTScore, BERTurk & COMET."
-    )
+    st.markdown("Translate any language into Turkish and evaluate (optional) with BLEU, BERTScore, BERTurk & COMET.")
 
     # Sidebar
     with st.sidebar:
         st.header("Settings")
-        metrics = st.multiselect(
-            "Select metrics",
+        metrics    = st.multiselect(
+            "Select metrics", 
             ["BLEU", "BERTScore", "BERTurk", "COMET"],
-            default=["BLEU", "BERTScore", "COMET"],
+            default=["BLEU", "BERTScore", "COMET"]
         )
         batch_size = st.slider("Batch size", 1, 32, 8)
-        mgr, ev = load_resources()
+        mgr, ev    = load_resources()
         display_model_info(mgr.get_info())
 
     # Tabs
@@ -358,16 +326,14 @@ def main():
             show_single_results(res, metrics)
 
     with tab2:
-        uploaded = st.file_uploader(
-            "Upload CSV with `src` & `ref_tr` columns", type=["csv"]
-        )
+        uploaded = st.file_uploader("Upload CSV with `src` & `ref_tr` columns", type=["csv"])
         if uploaded:
             with st.spinner("Processing file…"):
                 df_res = process_file(uploaded, mgr, ev, metrics, batch_size)
             st.markdown("### Batch Results")
             st.dataframe(df_res, use_container_width=True)
             show_batch_viz(df_res, metrics)
-            st.download_button("Download CSV", df_res.to_csv(index=False), "results.csv")
+            st.download_button("Download results as CSV", df_res.to_csv(index=False), "results.csv")
 
 
 if __name__ == "__main__":