Update app.py

app.py

@@ -12,7 +12,7 @@ if not HF_TOKEN:
 login(HF_TOKEN)
 
 # === 設定 ===
-BASE_MODEL = "Sakalti/template-4"  # 修正対象モデル名
+BASE_MODEL = "Sakalti/template-4"
 HF_REPO = "Sakalti/template-16"
 
 # === データ読み込み ===
@@ -22,9 +22,9 @@ dataset = load_dataset("Verah/JParaCrawl-Filtered-English-Japanese-Parallel-Corp
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 model = AutoModelForCausalLM.from_pretrained(BASE_MODEL)
 
-# === 超簡素トークナイズ関数 ===
+# === トークナイズ関数修正版 ===
 def preprocess(examples):
-    texts = [ex["en"] + " " + ex["ja"] for ex in examples]
+    texts = [en + " " + ja for en, ja in zip(examples["en"], examples["ja"])]
     tokenized = tokenizer(texts, max_length=256, truncation=True)
     tokenized["labels"] = tokenized["input_ids"].copy()
     return tokenized