update inference example

README.md

@@ -54,11 +54,15 @@ from transformers import (
 )
 from peft import PeftModel
 import torch
+from tqdm import tqdm
+import json
+import re
 
 # Hugging Face Token (recommended to set via environment variable)
 HF_TOKEN = "YOUR_HF_ACCESS_TOKEN"
 
 # Model and adapter IDs
+# base_model_id = "models/models--llm-jp--llm-jp-3-13b/snapshots/cd3823f4c1fcbb0ad2e2af46036ab1b0ca13192a"
 base_model_id = "llm-jp/llm-jp-3-13b"  # Base model
 adapter_id = "sasakipeter/llm-jp-3-13b-finetune"
 
@@ -90,39 +94,55 @@ tokenizer = AutoTokenizer.from_pretrained(
 
 # Integrate LoRA adapter into the base model
 model = PeftModel.from_pretrained(model, adapter_id, token=HF_TOKEN)
+model.config.use_cache = False
 ```
 
-### 4. Perform Inference
+### 4. Perform Inference on `[elyza-tasks-100](https://huggingface.co/datasets/elyza/ELYZA-tasks-100)`
 
 ```python
-# Example input prompt
-input_text = """次の文章を要約してください。
-
-日本は四季があり、春には桜が咲き、夏には暑さが続きます。秋には紅葉が美しく、冬には雪が降ります。"""
-
-# Format the input prompt
-prompt = f"""### 指示
-{input_text}
-### 回答
-"""
-
-# Tokenize input and move to the model's device
-tokenized_input = tokenizer(prompt, return_tensors="pt").to(model.device)
-
-# Generate output
-with torch.no_grad():
-    outputs = model.generate(
-        **tokenized_input,
-        max_new_tokens=100,
-        do_sample=False,
-        repetition_penalty=1.2,
-        pad_token_id=tokenizer.eos_token_id
-    )
-
-# Decode the output
-output = tokenizer.decode(outputs[0][tokenized_input.input_ids.size(1):], skip_special_tokens=True)
-print("Output:")
-print(output)
+# loading dataset
+datasets = []
+with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
+    item = ""
+    for line in f:
+        line = line.strip()
+        item += line
+        if item.endswith("}"):
+            datasets.append(json.loads(item))
+            item = ""
+
+# execute inference
+results = []
+for data in tqdm(datasets):
+
+    input_text = data["input"]
+
+    prompt = f"""### 指示
+    {input_text}
+    ### 回答
+    """
+
+    tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
+    attention_mask = torch.ones_like(tokenized_input)
+
+    with torch.no_grad():
+        outputs = model.generate(
+          tokenized_input,
+          attention_mask=attention_mask,
+          max_new_tokens=100,
+          do_sample=False,
+          repetition_penalty=1.2,
+          pad_token_id=tokenizer.eos_token_id
+        )[0]
+    output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
+
+    results.append({"task_id": data["task_id"], "input": input, "output": output})
+
+jsonl_id = re.sub(".*/", "", new_model_id)
+with open(f"./{jsonl_id}-outputs-validation.jsonl", 'w', encoding='utf-8') as f:
+    for result in results:
+        json.dump(result, f, ensure_ascii=False)  # ensure_ascii=False for handling non-ASCII characters
+        f.write('\n')
 ```
 
 ---