no padding

llm_toolkit/llm_utils.py

@@ -146,7 +146,7 @@ def eval_model(
             inputs = tokenizer(
                 batch_prompts,
                 return_tensors="pt",
-                padding=True,  # Ensure all inputs in the batch have the same length
+                padding=batch_size > 1,  # Ensure all inputs in the batch have the same length
             ).to(device)
 
             outputs = model.generate(
@@ -155,7 +155,7 @@ def eval_model(
                 repetition_penalty=repetition_penalty,
                 use_cache=False,
             )
-            outputs = outputs[:, inputs["input_ids"].shape[1] :]
+            outputs = outputs[:, inputs["input_ids"].shape[1]:]
             decoded_output = tokenizer.batch_decode(
                 outputs, skip_special_tokens=True
             )  # Skip special tokens for clean output

scripts/eval-epochs.sh

@@ -7,7 +7,7 @@ pwd
 
 export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
 export RESIZE_TOKEN_EMBEDDINGS=true
-export USING_LLAMA_FACTORY=true
+# export USING_LLAMA_FACTORY=true
 export USING_P1_PROMPT_TEMPLATE=false
 
 export ORG_NAME=$1

scripts/eval-mgtv-qwen2.5_4bit.sh

@@ -13,17 +13,18 @@ cat /etc/os-release
 lscpu
 grep MemTotal /proc/meminfo
 
+export USING_LLAMA_FACTORY=false
 export LOAD_IN_4BIT=true
 $BASEDIR/scripts/eval-epochs.sh Qwen Qwen2.5-72B-Instruct
 
 export START_NUM_SHOTS=5
 $BASEDIR/scripts/eval-shots_4bit.sh Qwen Qwen2.5-72B-Instruct
 
-
 export START_NUM_SHOTS=40
 $BASEDIR/scripts/eval-shots_4bit.sh shenzhi-wang Llama3.1-70B-Chinese-Chat
 
 export LOAD_IN_4BIT=false
+export USING_LLAMA_FACTORY=true
 $BASEDIR/scripts/eval-shots.sh internlm internlm2_5-7b-chat
 
 export START_NUM_SHOTS=50

scripts/eval-shots.sh

@@ -7,7 +7,7 @@ pwd
 
 export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
 export RESIZE_TOKEN_EMBEDDINGS=true
-export USING_LLAMA_FACTORY=true
+# export USING_LLAMA_FACTORY=true
 export USING_P1_PROMPT_TEMPLATE=false
 export LOAD_IN_4BIT=false