update

models/cpp_qwen2.py

@@ -134,12 +134,15 @@ class Qwen2Simulator(Simulator):
     def tokenize(self, text):
         return self.llm.tokenize(text.encode("utf-8"))
 
+    def detokenize(self, tokens):
+        return self.llm.detokenize(tokens).decode("utf-8")
+
     def strip_stoptokens(self, tokens):
         while tokens and tokens[0] in self.stop_tokens:
-            logger.info(f"striping {tokens[0]}")
+            logger.info(f"head-striping {tokens[0]} {self.llm.detokenize([tokens[0]])}")
             tokens.pop(0)
         while tokens and tokens[-1] in self.stop_tokens:
-            logger.info(f"striping {tokens[-1]}")
+            logger.info(f"tail-striping {tokens[-1]} {self.llm.detokenize([tokens[-1]])}")
             tokens.pop()
         return tokens