xu song
commited on
Commit
·
4c4af51
1
Parent(s):
123d4a3
update
Browse files- models/cpp_qwen2.py +2 -0
models/cpp_qwen2.py
CHANGED
|
@@ -88,8 +88,10 @@ class Qwen2Simulator(Simulator):
|
|
| 88 |
|
| 89 |
def strip_stoptokens(self, tokens):
|
| 90 |
while tokens and tokens[0] in self.stop_tokens:
|
|
|
|
| 91 |
tokens.pop(0)
|
| 92 |
while tokens and tokens[-1] in self.stop_tokens:
|
|
|
|
| 93 |
tokens.pop()
|
| 94 |
return tokens
|
| 95 |
|
|
|
|
| 88 |
|
| 89 |
def strip_stoptokens(self, tokens):
|
| 90 |
while tokens and tokens[0] in self.stop_tokens:
|
| 91 |
+
logger.info("striping", tokens[0])
|
| 92 |
tokens.pop(0)
|
| 93 |
while tokens and tokens[-1] in self.stop_tokens:
|
| 94 |
+
logger.info("striping", tokens[1])
|
| 95 |
tokens.pop()
|
| 96 |
return tokens
|
| 97 |
|