distinct
Browse files- distinct.py +1 -0
distinct.py
CHANGED
|
@@ -152,6 +152,7 @@ class distinct(evaluate.Measurement):
|
|
| 152 |
|
| 153 |
for prediction in predictions:
|
| 154 |
try:
|
|
|
|
| 155 |
tokens = list(tokenizer.tokenize(prediction))
|
| 156 |
print(tokens)
|
| 157 |
tokens_2grams = list(ngrams(list(tokenizer.tokenize(prediction)), 2, pad_left=True, left_pad_symbol='<s>'))
|
|
|
|
| 152 |
|
| 153 |
for prediction in predictions:
|
| 154 |
try:
|
| 155 |
+
print(prediction)
|
| 156 |
tokens = list(tokenizer.tokenize(prediction))
|
| 157 |
print(tokens)
|
| 158 |
tokens_2grams = list(ngrams(list(tokenizer.tokenize(prediction)), 2, pad_left=True, left_pad_symbol='<s>'))
|