distinct

distinct.py

@@ -153,6 +153,7 @@ class distinct(evaluate.Measurement):
         for prediction in predictions:          
             try:
                 print(prediction)
+                print(tokenizer.tokenize(prediction))
                 tokens = list(tokenizer.tokenize(prediction))
                 print(tokens)
                 tokens_2grams = list(ngrams(list(tokenizer.tokenize(prediction)), 2, pad_left=True, left_pad_symbol='<s>'))