add token output

app.py

@@ -39,6 +39,10 @@ def text2text(input_text):
     print( "Number of input tokens: " + str(input_num_tokens))
     print("Length of input: " + str(len(input_text)))
 
+    list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
+
+    print( "Tokens : " + ' '.join(list_of_tokens))
+
     # Does not seem to care if it goes over 512... humm...
     # To make it faster generate 100 tokens at a time
     # sampling mode.. don't greedily take the highest probability token every time. Helps it chat with some variation