Clip the exceeded value

model_repository/postprocessing/1/utils/gpt_token_encoder.py

@@ -152,7 +152,7 @@ class Encoder:
 
     def decode(self, tokens):
         text = "".join(
-            [self.decoder[token] for token in tokens]
+            [self.decoder[min(token, 50256)] for token in tokens]
         )
         text = bytearray([self.byte_decoder[c] for c in text]).decode(
             "utf-8", errors=self.errors

model_repository/preprocessing/1/model.py

@@ -11,11 +11,6 @@ from torch.nn.utils.rnn import pad_sequence
 from transformers import AutoTokenizer
 from word_list import to_word_list_format
 
-# GPT3 Related variables
-# Reference:
-#   https://github.com/NVIDIA/FasterTransformer/blob/main/sample/pytorch/gpt_sample.py
-MERGES_FILE = "gpt2-merges.txt"
-VOCAB_FILE = "gpt2-vocab.json"
 
 START_ID = 50256
 END_ID = 50256