Add exception for token overflow

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+*.pyc
+.cache

model_repository/postprocessing/1/utils/gpt_token_encoder.py

@@ -151,7 +151,9 @@ class Encoder:
         return bpe_tokens
 
     def decode(self, tokens):
-        text = "".join([self.decoder[token] for token in tokens])
+        text = "".join(
+            [self.decoder[min(token, 50256)] for token in tokens]
+        )
         text = bytearray([self.byte_decoder[c] for c in text]).decode(
             "utf-8", errors=self.errors
         )

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/blobs/69e076d9141589e88803bb7fffc48167082fc2e7

@@ -1 +0,0 @@
-{"   ": 50285, "                 ": 50271, "                            ": 50260, "                           ": 50261, "                ": 50272, "\t\t\t\t": 50292, "\t\t\t\t\t\t\t": 50289, "               ": 50273, "    ": 50284, "     ": 50283, "                         ": 50263, "                              ": 50258, "                   ": 50269, "                               ": 50257, "                       ": 50265, "             ": 50275, "                     ": 50267, "                  ": 50270, "          ": 50278, "  ": 50286, "            ": 50276, "                             ": 50259, "\t\t\t\t\t\t": 50290, "                    ": 50268, "         ": 50279, "\t\t\t\t\t\t\t\t\t": 50287, "\t\t\t": 50293, "                        ": 50264, "                      ": 50266, "           ": 50277, "\t\t\t\t\t": 50291, "\t\t": 50294, "       ": 50281, "              ": 50274, "\t\t\t\t\t\t\t\t": 50288, "      ": 50282, "                          ": 50262, "        ": 50280}

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/blobs/817762d631ad6f9c799f6b9dc713c46420e65546

@@ -1 +0,0 @@
-{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/blobs/b4f914dd9ab5282b31b0aafc5f6569151f910856

@@ -1 +0,0 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 2048, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "CodeGenTokenizer"}

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/refs/main

@@ -1 +0,0 @@
-40b7a3b6e99e73bdb497a14b740e7167b3413c74

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/snapshots/40b7a3b6e99e73bdb497a14b740e7167b3413c74/added_tokens.json

@@ -1 +0,0 @@
-../../blobs/69e076d9141589e88803bb7fffc48167082fc2e7

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/snapshots/40b7a3b6e99e73bdb497a14b740e7167b3413c74/merges.txt

@@ -1 +0,0 @@
-../../blobs/6636bda4a1fd7a63653dffb22683b8162c8de956

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/snapshots/40b7a3b6e99e73bdb497a14b740e7167b3413c74/special_tokens_map.json

@@ -1 +0,0 @@
-../../blobs/817762d631ad6f9c799f6b9dc713c46420e65546

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/snapshots/40b7a3b6e99e73bdb497a14b740e7167b3413c74/tokenizer.json

@@ -1 +0,0 @@
-../../blobs/e7c373e4b1189082ddcda33aaf2839bb1c04bb81

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/snapshots/40b7a3b6e99e73bdb497a14b740e7167b3413c74/tokenizer_config.json

@@ -1 +0,0 @@
-../../blobs/b4f914dd9ab5282b31b0aafc5f6569151f910856

model_repository/preprocessing/1/.cache/models--Salesforce--codegen-350M-mono/snapshots/40b7a3b6e99e73bdb497a14b740e7167b3413c74/vocab.json

@@ -1 +0,0 @@
-../../blobs/84ef7fb594b5c0979e48bdeddb60a0adef33df0b