Update handler.py

handler.py

@@ -21,7 +21,7 @@ class EndpointHandler():
             # 'mosaicml/mpt-7b',
             trust_remote_code=True,
             torch_dtype=torch.bfloat16,
-            max_seq_len=2048
+            max_seq_len=32000
             )
 
         self.tokenizer = transformers.AutoTokenizer.from_pretrained('EleutherAI/gpt-neox-20b')
@@ -48,7 +48,7 @@ class EndpointHandler():
             temperature=0.1,
             top_p=0.15,
             top_k=0,
-            max_new_tokens=64,
+            max_new_tokens=2048,
             repetition_penalty=1.1
         )