feat(sailor-chat): add sail/Sailor-4B-Chat with the same context length

main.py

@@ -29,6 +29,18 @@ engine_llama_3_2: LLM = LLM(
 )
 
 
+engine_sailor_chat: LLM = LLM(
+    model='sail/Sailor-4B-Chat',
+    revision="89a866a7041e6ec023dd462adeca8e28dd53c83e",
+    max_num_batched_tokens=512,    # Reduced for T4
+    max_num_seqs=16,               # Reduced for T4
+    gpu_memory_utilization=0.85,   # Slightly increased, adjust if needed
+    max_model_len=32768,
+    enforce_eager=True,            # Disable CUDA graph
+    dtype='auto',                  # Use 'half' if you want half precision
+)
+
+
 @app.get("/")
 def greet_json():
     cuda_info: dict[str, Any] = {}
@@ -49,7 +61,13 @@ def greet_json():
             {
                 "name": "meta-llama/Llama-3.2-3B-Instruct",
                 "revision": "0cb88a4f764b7a12671c53f0838cd831a0843b95",
-            }
+                "max_model_len": engine_llama_3_2.llm_engine.model_config.max_model_len,
+            },
+            {
+                "name": "sail/Sailor-4B-Chat",
+                "revision": "89a866a7041e6ec023dd462adeca8e28dd53c83e",
+                "max_model_len": engine_sailor_chat.llm_engine.model_config.max_model_len,
+            },
         ]
     }
 
@@ -85,3 +103,25 @@ def generate_text(request: GenerationRequest) -> list[RequestOutput] | dict[str,
         return {
             "error": str(e)
         }
+
+
+@app.post("/generate-sailor-chat")
+def generate_text(request: GenerationRequest) -> list[RequestOutput] | dict[str, str]:
+    try:
+        sampling_params: SamplingParams = SamplingParams(
+            temperature=request.temperature,
+            max_tokens=request.max_tokens,
+            logit_bias=request.logit_bias,
+        )
+
+        # Generate text
+        return engine_sailor_chat.generate(
+            prompts=request.prompt,
+            sampling_params=sampling_params
+        )
+
+    except Exception as e:
+        return {
+            "error": str(e)
+        }
+