metadata
base_model:
- Qwen/Qwen2.5-Coder-14B-Instruct
#!/usr/bin/env python3
import time
from vllm import LLM, SamplingParams
def main():
# Hard-coded model and tensor parallel configuration.
model_path = "miike-ai/qwen-14b-coder-fp8"
tensor_parallel_size = 1
# Define sampling parameters with an increased max_tokens and a stop string.
sampling_params = SamplingParams(
temperature=0.0,
top_p=0.95,
max_tokens=32000, # Increase this to allow longer responses.
stop=["\nUser:"], # Stop when the model outputs a new user marker.
)
print(f"Loading model '{model_path}' ...")
model = LLM(
model=model_path,
enforce_eager=True,
dtype="auto",
tensor_parallel_size=tensor_parallel_size,
)
print("Model loaded. You can now chat!")
print("Type 'exit' or 'quit' to end the conversation.\n")
conversation = ""
while True:
try:
user_input = input("User: ").strip()
except (KeyboardInterrupt, EOFError):
print("\nExiting chat.")
break
if user_input.lower() in {"exit", "quit"}:
print("Exiting chat.")
break
# Append the user's input to the conversation history.
conversation += f"User: {user_input}\nBot: "
print("Bot: ", end="", flush=True)
# Generate a response using the conversation history and sampling parameters.
response = model.generate(conversation, sampling_params=sampling_params)
# Extract the generated reply.
bot_reply = response[0].outputs[0].text.strip()
# Simulate streaming by printing one character at a time.
for char in bot_reply:
print(char, end="", flush=True)
time.sleep(0.02) # Adjust delay (in seconds) as desired.
print() # Newline after bot reply.
# Append the bot reply to conversation history.
conversation += bot_reply + "\n"
if __name__ == "__main__":
main()