| name: mistral-openorca | |
| mmap: true | |
| parameters: | |
| model: huggingface://TheBloke/Mistral-7B-OpenOrca-GGUF/mistral-7b-openorca.Q6_K.gguf | |
| temperature: 0.2 | |
| top_k: 40 | |
| top_p: 0.95 | |
| seed: -1 | |
| mirostat: 2 | |
| mirostat_eta: 1.0 | |
| mirostat_tau: 1.0 | |
| template: | |
| chat_message: | | |
| <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}} | |
| {{if .Content}}{{.Content}}{{end}} | |
| <|im_end|> | |
| chat: | | |
| {{.Input}} | |
| <|im_start|>assistant | |
| completion: | | |
| {{.Input}} | |
| context_size: 4096 | |
| f16: true | |
| stopwords: | |
| - <|im_end|> | |
| - <dummy32000> | |
| usage: | | |
| curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{ | |
| "model": "mistral-openorca", | |
| "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}] | |
| }' | |