3

.gitattributes

@@ -1,3 +1,4 @@
+*.gguf filter=lfs diff=lfs merge=lfs -text
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text

main.py

@@ -1,10 +1,6 @@
-import os
-
 from ctransformers import AutoModelForCausalLM
 from fastapi import FastAPI
 from pydantic import BaseModel
-import requests
-from huggingface_hub import hf_hub_download
 
 file_name = "zephyr-7b-beta.Q4_K_S.gguf"
 llm = AutoModelForCausalLM.from_pretrained(file_name,
@@ -26,5 +22,5 @@ async def stream(item: validation):
     E_INST = "</s>"
     user, assistant = "<|user|>", "<|assistant|>"
     prompt = f"{system_prompt}{E_INST}\n{user}\n{item.prompt}{E_INST}\n{assistant}\n"
-    # return llm(prompt)
-    return prompt
+
+    return llm(prompt)