Update app.py

app.py

@@ -1,19 +1,31 @@
-
 import os
-import gradio
+from dotenv import load_dotenv
+from huggingface_hub import login
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
+load_dotenv()
 
-os.system("gdown 1L0HmkMQ54le2Eel1r3aJVoxsOLFpTodL")
+API_TOKEN = os.environ.get("HUGGINGFACE_TOKEN")
 
-from langchain.llms.llamacpp import LlamaCpp
-model_path = "ggml-model-q4_0.bin"
-llm = LlamaCpp(model_path=model_path)
+login(API_TOKEN)
 
-def respond(input_text):
-    response = llm(input_text)
-    return str(response)
+model_id = "meta-llama/Llama-2-7b-chat-hf"
 
-demo = gradio.Interface(fn=respond, inputs='text', outputs='text')
+model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True, device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained(model_id)
 
-demo.launch()
+generate_text_pipeline = pipeline(
+    model=model, tokenizer=tokenizer,
+    return_full_text=True,
+    task='text-generation',
 
+    temperature=0.1,
+    max_new_tokens=512,
+    repetition_penalty=1.1  # without this output begins repeating
+)
+def get_results(text):
+    res = generate_text_pipeline(text)
+    return res[0]["generated_text"]
 
+iface = gr.Interface(fn=get_results, inputs="text", outputs="text")
+iface.launch()