Update app.py

app.py

@@ -1,42 +1,20 @@
-from transformers import AutoTokenizer, pipeline
-from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
+from ctransformers import AutoModelForCausalLM
 
-model_name = "TheBloke/Llama-2-7b-Chat-GPTQ"
-
-use_triton = False
-
-tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-
-model = AutoGPTQForCausalLM.from_quantized(model_name,
-        use_safetensors=True,
-        trust_remote_code=True,
-        device="cuda:0",
-        use_triton=use_triton,
-        quantize_config=None)
-
-#creating pipeline
-pipe = pipeline(
-    "text-generation",
-    model=model,
-    tokenizer=tokenizer,
+model = AutoModelForCausalLM.from_pretrained(
+    model_path_or_repo_id="TheBloke/Llama-2-7B-chat-GGML",
     max_new_tokens=512,
     temperature=0.6,
     top_p=0.95,
     repetition_penalty=1.15
 )
 
-from langchain.llms import HuggingFacePipeline
-from langchain.chains import LLMChain
-from langchain import PromptTemplate
-
-llm = HuggingFacePipeline(pipeline=pipe)
-
 system_message = """
 You are a helpful, respectful and honest assistant. Your job is to answer the users query as best as possible given the Web Page Content. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. If you DO NOT KNOW THE ANSWER DO NOT SHARE FALSE INFORMATION.
 You have been given scraped text content of a webpage under the section called "Web Page Content". Using this information answer the users query. However, if the webpage DOES NOT contain the answer to the query, you CAN answer based on your existing knowledge IF you are sure of the answer, but ALWAYS let the user know when doing so.
 """
 
-prompt_template='''[INST] <<SYS>>
+def generate_prompt(system_message, context, prompt):
+  prompt=f'''[INST] <<SYS>>
 {system_message}
 <</SYS>>
 
@@ -47,20 +25,24 @@ Web Page Content:
 
 {prompt} [/INST]'''
 
-
-chat = LLMChain(
-    llm=llm,
-    prompt=PromptTemplate.from_template(prompt_template),
-    verbose=True
-)
+  return prompt
 
 import requests
 from bs4 import BeautifulSoup
 import re
 
+def scraper(url):
+  req = requests.get(url)
+  soup = BeautifulSoup(req.content, "html.parser")
+  context = soup.get_text()
+  relevant_text = soup.get_text()
+  cleaned_text = re.sub(r'\s+', ' ', relevant_text).strip()
+
+  return cleaned_text
+
 def run(url, input):
   context = scraper(url)
-  response = chat.predict(system_message=system_message, context=context, prompt=input)
+  response = model(generate_prompt(system_message=system_message, context=context, prompt=input))
 
   return response