Sephfox commited on
Commit
b303379
·
verified ·
1 Parent(s): fd0702f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +13 -3
app.py CHANGED
@@ -109,10 +109,20 @@ class LLMResponder:
109
  self.llm = Llama(model_path="path/to/your/gguf/file.gguf", n_ctx=1024)
110
  self.backend = "llama_cpp"
111
  else:
112
- # Create a dummy config using LlamaConfig so that the model loads even if its config is missing a model_type key.
113
  dummy_config = LlamaConfig.from_dict({"model_type": "llama"})
114
- self.llm_tokenizer = AutoTokenizer.from_pretrained(model_name, config=dummy_config, trust_remote_code=True)
115
- self.llm_model = AutoModelForCausalLM.from_pretrained(model_name, config=dummy_config, trust_remote_code=True)
 
 
 
 
 
 
 
 
 
 
116
  self.backend = "transformers"
117
 
118
  def generate_response(self, prompt):
 
109
  self.llm = Llama(model_path="path/to/your/gguf/file.gguf", n_ctx=1024)
110
  self.backend = "llama_cpp"
111
  else:
112
+ # Create a dummy config using LlamaConfig so the model loads despite missing keys.
113
  dummy_config = LlamaConfig.from_dict({"model_type": "llama"})
114
+ try:
115
+ self.llm_tokenizer = AutoTokenizer.from_pretrained(model_name, config=dummy_config, trust_remote_code=True)
116
+ except Exception as e:
117
+ print("Error loading tokenizer from", model_name, "; using fallback tokenizer.")
118
+ fallback_model = "decapoda-research/llama-7b-hf"
119
+ self.llm_tokenizer = AutoTokenizer.from_pretrained(fallback_model, config=dummy_config, trust_remote_code=True)
120
+ try:
121
+ self.llm_model = AutoModelForCausalLM.from_pretrained(model_name, config=dummy_config, trust_remote_code=True)
122
+ except Exception as e:
123
+ print("Error loading model from", model_name, "; using fallback model.")
124
+ fallback_model = "decapoda-research/llama-7b-hf"
125
+ self.llm_model = AutoModelForCausalLM.from_pretrained(fallback_model, config=dummy_config, trust_remote_code=True)
126
  self.backend = "transformers"
127
 
128
  def generate_response(self, prompt):