bits and bytes + microsoft/phi-1_5

app.py

@@ -25,9 +25,23 @@ index = VectorStoreIndex.from_vector_store(vector_store, embed_model=embed_model
 # Utwórz silnik zapytań
 
 # Load the correct tokenizer and LLM
-from transformers import AutoTokenizer
-tokenizer = AutoTokenizer.from_pretrained("stabilityai/stablelm-zephyr-3b")
-llm = HuggingFaceLLM(model_name="stabilityai/stablelm-zephyr-3b", tokenizer=tokenizer)
+# from transformers import AutoTokenizer
+# tokenizer = AutoTokenizer.from_pretrained("stabilityai/stablelm-zephyr-3b")
+# llm = HuggingFaceLLM(model_name="stabilityai/stablelm-zephyr-3b", tokenizer=tokenizer)
+
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype="float16"
+)
+
+llm = HuggingFaceLLM(
+    model_name="microsoft/phi-1_5", # Mały model 1.3B
+    tokenizer=AutoTokenizer.from_pretrained("microsoft/phi-1_5"),
+    model_kwargs={
+        "quantization_config": quantization_config,
+        "device_map": "auto"
+    }
+)
 
 #Query Engine
 query_engine = index.as_query_engine(