Update README.md
Browse files
README.md
CHANGED
@@ -21,38 +21,6 @@ It is converted into 8-bit GGUF format from original repository https://huggingf
|
|
21 |
**Repository:** https://huggingface.co/defog/sqlcoder-7b-2
|
22 |
|
23 |
### Example usage
|
24 |
-
**
|
25 |
|
26 |
-
|
27 |
-
from huggingface_hub import hf_hub_download
|
28 |
-
|
29 |
-
YOUR MODEL DIRECTORY =
|
30 |
-
CONTEXT_LENGHT =
|
31 |
-
MAX_TOKENS =
|
32 |
-
BATCH_SIZE =
|
33 |
-
TEMPERATURE =
|
34 |
-
GPU_OFFLOAD =
|
35 |
-
|
36 |
-
def load_model(model_id, model_basename):
|
37 |
-
model_path = hf_hub_download(
|
38 |
-
repo_id=model_id,
|
39 |
-
filename=model_basename,
|
40 |
-
resume_download=True,
|
41 |
-
cache_dir="YOUR MODEL DIRECTORY",
|
42 |
-
)
|
43 |
-
kwargs = {
|
44 |
-
'model_path': model_path,
|
45 |
-
'n_ctx':CONTEXT_LENGHT,
|
46 |
-
'max_tokens':MAX_TOKENS,
|
47 |
-
'n_batch':BATCH_SIZE,
|
48 |
-
'n_gpu_layers':GPU_OFFLOAD,
|
49 |
-
'temperature':TEMPERATURE,
|
50 |
-
'verbose':True,
|
51 |
-
}
|
52 |
-
return LlamaCpp(**kwargs)
|
53 |
-
|
54 |
-
llm = load_model(
|
55 |
-
model_id="whoami02/defog-sqlcoder-2-GGUF",
|
56 |
-
model_basename="sqlcoder-7b-2.q8_0.gguf",
|
57 |
-
)
|
58 |
-
|
|
|
21 |
**Repository:** https://huggingface.co/defog/sqlcoder-7b-2
|
22 |
|
23 |
### Example usage
|
24 |
+
**With Llamacpp:**
|
25 |
|
26 |
+

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|