Upload README.md

README.md

@@ -40,13 +40,24 @@ In contrast to existing text-to-SQL models, the SQL generation is not contrained
 
 ## How to Use
 
+Setup llama.cpp:
+```shell
+CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python
+huggingface-cli download motherduckdb/DuckDB-NSQL-7B-v0.1-GGUF DuckDB-NSQL-7B-v0.1-q8_0.gguf --local-dir . --local-dir-use-symlinks False
+pip install wurlitzer
+```
+
 Example 1:
 
+
 ```python
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
-tokenizer = AutoTokenizer.from_pretrained("motherduckdb/DuckDB-NSQL-7B-v0.1")
-model = AutoModelForCausalLM.from_pretrained("motherduckdb/DuckDB-NSQL-7B-v0.1", torch_dtype=torch.bfloat16)
+## Setup - Llama.cpp
+from llama_cpp import Llama
+with pipes() as (out, err):
+    llama = Llama(
+        model_path="DuckDB-NSQL-7B-v0.1-q8_0.gguf",
+        n_ctx=2048,
+    )
 
 text = """### Instruction:
 Your task is to generate valid duckdb SQL to answer the following question.
@@ -59,20 +70,21 @@ create a new table called tmp from test.csv
 ### Response (use duckdb shorthand if possible):
 """
 
-input_ids = tokenizer(text, return_tensors="pt").input_ids
-
-generated_ids = model.generate(input_ids, max_length=500)
-print(tokenizer.decode(generated_ids[0], skip_special_tokens=True))
+with pipes() as (out, err):
+    pred = llama(text, temperature=0.1, max_tokens=500)
+print(pred["choices"][0]["text"])
 ```
 
 Example 2:
 
 ```python
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
-tokenizer = AutoTokenizer.from_pretrained("motherduckdb/DuckDB-NSQL-7B-v0.1")
-model = AutoModelForCausalLM.from_pretrained("motherduckdb/DuckDB-NSQL-7B-v0.1", torch_dtype=torch.bfloat16)
-
+from llama_cpp import Llama
+with pipes() as (out, err):
+    llama = Llama(
+        model_path="DuckDB-NSQL-7B-v0.1-q8_0.gguf",
+        n_ctx=2048,
+    )
+    
 text = """### Instruction:
 Your task is to generate valid duckdb SQL to answer the following question, given a duckdb database schema.
 
@@ -97,20 +109,21 @@ get all columns ending with _amount from taxi table
 
 ### Response (use duckdb shorthand if possible):"""
 
-input_ids = tokenizer(text, return_tensors="pt").input_ids
-
-generated_ids = model.generate(input_ids, max_length=500)
-print(tokenizer.decode(generated_ids[0], skip_special_tokens=True))
+with pipes() as (out, err):
+    pred = llama(text, temperature=0.1, max_tokens=500)
+print(pred["choices"][0]["text"])
 ```
 
 Example 3:
 
 ```python
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
-tokenizer = AutoTokenizer.from_pretrained("motherduckdb/DuckDB-NSQL-7B-v0.1")
-model = AutoModelForCausalLM.from_pretrained("motherduckdb/DuckDB-NSQL-7B-v0.1", torch_dtype=torch.bfloat16)
-
+from llama_cpp import Llama
+with pipes() as (out, err):
+    llama = Llama(
+        model_path="DuckDB-NSQL-7B-v0.1-q8_0.gguf",
+        n_ctx=2048,
+    )
+    
 text = """### Instruction:
 Your task is to generate valid duckdb SQL to answer the following question, given a duckdb database schema.
 
@@ -135,10 +148,9 @@ get longest trip in december 2022
 ### Response (use duckdb shorthand if possible):
 """
 
-input_ids = tokenizer(text, return_tensors="pt").input_ids
-
-generated_ids = model.generate(input_ids, max_length=500)
-print(tokenizer.decode(generated_ids[0], skip_special_tokens=True))
+with pipes() as (out, err):
+    pred = llama(text, temperature=0.1, max_tokens=500)
+print(pred["choices"][0]["text"])
 ```