File size: 708 Bytes
0f38ced 4523ba2 3ee8139 |
1 2 3 4 5 6 7 8 9 10 11 12 13 |
from unsloth import FastLanguageModel
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "aidando73/llama-3.3-70b-instruct-code-agent-fine-tune-v1",
max_seq_length = 2048,
dtype = "float16",
load_in_4bit = True,
)
FastLanguageModel.for_inference(model)
from transformers import TextStreamer
text_streamer = TextStreamer(tokenizer, skip_prompt = True)
input_ids = tokenizer.encode("<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nHello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", return_tensors = "pt").to("cuda")
_ = model.generate(input_ids, streamer = text_streamer, max_new_tokens = 128, pad_token_id = tokenizer.eos_token_id) |