| from unsloth import FastLanguageModel | |
| model, tokenizer = FastLanguageModel.from_pretrained( | |
| model_name = "aidando73/llama-3.3-70b-instruct-code-agent-fine-tune-v1", | |
| max_seq_length = 2048, | |
| dtype = "float16", | |
| load_in_4bit = True, | |
| ) | |
| FastLanguageModel.for_inference(model) | |
| from transformers import TextStreamer | |
| text_streamer = TextStreamer(tokenizer, skip_prompt = True) | |
| input_ids = tokenizer.encode("<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nHello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", return_tensors = "pt").to("cuda") | |
| _ = model.generate(input_ids, streamer = text_streamer, max_new_tokens = 128, pad_token_id = tokenizer.eos_token_id) |