Spaces:
Running
Running
File size: 2,319 Bytes
8f4e927 b38068f 8f4e927 1c6c674 a65ba38 30417d7 664e897 30417d7 34054e0 664e897 34054e0 664e897 a65ba38 664e897 34054e0 a65ba38 b38068f a65ba38 b38068f 8f4e927 34054e0 a65ba38 34054e0 8f4e927 34054e0 a65ba38 34054e0 a65ba38 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
import os
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from huggingface_hub import login
import time
import torch.quantization
# Directly assign your Hugging Face token here
hf_token = "your_hugging_face_api_token"
# Log in to Hugging Face
login(token=hf_token)
# Load the Mixtral-8x7B-Instruct model and tokenizer with authorization header
model_name = 'mistralai/Mistral-7B-Instruct-v0.3'
headers = {"Authorization": f"Bearer {hf_token}"}
# Ensure sentencepiece is installed
try:
import sentencepiece
except ImportError:
raise ImportError("The sentencepiece library is required for this tokenizer. Please install it with `pip install sentencepiece`.")
# Start time to measure execution time
start_time = time.time()
# Load tokenizer and model
tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token=hf_token)
model = AutoModelForCausalLM.from_pretrained(model_name, use_auth_token=hf_token)
# Quantize the model
quantized_model = torch.quantization.quantize_dynamic(model, {torch.nn.Linear}, dtype=torch.qint8)
# Check if a GPU is available and if not, fall back to CPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
quantized_model.to(device)
# Measure time for loading tokenizer, model, and quantization
loading_time = time.time() - start_time
print(f"Time taken to load tokenizer, model, and quantize: {loading_time:.2f} seconds")
# Example text input
text_input = "How did Tesla perform in Q1 2024?"
# Start time for inference
inference_start_time = time.time()
# Tokenize the input text
inputs = tokenizer(text_input, return_tensors="pt").to(device)
# Measure time for tokenization
tokenization_time = time.time() - inference_start_time
# Generate a response
outputs = quantized_model.generate(**inputs, max_length=150, do_sample=False)
# Measure time for inference
inference_time = time.time() - inference_start_time
print(f"Time taken for inference with quantized model: {inference_time:.2f} seconds")
# Decode the generated tokens to a readable string
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
# Print the response
print(f"Generated response: {response}")
# Total execution time
total_time = time.time() - start_time
print(f"Total execution time with quantized model: {total_time:.2f} seconds") |