File size: 21,153 Bytes
45be9e6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6363d4dd-5c18-4652-b682-bf54a468aae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from safetensors import safe_open\n",
    "from safetensors.torch import save_file\n",
    "\n",
    "# ๅŠ ่ฝฝ safetensors ๆ–‡ไปถ\n",
    "model_path = \"model.safetensors\"\n",
    "with safe_open(model_path, framework=\"pt\", device=\"cpu\") as f:\n",
    "    state_dict = {key: f.get_tensor(key) for key in f.keys()}\n",
    "\n",
    "# ๅฐ† BF16 ่ฝฌๆขไธบ FP16\n",
    "fp16_state_dict = {key: value.to(torch.float16) for key, value in state_dict.items()}\n",
    "\n",
    "# ไฟๅญ˜ไธบๆ–ฐ็š„ safetensors ๆ–‡ไปถ\n",
    "output_path = \"model_fp16.safetensors\"\n",
    "save_file(fp16_state_dict, output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9aa60eaa-883a-465e-958a-3280e0439f66",
   "metadata": {},
   "outputs": [
    {
     "ename": "PackageNotFoundError",
     "evalue": "No package metadata was found for auto-gptq",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:397\u001b[0m, in \u001b[0;36mDistribution.from_name\u001b[0;34m(cls, name)\u001b[0m\n\u001b[1;32m    396\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 397\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdiscover\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    398\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n",
      "\u001b[0;31mStopIteration\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mPackageNotFoundError\u001b[0m                      Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m model_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# ๅŠ ่ฝฝๆจกๅž‹๏ผŒๆ˜พๅผๆŒ‡ๅฎš torch_dtype\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mAutoModelForCausalLM\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtorch_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat16\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# ๆˆ– torch.float16๏ผŒไธŽๆƒ้‡ไธ€่‡ด\u001b[39;49;00m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdevice_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mauto\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# ๅฏ้€‰๏ผŒ่‡ชๅŠจๅˆ†้…่ฎพๅค‡\u001b[39;49;00m\n\u001b[1;32m     11\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;66;03m# ๅŠ ่ฝฝๅˆ†่ฏๅ™จ\u001b[39;00m\n\u001b[1;32m     14\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m AutoTokenizer\u001b[38;5;241m.\u001b[39mfrom_pretrained(model_path)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py:564\u001b[0m, in \u001b[0;36m_BaseAutoModelClass.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    562\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(config) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m    563\u001b[0m     model_class \u001b[38;5;241m=\u001b[39m _get_model_class(config, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping)\n\u001b[0;32m--> 564\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    565\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhub_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m    566\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    567\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    568\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnrecognized configuration class \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconfig\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m for this kind of AutoModel: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    569\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mModel type should be one of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(c\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mfor\u001b[39;00m\u001b[38;5;250m \u001b[39mc\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys())\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    570\u001b[0m )\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.12/site-packages/transformers/modeling_utils.py:3669\u001b[0m, in \u001b[0;36mPreTrainedModel.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   3666\u001b[0m     hf_quantizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   3668\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m hf_quantizer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 3669\u001b[0m     \u001b[43mhf_quantizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_environment\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3670\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtorch_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch_dtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3671\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfrom_tf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_tf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3672\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfrom_flax\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_flax\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3673\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdevice_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice_map\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3674\u001b[0m \u001b[43m        \u001b[49m\u001b[43mweights_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mweights_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3675\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3676\u001b[0m     torch_dtype \u001b[38;5;241m=\u001b[39m hf_quantizer\u001b[38;5;241m.\u001b[39mupdate_torch_dtype(torch_dtype)\n\u001b[1;32m   3677\u001b[0m     device_map \u001b[38;5;241m=\u001b[39m hf_quantizer\u001b[38;5;241m.\u001b[39mupdate_device_map(device_map)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.12/site-packages/transformers/quantizers/quantizer_gptq.py:52\u001b[0m, in \u001b[0;36mGptqHfQuantizer.validate_environment\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mvalidate_environment\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 52\u001b[0m     gptq_supports_cpu \u001b[38;5;241m=\u001b[39m version\u001b[38;5;241m.\u001b[39mparse(\u001b[43mimportlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmetadata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mversion\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mauto-gptq\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m) \u001b[38;5;241m>\u001b[39m version\u001b[38;5;241m.\u001b[39mparse(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m0.4.2\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     53\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m gptq_supports_cpu \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mis_available():\n\u001b[1;32m     54\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGPU is required to quantize or run quantize model.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:889\u001b[0m, in \u001b[0;36mversion\u001b[0;34m(distribution_name)\u001b[0m\n\u001b[1;32m    882\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mversion\u001b[39m(distribution_name):\n\u001b[1;32m    883\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Get the version string for the named package.\u001b[39;00m\n\u001b[1;32m    884\u001b[0m \n\u001b[1;32m    885\u001b[0m \u001b[38;5;124;03m    :param distribution_name: The name of the distribution package to query.\u001b[39;00m\n\u001b[1;32m    886\u001b[0m \u001b[38;5;124;03m    :return: The version string for the package as defined in the package's\u001b[39;00m\n\u001b[1;32m    887\u001b[0m \u001b[38;5;124;03m        \"Version\" metadata key.\u001b[39;00m\n\u001b[1;32m    888\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 889\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdistribution\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdistribution_name\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mversion\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:862\u001b[0m, in \u001b[0;36mdistribution\u001b[0;34m(distribution_name)\u001b[0m\n\u001b[1;32m    856\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdistribution\u001b[39m(distribution_name):\n\u001b[1;32m    857\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Get the ``Distribution`` instance for the named package.\u001b[39;00m\n\u001b[1;32m    858\u001b[0m \n\u001b[1;32m    859\u001b[0m \u001b[38;5;124;03m    :param distribution_name: The name of the distribution package as a string.\u001b[39;00m\n\u001b[1;32m    860\u001b[0m \u001b[38;5;124;03m    :return: A ``Distribution`` instance (or subclass thereof).\u001b[39;00m\n\u001b[1;32m    861\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 862\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mDistribution\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_name\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdistribution_name\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.12/importlib/metadata/__init__.py:399\u001b[0m, in \u001b[0;36mDistribution.from_name\u001b[0;34m(cls, name)\u001b[0m\n\u001b[1;32m    397\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mdiscover(name\u001b[38;5;241m=\u001b[39mname))\n\u001b[1;32m    398\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[0;32m--> 399\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m PackageNotFoundError(name)\n",
      "\u001b[0;31mPackageNotFoundError\u001b[0m: No package metadata was found for auto-gptq"
     ]
    }
   ],
   "source": [
    "\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "# ๆจกๅž‹่ทฏๅพ„\n",
    "model_path = \"./\"\n",
    "\n",
    "# ๅŠ ่ฝฝๆจกๅž‹๏ผŒๆ˜พๅผๆŒ‡ๅฎš torch_dtype\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_path,\n",
    "    torch_dtype=torch.float16,  # ๆˆ– torch.float16๏ผŒไธŽๆƒ้‡ไธ€่‡ด\n",
    "    device_map=\"auto\"  # ๅฏ้€‰๏ผŒ่‡ชๅŠจๅˆ†้…่ฎพๅค‡\n",
    ")\n",
    "\n",
    "# ๅŠ ่ฝฝๅˆ†่ฏๅ™จ\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "\n",
    "# ๆต‹่ฏ•ๆจกๅž‹\n",
    "input_text = \"Hello, how are you?\"\n",
    "inputs = tokenizer(input_text, return_tensors=\"pt\")\n",
    "outputs = model.generate(**inputs)\n",
    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9d15b55d-f5bc-40ad-80e5-81d421bd0592",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: http://mirrors.aliyun.com/pypi/simple\n",
      "Collecting auto-gptq\n",
      "  Downloading http://mirrors.aliyun.com/pypi/packages/90/e5/b22697903982284fe284568fb2663a2196694a8eee637f5cf4ccfe435a38/auto_gptq-0.7.1.tar.gz (126 kB)\n",
      "\u001b[2K     \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m126.1/126.1 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hDiscarding \u001b[4;34mhttp://mirrors.aliyun.com/pypi/packages/90/e5/b22697903982284fe284568fb2663a2196694a8eee637f5cf4ccfe435a38/auto_gptq-0.7.1.tar.gz#sha256=5c61ad380e9b4c603757c254765e9083a90a820cd0aff1b5d2c6f7fd96c85e80 (from http://mirrors.aliyun.com/pypi/simple/auto-gptq/) (requires-python:>=3.8.0)\u001b[0m: \u001b[33mRequested auto-gptq from http://mirrors.aliyun.com/pypi/packages/90/e5/b22697903982284fe284568fb2663a2196694a8eee637f5cf4ccfe435a38/auto_gptq-0.7.1.tar.gz#sha256=5c61ad380e9b4c603757c254765e9083a90a820cd0aff1b5d2c6f7fd96c85e80 has inconsistent version: expected '0.7.1', but metadata has '0.7.1+cu1241'\u001b[0m\n",
      "  Downloading http://mirrors.aliyun.com/pypi/packages/34/71/c3e73cf17681f6ff4754ef8f4cb8b67af3def230fc8711eac1250bbd78d5/auto_gptq-0.7.0.tar.gz (124 kB)\n",
      "\u001b[2K     \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m124.6/124.6 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hDiscarding \u001b[4;34mhttp://mirrors.aliyun.com/pypi/packages/34/71/c3e73cf17681f6ff4754ef8f4cb8b67af3def230fc8711eac1250bbd78d5/auto_gptq-0.7.0.tar.gz#sha256=50a5396fae2db5a19446b3198ef0e86ee520846b881db47bdbf4eb9260eac723 (from http://mirrors.aliyun.com/pypi/simple/auto-gptq/) (requires-python:>=3.8.0)\u001b[0m: \u001b[33mRequested auto-gptq from http://mirrors.aliyun.com/pypi/packages/34/71/c3e73cf17681f6ff4754ef8f4cb8b67af3def230fc8711eac1250bbd78d5/auto_gptq-0.7.0.tar.gz#sha256=50a5396fae2db5a19446b3198ef0e86ee520846b881db47bdbf4eb9260eac723 has inconsistent version: expected '0.7.0', but metadata has '0.7.0+cu1241'\u001b[0m\n",
      "  Downloading http://mirrors.aliyun.com/pypi/packages/49/af/02b66e55dfd9aeb0ece923843043724ed7432ec0c649ea0f3b9fa1dd90c6/auto_gptq-0.6.0.tar.gz (120 kB)\n",
      "\u001b[2K     \u001b[90mโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”\u001b[0m \u001b[32m121.0/121.0 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25lerror\n",
      "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
      "  \n",
      "  \u001b[31mร—\u001b[0m \u001b[32mpython setup.py egg_info\u001b[0m did not run successfully.\n",
      "  \u001b[31mโ”‚\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
      "  \u001b[31mโ•ฐโ”€>\u001b[0m \u001b[31m[20 lines of output]\u001b[0m\n",
      "  \u001b[31m   \u001b[0m python: can't open file '/tmp/pip-install-9tzda5pv/auto-gptq_60a9cac9f2ef4bb89bb1ce3f0d0126b0/./autogptq_extension/qigen/generate.py': [Errno 2] No such file or directory\n",
      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
      "  \u001b[31m   \u001b[0m   File \"/tmp/pip-install-9tzda5pv/auto-gptq_60a9cac9f2ef4bb89bb1ce3f0d0126b0/setup.py\", line 109, in <module>\n",
      "  \u001b[31m   \u001b[0m     subprocess.check_output([\"python\", \"./autogptq_extension/qigen/generate.py\", \"--module\", \"--search\", \"--p\", str(p)])\n",
      "  \u001b[31m   \u001b[0m   File \"/root/miniconda3/lib/python3.12/subprocess.py\", line 466, in check_output\n",
      "  \u001b[31m   \u001b[0m     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,\n",
      "  \u001b[31m   \u001b[0m            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  \u001b[31m   \u001b[0m   File \"/root/miniconda3/lib/python3.12/subprocess.py\", line 571, in run\n",
      "  \u001b[31m   \u001b[0m     raise CalledProcessError(retcode, process.args,\n",
      "  \u001b[31m   \u001b[0m subprocess.CalledProcessError: Command '['python', './autogptq_extension/qigen/generate.py', '--module', '--search', '--p', '48']' returned non-zero exit status 2.\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m During handling of the above exception, another exception occurred:\n",
      "  \u001b[31m   \u001b[0m \n",
      "  \u001b[31m   \u001b[0m Traceback (most recent call last):\n",
      "  \u001b[31m   \u001b[0m   File \"<string>\", line 2, in <module>\n",
      "  \u001b[31m   \u001b[0m   File \"<pip-setuptools-caller>\", line 34, in <module>\n",
      "  \u001b[31m   \u001b[0m   File \"/tmp/pip-install-9tzda5pv/auto-gptq_60a9cac9f2ef4bb89bb1ce3f0d0126b0/setup.py\", line 111, in <module>\n",
      "  \u001b[31m   \u001b[0m     raise Exception(f\"Generating QiGen kernels failed with the error shown above.\")\n",
      "  \u001b[31m   \u001b[0m Exception: Generating QiGen kernels failed with the error shown above.\n",
      "  \u001b[31m   \u001b[0m Generating qigen kernels...\n",
      "  \u001b[31m   \u001b[0m \u001b[31m[end of output]\u001b[0m\n",
      "  \n",
      "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "\u001b[1;31merror\u001b[0m: \u001b[1mmetadata-generation-failed\u001b[0m\n",
      "\n",
      "\u001b[31mร—\u001b[0m Encountered error while generating package metadata.\n",
      "\u001b[31mโ•ฐโ”€>\u001b[0m See above for output.\n",
      "\n",
      "\u001b[1;35mnote\u001b[0m: This is an issue with the package mentioned above, not pip.\n",
      "\u001b[1;36mhint\u001b[0m: See above for details.\n",
      "\u001b[?25h"
     ]
    }
   ],
   "source": [
    "!pip install auto-gptq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8c14f99-2926-433f-8249-58cc123ce73d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}