Not-Grim-Refer commited on
Commit
f6a03c5
1 Parent(s): 622c9e8

Upload 6 files

Browse files
Files changed (6) hide show
  1. README.md +7 -7
  2. app.py +380 -48
  3. flake8.txt +21 -0
  4. gitattributes.txt +35 -0
  5. gitignore.txt +0 -0
  6. requirements.txt +3 -3
README.md CHANGED
@@ -1,12 +1,12 @@
1
  ---
2
- title: FalconInstructQA
3
- emoji: 😻
4
- colorFrom: red
5
- colorTo: gray
6
- sdk: streamlit
7
- sdk_version: 1.21.0
8
  app_file: app.py
9
- pinned: false
10
  ---
11
 
12
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: chatglm2 6b int4
3
+ emoji: 🌖
4
+ colorFrom: purple
5
+ colorTo: blue
6
+ sdk: gradio
7
+ sdk_version: 3.35.2
8
  app_file: app.py
9
+ pinned: true
10
  ---
11
 
12
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py CHANGED
@@ -1,54 +1,386 @@
1
- import streamlit as st
2
- from transformers import AutoModel, AutoTokenizer
 
 
 
 
 
 
 
 
 
 
 
 
3
  import mdtex2html
4
- from utils import load_model_on_gpus
 
 
 
 
 
 
 
 
 
 
5
 
6
- st.set_page_config(page_title="ChatGLM2-6B", page_icon=":robot:")
 
7
 
8
- st.header("ChatGLM2-6B")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
 
10
- tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True)
11
- model = AutoModel.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True).cuda()
12
- # Load model on multiple GPUs
13
- #model = load_model_on_gpus("THUDM/chatglm2-6b", num_gpus=2)
14
  model = model.eval()
15
 
16
- def postprocess(chat):
17
- for i, (user, response) in enumerate(chat):
18
- chat[i] = (mdtex2html.convert(user), mdtex2html.convert(response))
19
- return chat
20
-
21
- user_input = st.text_area("Input:", height=200, placeholder="Ask me anything!")
22
- if user_input:
23
- history = st.session_state.get('history', [])
24
-
25
- max_length = st.slider("Max Length:", 0, 32768, 8192, 1)
26
- top_p = st.slider("Top P:", 0.0, 1.0, 0.8, 0.01)
27
- temperature = st.slider("Temperature:", 0.0, 1.0, 0.95, 0.01)
28
-
29
- if 'past_key_values' not in st.session_state:
30
- st.session_state['past_key_values'] = None
31
-
32
- with st.spinner("Thinking..."):
33
- response = model.generate(tokenizer.encode(user_input),
34
- max_length=max_length,
35
- top_p=top_p,
36
- temperature=temperature,
37
- return_dict_in_generate=True,
38
- output_scores=True,
39
- return_past_key_values=True,
40
- past_key_values=st.session_state.past_key_values)
41
-
42
- st.session_state.past_key_values = response.past_key_values
43
-
44
- history.append((user_input, response.sequences[0]))
45
- history = postprocess(history)
46
-
47
- for user, chatbot in history:
48
- message = f"**Human:** {user}" if user else ""
49
- response = f"**AI:** {chatbot}" if chatbot else ""
50
- st.markdown(message + response, unsafe_allow_html=True)
51
-
52
- if st.button("Clear History"):
53
- st.session_state['history'] = []
54
- st.session_state['past_key_values'] = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Credit to https://github.com/THUDM/ChatGLM2-6B/blob/main/web_demo.py while mistakes are mine."""
2
+ # pylint: disable=broad-exception-caught, redefined-outer-name, missing-function-docstring, missing-module-docstring, too-many-arguments, line-too-long, invalid-name, redefined-builtin, redefined-argument-from-local
3
+ # import gradio as gr
4
+
5
+ # model_name = "models/THUDM/chatglm2-6b-int4"
6
+ # gr.load(model_name).lauch()
7
+
8
+ # %%writefile demo-4bit.py
9
+
10
+ import os
11
+ import time
12
+ from textwrap import dedent
13
+
14
+ import gradio as gr
15
  import mdtex2html
16
+ import torch
17
+ from loguru import logger
18
+ from transformers import AutoModel, AutoTokenizer
19
+
20
+ # fix timezone in Linux
21
+ os.environ["TZ"] = "Asia/Shanghai"
22
+ try:
23
+ time.tzset() # type: ignore # pylint: disable=no-member
24
+ except Exception:
25
+ # Windows
26
+ logger.warning("Windows, cant run time.tzset()")
27
 
28
+ # model_name = "THUDM/chatglm2-6b" # 7x?G
29
+ model_name = "THUDM/chatglm2-6b-int4" # 3.9G
30
 
31
+ RETRY_FLAG = False
32
+
33
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
34
+
35
+ # model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda()
36
+
37
+ # 4/8 bit
38
+ # model = AutoModel.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True).quantize(4).cuda()
39
+
40
+ has_cuda = torch.cuda.is_available()
41
+ # has_cuda = False # force cpu
42
+
43
+ if has_cuda:
44
+ if model_name.endswith("int4"):
45
+ model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda()
46
+ else:
47
+ model = (
48
+ AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half()
49
+ )
50
+ else:
51
+ model = AutoModel.from_pretrained(
52
+ model_name, trust_remote_code=True
53
+ ).float() # .half().float(), .float() required for CPU
54
 
 
 
 
 
55
  model = model.eval()
56
 
57
+ _ = """Override Chatbot.postprocess"""
58
+
59
+
60
+ def postprocess(self, y):
61
+ if y is None:
62
+ return []
63
+ for i, (message, response) in enumerate(y):
64
+ y[i] = (
65
+ None if message is None else mdtex2html.convert((message)),
66
+ None if response is None else mdtex2html.convert(response),
67
+ )
68
+ return y
69
+
70
+
71
+ gr.Chatbot.postprocess = postprocess
72
+
73
+
74
+ def parse_text(text):
75
+ """Copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/."""
76
+ lines = text.split("\n")
77
+ lines = [line for line in lines if line != ""]
78
+ count = 0
79
+ for i, line in enumerate(lines):
80
+ if "```" in line:
81
+ count += 1
82
+ items = line.split("`")
83
+ if count % 2 == 1:
84
+ lines[i] = f'<pre><code class="language-{items[-1]}">'
85
+ else:
86
+ lines[i] = "<br></code></pre>"
87
+ else:
88
+ if i > 0:
89
+ if count % 2 == 1:
90
+ line = line.replace("`", r"\`")
91
+ line = line.replace("<", "&lt;")
92
+ line = line.replace(">", "&gt;")
93
+ line = line.replace(" ", "&nbsp;")
94
+ line = line.replace("*", "&ast;")
95
+ line = line.replace("_", "&lowbar;")
96
+ line = line.replace("-", "&#45;")
97
+ line = line.replace(".", "&#46;")
98
+ line = line.replace("!", "&#33;")
99
+ line = line.replace("(", "&#40;")
100
+ line = line.replace(")", "&#41;")
101
+ line = line.replace("$", "&#36;")
102
+ lines[i] = "<br>" + line
103
+ text = "".join(lines)
104
+ return text
105
+
106
+
107
+ def predict(
108
+ RETRY_FLAG, input, chatbot, max_length, top_p, temperature, history, past_key_values
109
+ ):
110
+ try:
111
+ chatbot.append((parse_text(input), ""))
112
+ except Exception as exc:
113
+ logger.error(exc)
114
+ logger.debug(f"{chatbot=}")
115
+ _ = """
116
+ if chatbot:
117
+ chatbot[-1] = (parse_text(input), str(exc))
118
+ yield chatbot, history, past_key_values
119
+ # """
120
+ yield chatbot, history, past_key_values
121
+
122
+ for response, history, past_key_values in model.stream_chat(
123
+ tokenizer,
124
+ input,
125
+ history,
126
+ past_key_values=past_key_values,
127
+ return_past_key_values=True,
128
+ max_length=max_length,
129
+ top_p=top_p,
130
+ temperature=temperature,
131
+ ):
132
+ chatbot[-1] = (parse_text(input), parse_text(response))
133
+
134
+ yield chatbot, history, past_key_values
135
+
136
+
137
+ def trans_api(input, max_length=4096, top_p=0.8, temperature=0.2):
138
+ if max_length < 10:
139
+ max_length = 4096
140
+ if top_p < 0.1 or top_p > 1:
141
+ top_p = 0.85
142
+ if temperature <= 0 or temperature > 1:
143
+ temperature = 0.01
144
+ try:
145
+ res, _ = model.chat(
146
+ tokenizer,
147
+ input,
148
+ history=[],
149
+ past_key_values=None,
150
+ max_length=max_length,
151
+ top_p=top_p,
152
+ temperature=temperature,
153
+ )
154
+ # logger.debug(f"{res=} \n{_=}")
155
+ except Exception as exc:
156
+ logger.error(f"{exc=}")
157
+ res = str(exc)
158
+
159
+ return res
160
+
161
+
162
+ def reset_user_input():
163
+ return gr.update(value="")
164
+
165
+
166
+ def reset_state():
167
+ return [], [], None
168
+
169
+
170
+ # Delete last turn
171
+ def delete_last_turn(chat, history):
172
+ if chat and history:
173
+ chat.pop(-1)
174
+ history.pop(-1)
175
+ return chat, history
176
+
177
+
178
+ # Regenerate response
179
+ def retry_last_answer(
180
+ user_input, chatbot, max_length, top_p, temperature, history, past_key_values
181
+ ):
182
+ if chatbot and history:
183
+ # Removing the previous conversation from chat
184
+ chatbot.pop(-1)
185
+ # Setting up a flag to capture a retry
186
+ RETRY_FLAG = True
187
+ # Getting last message from user
188
+ user_input = history[-1][0]
189
+ # Removing bot response from the history
190
+ history.pop(-1)
191
+
192
+ yield from predict(
193
+ RETRY_FLAG, # type: ignore
194
+ user_input,
195
+ chatbot,
196
+ max_length,
197
+ top_p,
198
+ temperature,
199
+ history,
200
+ past_key_values,
201
+ )
202
+
203
+
204
+ with gr.Blocks(title="ChatGLM2-6B-int4", theme=gr.themes.Soft(text_size="sm")) as demo:
205
+ # gr.HTML("""<h1 align="center">ChatGLM2-6B-int4</h1>""")
206
+ gr.HTML(
207
+ """<center><a href="https://huggingface.co/spaces/mikeee/chatglm2-6b-4bit?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>To avoid the queue and for faster inference Duplicate this Space and upgrade to GPU</center>"""
208
+ )
209
+
210
+ with gr.Accordion("🎈 Info", open=False):
211
+ _ = f"""
212
+ ## {model_name}
213
+
214
+ Try to refresh the browser and try again when occasionally an error occurs.
215
+
216
+ With a GPU, a query takes from a few seconds to a few tens of seconds, dependent on the number of words/characters
217
+ the question and responses contain. The quality of the responses varies quite a bit it seems. Even the same
218
+ question with the same parameters, asked at different times, can result in quite different responses.
219
+
220
+ * Low temperature: responses will be more deterministic and focused; High temperature: responses more creative.
221
+
222
+ * Suggested temperatures -- translation: up to 0.3; chatting: > 0.4
223
+
224
+ * Top P controls dynamic vocabulary selection based on context.
225
+
226
+ For a table of example values for different scenarios, refer to [this](https://community.openai.com/t/cheat-sheet-mastering-temperature-and-top-p-in-chatgpt-api-a-few-tips-and-tricks-on-controlling-the-creativity-deterministic-output-of-prompt-responses/172683)
227
+
228
+ If the instance is not on a GPU (T4), it will be very slow. You can try to run the colab notebook [chatglm2-6b-4bit colab notebook](https://colab.research.google.com/drive/1WkF7kOjVCcBBatDHjaGkuJHnPdMWNtbW?usp=sharing) for a spin.
229
+
230
+ The T4 GPU is sponsored by a community GPU grant from Huggingface. Thanks a lot!
231
+ """
232
+ gr.Markdown(dedent(_))
233
+ chatbot = gr.Chatbot()
234
+ with gr.Row():
235
+ with gr.Column(scale=4):
236
+ with gr.Column(scale=12):
237
+ user_input = gr.Textbox(
238
+ show_label=False,
239
+ placeholder="Input...",
240
+ ).style(container=False)
241
+ RETRY_FLAG = gr.Checkbox(value=False, visible=False)
242
+ with gr.Column(min_width=32, scale=1):
243
+ with gr.Row():
244
+ submitBtn = gr.Button("Submit", variant="primary")
245
+ deleteBtn = gr.Button("Delete last turn", variant="secondary")
246
+ retryBtn = gr.Button("Regenerate", variant="secondary")
247
+ with gr.Column(scale=1):
248
+ emptyBtn = gr.Button("Clear History")
249
+ max_length = gr.Slider(
250
+ 0,
251
+ 32768,
252
+ value=8192,
253
+ step=1.0,
254
+ label="Maximum length",
255
+ interactive=True,
256
+ )
257
+ top_p = gr.Slider(
258
+ 0, 1, value=0.85, step=0.01, label="Top P", interactive=True
259
+ )
260
+ temperature = gr.Slider(
261
+ 0.01, 1, value=0.95, step=0.01, label="Temperature", interactive=True
262
+ )
263
+
264
+ history = gr.State([])
265
+ past_key_values = gr.State(None)
266
+
267
+ user_input.submit(
268
+ predict,
269
+ [
270
+ RETRY_FLAG,
271
+ user_input,
272
+ chatbot,
273
+ max_length,
274
+ top_p,
275
+ temperature,
276
+ history,
277
+ past_key_values,
278
+ ],
279
+ [chatbot, history, past_key_values],
280
+ show_progress="full",
281
+ )
282
+ submitBtn.click(
283
+ predict,
284
+ [
285
+ RETRY_FLAG,
286
+ user_input,
287
+ chatbot,
288
+ max_length,
289
+ top_p,
290
+ temperature,
291
+ history,
292
+ past_key_values,
293
+ ],
294
+ [chatbot, history, past_key_values],
295
+ show_progress="full",
296
+ api_name="predict",
297
+ )
298
+ submitBtn.click(reset_user_input, [], [user_input])
299
+
300
+ emptyBtn.click(
301
+ reset_state, outputs=[chatbot, history, past_key_values], show_progress="full"
302
+ )
303
+
304
+ retryBtn.click(
305
+ retry_last_answer,
306
+ inputs=[
307
+ user_input,
308
+ chatbot,
309
+ max_length,
310
+ top_p,
311
+ temperature,
312
+ history,
313
+ past_key_values,
314
+ ],
315
+ # outputs = [chatbot, history, last_user_message, user_message]
316
+ outputs=[chatbot, history, past_key_values],
317
+ )
318
+ deleteBtn.click(delete_last_turn, [chatbot, history], [chatbot, history])
319
+
320
+ with gr.Accordion("Example inputs", open=True):
321
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
322
+ examples = gr.Examples(
323
+ examples=[
324
+ ["What NFL team won the Super Bowl in the year Justin Bieber was born? "],
325
+ ["What NFL team won the Super Bowl in the year Justin Bieber was born? Think step by step."],
326
+ ["Explain the plot of Cinderella in a sentence."],
327
+ [
328
+ "How long does it take to become proficient in French, and what are the best methods for retaining information?"
329
+ ],
330
+ ["What are some common mistakes to avoid when writing code?"],
331
+ ["Build a prompt to generate a beautiful portrait of a horse"],
332
+ ["Suggest four metaphors to describe the benefits of AI"],
333
+ ["Write a pop song about leaving home for the sandy beaches."],
334
+ ["Write a summary demonstrating my ability to tame lions"],
335
+ ["鲁迅和周树人什么关系"],
336
+ ["从前有一头牛,这头牛后面有什么?"],
337
+ ["正无穷大加一大于正无穷大吗?"],
338
+ ["正无穷大加正无穷大大于正无穷大吗?"],
339
+ ["-2的平方根等于什么"],
340
+ ["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"],
341
+ ["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"],
342
+ ["鲁迅和周树人什么关系 用英文回答"],
343
+ ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
344
+ [f"{etext} 翻成中文,列出3个版本"],
345
+ [f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本"],
346
+ ["js 判断一个数是不是质数"],
347
+ ["js 实现python 的 range(10)"],
348
+ ["js 实现python 的 [*(range(10)]"],
349
+ ["假定 1 + 2 = 4, 试求 7 + 8"],
350
+ ["Erkläre die Handlung von Cinderella in einem Satz."],
351
+ ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"],
352
+ ],
353
+ inputs=[user_input],
354
+ examples_per_page=30,
355
+ )
356
+
357
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
358
+ input_text = gr.Text()
359
+ tr_btn = gr.Button("Go", variant="primary")
360
+ out_text = gr.Text()
361
+ tr_btn.click(
362
+ trans_api,
363
+ [input_text, max_length, top_p, temperature],
364
+ out_text,
365
+ # show_progress="full",
366
+ api_name="tr",
367
+ )
368
+ _ = """
369
+ input_text.submit(
370
+ trans_api,
371
+ [input_text, max_length, top_p, temperature],
372
+ out_text,
373
+ show_progress="full",
374
+ api_name="tr1",
375
+ )
376
+ # """
377
+
378
+ # demo.queue().launch(share=False, inbrowser=True)
379
+ # demo.queue().launch(share=True, inbrowser=True, debug=True)
380
+
381
+ # concurrency_count > 1 requires more memory, max_size: queue size
382
+ # T4 medium: 30GB, model size: ~4G concurrency_count = 6
383
+ # leave one for api access
384
+ # reduce to 5 if OOM occurs to often
385
+
386
+ demo.queue(concurrency_count=6, max_size=30).launch(debug=True)
flake8.txt ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [flake8]
2
+ ignore =
3
+ # E203 whitespace before ':'
4
+ E203
5
+ D203,
6
+ # line too long
7
+ E501
8
+ per-file-ignores =
9
+ # imported but unused
10
+ # __init__.py: F401
11
+ test_*.py: F401
12
+ exclude =
13
+ .git,
14
+ __pycache__,
15
+ docs/source/conf.py,
16
+ old,
17
+ build,
18
+ dist,
19
+ .venv
20
+ pad*.py
21
+ max-complexity = 25
gitattributes.txt ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
gitignore.txt ADDED
File without changes
requirements.txt CHANGED
@@ -1,9 +1,9 @@
1
- streamlit
2
  protobuf
3
- transformers
4
  cpm_kernels
5
  torch>=2.0
 
6
  mdtex2html
7
  sentencepiece
8
  accelerate
9
- loguru
 
 
1
  protobuf
2
+ transformers==4.30.2
3
  cpm_kernels
4
  torch>=2.0
5
+ # gradio
6
  mdtex2html
7
  sentencepiece
8
  accelerate
9
+ loguru