Update README.md
Browse files
README.md
CHANGED
@@ -62,7 +62,106 @@ MT-Bench-TC
|
|
62 |
|
63 |
**Dependiency**
|
64 |
|
|
|
|
|
|
|
|
|
|
|
|
|
65 |
```
|
66 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
67 |
```
|
68 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
62 |
|
63 |
**Dependiency**
|
64 |
|
65 |
+
Install `mtkresearch` package
|
66 |
+
|
67 |
+
```
|
68 |
+
git clone https://github.com/mtkresearch/mtkresearch.git
|
69 |
+
cd mtkresearch
|
70 |
+
pip install -e .
|
71 |
```
|
72 |
+
|
73 |
+
**Hosting by VLLM**
|
74 |
+
|
75 |
+
```python
|
76 |
+
from vllm import LLM, SamplingParams
|
77 |
+
|
78 |
+
llm = LLM(
|
79 |
+
model='MediaTek-Research/Breeze-7B-FC-v1_0',
|
80 |
+
tensor_parallel_size=num_gpu, # number of gpus
|
81 |
+
gpu_memory_utilization=0.7
|
82 |
+
)
|
83 |
+
|
84 |
+
instance_end_token_id = llm.get_tokenizer().convert_token_to_ids('<|im_end|>')
|
85 |
+
params = SamplingParams(
|
86 |
+
temperature=0.01,
|
87 |
+
top_p=0.01,
|
88 |
+
max_tokens=4096,
|
89 |
+
repetition_penalty=1.1,
|
90 |
+
stop_token_ids=[instance_end_token_id]
|
91 |
+
)
|
92 |
+
|
93 |
+
def _inference(prompt, llm, params):
|
94 |
+
return llm.generate(prompt, params)[0].outputs[0].text
|
95 |
+
|
96 |
+
```
|
97 |
+
|
98 |
+
**Instruction following**
|
99 |
+
|
100 |
+
```python
|
101 |
+
from mtkresearch.llm.prompt import MRPromptV2
|
102 |
+
|
103 |
+
sys_prompt = 'You are a helpful AI assistant built by MediaTek Research. The user you are helping speaks Traditional Chinese and comes from Taiwan.'
|
104 |
+
|
105 |
+
prompt_engine = MRPromptV2()
|
106 |
+
|
107 |
+
conversations = [
|
108 |
+
{"role": "system", "content": sys_prompt},
|
109 |
+
{"role": "user", "content": "請問什麼是深度學習?"},
|
110 |
+
]
|
111 |
+
|
112 |
+
prompt = prompt_engine.get_prompt(conversations)
|
113 |
+
|
114 |
+
|
115 |
+
output_str = _inference(prompt, llm, params)
|
116 |
+
result = prompt_engine.parse_generated_str(output_str)
|
117 |
+
|
118 |
+
print(result) #
|
119 |
```
|
120 |
|
121 |
+
**Function Calling**
|
122 |
+
|
123 |
+
```python
|
124 |
+
from mtkresearch.llm.prompt import MRPromptV2
|
125 |
+
|
126 |
+
sys_prompt = 'You are a helpful AI assistant built by MediaTek Research. The user you are helping speaks Traditional Chinese and comes from Taiwan.'
|
127 |
+
|
128 |
+
functions = [
|
129 |
+
{
|
130 |
+
"name": "get_current_weather",
|
131 |
+
"description": "Get the current weather in a given location",
|
132 |
+
"parameters": {
|
133 |
+
"type": "object",
|
134 |
+
"properties": {
|
135 |
+
"location": {
|
136 |
+
"type": "string",
|
137 |
+
"description": "The city and state, e.g. San Francisco, CA"
|
138 |
+
},
|
139 |
+
"unit": {
|
140 |
+
"type": "string",
|
141 |
+
"enum": ["celsius", "fahrenheit"]
|
142 |
+
}
|
143 |
+
},
|
144 |
+
"required": ["location"]
|
145 |
+
}
|
146 |
+
}
|
147 |
+
]
|
148 |
+
|
149 |
+
prompt_engine = MRPromptV2()
|
150 |
+
|
151 |
+
# stage 1: query
|
152 |
+
conversations = [
|
153 |
+
{"role": "user", "content": "台北目前溫度是攝氏幾度?"},
|
154 |
+
]
|
155 |
+
|
156 |
+
prompt = prompt_engine.get_prompt(conversations, functions=functions)
|
157 |
+
|
158 |
+
output_str = _inference(prompt, llm, params)
|
159 |
+
result = prompt_engine.parse_generated_str(output_str)
|
160 |
+
|
161 |
+
print(result) #
|
162 |
+
|
163 |
+
# stage 2: execute called functions
|
164 |
+
|
165 |
+
# stage 3: put executed results
|
166 |
+
|
167 |
+
```
|