Update README.md
Browse files
README.md
CHANGED
@@ -6,36 +6,19 @@ The quantized version of [stablelm-zephyr-3b](https://huggingface.co/stabilityai
|
|
6 |
|
7 |
You can use it like this (steps taken from the above link):
|
8 |
|
9 |
-
```
|
10 |
-
|
11 |
-
import sys
|
12 |
-
|
13 |
-
|
14 |
-
genai_llm_bench = Path("openvino.genai/llm_bench/python")
|
15 |
-
|
16 |
-
if not genai_llm_bench.exists():
|
17 |
-
!git clone https://github.com/openvinotoolkit/openvino.genai.git
|
18 |
-
|
19 |
-
sys.path.append(str(genai_llm_bench))
|
20 |
-
```
|
21 |
-
|
22 |
-
then
|
23 |
-
|
24 |
-
```
|
25 |
-
pip install -q --extra-index-url https://download.pytorch.org/whl/cpu -r ./openvino.genai/llm_bench/python/requirements.txt
|
26 |
-
pip uninstall -q -y openvino openvino-dev openvino-nightly
|
27 |
-
pip install -q openvino-nightly
|
28 |
```
|
29 |
|
30 |
then
|
31 |
|
32 |
```python
|
33 |
-
from utils.ov_model_classes import register_normalized_configs
|
34 |
from optimum.intel.openvino import OVModelForCausalLM
|
35 |
from transformers import AutoConfig, AutoTokenizer
|
|
|
36 |
|
37 |
-
|
38 |
-
|
39 |
|
40 |
model_path = 'Mihaiii/stablelm-zephyr-3b-OV_FP14-4BIT'
|
41 |
model = OVModelForCausalLM.from_pretrained(model_path, compile=False, config=AutoConfig.from_pretrained(model_path, trust_remote_code=True), stateful=True)
|
|
|
6 |
|
7 |
You can use it like this (steps taken from the above link):
|
8 |
|
9 |
+
```bash
|
10 |
+
pip install -q git+https://github.com/huggingface/optimum-intel.git@e22a2ac26b3a6c7854da956d538f784ebeca879b onnx openvino-nightly
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
11 |
```
|
12 |
|
13 |
then
|
14 |
|
15 |
```python
|
|
|
16 |
from optimum.intel.openvino import OVModelForCausalLM
|
17 |
from transformers import AutoConfig, AutoTokenizer
|
18 |
+
from optimum.utils import NormalizedTextConfig, NormalizedConfigManager
|
19 |
|
20 |
+
NormalizedConfigManager._conf['stablelm_epoch'] = NormalizedTextConfig.with_args(num_layers='num_hidden_layers', num_attention_heads='num_attention_heads')
|
21 |
+
NormalizedConfigManager._conf['stablelm-epoch'] = NormalizedTextConfig.with_args(num_layers='num_hidden_layers', num_attention_heads='num_attention_heads')
|
22 |
|
23 |
model_path = 'Mihaiii/stablelm-zephyr-3b-OV_FP14-4BIT'
|
24 |
model = OVModelForCausalLM.from_pretrained(model_path, compile=False, config=AutoConfig.from_pretrained(model_path, trust_remote_code=True), stateful=True)
|