Hugging Face's logo

aws-neuron
/
optimum-neuron-cache

Model card Files Community
276
dacorvo HF staff commited on
Commit
687da09
•
1 Parent(s): a416547

Add configuration for granite models

Browse files
Files changed (1) hide show
  1. inference-cache-config/granite.json +66 -0
inference-cache-config/granite.json ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "ibm-granite/granite-3.1-2b-instruct": [
3
+ {
4
+ "batch_size": 1,
5
+ "sequence_length": 4096,
6
+ "num_cores": 2,
7
+ "auto_cast_type": "bf16"
8
+ },
9
+ {
10
+ "batch_size": 4,
11
+ "sequence_length": 4096,
12
+ "num_cores": 2,
13
+ "auto_cast_type": "bf16"
14
+ },
15
+ {
16
+ "batch_size": 8,
17
+ "sequence_length": 4096,
18
+ "num_cores": 2,
19
+ "auto_cast_type": "bf16"
20
+ },
21
+ {
22
+ "batch_size": 1,
23
+ "sequence_length": 4096,
24
+ "num_cores": 8,
25
+ "auto_cast_type": "bf16"
26
+ },
27
+ {
28
+ "batch_size": 32,
29
+ "sequence_length": 4096,
30
+ "num_cores": 8,
31
+ "auto_cast_type": "bf16"
32
+ }
33
+ ],
34
+ "ibm-granite/granite-3.1-8b-instruct": [
35
+ {
36
+ "batch_size": 1,
37
+ "sequence_length": 4096,
38
+ "num_cores": 2,
39
+ "auto_cast_type": "bf16"
40
+ },
41
+ {
42
+ "batch_size": 4,
43
+ "sequence_length": 4096,
44
+ "num_cores": 2,
45
+ "auto_cast_type": "bf16"
46
+ },
47
+ {
48
+ "batch_size": 8,
49
+ "sequence_length": 4096,
50
+ "num_cores": 2,
51
+ "auto_cast_type": "bf16"
52
+ },
53
+ {
54
+ "batch_size": 1,
55
+ "sequence_length": 4096,
56
+ "num_cores": 8,
57
+ "auto_cast_type": "bf16"
58
+ },
59
+ {
60
+ "batch_size": 32,
61
+ "sequence_length": 4096,
62
+ "num_cores": 8,
63
+ "auto_cast_type": "bf16"
64
+ }
65
+ ]
66
+ }