File size: 944 Bytes
2fc36fb 12857bc 2fc36fb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
{
"api_key": null,
"verify_url": "http://johnrachwan.pythonanywhere.com",
"smash_config": {
"pruners": "None",
"factorizers": "None",
"quantizers": "['llm-int8']",
"compilers": "None",
"task": "text_text_generation",
"device": "cuda",
"cache_dir": "/ceph/hdd/staff/charpent/.cache/modelsiqmlou71",
"batch_size": 1,
"model_name": "TinyLlama/TinyLlama-1.1B-intermediate-step-1195k-token-2.5T",
"pruning_ratio": 0.0,
"n_quantization_bits": 8,
"output_deviation": 0.005,
"max_batch_size": 1,
"qtype_weight": "torch.qint8",
"qtype_activation": "torch.quint8",
"qobserver": "<class 'torch.ao.quantization.observer.MinMaxObserver'>",
"qscheme": "torch.per_tensor_symmetric",
"qconfig": "x86",
"group_size": 128,
"damp_percent": 0.1,
"save_load_fn": "bitsandbytes"
}
} |