metadata
license: cc-by-nc-4.0
language:
- ro
base_model: OpenLLM-Ro/RoLlama3-8b-Instruct-2024-06-28
datasets:
- OpenLLM-Ro/ro_sft_alpaca
- OpenLLM-Ro/ro_sft_alpaca_gpt4
- OpenLLM-Ro/ro_sft_dolly
- OpenLLM-Ro/ro_sft_selfinstruct_gpt4
- OpenLLM-Ro/ro_sft_norobots
- OpenLLM-Ro/ro_sft_orca
- OpenLLM-Ro/ro_sft_camel
tags:
- llama-cpp
- gguf-my-repo
model-index:
- name: OpenLLM-Ro/RoLlama3-8b-Instruct-2024-06-28
results:
- task:
type: text-generation
dataset:
name: RoMT-Bench
type: RoMT-Bench
metrics:
- type: Score
value: 5.15
name: Score
- type: Score
value: 6.03
name: First turn
- type: Score
value: 4.28
name: Second turn
- task:
type: text-generation
dataset:
name: RoCulturaBench
type: RoCulturaBench
metrics:
- type: Score
value: 3.71
name: Score
- task:
type: text-generation
dataset:
name: Romanian_Academic_Benchmarks
type: Romanian_Academic_Benchmarks
metrics:
- type: accuracy
value: 50.56
name: Average accuracy
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_arc_challenge
type: OpenLLM-Ro/ro_arc_challenge
metrics:
- type: accuracy
value: 44.7
name: Average accuracy
- type: accuracy
value: 41.9
name: 0-shot
- type: accuracy
value: 44.3
name: 1-shot
- type: accuracy
value: 44.56
name: 3-shot
- type: accuracy
value: 45.5
name: 5-shot
- type: accuracy
value: 46.1
name: 10-shot
- type: accuracy
value: 45.84
name: 25-shot
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_mmlu
type: OpenLLM-Ro/ro_mmlu
metrics:
- type: accuracy
value: 52.19
name: Average accuracy
- type: accuracy
value: 50.85
name: 0-shot
- type: accuracy
value: 51.24
name: 1-shot
- type: accuracy
value: 53.3
name: 3-shot
- type: accuracy
value: 53.39
name: 5-shot
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_winogrande
type: OpenLLM-Ro/ro_winogrande
metrics:
- type: accuracy
value: 67.23
name: Average accuracy
- type: accuracy
value: 65.19
name: 0-shot
- type: accuracy
value: 66.54
name: 1-shot
- type: accuracy
value: 67.88
name: 3-shot
- type: accuracy
value: 69.3
name: 5-shot
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_hellaswag
type: OpenLLM-Ro/ro_hellaswag
metrics:
- type: accuracy
value: 57.69
name: Average accuracy
- type: accuracy
value: 56.12
name: 0-shot
- type: accuracy
value: 57.37
name: 1-shot
- type: accuracy
value: 57.92
name: 3-shot
- type: accuracy
value: 58.18
name: 5-shot
- type: accuracy
value: 58.85
name: 10-shot
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_gsm8k
type: OpenLLM-Ro/ro_gsm8k
metrics:
- type: accuracy
value: 30.23
name: Average accuracy
- type: accuracy
value: 29.42
name: 1-shot
- type: accuracy
value: 30.02
name: 3-shot
- type: accuracy
value: 31.24
name: 5-shot
- task:
type: text-generation
dataset:
name: OpenLLM-Ro/ro_truthfulqa
type: OpenLLM-Ro/ro_truthfulqa
metrics:
- type: accuracy
value: 51.34
name: Average accuracy
- task:
type: text-generation
dataset:
name: LaRoSeDa_binary
type: LaRoSeDa_binary
metrics:
- type: macro-f1
value: 97.52
name: Average macro-f1
- type: macro-f1
value: 97.43
name: 0-shot
- type: macro-f1
value: 96.6
name: 1-shot
- type: macro-f1
value: 97.9
name: 3-shot
- type: macro-f1
value: 98.13
name: 5-shot
- task:
type: text-generation
dataset:
name: LaRoSeDa_multiclass
type: LaRoSeDa_multiclass
metrics:
- type: macro-f1
value: 67.41
name: Average macro-f1
- type: macro-f1
value: 63.77
name: 0-shot
- type: macro-f1
value: 68.91
name: 1-shot
- type: macro-f1
value: 66.36
name: 3-shot
- type: macro-f1
value: 70.61
name: 5-shot
- task:
type: text-generation
dataset:
name: LaRoSeDa_binary_finetuned
type: LaRoSeDa_binary_finetuned
metrics:
- type: macro-f1
value: 94.15
name: Average macro-f1
- task:
type: text-generation
dataset:
name: LaRoSeDa_multiclass_finetuned
type: LaRoSeDa_multiclass_finetuned
metrics:
- type: macro-f1
value: 87.13
name: Average macro-f1
- task:
type: text-generation
dataset:
name: WMT_EN-RO
type: WMT_EN-RO
metrics:
- type: bleu
value: 24.01
name: Average bleu
- type: bleu
value: 6.92
name: 0-shot
- type: bleu
value: 29.33
name: 1-shot
- type: bleu
value: 29.79
name: 3-shot
- type: bleu
value: 30.02
name: 5-shot
- task:
type: text-generation
dataset:
name: WMT_RO-EN
type: WMT_RO-EN
metrics:
- type: bleu
value: 27.36
name: Average bleu
- type: bleu
value: 4.5
name: 0-shot
- type: bleu
value: 30.3
name: 1-shot
- type: bleu
value: 36.96
name: 3-shot
- type: bleu
value: 37.7
name: 5-shot
- task:
type: text-generation
dataset:
name: WMT_EN-RO_finetuned
type: WMT_EN-RO_finetuned
metrics:
- type: bleu
value: 26.53
name: Average bleu
- task:
type: text-generation
dataset:
name: WMT_RO-EN_finetuned
type: WMT_RO-EN_finetuned
metrics:
- type: bleu
value: 40.36
name: Average bleu
- task:
type: text-generation
dataset:
name: XQuAD
type: XQuAD
metrics:
- type: exact_match
value: 39.43
name: Average exact_match
- type: f1
value: 59.5
name: Average f1
- task:
type: text-generation
dataset:
name: XQuAD_finetuned
type: XQuAD_finetuned
metrics:
- type: exact_match
value: 44.45
name: Average exact_match
- type: f1
value: 59.76
name: Average f1
- task:
type: text-generation
dataset:
name: STS
type: STS
metrics:
- type: spearman
value: 77.2
name: Average spearman
- type: pearson
value: 77.87
name: Average pearson
- task:
type: text-generation
dataset:
name: STS_finetuned
type: STS_finetuned
metrics:
- type: spearman
value: 85.8
name: Average spearman
- type: pearson
value: 86.05
name: Average pearson
- task:
type: text-generation
dataset:
name: XQuAD_EM
type: XQuAD_EM
metrics:
- type: exact_match
value: 4.45
name: 0-shot
- type: exact_match
value: 48.24
name: 1-shot
- type: exact_match
value: 52.03
name: 3-shot
- type: exact_match
value: 53.03
name: 5-shot
- task:
type: text-generation
dataset:
name: XQuAD_F1
type: XQuAD_F1
metrics:
- type: f1
value: 26.08
name: 0-shot
- type: f1
value: 68.4
name: 1-shot
- type: f1
value: 71.92
name: 3-shot
- type: f1
value: 71.6
name: 5-shot
- task:
type: text-generation
dataset:
name: STS_Spearman
type: STS_Spearman
metrics:
- type: spearman
value: 77.76
name: 1-shot
- type: spearman
value: 76.72
name: 3-shot
- type: spearman
value: 77.12
name: 5-shot
- task:
type: text-generation
dataset:
name: STS_Pearson
type: STS_Pearson
metrics:
- type: pearson
value: 77.83
name: 1-shot
- type: pearson
value: 77.64
name: 3-shot
- type: pearson
value: 78.13
name: 5-shot
vladciocan88/RoLlama3-8b-Instruct-2024-06-28-Q8_0-GGUF
This model was converted to GGUF format from OpenLLM-Ro/RoLlama3-8b-Instruct-2024-06-28 using llama.cpp via the ggml.ai's GGUF-my-repo space.
Refer to the original model card for more details on the model.
Use with llama.cpp
Install llama.cpp through brew (works on Mac and Linux)
brew install llama.cpp
Invoke the llama.cpp server or the CLI.
CLI:
llama-cli --hf-repo vladciocan88/RoLlama3-8b-Instruct-2024-06-28-Q8_0-GGUF --hf-file rollama3-8b-instruct-2024-06-28-q8_0.gguf -p "The meaning to life and the universe is"
Server:
llama-server --hf-repo vladciocan88/RoLlama3-8b-Instruct-2024-06-28-Q8_0-GGUF --hf-file rollama3-8b-instruct-2024-06-28-q8_0.gguf -c 2048
Note: You can also use this checkpoint directly through the usage steps listed in the Llama.cpp repo as well.
Step 1: Clone llama.cpp from GitHub.
git clone https://github.com/ggerganov/llama.cpp
Step 2: Move into the llama.cpp folder and build it with LLAMA_CURL=1 flag along with other hardware-specific flags (for ex: LLAMA_CUDA=1 for Nvidia GPUs on Linux).
cd llama.cpp && LLAMA_CURL=1 make
Step 3: Run inference through the main binary.
./llama-cli --hf-repo vladciocan88/RoLlama3-8b-Instruct-2024-06-28-Q8_0-GGUF --hf-file rollama3-8b-instruct-2024-06-28-q8_0.gguf -p "The meaning to life and the universe is"
or
./llama-server --hf-repo vladciocan88/RoLlama3-8b-Instruct-2024-06-28-Q8_0-GGUF --hf-file rollama3-8b-instruct-2024-06-28-q8_0.gguf -c 2048