model update
Browse files
README.md
CHANGED
@@ -33,62 +33,43 @@ model-index:
|
|
33 |
metrics:
|
34 |
- name: BLEU4
|
35 |
type: bleu4
|
36 |
-
value:
|
37 |
- name: ROUGE-L
|
38 |
type: rouge-l
|
39 |
-
value:
|
40 |
- name: METEOR
|
41 |
type: meteor
|
42 |
-
value:
|
43 |
- name: BERTScore
|
44 |
type: bertscore
|
45 |
-
value:
|
46 |
- name: MoverScore
|
47 |
type: moverscore
|
48 |
-
value:
|
49 |
-
- name: QAAlignedF1Score (BERTScore)
|
50 |
-
type:
|
51 |
-
value:
|
52 |
-
- name: QAAlignedRecall (BERTScore)
|
53 |
-
type:
|
54 |
-
value:
|
55 |
-
- name: QAAlignedPrecision (BERTScore)
|
56 |
-
type:
|
57 |
-
value:
|
58 |
-
- name: QAAlignedF1Score (MoverScore)
|
59 |
-
type:
|
60 |
-
value:
|
61 |
-
- name: QAAlignedRecall (MoverScore)
|
62 |
-
type:
|
63 |
-
value:
|
64 |
-
- name: QAAlignedPrecision (MoverScore)
|
65 |
-
type:
|
66 |
-
value:
|
67 |
---
|
68 |
|
69 |
# Model Card of `lmqg/mbart-large-cc25-koquad`
|
70 |
-
This model is fine-tuned version of [facebook/mbart-large-cc25](https://huggingface.co/facebook/mbart-large-cc25) for question generation task on the
|
71 |
-
[lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) (dataset_name: default) via [`lmqg`](https://github.com/asahi417/lm-question-generation).
|
72 |
|
73 |
|
74 |
-
Please cite our paper if you use the model ([https://arxiv.org/abs/2210.03992](https://arxiv.org/abs/2210.03992)).
|
75 |
-
|
76 |
-
```
|
77 |
-
|
78 |
-
@inproceedings{ushio-etal-2022-generative,
|
79 |
-
title = "{G}enerative {L}anguage {M}odels for {P}aragraph-{L}evel {Q}uestion {G}eneration",
|
80 |
-
author = "Ushio, Asahi and
|
81 |
-
Alva-Manchego, Fernando and
|
82 |
-
Camacho-Collados, Jose",
|
83 |
-
booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
|
84 |
-
month = dec,
|
85 |
-
year = "2022",
|
86 |
-
address = "Abu Dhabi, U.A.E.",
|
87 |
-
publisher = "Association for Computational Linguistics",
|
88 |
-
}
|
89 |
-
|
90 |
-
```
|
91 |
-
|
92 |
### Overview
|
93 |
- **Language model:** [facebook/mbart-large-cc25](https://huggingface.co/facebook/mbart-large-cc25)
|
94 |
- **Language:** ko
|
@@ -100,42 +81,52 @@ Please cite our paper if you use the model ([https://arxiv.org/abs/2210.03992](h
|
|
100 |
### Usage
|
101 |
- With [`lmqg`](https://github.com/asahi417/lm-question-generation#lmqg-language-model-for-question-generation-)
|
102 |
```python
|
103 |
-
|
104 |
from lmqg import TransformersQG
|
|
|
105 |
# initialize model
|
106 |
-
model = TransformersQG(language=
|
|
|
107 |
# model prediction
|
108 |
-
|
109 |
|
110 |
```
|
111 |
|
112 |
- With `transformers`
|
113 |
```python
|
114 |
-
|
115 |
from transformers import pipeline
|
116 |
-
|
117 |
-
pipe = pipeline("text2text-generation",
|
118 |
-
|
119 |
-
question = pipe('1990년 영화 《 <hl> 남부군 <hl> 》에서 단역으로 영화배우 첫 데뷔에 이어 같은 해 KBS 드라마 《지구인》에서 단역으로 출연하였고 이듬해 MBC 《여명의 눈동자》를 통해 단역으로 출연하였다.')
|
120 |
|
121 |
```
|
122 |
|
123 |
-
## Evaluation
|
124 |
|
125 |
|
126 |
-
|
127 |
|
128 |
-
|
|
129 |
-
|
130 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
131 |
|
132 |
|
133 |
-
|
134 |
|
135 |
-
|
|
136 |
-
|
137 |
-
|
|
138 |
-
|
|
|
|
|
|
|
|
|
139 |
|
140 |
|
141 |
|
@@ -162,7 +153,6 @@ The full configuration can be found at [fine-tuning config file](https://hugging
|
|
162 |
|
163 |
## Citation
|
164 |
```
|
165 |
-
|
166 |
@inproceedings{ushio-etal-2022-generative,
|
167 |
title = "{G}enerative {L}anguage {M}odels for {P}aragraph-{L}evel {Q}uestion {G}eneration",
|
168 |
author = "Ushio, Asahi and
|
|
|
33 |
metrics:
|
34 |
- name: BLEU4
|
35 |
type: bleu4
|
36 |
+
value: 10.92
|
37 |
- name: ROUGE-L
|
38 |
type: rouge-l
|
39 |
+
value: 27.76
|
40 |
- name: METEOR
|
41 |
type: meteor
|
42 |
+
value: 30.23
|
43 |
- name: BERTScore
|
44 |
type: bertscore
|
45 |
+
value: 83.89
|
46 |
- name: MoverScore
|
47 |
type: moverscore
|
48 |
+
value: 82.95
|
49 |
+
- name: QAAlignedF1Score (BERTScore) [Gold Answer]
|
50 |
+
type: qa_aligned_f1_score_bertscore_gold_answer
|
51 |
+
value: 88.18
|
52 |
+
- name: QAAlignedRecall (BERTScore) [Gold Answer]
|
53 |
+
type: qa_aligned_recall_bertscore_gold_answer
|
54 |
+
value: 88.15
|
55 |
+
- name: QAAlignedPrecision (BERTScore) [Gold Answer]
|
56 |
+
type: qa_aligned_precision_bertscore_gold_answer
|
57 |
+
value: 88.22
|
58 |
+
- name: QAAlignedF1Score (MoverScore) [Gold Answer]
|
59 |
+
type: qa_aligned_f1_score_moverscore_gold_answer
|
60 |
+
value: 85.53
|
61 |
+
- name: QAAlignedRecall (MoverScore) [Gold Answer]
|
62 |
+
type: qa_aligned_recall_moverscore_gold_answer
|
63 |
+
value: 85.46
|
64 |
+
- name: QAAlignedPrecision (MoverScore) [Gold Answer]
|
65 |
+
type: qa_aligned_precision_moverscore_gold_answer
|
66 |
+
value: 85.62
|
67 |
---
|
68 |
|
69 |
# Model Card of `lmqg/mbart-large-cc25-koquad`
|
70 |
+
This model is fine-tuned version of [facebook/mbart-large-cc25](https://huggingface.co/facebook/mbart-large-cc25) for question generation task on the [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) (dataset_name: default) via [`lmqg`](https://github.com/asahi417/lm-question-generation).
|
|
|
71 |
|
72 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
73 |
### Overview
|
74 |
- **Language model:** [facebook/mbart-large-cc25](https://huggingface.co/facebook/mbart-large-cc25)
|
75 |
- **Language:** ko
|
|
|
81 |
### Usage
|
82 |
- With [`lmqg`](https://github.com/asahi417/lm-question-generation#lmqg-language-model-for-question-generation-)
|
83 |
```python
|
|
|
84 |
from lmqg import TransformersQG
|
85 |
+
|
86 |
# initialize model
|
87 |
+
model = TransformersQG(language="ko", model="lmqg/mbart-large-cc25-koquad")
|
88 |
+
|
89 |
# model prediction
|
90 |
+
questions = model.generate_q(list_context="1990년 영화 《 남부군 》에서 단역으로 영화배우 첫 데뷔에 이어 같은 해 KBS 드라마 《지구인》에서 단역으로 출연하였고 이듬해 MBC 《여명의 눈동자》를 통해 단역으로 출연하였다.", list_answer="남부군")
|
91 |
|
92 |
```
|
93 |
|
94 |
- With `transformers`
|
95 |
```python
|
|
|
96 |
from transformers import pipeline
|
97 |
+
|
98 |
+
pipe = pipeline("text2text-generation", "lmqg/mbart-large-cc25-koquad")
|
99 |
+
output = pipe("1990년 영화 《 <hl> 남부군 <hl> 》에서 단역으로 영화배우 첫 데뷔에 이어 같은 해 KBS 드라마 《지구인》에서 단역으로 출연하였고 이듬해 MBC 《여명의 눈동자》를 통해 단역으로 출연하였다.")
|
|
|
100 |
|
101 |
```
|
102 |
|
103 |
+
## Evaluation
|
104 |
|
105 |
|
106 |
+
- ***Metric (Question Generation)***: [raw metric file](https://huggingface.co/lmqg/mbart-large-cc25-koquad/raw/main/eval/metric.first.sentence.paragraph_answer.question.lmqg_qg_koquad.default.json)
|
107 |
|
108 |
+
| | Score | Type | Dataset |
|
109 |
+
|:-----------|--------:|:--------|:-----------------------------------------------------------------|
|
110 |
+
| BERTScore | 83.89 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
111 |
+
| Bleu_1 | 26.92 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
112 |
+
| Bleu_2 | 19.57 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
113 |
+
| Bleu_3 | 14.52 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
114 |
+
| Bleu_4 | 10.92 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
115 |
+
| METEOR | 30.23 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
116 |
+
| MoverScore | 82.95 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
117 |
+
| ROUGE_L | 27.76 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
118 |
|
119 |
|
120 |
+
- ***Metric (Question & Answer Generation)***: QAG metrics are computed with *the gold answer* and generated question on it for this model, as the model cannot provide an answer. [raw metric file](https://huggingface.co/lmqg/mbart-large-cc25-koquad/raw/main/eval/metric.first.answer.paragraph.questions_answers.lmqg_qg_koquad.default.json)
|
121 |
|
122 |
+
| | Score | Type | Dataset |
|
123 |
+
|:--------------------------------|--------:|:--------|:-----------------------------------------------------------------|
|
124 |
+
| QAAlignedF1Score (BERTScore) | 88.18 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
125 |
+
| QAAlignedF1Score (MoverScore) | 85.53 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
126 |
+
| QAAlignedPrecision (BERTScore) | 88.22 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
127 |
+
| QAAlignedPrecision (MoverScore) | 85.62 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
128 |
+
| QAAlignedRecall (BERTScore) | 88.15 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
129 |
+
| QAAlignedRecall (MoverScore) | 85.46 | default | [lmqg/qg_koquad](https://huggingface.co/datasets/lmqg/qg_koquad) |
|
130 |
|
131 |
|
132 |
|
|
|
153 |
|
154 |
## Citation
|
155 |
```
|
|
|
156 |
@inproceedings{ushio-etal-2022-generative,
|
157 |
title = "{G}enerative {L}anguage {M}odels for {P}aragraph-{L}evel {Q}uestion {G}eneration",
|
158 |
author = "Ushio, Asahi and
|