Asier Gutiérrez Fandiño commited on
Commit
bcd52ae
1 Parent(s): 446a412

Initial commit

Browse files
README.md ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - es
4
+ license: apache-2.0
5
+ tags:
6
+ - "national library of spain"
7
+ - "spanish"
8
+ - "bne"
9
+ - "capitel"
10
+ - "pos"
11
+ datasets:
12
+ - "bne"
13
+ - "capitel"
14
+ metrics:
15
+ - "f1"
16
+ widget:
17
+ - text: "Festival de San Sebastián: Johnny Depp recibirá el premio Donostia en pleno rifirrafe judicial con Amber Heard"
18
+ - text: "El alcalde de Vigo, Abel Caballero, ha comenzado a colocar las luces de Navidad en agosto."
19
+ - text: "Gracias a los datos de la BNE, se ha podido lograr este modelo del lenguaje."
20
+ - text: "El Tribunal Superior de Justicia se pronunció ayer: \"Hay base legal dentro del marco jurídico actual\"."
21
+
22
+ ---
23
+
24
+ # Spanish RoBERTa-large trained on BNE finetuned for CAPITEL Part of Speech (POS) dataset
25
+ RoBERTa-large-bne is a transformer-based masked language model for the Spanish language. It is based on the [RoBERTa](https://arxiv.org/abs/1907.11692) large model and has been pre-trained using the largest Spanish corpus known to date, with a total of 570GB of clean and deduplicated text processed for this work, compiled from the web crawlings performed by the [National Library of Spain (Biblioteca Nacional de España)](http://www.bne.es/en/Inicio/index.html) from 2009 to 2019.
26
+
27
+ Original pre-trained model can be found here: https://huggingface.co/BSC-TeMU/roberta-large-bne
28
+
29
+ ## Dataset
30
+ The dataset used is the one from the [CAPITEL competition at IberLEF 2020](https://sites.google.com/view/capitel2020) (sub-task 2).
31
+
32
+ ## Evaluation and results
33
+ F1 Score: 0.9851 (average of 5 runs).
34
+
35
+ For evaluation details visit our [GitHub repository](https://github.com/PlanTL-SANIDAD/lm-spanish).
36
+
37
+
38
+ ## Citing
39
+ Check out our paper for all the details: https://arxiv.org/abs/2107.07253
40
+ ```
41
+ @misc{gutierrezfandino2021spanish,
42
+ title={Spanish Language Models},
43
+ author={Asier Gutiérrez-Fandiño and Jordi Armengol-Estapé and Marc Pàmies and Joan Llop-Palao and Joaquín Silveira-Ocampo and Casimiro Pio Carrino and Aitor Gonzalez-Agirre and Carme Armentano-Oller and Carlos Rodriguez-Penagos and Marta Villegas},
44
+ year={2021},
45
+ eprint={2107.07253},
46
+ archivePrefix={arXiv},
47
+ primaryClass={cs.CL}
48
+ }
49
+ ```
config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "../models/bne-large",
3
+ "architectures": [
4
+ "RobertaForTokenClassification"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.0,
7
+ "bos_token_id": 0,
8
+ "eos_token_id": 2,
9
+ "finetuning_task": "pos",
10
+ "gradient_checkpointing": false,
11
+ "hidden_act": "gelu",
12
+ "hidden_dropout_prob": 0.0,
13
+ "hidden_size": 1024,
14
+ "id2label": {
15
+ "0": "NUM",
16
+ "1": "VERB",
17
+ "2": "X",
18
+ "3": "ADV",
19
+ "4": "PRON",
20
+ "5": "INTJ",
21
+ "6": "ADJ",
22
+ "7": "PUNCT",
23
+ "8": "ADP",
24
+ "9": "SCONJ",
25
+ "10": "NOUN",
26
+ "11": "PROPN",
27
+ "12": "CCONJ",
28
+ "13": "DET"
29
+ },
30
+ "initializer_range": 0.02,
31
+ "intermediate_size": 4096,
32
+ "label2id": {
33
+ "NUM": 0,
34
+ "VERB": 1,
35
+ "X": 2,
36
+ "ADV": 3,
37
+ "PRON": 4,
38
+ "INTJ": 5,
39
+ "ADJ": 6,
40
+ "PUNCT": 7,
41
+ "ADP": 8,
42
+ "SCONJ": 9,
43
+ "NOUN": 10,
44
+ "PROPN": 11,
45
+ "CCONJ": 12,
46
+ "DET": 13
47
+ },
48
+ "layer_norm_eps": 1e-05,
49
+ "max_position_embeddings": 514,
50
+ "model_type": "roberta",
51
+ "num_attention_heads": 16,
52
+ "num_hidden_layers": 24,
53
+ "pad_token_id": 1,
54
+ "position_embedding_type": "absolute",
55
+ "transformers_version": "4.6.1",
56
+ "type_vocab_size": 1,
57
+ "use_cache": true,
58
+ "vocab_size": 50262
59
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cf7c94c33ae66954fe4667caa96a3964d09d144bce679301e170ced3b2b1096
3
+ size 1417449073
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": true, "errors": "replace", "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "max_len": 512, "special_tokens_map_file": null, "name_or_path": "../models/bne-large"}
vocab.json ADDED
The diff for this file is too large to render. See raw diff