Upload folder using huggingface_hub

README.md

@@ -0,0 +1,278 @@
+---
+language: fr
+license: mit
+tags:
+- deberta-v2
+- text-classification
+- nli
+base_model: almanach/camembertav2-base
+datasets:
+- FLUE-XNLI
+metrics:
+- accuracy
+pipeline_tag: text-classification
+library_name: transformers
+model-index:
+- name: almanach/camembertav2-base-xnli
+  results:
+  - task:
+      type: text-classification
+      name: Natural Language Inference
+    dataset:
+      type: flue-XNLI
+      name: FLUE-XNLI
+    metrics:
+    - name: accuracy
+      type: accuracy
+      value: 0.85582
+      verified: false
+---
+
+# Model Card for almanach/camembertav2-base-xnli
+
+almanach/camembertav2-base-xnli is a deberta-v2 model for text classification. It is trained on the FLUE-XNLI dataset for the task of Natural Language Inference. The model achieves an accuracy of 0.85582 on the FLUE-XNLI dataset.
+
+The model is part of the almanach/camembertav2-base family of model finetunes.
+
+## Model Details
+
+### Model Description
+
+- **Developed by:** Wissam Antoun (Phd Student at Almanach, Inria-Paris)
+- **Model type:** deberta-v2
+- **Language(s) (NLP):** French
+- **License:** MIT
+- **Finetuned from model [optional]:** almanach/camembertav2-base
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** https://github.com/WissamAntoun/camemberta
+- **Paper:** https://arxiv.org/abs/2411.08868
+
+## Uses
+
+The model can be used for text classification tasks in French for Natural Language Inference.
+
+## Bias, Risks, and Limitations
+
+The model may exhibit biases based on the training data. The model may not generalize well to other datasets or tasks. The model may also have limitations in terms of the data it was trained on.
+
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+```python
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline
+
+model = AutoModelForSequenceClassification.from_pretrained("almanach/camembertav2-base-xnli")
+tokenizer = AutoTokenizer.from_pretrained("almanach/camembertav2-base-xnli")
+
+classifier = pipeline("text-classification", model=model, tokenizer=tokenizer)
+
+classifier({
+    "text": "Le livre est très intéressant et j'ai appris beaucoup de choses.",
+    "text_pair": "Le livre est très ennuyeux et je n'ai rien appris.",
+})
+```
+
+
+## Training Details
+
+### Training Data
+
+The model is trained on the FLUE-XNLI dataset.
+
+- Dataset Name: FLUE-XNLI
+- Dataset Size:
+    - Train: 49399
+    - Dev: 1988
+    - Test: 2000
+
+
+### Training Procedure
+
+Model trained with the run_xnli.py script from the huggingface repository.
+
+
+
+#### Training Hyperparameters
+
+```yml
+accelerator_config: '{''split_batches'': False, ''dispatch_batches'': None, ''even_batches'':
+  True, ''use_seedable_sampler'': True, ''non_blocking'': False, ''gradient_accumulation_kwargs'':
+  None}'
+adafactor: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+auto_find_batch_size: false
+base_model: camembertv2
+base_model_name: camembertav2-base-bf16-p2-17000
+batch_eval_metrics: false
+bf16: false
+bf16_full_eval: false
+data_seed: 1.0
+dataloader_drop_last: false
+dataloader_num_workers: 0
+dataloader_persistent_workers: false
+dataloader_pin_memory: true
+dataloader_prefetch_factor: .nan
+ddp_backend: .nan
+ddp_broadcast_buffers: .nan
+ddp_bucket_cap_mb: .nan
+ddp_find_unused_parameters: .nan
+ddp_timeout: 1800
+debug: '[]'
+deepspeed: .nan
+disable_tqdm: false
+dispatch_batches: .nan
+do_eval: true
+do_predict: false
+do_train: true
+epoch: 10.0
+eval_accumulation_steps: 4
+eval_accuracy: 0.8558232931726908
+eval_delay: 0
+eval_do_concat_batches: true
+eval_loss: 0.4229515790939331
+eval_on_start: false
+eval_runtime: 5.5233
+eval_samples: 2490
+eval_samples_per_second: 450.82
+eval_steps: .nan
+eval_steps_per_second: 56.488
+eval_strategy: epoch
+eval_use_gather_object: false
+evaluation_strategy: epoch
+fp16: false
+fp16_backend: auto
+fp16_full_eval: false
+fp16_opt_level: O1
+fsdp: '[]'
+fsdp_config: '{''min_num_params'': 0, ''xla'': False, ''xla_fsdp_v2'': False, ''xla_fsdp_grad_ckpt'':
+  False}'
+fsdp_min_num_params: 0
+fsdp_transformer_layer_cls_to_wrap: .nan
+full_determinism: false
+gradient_accumulation_steps: 4
+gradient_checkpointing: false
+gradient_checkpointing_kwargs: .nan
+greater_is_better: true
+group_by_length: false
+half_precision_backend: auto
+hub_always_push: false
+hub_model_id: .nan
+hub_private_repo: false
+hub_strategy: every_save
+hub_token: <HUB_TOKEN>
+ignore_data_skip: false
+include_inputs_for_metrics: false
+include_num_input_tokens_seen: false
+include_tokens_per_second: false
+jit_mode_eval: false
+label_names: .nan
+label_smoothing_factor: 0.0
+learning_rate: 1.0e-05
+length_column_name: length
+load_best_model_at_end: true
+local_rank: 0
+log_level: debug
+log_level_replica: warning
+log_on_each_node: true
+logging_dir: /scratch/camembertv2/runs/results/xnli/camembertav2-base-bf16-p2-17000/max_seq_length-160-gradient_accumulation_steps-4-precision-fp32-learning_rate-1e-05-epochs-10-lr_scheduler-polynomial-warmup_steps-0.1/SEED-1/logs
+logging_first_step: false
+logging_nan_inf_filter: true
+logging_steps: 100
+logging_strategy: steps
+lr_scheduler_kwargs: '{}'
+lr_scheduler_type: polynomial
+max_grad_norm: 1.0
+max_steps: -1
+metric_for_best_model: accuracy
+mp_parameters: .nan
+name: camembertv2/runs/results/xnli/camembertav2-base-bf16-p2-17000/max_seq_length-160-gradient_accumulation_steps-4-precision-fp32-learning_rate-1e-05-epochs-10-lr_scheduler-polynomial-warmup_steps-0.1
+neftune_noise_alpha: .nan
+no_cuda: false
+num_train_epochs: 10.0
+optim: adamw_torch
+optim_args: .nan
+optim_target_modules: .nan
+output_dir: /scratch/camembertv2/runs/results/xnli/camembertav2-base-bf16-p2-17000/max_seq_length-160-gradient_accumulation_steps-4-precision-fp32-learning_rate-1e-05-epochs-10-lr_scheduler-polynomial-warmup_steps-0.1/SEED-1
+overwrite_output_dir: false
+past_index: -1
+per_device_eval_batch_size: 8
+per_device_train_batch_size: 8
+per_gpu_eval_batch_size: .nan
+per_gpu_train_batch_size: .nan
+prediction_loss_only: false
+push_to_hub: false
+push_to_hub_model_id: .nan
+push_to_hub_organization: .nan
+push_to_hub_token: <PUSH_TO_HUB_TOKEN>
+ray_scope: last
+remove_unused_columns: true
+report_to: '[''tensorboard'']'
+restore_callback_states_from_checkpoint: false
+resume_from_checkpoint: .nan
+run_name: /scratch/camembertv2/runs/results/xnli/camembertav2-base-bf16-p2-17000/max_seq_length-160-gradient_accumulation_steps-4-precision-fp32-learning_rate-1e-05-epochs-10-lr_scheduler-polynomial-warmup_steps-0.1/SEED-1
+save_on_each_node: false
+save_only_model: false
+save_safetensors: true
+save_steps: 500
+save_strategy: epoch
+save_total_limit: .nan
+seed: 1
+skip_memory_metrics: true
+split_batches: .nan
+tf32: .nan
+torch_compile: true
+torch_compile_backend: inductor
+torch_compile_mode: .nan
+torch_empty_cache_steps: .nan
+torchdynamo: .nan
+total_flos: 1.6169329248859574e+17
+tpu_metrics_debug: false
+tpu_num_cores: .nan
+train_loss: 0.2765254594356482
+train_runtime: 31314.1287
+train_samples: 392702
+train_samples_per_second: 125.407
+train_steps_per_second: 3.919
+use_cpu: false
+use_ipex: false
+use_legacy_prediction_loop: false
+use_mps_device: false
+warmup_ratio: 0.1
+warmup_steps: 0
+weight_decay: 0.0
+
+```
+
+#### Results
+
+**Accuracy:** 0.85582
+
+## Technical Specifications
+
+### Model Architecture and Objective
+
+deberta-v2 for sequence classification.
+
+## Citation
+
+**BibTeX:**
+
+```bibtex
+@misc{antoun2024camembert20smarterfrench,
+      title={CamemBERT 2.0: A Smarter French Language Model Aged to Perfection},
+      author={Wissam Antoun and Francis Kulumba and Rian Touchent and Éric de la Clergerie and Benoît Sagot and Djamé Seddah},
+      year={2024},
+      eprint={2411.08868},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2411.08868},
+}
+```

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.8558232931726908,
+    "eval_loss": 0.4229515790939331,
+    "eval_runtime": 5.5233,
+    "eval_samples": 2490,
+    "eval_samples_per_second": 450.82,
+    "eval_steps_per_second": 56.488,
+    "total_flos": 1.6169329248859574e+17,
+    "train_loss": 0.2765254594356482,
+    "train_runtime": 31314.1287,
+    "train_samples": 392702,
+    "train_samples_per_second": 125.407,
+    "train_steps_per_second": 3.919
+}

config.json

@@ -0,0 +1,52 @@
+{
+  "_name_or_path": "/scratch/camembertv2/runs/models/camembertav2-base-bf16/post/ckpt-p2-17000/pt/discriminator/",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 1,
+  "conv_act": "gelu",
+  "conv_kernel_size": 0,
+  "embedding_size": 768,
+  "eos_token_id": 2,
+  "finetuning_task": "xnli",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "entailment",
+    "1": "neutral",
+    "2": "contradiction"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "contradiction": 2,
+    "entailment": 0,
+    "neutral": 1
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 1024,
+  "max_relative_positions": -1,
+  "model_name": "camembertav2-base-bf16",
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "type_vocab_size": 0,
+  "vocab_size": 32768
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.8558232931726908,
+    "eval_loss": 0.4229515790939331,
+    "eval_runtime": 5.5233,
+    "eval_samples": 2490,
+    "eval_samples_per_second": 450.82,
+    "eval_steps_per_second": 56.488
+}

logs/events.out.tfevents.1724688665.nefgpu47.443271.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:491896fe7e592d18209d695d32e5834cad106afb3a6d248155deaf5e5738067a
+size 272855

logs/events.out.tfevents.1724719985.nefgpu47.443271.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d451b2ab94b2202f6e45af5134074148690f544bd7649ab294fe7543e245e760
+size 369

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:202a9488c158606656127f8347c0bebc788dddcd996bf129a6cfa07c437a6fa9
+size 444862452

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "errors": "replace",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 10.0,
+    "total_flos": 1.6169329248859574e+17,
+    "train_loss": 0.2765254594356482,
+    "train_runtime": 31314.1287,
+    "train_samples": 392702,
+    "train_samples_per_second": 125.407,
+    "train_steps_per_second": 3.919
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a6d0753c3f04e1bd007a6f65c6a9d5f962d36212202182e63d1b9eb784fa4be
+size 5560